{"id":"W4389403907","doi":"10.2139/ssrn.4583531","title":"Legalbench: A Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models","year":2023,"lang":"en","type":"article","venue":"SSRN Electronic Journal","topic":"Artificial Intelligence in Law","field":"Social Sciences","cited_by":135,"is_retracted":false,"has_abstract":false,"ca_institutions":"York University; University of Toronto","funders":"","keywords":"Benchmark (surveying); Vocabulary; Process (computing); Computer science; Case-based reasoning; Knowledge management; Engineering ethics; Psychology; Data science; Artificial intelligence; Linguistics; Engineering; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.008461582,0.000167561,0.000257121,0.0002930397,0.001085978,0.0002419836,0.0005109696,0.0001420837,0.0000474483],"category_scores_gemma":[0.001169184,0.0001734595,0.0001270562,0.001442776,0.0001236873,0.001022204,0.0000541585,0.001194625,0.0000383388],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001678661,"about_ca_system_score_gemma":0.004802803,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003875599,"about_ca_topic_score_gemma":0.2193678,"domain_scores_codex":[0.9949349,0.0002974279,0.0004194822,0.000296932,0.0005816479,0.003469611],"domain_scores_gemma":[0.9990381,0.0002589619,0.000173739,0.0001414961,0.0002520195,0.0001356663],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00006088938,0.00004378827,0.0006924658,0.000004647932,0.00003598486,0.00001842092,0.02312826,0.001767259,0.0003961502,0.9692834,0.0002242269,0.004344523],"study_design_scores_gemma":[0.0006933636,0.0002650605,0.0001831646,0.0001533633,0.00003324947,0.0000295773,0.3114092,0.01614775,0.0008467508,0.6441826,0.0254463,0.0006096165],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9305657,0.002275183,0.03486874,0.003498901,0.0008179016,0.000936484,0.00002799631,0.0002408233,0.02676824],"genre_scores_gemma":[0.9942811,0.0009222156,0.0002616236,0.0000675335,0.0006166955,0.00004649221,0.000005902709,0.00003156083,0.003766875],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3251007,"threshold_uncertainty_score":0.8519965,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0462981910877303,"score_gpt":0.3537987956978807,"score_spread":0.3075006046101504,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}