{"id":"W4411638719","doi":"10.18653/v1/2024.eacl-tutorials.2","title":"Item Response Theory for Natural Language Processing","year":2024,"lang":"en","type":"article","venue":"","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Atomic Energy of Canada Limited; York University; Institute for Catastrophic Loss Reduction; University of Notre Dame","keywords":"Computer science; Natural (archaeology); Natural language processing","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01882351,0.00008186016,0.0001437923,0.0004552123,0.0001128986,0.0005691088,0.0004162554,0.00004248319,0.0003005078],"category_scores_gemma":[0.2123037,0.00004459497,0.0001013848,0.00168834,0.00004117478,0.0002189867,0.00007066655,0.0001093121,0.00007429183],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001766593,"about_ca_system_score_gemma":0.00006387697,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000002056286,"about_ca_topic_score_gemma":9.969754e-7,"domain_scores_codex":[0.9982248,0.0004517246,0.0003002081,0.0003794273,0.0004331533,0.0002107007],"domain_scores_gemma":[0.8759565,0.1236063,0.00004848346,0.0002424815,0.0001037462,0.00004249892],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0002090869,0.000004248525,0.0001689316,0.00001166261,0.000004682709,0.00001104082,0.0007184685,0.000001844559,0.002214683,0.00365896,0.008049774,0.9849466],"study_design_scores_gemma":[0.0007606603,0.0003330527,0.02935335,0.0002329612,0.00003662758,0.0001497891,0.03257737,0.2145897,0.005939503,0.3945468,0.3207463,0.0007338684],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4943576,0.02786505,0.4466675,0.002304447,0.002546234,0.0003154653,0.00001169035,0.0006371642,0.0252949],"genre_scores_gemma":[0.9036379,0.000001686329,0.05522826,0.0003030144,0.0001418649,0.00001165903,5.957839e-7,0.00000927037,0.04066574],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9842128,"threshold_uncertainty_score":0.7943314,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4064670903822485,"score_gpt":0.5454917346558178,"score_spread":0.1390246442735693,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}