{"id":"W1972821056","doi":"10.1145/2009916.2010134","title":"Measuring assessor accuracy","year":2011,"lang":"en","type":"article","venue":"","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Institute of Standards and Technology","keywords":"NIST; Relevance (law); Computer science; Information retrieval; Natural language processing","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0001862945,0.00004145101,0.00004444186,0.00005030284,0.00006168982,0.00007860651,0.000485837,0.00001863233,0.0002937038],"category_scores_gemma":[0.00005485752,0.00003074451,0.00002570032,0.0001664936,0.000009159175,0.001153183,0.0001277188,0.00005723038,0.0006793406],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000103655,"about_ca_system_score_gemma":0.00003300473,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002196409,"about_ca_topic_score_gemma":0.000001335893,"domain_scores_codex":[0.9994476,0.00001246116,0.0001204691,0.00007786826,0.0002015307,0.0001400534],"domain_scores_gemma":[0.9995894,0.00002244989,0.00002781266,0.0002172715,0.00008226557,0.00006084304],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.000005874433,0.0001207434,0.006917713,0.00001146931,0.000008534405,0.00001441858,0.005470211,0.000001567978,0.001373054,0.6425682,0.001181426,0.3423268],"study_design_scores_gemma":[0.001034189,0.0003122118,0.3303661,0.0000247148,0.000009216823,0.00005786972,0.0004494731,0.0493113,0.5742882,0.009655505,0.03359725,0.0008938739],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0456535,0.000005504781,0.7226473,0.0001313713,0.0001969936,0.00008530833,2.258175e-7,0.0002643421,0.2310154],"genre_scores_gemma":[0.9470664,0.000001263071,0.05164209,0.0002086376,0.00001038011,0.000004412153,2.299766e-7,0.000001531138,0.001065099],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9014128,"threshold_uncertainty_score":0.8731776,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1940061020223355,"score_gpt":0.2848193298977675,"score_spread":0.09081322787543206,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}