{"id":"W1972821056","doi":"10.1145/2009916.2010134","title":"Measuring assessor accuracy","year":2011,"lang":"en","type":"article","venue":"","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Institute of Standards and Technology","keywords":"NIST; Relevance (law); Computer science; Information retrieval; Natural language processing","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.06918733,0.001029948,0.001396236,0.006401543,0.001674756,0.00327218,0.001290465,0.002041823,0.002574544],"category_scores_gemma":[0.252857,0.0004882754,0.001431778,0.003412407,0.001198883,0.00344248,0.003420168,0.001253213,0.002492111],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009597442,"about_ca_system_score_gemma":0.001251056,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001988586,"about_ca_topic_score_gemma":0.002380478,"domain_scores_codex":[0.8793525,0.0454029,0.01942037,0.009528299,0.0439507,0.002345361],"domain_scores_gemma":[0.6805222,0.1536072,0.03200283,0.03552586,0.09527117,0.003070798],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001997031,0.0005480858,0.602531,0.001326123,0.001156096,0.0002088216,0.009887719,0.002943033,0.01696543,0.006305684,0.009380893,0.3467501],"study_design_scores_gemma":[0.000314577,0.003346127,0.8071837,0.001058051,0.001333807,0.002308526,0.006948039,0.03103643,0.06224643,0.01803247,0.06554258,0.0006493146],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6349578,0.003743884,0.2996407,0.001189593,0.0006988319,0.002247901,0.002979892,0.001609283,0.05293217],"genre_scores_gemma":[0.9058206,0.0006363259,0.08599357,0.0004072541,0.0001967242,0.001278124,0.001364507,0.0002099983,0.00409284],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.06918733,"threshold_uncertainty_score":0.3659019,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1940061020223355,"score_gpt":0.2848193298977675,"score_spread":0.09081322787543206,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}