{"id":"W1967160681","doi":"10.1145/2348283.2348515","title":"Time to judge relevance as an indicator of assessor error","year":2012,"lang":"en","type":"article","venue":"","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Institute of Standards and Technology","keywords":"Relevance (law); Adjudication; Computer science; Information retrieval; Psychology; Law; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0004266457,0.00006550366,0.0001037582,0.0001057765,0.0000471498,0.00003892606,0.0005761847,0.00003921784,0.0008914198],"category_scores_gemma":[0.00008261227,0.00005135561,0.00002861365,0.0003227807,0.00002480432,0.001546122,0.0001542829,0.00006738817,0.006709211],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002010355,"about_ca_system_score_gemma":0.00006308623,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008967205,"about_ca_topic_score_gemma":4.560397e-7,"domain_scores_codex":[0.9990301,0.00003068507,0.0002124267,0.0001082016,0.0003613916,0.0002572507],"domain_scores_gemma":[0.9991795,0.0000410628,0.00006416775,0.0003640868,0.00008337328,0.0002678053],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0001172247,0.002389572,0.01886255,0.0001158825,0.00004862503,0.000008297313,0.02967675,0.00005773226,0.06346133,0.3975921,0.04536641,0.4423035],"study_design_scores_gemma":[0.001116149,0.002015528,0.1953277,0.00003949658,0.00001973891,0.0000442123,0.0005152717,0.02600377,0.449557,0.0006267721,0.3235768,0.001157574],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9324782,0.00001187911,0.04388673,0.0009109223,0.0001628151,0.0002583854,0.000004708739,0.0001539701,0.0221324],"genre_scores_gemma":[0.9593543,5.44998e-7,0.03320804,0.000968422,0.00003513989,0.00000990549,0.000002662091,0.000004173003,0.006416784],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4411459,"threshold_uncertainty_score":0.9940642,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02733602163818724,"score_gpt":0.3158562752420382,"score_spread":0.288520253603851,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}