{"id":"W1967160681","doi":"10.1145/2348283.2348515","title":"Time to judge relevance as an indicator of assessor error","year":2012,"lang":"en","type":"article","venue":"","topic":"Information Retrieval and Search Behavior","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Institute of Standards and Technology","keywords":"Relevance (law); Adjudication; Computer science; Information retrieval; Psychology; Law; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.05026738,0.001114967,0.001351289,0.003546632,0.001408145,0.003001183,0.001369304,0.001559359,0.003358875],"category_scores_gemma":[0.3903825,0.0007606289,0.0008796402,0.002156205,0.001222306,0.002870892,0.002215399,0.002449148,0.001172854],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001208463,"about_ca_system_score_gemma":0.001715494,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002872945,"about_ca_topic_score_gemma":0.00344827,"domain_scores_codex":[0.9008536,0.0545864,0.01658457,0.005921143,0.02021431,0.001839943],"domain_scores_gemma":[0.3431115,0.50682,0.07029624,0.02469417,0.04973133,0.005346734],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.01177649,0.001969653,0.5285074,0.002791057,0.001304495,0.001368556,0.04669046,0.005299815,0.06864433,0.002509365,0.008689413,0.3204491],"study_design_scores_gemma":[0.000810777,0.01151565,0.818916,0.001151208,0.0009373243,0.003367849,0.02211623,0.03879525,0.06112926,0.008033864,0.03198684,0.001239726],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9214885,0.002335694,0.06386217,0.001051106,0.0005530322,0.0008745139,0.001053161,0.00105425,0.007727703],"genre_scores_gemma":[0.9526306,0.0005035677,0.04176848,0.0004256222,0.0001709668,0.0009182793,0.0008478791,0.0002276662,0.002507007],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.05026738,"threshold_uncertainty_score":0.2658425,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02733602163818724,"score_gpt":0.3158562752420382,"score_spread":0.288520253603851,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}