{"id":"W2521312816","doi":"10.1007/s10459-016-9711-8","title":"Inter-rater variability as mutual disagreement: identifying raters’ divergent points of view","year":2016,"lang":"en","type":"article","venue":"Advances in Health Sciences Education","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":43,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of British Columbia; University of Northern British Columbia","funders":"National Board of Medical Examiners","keywords":"Psychology; Variation (astronomy); Salient; Variance (accounting); Inter-rater reliability; Cognitive psychology; Point (geometry); Applied psychology; Cognition; Social psychology; Computer science; Rating scale; Artificial intelligence; Developmental psychology; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.002154843,0.00008897278,0.0002439262,0.0001030749,0.00007601015,0.00001043104,0.0001384489,0.00003067942,0.0004561337],"category_scores_gemma":[0.01469132,0.00005494567,0.00004199415,0.0003541768,0.0003353389,0.0003326057,0.00004895802,0.00006211362,0.00006158787],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00018141,"about_ca_system_score_gemma":0.0007577951,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003697292,"about_ca_topic_score_gemma":0.00007758413,"domain_scores_codex":[0.9982868,0.0001614016,0.000638695,0.0003597745,0.0002955319,0.0002578236],"domain_scores_gemma":[0.9959047,0.003309402,0.0002761065,0.0002354723,0.00009568496,0.0001785927],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0000399447,0.0004276934,0.7161919,0.0002324378,0.000002657568,3.4652e-7,0.000738293,0.000002934316,0.00005469798,0.002170574,0.0001425743,0.2799959],"study_design_scores_gemma":[0.002150849,0.002324446,0.9034349,0.02875166,0.0000420152,0.00002453711,0.002375803,0.0001090197,0.002680884,0.04127236,0.01646288,0.0003706341],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9797896,0.003074748,0.0016734,0.01051455,0.002527499,0.0005228213,0.000004915165,0.0000147792,0.001877713],"genre_scores_gemma":[0.9921947,0.004509302,0.001739839,0.001180344,0.00008447683,0.00003553847,0.000004929683,0.000003712143,0.0002472293],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2796253,"threshold_uncertainty_score":0.9936084,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0390627408393996,"score_gpt":0.4441012497172203,"score_spread":0.4050385088778207,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}