{"id":"W2521312816","doi":"10.1007/s10459-016-9711-8","title":"Inter-rater variability as mutual disagreement: identifying raters’ divergent points of view","year":2016,"lang":"en","type":"article","venue":"Advances in Health Sciences Education","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":43,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of British Columbia; University of Northern British Columbia","funders":"National Board of Medical Examiners","keywords":"Psychology; Variation (astronomy); Salient; Variance (accounting); Inter-rater reliability; Cognitive psychology; Point (geometry); Applied psychology; Cognition; Social psychology; Computer science; Rating scale; Artificial intelligence; Developmental psychology; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2416619,0.001949625,0.002925227,0.008776806,0.002147849,0.005879176,0.003495236,0.002617569,0.001218048],"category_scores_gemma":[0.5381776,0.00128499,0.003538693,0.007178061,0.003973589,0.005519067,0.006312871,0.003413556,0.0007503036],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001823303,"about_ca_system_score_gemma":0.001765738,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001403301,"about_ca_topic_score_gemma":0.003710232,"domain_scores_codex":[0.6211705,0.2341853,0.06070944,0.02841858,0.0532553,0.002260975],"domain_scores_gemma":[0.2264541,0.6386517,0.05069054,0.03650728,0.04613277,0.001563675],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"qualitative","study_design_scores_codex":[0.005647433,0.0006620065,0.6497993,0.003843605,0.01374147,0.00126142,0.06278775,0.01345386,0.0117888,0.017743,0.009273054,0.2099982],"study_design_scores_gemma":[0.0007454171,0.001983823,0.5592694,0.00262136,0.005868777,0.004409339,0.03189432,0.2773904,0.02265808,0.07120626,0.02018472,0.00176821],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"methods","genre_scores_codex":[0.5047954,0.002453193,0.4720263,0.0008286012,0.000904955,0.003870795,0.002565712,0.001044381,0.01151065],"genre_scores_gemma":[0.8897488,0.000223956,0.1036421,0.000166359,0.0001684544,0.003165674,0.001659231,0.0004205806,0.000804815],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7583381,"threshold_uncertainty_score":0.9351659,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0390627408393996,"score_gpt":0.4441012497172203,"score_spread":0.4050385088778207,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}