{"id":"W2024888162","doi":"10.1007/s10459-013-9482-4","title":"Effect of clinically discriminating, evidence-based checklist items on the reliability of scores from an Internal Medicine residency OSCE","year":2014,"lang":"en","type":"article","venue":"Advances in Health Sciences Education","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":24,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Alberta","funders":"","keywords":"Checklist; Generalizability theory; Reliability (semiconductor); Medicine; Inter-rater reliability; Summative assessment; Psychology; Rating scale; Formative assessment","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.07933638,0.00136747,0.001626589,0.001321004,0.001119334,0.002133181,0.001225887,0.002382722,0.001244077],"category_scores_gemma":[0.3504187,0.001237224,0.002736667,0.001006059,0.002625962,0.002190926,0.00247535,0.003245513,0.0004589724],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001331129,"about_ca_system_score_gemma":0.00208788,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001841038,"about_ca_topic_score_gemma":0.003244176,"domain_scores_codex":[0.9161326,0.0534631,0.009414308,0.006756252,0.01275704,0.001476611],"domain_scores_gemma":[0.2744651,0.6564956,0.02990197,0.01365096,0.02066622,0.004820103],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.04439069,0.005848854,0.8941962,0.000361281,0.003047359,0.0001426438,0.004339063,0.001688107,0.004639972,0.0002142533,0.001163049,0.03996862],"study_design_scores_gemma":[0.00208548,0.02343549,0.9628733,0.0001908819,0.001733589,0.0001962134,0.0008140071,0.004694411,0.00283638,0.0002185102,0.0007866483,0.0001350768],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9976559,0.0002550338,0.0007813674,0.0001885995,0.0001209054,0.0001261131,0.00009027003,0.00002521877,0.0007564627],"genre_scores_gemma":[0.9970738,0.00008592886,0.001949623,0.0001314832,0.00005349828,0.000167608,0.0002223619,0.00002876072,0.0002871047],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9206636,"threshold_uncertainty_score":0.4195759,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03959902299030457,"score_gpt":0.4776864808113299,"score_spread":0.4380874578210254,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}