{"id":"W2022108487","doi":"10.1097/acm.0b013e3182a36c3d","title":"Do In-Training Evaluation Reports Deserve Their Bad Reputations? A Study of the Reliability and Predictive Ability of ITER Scores and Narrative Comments","year":2013,"lang":"en","type":"article","venue":"Academic Medicine","topic":"Innovations in Medical Education","field":"Medicine","cited_by":76,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia; Mount Sinai Hospital","funders":"","keywords":"Reliability (semiconductor); Inter-rater reliability; Predictive validity; Variance (accounting); Psychology; Validity; Narrative; Clinical psychology; Medical education; Medicine; Rating scale; Psychometrics; Developmental psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.03164717,0.0005359259,0.0004507371,0.003004006,0.0005609508,0.001643215,0.0007570047,0.0004873684,0.001174739],"category_scores_gemma":[0.2393621,0.0003221394,0.0004913576,0.00209315,0.001320808,0.001445134,0.001935188,0.0007126059,0.0004372364],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001008348,"about_ca_system_score_gemma":0.0009982407,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003038957,"about_ca_topic_score_gemma":0.003770137,"domain_scores_codex":[0.9774715,0.009868656,0.003266727,0.001235417,0.007294426,0.0008633031],"domain_scores_gemma":[0.6261396,0.1577363,0.1434626,0.01344253,0.05507406,0.004144942],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0001579085,0.00003512331,0.9803137,0.00007201791,0.00007985184,0.00005634688,0.002557615,0.00009438702,0.0001930275,0.00005600273,0.0004951383,0.01588884],"study_design_scores_gemma":[0.000008071962,0.0002256967,0.9922972,0.0001403201,0.00005592573,0.0002216435,0.003841577,0.0009936338,0.0006603617,0.0000962036,0.001430738,0.00002854912],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9960874,0.0003366151,0.001234805,0.0002918224,0.0000695003,0.00005185256,0.0002774294,0.00004151399,0.001609115],"genre_scores_gemma":[0.9984112,0.0001275719,0.0006943828,0.00007762741,0.00006343642,0.00003263531,0.0002416561,0.00001657382,0.0003347642],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9683529,"threshold_uncertainty_score":0.1673682,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06121270578846461,"score_gpt":0.3947465064594388,"score_spread":0.3335338006709742,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}