{"id":"W2767278518","doi":"10.4300/jgme-d-17-00086.1","title":"Nuance and Noise: Lessons Learned From Longitudinal Aggregated Assessment Data","year":2017,"lang":"en","type":"article","venue":"Journal of Graduate Medical Education","topic":"Innovations in Medical Education","field":"Medicine","cited_by":38,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"Multilevel model; Variance (accounting); Medical education; Medicine; Scale (ratio); Rating scale; Psychology; Family medicine; MEDLINE; Computer science; Machine learning; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1871836,0.002577304,0.003455646,0.004807432,0.002055811,0.00839102,0.007090984,0.004086946,0.002921809],"category_scores_gemma":[0.4459154,0.002265838,0.004992362,0.005464238,0.004473106,0.00867983,0.004597932,0.008406365,0.001043159],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00314887,"about_ca_system_score_gemma":0.006289258,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.05449361,"about_ca_topic_score_gemma":0.04900365,"domain_scores_codex":[0.8981763,0.08111346,0.004979171,0.00913386,0.005731592,0.0008656295],"domain_scores_gemma":[0.4024771,0.5289392,0.0115602,0.03898483,0.01595694,0.002081794],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0005741625,0.0004538983,0.5379706,0.00245975,0.005683437,0.001971831,0.01383388,0.04416291,0.0006770993,0.02733746,0.03050941,0.3343656],"study_design_scores_gemma":[0.0002778181,0.001035084,0.1867341,0.006086956,0.002549125,0.002083026,0.01060243,0.4218198,0.001217216,0.3006302,0.06616133,0.0008029367],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08093242,0.01716289,0.8612766,0.0256612,0.002150964,0.001406664,0.00432646,0.002493709,0.004589071],"genre_scores_gemma":[0.5256125,0.008285079,0.449622,0.004992987,0.002189494,0.002043999,0.004500322,0.0008501172,0.001903509],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.1871836,"threshold_uncertainty_score":0.9899333,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2308791818014397,"score_gpt":0.4963601855767496,"score_spread":0.2654810037753099,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}