{"id":"W3030162189","doi":"10.1186/s12909-020-02077-6","title":"Exploring assessor cognition as a source of score variability in a performance assessment of practice-based competencies","year":2020,"lang":"en","type":"article","venue":"BMC Medical Education","topic":"Medical Education and Admissions","field":"Medicine","cited_by":11,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"University of Alberta","keywords":"Generalizability theory; Psychology; Cognition; Reliability (semiconductor); Applied psychology; Perspective (graphical); Think aloud protocol; Medical education; Clinical psychology; Computer science; Developmental psychology; Medicine; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.07977366,0.0005061962,0.0005566407,0.002572909,0.0009635643,0.002830323,0.001041025,0.0007672741,0.0009063576],"category_scores_gemma":[0.2170262,0.0003514763,0.001100333,0.001752409,0.003108739,0.001581054,0.002858636,0.001185124,0.0001201812],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002420754,"about_ca_system_score_gemma":0.002769827,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0020587,"about_ca_topic_score_gemma":0.002831335,"domain_scores_codex":[0.9101416,0.05808458,0.006525209,0.003659637,0.02060379,0.0009851304],"domain_scores_gemma":[0.5038555,0.4262804,0.03805766,0.01577941,0.01511479,0.0009121906],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0006392674,0.0003116181,0.8434148,0.0006412388,0.0005817097,0.0002740294,0.06163286,0.00133931,0.00390939,0.001132587,0.0002432761,0.08587985],"study_design_scores_gemma":[0.00004375392,0.001752548,0.9573047,0.0004165483,0.0003589195,0.0007768359,0.02015804,0.005807517,0.008791568,0.002886856,0.001592703,0.0001099994],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9750132,0.000257327,0.02221195,0.0001770403,0.00001717504,0.0001424946,0.00005939034,0.00004024419,0.002081259],"genre_scores_gemma":[0.9943874,0.00004950661,0.005263485,0.00003178416,0.000009210003,0.00009770306,0.00003865453,0.00001397016,0.0001080591],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.07977366,"threshold_uncertainty_score":0.4218885,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1713686131804195,"score_gpt":0.41054813458489,"score_spread":0.2391795214044705,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}