{"id":"W3030162189","doi":"10.1186/s12909-020-02077-6","title":"Exploring assessor cognition as a source of score variability in a performance assessment of practice-based competencies","year":2020,"lang":"en","type":"article","venue":"BMC Medical Education","topic":"Medical Education and Admissions","field":"Medicine","cited_by":11,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"University of Alberta","keywords":"Generalizability theory; Psychology; Cognition; Reliability (semiconductor); Applied psychology; Perspective (graphical); Think aloud protocol; Medical education; Clinical psychology; Computer science; Developmental psychology; Medicine; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001425117,0.0001041727,0.0003396237,0.0001143807,0.00003117818,0.000005051407,0.0001046977,0.00009134527,0.002726671],"category_scores_gemma":[0.09122614,0.00008887295,0.00006315691,0.0005181035,0.0001470176,0.0001750523,0.00002531015,0.0003559544,0.000007848471],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008148468,"about_ca_system_score_gemma":0.02301591,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00008682108,"about_ca_topic_score_gemma":0.000005238131,"domain_scores_codex":[0.9976707,0.0003148973,0.0006840721,0.0002329537,0.0009548495,0.0001426005],"domain_scores_gemma":[0.9966171,0.001434243,0.0003171756,0.0002136137,0.0004066001,0.001011267],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001286732,0.02490422,0.6988434,0.01470174,0.00010589,0.000006278469,0.01479332,0.0002367734,0.009547327,0.005590865,0.002276128,0.2277073],"study_design_scores_gemma":[0.005293705,0.001654581,0.8170279,0.00756624,0.0003777453,0.0000358166,0.02142293,0.1241743,0.009704503,0.000126203,0.01224591,0.0003701324],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9634797,0.00004006914,0.009384882,0.02277185,0.0002808156,0.0004688979,0.000001138698,0.00002875928,0.003543912],"genre_scores_gemma":[0.9801796,0.00005982651,0.01424217,0.005111705,0.0001609171,0.000136207,0.00006738566,0.00001008768,0.00003209883],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2273372,"threshold_uncertainty_score":0.998185,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1713686131804195,"score_gpt":0.41054813458489,"score_spread":0.2391795214044705,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}