{"id":"W2478762182","doi":"10.4300/jgme-d-15-00751.1","title":"Assessing the Reliability of Performance Assessment Scores: Some Considerations in Selecting an Appropriate Framework","year":2016,"lang":"en","type":"article","venue":"Journal of Graduate Medical Education","topic":"Innovations in Medical Education","field":"Medicine","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"Medical Council of Canada","funders":"","keywords":"Accreditation; Observational study; Reliability (semiconductor); Graduate medical education; Sample (material); Psychology; Test (biology); Process (computing); Computer science; Educational measurement; Domain (mathematical analysis); Sample size determination; Medical education; Applied psychology; Medicine; Statistics; Curriculum; Mathematics; Pedagogy","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.005497138,0.0001201349,0.0003307629,0.0003117662,0.0001398139,0.00003723176,0.0001721712,0.0001598039,0.000183312],"category_scores_gemma":[0.02644337,0.00006406364,0.00006766031,0.0006121605,0.0003185039,0.0008382621,0.00002272563,0.0009722832,0.000002275019],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004294851,"about_ca_system_score_gemma":0.006728488,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002305312,"about_ca_topic_score_gemma":0.000002775045,"domain_scores_codex":[0.9967187,0.000312842,0.001309086,0.0001713711,0.00128293,0.000205059],"domain_scores_gemma":[0.99655,0.0006984246,0.0008550459,0.0003554346,0.00139612,0.0001450195],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00008674386,0.00456088,0.3780935,0.0004893282,0.00007690012,0.000008692372,0.00431182,0.00005937074,0.003487337,0.01392312,0.003374842,0.5915275],"study_design_scores_gemma":[0.001500962,0.0006689393,0.933439,0.005903954,0.0001326793,0.0004006658,0.004054178,0.003648179,0.001963849,0.04765512,0.0004475563,0.0001849727],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.929284,0.00006497229,0.006773254,0.0620378,0.001454619,0.0002428728,2.891719e-7,0.000009427256,0.0001328115],"genre_scores_gemma":[0.9689615,0.0001459061,0.0280048,0.001945873,0.0008846162,0.00002004005,0.000003980206,0.00001394181,0.00001938872],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5913426,"threshold_uncertainty_score":0.9989024,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04909405547366617,"score_gpt":0.4256244603539484,"score_spread":0.3765304048802822,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}