{"id":"W2341010248","doi":"10.1177/0163278715605358","title":"Using Automated Scoring to Evaluate Written Responses in English and French on a High-Stakes Clinical Competency Examination","year":2015,"lang":"en","type":"article","venue":"Evaluation & the Health Professions","topic":"Topic Modeling","field":"Computer Science","cited_by":29,"is_retracted":false,"has_abstract":true,"ca_institutions":"Medical Council of Canada; University of Alberta","funders":"","keywords":"Reliability (semiconductor); Computer science; Stage (stratigraphy); Artificial intelligence; Natural language processing; Machine learning; Medical education; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02472413,0.0001435911,0.0002513095,0.0002915477,0.0004350209,0.00008071755,0.0004135164,0.00009024476,0.000006940654],"category_scores_gemma":[0.004626111,0.0001070976,0.00002367801,0.0006442445,0.00003660342,0.0003505099,0.0002619512,0.0003733555,0.00001690986],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003996163,"about_ca_system_score_gemma":0.00190256,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003620968,"about_ca_topic_score_gemma":0.0002046623,"domain_scores_codex":[0.990899,0.006439669,0.0008124844,0.00051456,0.0009741726,0.0003600595],"domain_scores_gemma":[0.997155,0.001016417,0.0002799221,0.0006547849,0.0006488694,0.0002450464],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002498534,0.0009157947,0.04606875,0.0001723596,0.0000486635,0.00001074794,0.2239785,0.09749638,0.0003017388,0.03811464,0.004803735,0.5878389],"study_design_scores_gemma":[0.0009312623,0.0002233674,0.1956016,0.0003692461,0.0000072313,0.000001543951,0.001018688,0.7986625,0.000007824549,0.003001827,0.00007117724,0.0001037157],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9359041,0.0001630628,0.04517967,0.01475417,0.001849411,0.001670988,0.000002431377,0.0003077133,0.0001684982],"genre_scores_gemma":[0.9526242,0.00001025455,0.04553491,0.001446029,0.0001621856,0.0001312918,0.000004415741,0.00001095353,0.00007576171],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7011662,"threshold_uncertainty_score":0.856894,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4259430117257569,"score_gpt":0.5111785923759283,"score_spread":0.08523558065017145,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}