{"id":"W2341010248","doi":"10.1177/0163278715605358","title":"Using Automated Scoring to Evaluate Written Responses in English and French on a High-Stakes Clinical Competency Examination","year":2015,"lang":"en","type":"article","venue":"Evaluation & the Health Professions","topic":"Topic Modeling","field":"Computer Science","cited_by":29,"is_retracted":false,"has_abstract":true,"ca_institutions":"Medical Council of Canada; University of Alberta","funders":"","keywords":"Reliability (semiconductor); Computer science; Stage (stratigraphy); Artificial intelligence; Natural language processing; Machine learning; Medical education; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01581016,0.0009760304,0.000615637,0.004175251,0.0006104189,0.001759268,0.0008516411,0.0007321174,0.001488677],"category_scores_gemma":[0.04321626,0.0002118599,0.0005277143,0.001704709,0.0004591133,0.0008437664,0.001461399,0.0005993485,0.0009567674],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001112568,"about_ca_system_score_gemma":0.001921854,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01051874,"about_ca_topic_score_gemma":0.02180839,"domain_scores_codex":[0.9780075,0.01524307,0.001183803,0.001559166,0.003473183,0.0005332525],"domain_scores_gemma":[0.9489926,0.02977962,0.00484361,0.002551758,0.01302304,0.0008094408],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0006036446,0.0006139239,0.200064,0.0003416812,0.0002860421,0.0002305014,0.003582238,0.01811718,0.02887038,0.001505611,0.004552653,0.7412322],"study_design_scores_gemma":[0.0002335689,0.001678869,0.3849927,0.0001893383,0.0002378257,0.0009300447,0.003860703,0.5331036,0.05838478,0.004374868,0.0116157,0.0003978403],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6752609,0.0002620663,0.3111517,0.0003782534,0.00008414534,0.001328178,0.001117901,0.002954664,0.007462376],"genre_scores_gemma":[0.8023966,0.0001104709,0.1936503,0.00009104976,0.00005532152,0.0006816608,0.001237954,0.00008106016,0.0016955],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01581016,"threshold_uncertainty_score":0.0836131,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4259430117257569,"score_gpt":0.5111785923759283,"score_spread":0.08523558065017145,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}