{"id":"W4390988578","doi":"10.1097/acm.0000000000005634","title":"Using Natural Language Processing to Evaluate the Quality of Supervisor Narrative Comments in Competency-Based Medical Education","year":2024,"lang":"en","type":"article","venue":"Academic Medicine","topic":"Innovations in Medical Education","field":"Medicine","cited_by":31,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University; University of Saskatchewan; Royal College of Physicians and Surgeons of Canada; McMaster University Medical Centre","funders":"","keywords":"Supervisor; Narrative; Quality (philosophy); Medical education; Natural (archaeology); Psychology; Computer science; Natural language processing; Medicine; Linguistics; Political science; History","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02467487,0.0006809874,0.000419891,0.003264426,0.0005535658,0.002262266,0.0008065985,0.0007313874,0.001323803],"category_scores_gemma":[0.1414972,0.0002665348,0.0007467132,0.001528015,0.0008517096,0.001987892,0.00128524,0.0009294022,0.0005840668],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001944734,"about_ca_system_score_gemma":0.002128032,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005441063,"about_ca_topic_score_gemma":0.006808077,"domain_scores_codex":[0.9799392,0.01359736,0.001998913,0.001663727,0.002548683,0.0002520661],"domain_scores_gemma":[0.7429317,0.2044187,0.02909781,0.003401413,0.01899799,0.001152234],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001577122,0.001615525,0.7323862,0.0008306302,0.000290556,0.0003759751,0.00984744,0.01744558,0.005297484,0.0006589149,0.002304452,0.2273701],"study_design_scores_gemma":[0.0002017486,0.001834704,0.5743736,0.0006336528,0.0001809151,0.0004457777,0.006550343,0.3977361,0.011581,0.003276527,0.002993724,0.0001918603],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9411212,0.0001352601,0.05402249,0.0003113628,0.00003052242,0.0009682033,0.000973174,0.0003742737,0.002063541],"genre_scores_gemma":[0.9451134,0.00006377343,0.05256248,0.00008318836,0.00001743234,0.0006410535,0.001127734,0.00002242793,0.0003684801],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02467487,"threshold_uncertainty_score":0.1304948,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08285630765382505,"score_gpt":0.5130708602996962,"score_spread":0.4302145526458711,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}