{"id":"W4391941699","doi":"10.1080/0142159x.2024.2316223","title":"Twelve tips for Natural Language Processing in medical education program evaluation","year":2024,"lang":"en","type":"article","venue":"Medical Teacher","topic":"Topic Modeling","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto; Centre for Addiction and Mental Health","funders":"","keywords":"Workflow; Computer science; Preprocessor; Process (computing); Medical education; Data science; Artificial intelligence; Medicine; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.3469382,0.003294075,0.002949535,0.0110812,0.00766438,0.01939731,0.006552052,0.0101138,0.007325958],"category_scores_gemma":[0.5261205,0.002799711,0.002940689,0.007308522,0.01214062,0.02755065,0.01723563,0.02337818,0.005447766],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01095098,"about_ca_system_score_gemma":0.0321515,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003189266,"about_ca_topic_score_gemma":0.007579497,"domain_scores_codex":[0.5886322,0.2867737,0.05924098,0.005610201,0.05428374,0.005459112],"domain_scores_gemma":[0.3333634,0.4979902,0.02009615,0.03560052,0.09795351,0.01499621],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0004297892,0.0009297427,0.004268557,0.004143019,0.0001656248,0.000541084,0.01521521,0.002673844,0.002122389,0.03726256,0.1177906,0.8144575],"study_design_scores_gemma":[0.0008128714,0.002487895,0.01081036,0.03777139,0.0005149309,0.002813607,0.0487244,0.01885719,0.008780272,0.3733361,0.4935069,0.001584203],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.009696018,0.009505946,0.5599523,0.3833366,0.004506053,0.00638133,0.00056288,0.007616173,0.01844274],"genre_scores_gemma":[0.02598221,0.004197545,0.951167,0.01042807,0.0007151503,0.004351807,0.0002450221,0.0008831049,0.002030097],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.3469382,"threshold_uncertainty_score":0.8053415,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0370885945004089,"score_gpt":0.4096117849988083,"score_spread":0.3725231904983994,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}