{"id":"W4394689344","doi":"10.2196/54067","title":"Using ChatGPT in Psychiatry to Design Script Concordance Tests in Undergraduate Medical Education: Mixed Methods Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":30,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université Laval; Université de Montréal","funders":"","keywords":"Concordance; Medical education; Psychology; Expert opinion; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003303281,0.0002400088,0.0004082314,0.0006250306,0.00008230483,0.00006207677,0.0002469507,0.0003285662,0.0005849904],"category_scores_gemma":[0.005501824,0.0002241143,0.00006191064,0.002155504,0.0001012724,0.0002089208,0.00004856923,0.00104181,0.0001266451],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008416772,"about_ca_system_score_gemma":0.03372151,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002922487,"about_ca_topic_score_gemma":0.002331749,"domain_scores_codex":[0.99597,0.0007863236,0.001093035,0.000680439,0.001015093,0.0004551631],"domain_scores_gemma":[0.9978157,0.0004841672,0.00008757035,0.0004241668,0.0002241894,0.0009641911],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001045903,0.005142668,0.05896003,0.0003612873,0.00001861946,0.00002137545,0.010926,0.00001230654,0.00007352631,0.001428839,0.02312891,0.8998219],"study_design_scores_gemma":[0.00170387,0.004567134,0.4939471,0.04648188,0.0003771268,0.00131263,0.2118771,0.09046851,0.001706158,0.08141828,0.06347904,0.00266111],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8158934,0.003460686,0.01410447,0.1453381,0.01809948,0.002702954,5.392203e-7,0.0001099281,0.0002904284],"genre_scores_gemma":[0.9565801,0.0001327463,0.03013179,0.009605203,0.002010714,0.001193588,0.00002409617,0.00004648362,0.0002752973],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8971608,"threshold_uncertainty_score":0.9717564,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.160236527378206,"score_gpt":0.5630832943110788,"score_spread":0.4028467669328728,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}