{"id":"W4417309864","doi":"10.2196/81673","title":"AI- vs Human-Based Assessment of Medical Interview Transcripts in a Generative AI–Simulated Patient System: Cross-Sectional Validation Study","year":2025,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Generalizability theory; Consistency (knowledge bases); Matching (statistics); Generative grammar; Psychometrics; Generative model","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001825348,0.0001808402,0.0004204026,0.0004896829,0.0001318985,0.00004036915,0.0001868626,0.0003808664,0.001006751],"category_scores_gemma":[0.0007074045,0.0001614658,0.0001058757,0.0007719818,0.0001341398,0.0001201367,0.00002391999,0.0006743504,0.000008855439],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008556623,"about_ca_system_score_gemma":0.008477302,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001866234,"about_ca_topic_score_gemma":0.0003620616,"domain_scores_codex":[0.9956419,0.0005633854,0.001515672,0.0004310201,0.001613385,0.0002346782],"domain_scores_gemma":[0.9981969,0.0001801726,0.0002081458,0.0003079748,0.0008009585,0.0003058204],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0002064495,0.01032933,0.9478287,0.001758375,0.00007504874,0.00001278576,0.004920314,0.0001904951,0.000199016,0.002170593,0.001710672,0.03059827],"study_design_scores_gemma":[0.001070166,0.001593126,0.938178,0.003838402,0.00008477578,0.0000147239,0.007408047,0.044027,0.002568504,0.000324874,0.0006748091,0.0002175618],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9739544,0.0001166268,0.002988298,0.01853576,0.002048963,0.002034151,0.000003158397,0.00005229918,0.0002662987],"genre_scores_gemma":[0.9932783,0.000008928505,0.00005922411,0.005413227,0.0002569069,0.0006382099,0.0002550861,0.00001398125,0.00007616125],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.04383651,"threshold_uncertainty_score":0.9999065,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1288360537981913,"score_gpt":0.550083557258638,"score_spread":0.4212475034604467,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}