{"id":"W4400549596","doi":"10.2196/51282","title":"Assessing GPT-4’s Performance in Delivering Medical Advice: Comparative Analysis With Human Experts","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":34,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Medical diagnosis; CLARITY; Computer science; Proxy (statistics); Grading (engineering); Health care; Vocabulary; Medical education; Medicine; Psychology; Machine learning; Pathology; Engineering","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.03226342,0.001072909,0.001001062,0.002953024,0.0007369421,0.00226337,0.001394933,0.002214006,0.002515347],"category_scores_gemma":[0.1294145,0.0003982907,0.0008923328,0.001538431,0.001242024,0.002172815,0.002911167,0.001073185,0.001423072],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001208493,"about_ca_system_score_gemma":0.001453895,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004185391,"about_ca_topic_score_gemma":0.003741385,"domain_scores_codex":[0.973724,0.01785214,0.001863892,0.002648809,0.00315743,0.0007538231],"domain_scores_gemma":[0.7854266,0.1799962,0.007731235,0.006224866,0.01659175,0.004029324],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.009284744,0.002842256,0.5072212,0.005305937,0.001551308,0.001772695,0.05074759,0.02989092,0.01511942,0.001622056,0.01759543,0.3570464],"study_design_scores_gemma":[0.001860444,0.01417409,0.5839141,0.001598945,0.001348331,0.003291482,0.02801859,0.3039543,0.01723816,0.006541167,0.03724903,0.0008114002],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9751267,0.0008356483,0.0140279,0.0006857255,0.0001082571,0.0006783201,0.0008932912,0.0009565445,0.006687661],"genre_scores_gemma":[0.9683948,0.0004589235,0.02702615,0.000527097,0.00009681932,0.0005474809,0.001519926,0.0001361172,0.00129268],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03226342,"threshold_uncertainty_score":0.1706273,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1181307132958916,"score_gpt":0.5247994825716034,"score_spread":0.4066687692757117,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}