{"id":"W4403177786","doi":"10.2196/56762","title":"Evaluating AI Competence in Specialized Medicine: Comparative Analysis of ChatGPT and Neurologists in a Neurology Specialist Examination in Spain","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Preprint; Competence (human resources); Neurology; Medical education; Psychology; Medicine; Psychiatry; Computer science; World Wide Web","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002422693,0.0001301659,0.0005511351,0.001705788,0.00002155462,0.00001196738,0.00008614108,0.0002053506,0.0005226013],"category_scores_gemma":[0.002535339,0.0001178388,0.00003890957,0.002817686,0.0003151463,0.0001169961,0.00002222493,0.0005799179,0.0000053048],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001634236,"about_ca_system_score_gemma":0.001164407,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005473431,"about_ca_topic_score_gemma":0.01382904,"domain_scores_codex":[0.9972938,0.0006168569,0.0009428569,0.0004197235,0.0005065166,0.0002202943],"domain_scores_gemma":[0.9985272,0.0009048725,0.0001228076,0.0001652371,0.0001381577,0.0001417306],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0004459143,0.001101436,0.5650567,0.000504104,0.00004468,0.00006177206,0.05787788,0.0001390441,0.0005819817,0.006387604,0.0003170134,0.3674819],"study_design_scores_gemma":[0.0001859983,0.0004147279,0.8916828,0.0005196712,0.00006578293,0.00001285203,0.002023678,0.1035183,0.00004596652,0.001112628,0.0003392875,0.00007824046],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9650637,0.0007562715,0.00006225994,0.03203047,0.0007089578,0.0006750318,0.000001149536,0.00001659393,0.0006855761],"genre_scores_gemma":[0.9971203,0.0001962382,0.00006779303,0.001928935,0.0004037251,0.0001592733,0.0000874226,0.000006905202,0.00002945345],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3674036,"threshold_uncertainty_score":0.8274227,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1737585601633037,"score_gpt":0.5358228037959748,"score_spread":0.3620642436326711,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}