{"id":"W4400742652","doi":"10.1101/2024.07.16.24310297","title":"Evaluating AI Proficiency in Nuclear Cardiology: Large Language Models take on the Board Preparation Exam","year":2024,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Calgary","funders":"National Heart, Lung, and Blood Institute; National Institutes of Health","keywords":"Computer science; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01058854,0.0009041448,0.0006034545,0.0008174941,0.0003008363,0.002031182,0.0009271747,0.0009693847,0.003906205],"category_scores_gemma":[0.05451187,0.0002774889,0.0009296121,0.0003796628,0.0006624938,0.001713763,0.00216811,0.001212761,0.002328384],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008519738,"about_ca_system_score_gemma":0.0009493165,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001599692,"about_ca_topic_score_gemma":0.002134712,"domain_scores_codex":[0.9935488,0.003611831,0.0005395639,0.0008514464,0.001042864,0.0004054001],"domain_scores_gemma":[0.9571254,0.02981414,0.005121628,0.002048233,0.002925516,0.002965042],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.002377461,0.00307275,0.8701099,0.0002808416,0.0002550514,0.0003680236,0.006652896,0.007645677,0.004483052,0.0003605351,0.005214847,0.09917898],"study_design_scores_gemma":[0.0002845339,0.007992979,0.9182771,0.0002966338,0.0002245353,0.0009860785,0.005547637,0.04564916,0.010627,0.001966906,0.00796141,0.0001859279],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9943054,0.00008332074,0.002143816,0.0001653608,0.00002184847,0.0001270378,0.0001867084,0.0001038905,0.002862622],"genre_scores_gemma":[0.9959274,0.00008768138,0.002530935,0.0001076218,0.00001767729,0.0001425935,0.0004604811,0.00003361849,0.0006920684],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01058854,"threshold_uncertainty_score":0.05599821,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2424429298615566,"score_gpt":0.490345726704285,"score_spread":0.2479027968427284,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}