{"id":"W4405018172","doi":"10.2196/58898","title":"Factors Associated With the Accuracy of Large Language Models in Basic Medical Science Examinations: Cross-Sectional Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Education","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Preprint; Cross-sectional study; Medical science; Psychology; Computer science; Artificial intelligence; Medical education; Medicine; World Wide Web; Pathology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01135948,0.0004897313,0.0005511236,0.001326318,0.0004797001,0.001695491,0.0006633372,0.001057516,0.003017215],"category_scores_gemma":[0.0457612,0.0005662474,0.001504569,0.00107669,0.0005875167,0.001937756,0.0008827704,0.001913929,0.0006830843],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000448099,"about_ca_system_score_gemma":0.0005460386,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003063797,"about_ca_topic_score_gemma":0.002578744,"domain_scores_codex":[0.995736,0.001933852,0.0006885762,0.0006838901,0.0006554463,0.0003021614],"domain_scores_gemma":[0.9548195,0.02856483,0.009444999,0.002572106,0.003327634,0.001270912],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0000510195,0.00007141915,0.9984097,0.0000140051,0.00006165405,0.00003047376,0.0002367147,0.0001025851,0.00003865865,0.00002009542,0.0000827125,0.0008809992],"study_design_scores_gemma":[0.000008656914,0.000443431,0.9923419,0.00005721328,0.000190604,0.000432614,0.001619982,0.004081723,0.0001725916,0.0001537802,0.0004779974,0.00001959723],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9985279,0.0002736716,0.0004531703,0.0001188609,0.000009783906,0.00002086473,0.0002433693,0.000008384971,0.0003440425],"genre_scores_gemma":[0.99918,0.00009027362,0.0002431526,0.00002749362,0.00001041727,0.00001696041,0.0002565251,0.00000672516,0.0001684635],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01135948,"threshold_uncertainty_score":0.0600754,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1567278805114249,"score_gpt":0.5171832087574989,"score_spread":0.360455328246074,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}