{"id":"W3207657503","doi":"10.2196/32507","title":"Assessing the Performance of a New Artificial Intelligence–Driven Diagnostic Support Tool Using Medical Board Exam Simulations: Clinical Vignette Study","year":2021,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Clinical Reasoning and Diagnostic Skills","field":"Medicine","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Medical diagnosis; Medicine; Diagnostic test; Clinical decision support system; Decision support system; Diagnostic accuracy; Vignette; Medical physics; Emergency medicine; Artificial intelligence; Psychology; Computer science; Radiology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00748775,0.0006092776,0.000479519,0.001876426,0.0004617506,0.001066422,0.0008108052,0.001078805,0.001381021],"category_scores_gemma":[0.04932622,0.0004132848,0.0006216813,0.0007349807,0.0006108787,0.001012199,0.001130293,0.0005810922,0.0004326952],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001457899,"about_ca_system_score_gemma":0.0008294825,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001421729,"about_ca_topic_score_gemma":0.001921154,"domain_scores_codex":[0.995221,0.003286519,0.0004423657,0.0003856311,0.0004670215,0.0001974809],"domain_scores_gemma":[0.9551153,0.03194937,0.003656247,0.002082511,0.004589093,0.002607457],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.009103131,0.02625412,0.796262,0.0007833362,0.0008249212,0.002868793,0.01199839,0.0275221,0.007022752,0.001016148,0.002614528,0.1137298],"study_design_scores_gemma":[0.001814958,0.08108632,0.7268533,0.0002807952,0.0005131308,0.005112958,0.007564465,0.1478764,0.01945403,0.0008840387,0.008155677,0.0004040436],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9985259,0.00003521425,0.0007491904,0.00004819841,0.00000464874,0.0002293469,0.00005120859,0.00001518709,0.0003411013],"genre_scores_gemma":[0.9928751,0.00009229535,0.006325631,0.00006622532,0.00001602578,0.0001890757,0.0002271712,0.00000537857,0.0002032146],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.00748775,"threshold_uncertainty_score":0.03959948,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1227083511359059,"score_gpt":0.4719615238526976,"score_spread":0.3492531727167917,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}