{"id":"W4378976945","doi":"10.1101/2023.05.30.23290758","title":"Evaluating ChatGPT-4 in Otolaryngology–Head and Neck Surgery Board Examination using the CVSA Model","year":2023,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":16,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Otorhinolaryngology; Concordance; Head and neck surgery; Medicine; Head and neck; SAFER; Sample (material); Consistency (knowledge bases); Medical physics; Multiple choice; Medical education; Surgery; Artificial intelligence; Computer science; Internal medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0211046,0.001203471,0.0009418439,0.002518957,0.0007066409,0.002920412,0.001645199,0.001547814,0.001848429],"category_scores_gemma":[0.08010988,0.0004137655,0.001702695,0.001290222,0.000719121,0.001616968,0.001961917,0.001719682,0.0006771074],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004390779,"about_ca_system_score_gemma":0.003488932,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.03233471,"about_ca_topic_score_gemma":0.01925569,"domain_scores_codex":[0.9888843,0.006791279,0.0009318042,0.001292867,0.001761295,0.0003384314],"domain_scores_gemma":[0.885861,0.09385222,0.003552635,0.003549426,0.01171679,0.001468055],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","study_design_scores_codex":[0.003119338,0.002211428,0.3014829,0.001105645,0.001063241,0.0004149941,0.002866875,0.4383607,0.004436201,0.003515906,0.006190011,0.2352327],"study_design_scores_gemma":[0.00006671017,0.0006106053,0.02214238,0.0000784392,0.0001270713,0.00008188232,0.0003361482,0.97198,0.00215903,0.001122053,0.001241631,0.00005406014],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9272295,0.0003735156,0.06331161,0.0006223194,0.00009628648,0.0008787265,0.001191627,0.001657535,0.004638867],"genre_scores_gemma":[0.9589553,0.00007494634,0.0376324,0.0001077272,0.00002168314,0.0004443659,0.00180757,0.00005530246,0.0009006445],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03233471,"threshold_uncertainty_score":0.1116132,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4489275324866818,"score_gpt":0.4892135576655389,"score_spread":0.04028602517885704,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}