{"id":"W4394759098","doi":"10.5489/cuaj.8526","title":"Comprehensive analysis of the performance of GPT-3.5 and GPT-4 on the American Urological Association self-assessment study program exams from 2012-2023.","year":2023,"lang":"en","type":"article","venue":"PubMed","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"Saint John Regional Hospital; Dalhousie University","funders":"","keywords":"Percentile; Confidence interval; Odds ratio; Medicine; Internal medicine; Mathematics; Statistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005699518,0.0003390119,0.0003314559,0.00219441,0.0003258236,0.0006777579,0.0004816284,0.0004695707,0.001212338],"category_scores_gemma":[0.01570353,0.0001563186,0.0006492878,0.00144571,0.0002589315,0.0007046682,0.001164706,0.0005368801,0.0006295296],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006976743,"about_ca_system_score_gemma":0.0009832076,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005935634,"about_ca_topic_score_gemma":0.009512915,"domain_scores_codex":[0.9978004,0.0005539749,0.0002369458,0.0002691975,0.000890359,0.0002490995],"domain_scores_gemma":[0.9827678,0.004545613,0.00537927,0.0008438848,0.004683653,0.001779806],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002011308,0.0002033484,0.9832031,0.00003196263,0.0000828925,0.00003783752,0.000255826,0.0003917513,0.0003002687,0.0000371819,0.001596233,0.01365848],"study_design_scores_gemma":[0.000003905262,0.0001562202,0.9986033,0.000005004972,0.00001064269,0.0000323533,0.00006327538,0.0004749416,0.0001513041,0.00001330101,0.0004827,0.000003034196],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9945029,0.0002178808,0.0003811377,0.0001319617,0.00002337171,0.0001020415,0.002392055,0.00005749464,0.002191135],"genre_scores_gemma":[0.9956241,0.00007017687,0.0004928929,0.00004487333,0.00001610616,0.00008254248,0.002966891,0.00001241226,0.0006900555],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.005935634,"threshold_uncertainty_score":0.03014231,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1203422909210264,"score_gpt":0.3890914971326901,"score_spread":0.2687492062116638,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}