{"id":"W4399545971","doi":"10.5489/cuaj.8800","title":"Performance of artificial intelligence on a simulated Canadian urology board exam","year":2024,"lang":"en","type":"article","venue":"Canadian Urological Association Journal","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"Kingston General Hospital; Queen's University","funders":"","keywords":"Board certification; Urology; Multiple choice; Percentile; Medicine; Certification; Specialty; Educational measurement; Internal medicine; Medical education; Psychology; Curriculum; Family medicine; Residency training; Mathematics; Continuing education; Statistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.001915506,0.0007252223,0.0003729936,0.0009150648,0.0008931741,0.0009340686,0.0008318269,0.0005663818,0.003926429],"category_scores_gemma":[0.01135143,0.0002148036,0.0004271259,0.0007556673,0.0006102764,0.0002688212,0.001050685,0.0004128197,0.0007113032],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.005322944,"about_ca_system_score_gemma":0.005970545,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.2504965,"about_ca_topic_score_gemma":0.2987876,"domain_scores_codex":[0.9983568,0.0003672068,0.00009973573,0.0002500216,0.0006457519,0.0002805372],"domain_scores_gemma":[0.9948421,0.001406841,0.0006744774,0.000274549,0.001435237,0.001366819],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002728512,0.002662844,0.7441181,0.0003729333,0.0003392113,0.0007684614,0.002987127,0.03507935,0.006868571,0.0006389195,0.01117243,0.1922635],"study_design_scores_gemma":[0.0001793583,0.00330631,0.9457803,0.00007653751,0.00006431823,0.0003975209,0.001050709,0.03360996,0.006060236,0.0002995439,0.009070112,0.0001050612],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9940501,0.00007948206,0.0004495894,0.0001332594,0.00001875257,0.0001202447,0.0003357237,0.0000755521,0.004737298],"genre_scores_gemma":[0.9950566,0.00008332502,0.001990925,0.00007150008,0.000008390542,0.00003978345,0.0007852466,0.000009245338,0.0019548],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9980845,"threshold_uncertainty_score":0.4980769,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08688040358933152,"score_gpt":0.3467622131964259,"score_spread":0.2598818096070944,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}