{"id":"W4417194449","doi":"10.1177/19160216251387618","title":"Performance of Artificial Intelligence Chatbot on the Canadian Otolaryngology in-Training Exam: Unlocking Insights on the Intersection of Technology and Education","year":2025,"lang":"en","type":"article","venue":"Journal of Otolaryngology - Head and Neck Surgery","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University Health Centre; Montreal Children's Hospital; McGill University","funders":"","keywords":"Percentile; Percentile rank; Grading (engineering); Subspecialty; Confidence interval; Checklist; Rank (graph theory)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.006887283,0.0006734334,0.00047987,0.001723954,0.001778281,0.002242387,0.001185105,0.0008475528,0.003781534],"category_scores_gemma":[0.04689115,0.0003239654,0.0004297849,0.001036697,0.001149992,0.001404299,0.002227465,0.0007609327,0.0009098289],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.007853314,"about_ca_system_score_gemma":0.01085638,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.3754129,"about_ca_topic_score_gemma":0.5461556,"domain_scores_codex":[0.9943103,0.001765419,0.0003123398,0.0006813772,0.002082264,0.0008483176],"domain_scores_gemma":[0.9669182,0.01526963,0.003683559,0.001703503,0.007790959,0.004634229],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.002298815,0.001563454,0.7944448,0.0004396953,0.0001486595,0.0005293775,0.01506095,0.004289453,0.0049083,0.00071771,0.01184594,0.1637528],"study_design_scores_gemma":[0.0001148964,0.001925658,0.9571851,0.0001362653,0.00008359551,0.0002402976,0.007567134,0.01729669,0.004324979,0.0006067489,0.01037097,0.0001476036],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9904165,0.0001212166,0.0008799512,0.000370291,0.00003608171,0.0001867139,0.000469715,0.0002026002,0.007316919],"genre_scores_gemma":[0.9938117,0.0001095318,0.00242894,0.0001648683,0.00002093207,0.00008555278,0.0006111714,0.00003213619,0.002735063],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9931127,"threshold_uncertainty_score":0.7464556,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.09020799409340866,"score_gpt":0.3439170235156193,"score_spread":0.2537090294222106,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}