{"id":"W4402966708","doi":"10.2196/55648","title":"Accuracy of a Commercial Large Language Model (ChatGPT) to Perform Disaster Triage of Simulated Patients Using the Simple Triage and Rapid Treatment (START) Protocol: Gage Repeatability and Reproducibility Study","year":2024,"lang":"en","type":"article","venue":"Journal of Medical Internet Research","topic":"Disaster Response and Management","field":"Health Professions","cited_by":18,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Repeatability; Triage; Reproducibility; Protocol (science); Computer science; Medicine; Medical emergency; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"simulation_or_modeling","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"bench_or_experimental","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03056831,0.0001678562,0.0005984469,0.000293123,0.0001648239,0.00004496826,0.0004276685,0.0001198404,0.0006812423],"category_scores_gemma":[0.009739726,0.00008961685,0.0001161696,0.0003009438,0.0002021565,0.0001922986,0.001099328,0.001036329,0.000002487821],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002410059,"about_ca_system_score_gemma":0.0004965886,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001036584,"about_ca_topic_score_gemma":0.0004517047,"domain_scores_codex":[0.9921784,0.003633551,0.001539443,0.0005202055,0.001683277,0.0004451646],"domain_scores_gemma":[0.9951528,0.002712852,0.0003119251,0.001086381,0.0004073459,0.0003287539],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.03073419,0.01179877,0.1563954,0.004734237,0.001115114,0.0003160743,0.5722049,0.00005108878,0.0007039121,0.0001753856,0.009422626,0.2123483],"study_design_scores_gemma":[0.07139236,0.04486195,0.1356135,0.008793791,0.0008503243,0.00002966601,0.3332096,0.3353168,0.001755216,0.001143995,0.06613553,0.0008972557],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9717947,0.0001108542,0.0002347941,0.0009194757,0.000115299,0.02662913,0.00004675844,0.000008181957,0.0001408616],"genre_scores_gemma":[0.9983861,0.0000241176,0.00002775052,0.00008770152,0.0001350673,0.0008845858,0.000003653471,0.00001915055,0.0004319051],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3352657,"threshold_uncertainty_score":0.9986017,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2537773929940432,"score_gpt":0.5869407438634208,"score_spread":0.3331633508693776,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}