{"id":"W4402966708","doi":"10.2196/55648","title":"Accuracy of a Commercial Large Language Model (ChatGPT) to Perform Disaster Triage of Simulated Patients Using the Simple Triage and Rapid Treatment (START) Protocol: Gage Repeatability and Reproducibility Study","year":2024,"lang":"en","type":"article","venue":"Journal of Medical Internet Research","topic":"Disaster Response and Management","field":"Health Professions","cited_by":18,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Repeatability; Triage; Reproducibility; Protocol (science); Computer science; Medicine; Medical emergency; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"simulation_or_modeling","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"bench_or_experimental","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0928623,0.00103216,0.0007907705,0.00130669,0.0008130106,0.002465249,0.001628553,0.001155858,0.001887964],"category_scores_gemma":[0.3421813,0.0007441824,0.001513833,0.0007254861,0.001527763,0.00179431,0.002821747,0.001396184,0.001701384],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001736691,"about_ca_system_score_gemma":0.002293946,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001940243,"about_ca_topic_score_gemma":0.002092459,"domain_scores_codex":[0.9173056,0.05743304,0.008734015,0.007479272,0.008034605,0.001013526],"domain_scores_gemma":[0.6658143,0.2157865,0.02626096,0.04709524,0.04239966,0.002643374],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.01705872,0.004588583,0.4473488,0.002743477,0.002007211,0.0007470349,0.03128105,0.0341579,0.02512113,0.002900157,0.02472742,0.4073185],"study_design_scores_gemma":[0.002602851,0.02432427,0.5162164,0.001732702,0.001991264,0.001772985,0.0088732,0.3255747,0.07471126,0.0057971,0.03507039,0.001332911],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8674444,0.000306742,0.1149698,0.000593944,0.0005294589,0.005477339,0.001687327,0.005274123,0.003716867],"genre_scores_gemma":[0.9072877,0.0001135286,0.08420731,0.0005100498,0.00009488436,0.004431311,0.001443521,0.0008682327,0.001043509],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0928623,"threshold_uncertainty_score":0.4911087,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2537773929940432,"score_gpt":0.5869407438634208,"score_spread":0.3331633508693776,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}