{"id":"W4394579747","doi":"10.2196/55318","title":"An Empirical Evaluation of Prompting Strategies for Large Language Models in Zero-Shot Clinical Natural Language Processing: Algorithm Development and Validation Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":162,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"National Center for Advancing Translational Sciences; U.S. National Library of Medicine; National Institutes of Health","keywords":"Computer science; Natural language processing; Artificial intelligence; Heuristic; Context (archaeology); Task (project management); Relationship extraction; Machine learning; Information extraction","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02173532,0.00199798,0.001557944,0.001186095,0.0006535358,0.001488542,0.002220594,0.002271382,0.002024983],"category_scores_gemma":[0.0621652,0.0008583414,0.001174816,0.0008503429,0.0009384629,0.002727245,0.002249957,0.003797875,0.0007696046],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002242781,"about_ca_system_score_gemma":0.003017307,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005626713,"about_ca_topic_score_gemma":0.005497971,"domain_scores_codex":[0.9931878,0.004395889,0.0005053881,0.001163237,0.0005380362,0.0002096171],"domain_scores_gemma":[0.9175776,0.07296889,0.001353819,0.002900098,0.004379707,0.0008198203],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003807,0.003255352,0.015639,0.001710731,0.0006943344,0.0002946357,0.001008136,0.5364739,0.006905041,0.003814475,0.008165665,0.4182318],"study_design_scores_gemma":[0.00025115,0.0006988326,0.00141277,0.00005980899,0.0001089721,0.00008022798,0.0001418519,0.9909633,0.003779464,0.001652586,0.0008227713,0.00002831815],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5592039,0.005422083,0.4165815,0.00116401,0.0003742436,0.002041454,0.001347743,0.01069498,0.003170098],"genre_scores_gemma":[0.715293,0.0007514203,0.2784394,0.000397218,0.0000577765,0.001175803,0.002628023,0.0002754665,0.0009817702],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02173532,"threshold_uncertainty_score":0.1149487,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2913679776604406,"score_gpt":0.5726627397999422,"score_spread":0.2812947621395016,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}