{"id":"W4394579747","doi":"10.2196/55318","title":"An Empirical Evaluation of Prompting Strategies for Large Language Models in Zero-Shot Clinical Natural Language Processing: Algorithm Development and Validation Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":162,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"National Center for Advancing Translational Sciences; U.S. National Library of Medicine; National Institutes of Health","keywords":"Computer science; Natural language processing; Artificial intelligence; Heuristic; Context (archaeology); Task (project management); Relationship extraction; Machine learning; Information extraction","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004391661,0.0001101547,0.0002548617,0.0001653212,0.00005688392,0.00006682854,0.00007220377,0.0001653211,0.00002070973],"category_scores_gemma":[0.0003411733,0.00008479851,0.00003095441,0.0002279425,0.00004498681,0.0005732632,0.00002422956,0.00034404,0.000002634912],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000944765,"about_ca_system_score_gemma":0.00174938,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002750584,"about_ca_topic_score_gemma":0.00006605112,"domain_scores_codex":[0.9975318,0.0001033166,0.001175014,0.0001447609,0.0008357737,0.0002093158],"domain_scores_gemma":[0.9991419,0.0001868027,0.0001261275,0.0001225968,0.0002722704,0.0001503179],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003386458,0.0004997615,0.00289046,0.0007746319,0.00001657171,0.000003782858,0.2210771,0.00001898869,0.000009008902,0.00003839014,0.0000544675,0.7745829],"study_design_scores_gemma":[0.0002819856,0.0003134517,0.002342795,0.0004053786,0.00004359762,0.000008653274,0.1929663,0.8029857,0.0002788335,0.0002542336,0.00004086566,0.00007829987],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9794784,0.0005317995,0.01791421,0.0001726819,0.0001831867,0.001569208,0.000003655848,0.00005342894,0.00009345898],"genre_scores_gemma":[0.9900499,0.000007269221,0.009185138,0.0001639491,0.0001687326,0.0002343777,0.0001687326,0.00001104365,0.00001084104],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8029667,"threshold_uncertainty_score":0.3457983,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2913679776604406,"score_gpt":0.5726627397999422,"score_spread":0.2812947621395016,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}