{"id":"W4391973028","doi":"10.1016/j.compbiomed.2024.108189","title":"A comprehensive evaluation of large Language models on benchmark biomedical text processing tasks","year":2024,"lang":"en","type":"article","venue":"Computers in Biology and Medicine","topic":"Topic Modeling","field":"Computer Science","cited_by":97,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University","funders":"Natural Sciences and Engineering Research Council of Canada; York University","keywords":"Benchmark (surveying); Computer science; Task (project management); Set (abstract data type); Domain (mathematical analysis); Work (physics); Artificial intelligence; Engineering; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008616223,0.0001012419,0.0002183357,0.0002736967,0.00003386553,0.000009080737,0.0002269256,0.00009778896,0.000008360843],"category_scores_gemma":[0.00003990783,0.00007313013,0.00001717094,0.0002741185,0.0001603342,0.00009257568,0.0001276276,0.0001724236,0.000001211692],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000324276,"about_ca_system_score_gemma":0.00007464048,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002045363,"about_ca_topic_score_gemma":0.000002706544,"domain_scores_codex":[0.998883,0.0001400408,0.0002487071,0.0003570143,0.0001952338,0.0001759927],"domain_scores_gemma":[0.9994388,0.0002129067,0.00004182719,0.0001927334,0.00006137478,0.00005234922],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001197041,0.00006468674,0.0001584979,0.0001977739,0.00002497691,0.00003239114,0.009880077,0.0006747023,0.002150564,0.1032004,0.0005645319,0.8830395],"study_design_scores_gemma":[0.0006970135,0.0001862399,0.0005003799,0.0005981071,0.00001092704,0.00001832118,0.0001573222,0.9753846,0.00003913436,0.02175251,0.0005861997,0.00006923918],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2047244,0.01046209,0.7809845,0.002135904,0.0008285862,0.000176181,0.000002211984,0.00005661347,0.0006294486],"genre_scores_gemma":[0.9890683,0.00006488923,0.01006698,0.0006281615,0.0001398901,0.000007739523,0.00001553622,0.000003470973,0.000004994003],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9747099,"threshold_uncertainty_score":0.298216,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04517738477580747,"score_gpt":0.377081677449967,"score_spread":0.3319042926741596,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}