{"id":"W4385572482","doi":"10.18653/v1/2023.bionlp-1.30","title":"Evaluation of ChatGPT on Biomedical Tasks: A Zero-Shot Comparison with Fine-Tuned Generative Transformers","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":34,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University","funders":"Natural Sciences and Engineering Research Council of Canada; York University","keywords":"Automatic summarization; Computer science; Transformer; Generative grammar; Benchmark (surveying); Artificial intelligence; Relationship extraction; Domain (mathematical analysis); Language model; Training set; One shot; Machine learning; Natural language processing; Information extraction; Engineering; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004896292,0.001981281,0.001733882,0.001606858,0.001001205,0.001679306,0.003542881,0.002936835,0.005460963],"category_scores_gemma":[0.01638807,0.0006425413,0.00130658,0.001051031,0.00133519,0.004516492,0.003330657,0.003203398,0.002913623],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001699719,"about_ca_system_score_gemma":0.002090403,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008843049,"about_ca_topic_score_gemma":0.01176099,"domain_scores_codex":[0.9974184,0.001058183,0.0001542517,0.0008279659,0.0003620243,0.0001791855],"domain_scores_gemma":[0.9920116,0.005794298,0.0001337622,0.0009999044,0.0006094717,0.0004509577],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.004321202,0.001857125,0.00749696,0.00346751,0.001181871,0.001270524,0.001053137,0.2285023,0.02648092,0.006345404,0.05558071,0.6624423],"study_design_scores_gemma":[0.0005826078,0.001753742,0.003510376,0.0001616452,0.0003569526,0.0009687196,0.0005766038,0.9441695,0.01918533,0.0122021,0.01640747,0.0001249757],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4488899,0.02113681,0.4101647,0.002782722,0.002512562,0.001547202,0.009759415,0.07571449,0.02749216],"genre_scores_gemma":[0.8187223,0.00221269,0.1352032,0.00172571,0.0003482831,0.0009645355,0.0281838,0.002871855,0.009767715],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.008843049,"threshold_uncertainty_score":0.0258944,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1115696166550883,"score_gpt":0.3477993786376039,"score_spread":0.2362297619825156,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}