{"id":"W4404782882","doi":"10.18653/v1/2024.emnlp-main.764","title":"A Systematic Survey and Critical Review on Evaluating Large Language Models: Challenges, Limitations, and Recommendations","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":50,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; Compute Canada","keywords":"Computer science; Data science; Management science; Engineering","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00280593,0.00008672179,0.0001799425,0.00006731205,0.00008260742,0.0001804018,0.0001331049,0.00002696583,0.000007177078],"category_scores_gemma":[0.001383007,0.0000682007,0.00001926304,0.0001220164,0.000009402408,0.0003595308,0.0001171757,0.0001002727,0.00001886569],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00001755857,"about_ca_system_score_gemma":0.00002879591,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001363287,"about_ca_topic_score_gemma":0.00007377688,"domain_scores_codex":[0.9986533,0.0004417973,0.0002661218,0.0003288145,0.0001729879,0.0001369765],"domain_scores_gemma":[0.9973989,0.002162509,0.00001978719,0.000299217,0.00005935694,0.00006028541],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[3.215489e-7,0.00002246718,0.000002120635,0.0265576,0.00001787345,0.000005518232,0.002131265,0.00001334354,0.000002608693,0.9279278,0.0004171116,0.04290198],"study_design_scores_gemma":[0.00005037149,0.00003545508,0.00003047421,0.01502959,0.0000177944,0.00001894237,0.0002389672,0.979029,6.359233e-7,0.005415425,0.00004224241,0.0000911075],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0004939535,0.3334924,0.6427769,0.02027511,0.0001777991,0.0004400241,0.000008245943,0.0002198187,0.002115741],"genre_scores_gemma":[0.8217841,0.05886148,0.116411,0.002345563,0.00004967051,0.0001680314,0.00001688756,0.00002007284,0.0003431898],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9790156,"threshold_uncertainty_score":0.2781144,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3056448804553619,"score_gpt":0.4092377600908385,"score_spread":0.1035928796354765,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}