{"id":"W4399785296","doi":"10.1016/j.ipm.2024.103809","title":"Are LLMs good at structured outputs? A benchmark for evaluating structured output capabilities in LLMs","year":2024,"lang":"en","type":"article","venue":"Information Processing & Management","topic":"Imbalanced Data Classification Techniques","field":"Computer Science","cited_by":60,"is_retracted":false,"has_abstract":false,"ca_institutions":"McGill University","funders":"National Natural Science Foundation of China","keywords":"Benchmark (surveying); Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0106487,0.001232775,0.0007879948,0.002847513,0.0008064102,0.003162939,0.001570593,0.00215718,0.005366604],"category_scores_gemma":[0.08095416,0.0003267534,0.0006367647,0.002374233,0.001364825,0.006992212,0.003142555,0.001428085,0.002051217],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001233375,"about_ca_system_score_gemma":0.0020072,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003063819,"about_ca_topic_score_gemma":0.002946889,"domain_scores_codex":[0.9924003,0.003365753,0.0007977803,0.0009822297,0.002005003,0.0004489078],"domain_scores_gemma":[0.9277063,0.04905203,0.003221053,0.009919933,0.007881326,0.002219389],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.009246466,0.002138838,0.08103255,0.002611279,0.000929056,0.0005245943,0.001717322,0.2696316,0.01611589,0.02821601,0.04249147,0.5453449],"study_design_scores_gemma":[0.0005175144,0.002837252,0.01406865,0.0003486695,0.0001779772,0.0002375057,0.001501293,0.8925801,0.03027677,0.04033659,0.01701857,0.00009903836],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8347823,0.003132547,0.1144588,0.003173382,0.0003580886,0.0004639877,0.008036984,0.01497299,0.02062103],"genre_scores_gemma":[0.9134699,0.0003505881,0.07550642,0.0002985371,0.00009705407,0.0001975103,0.007630764,0.0005103105,0.001938956],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0106487,"threshold_uncertainty_score":0.05631638,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02699792293706213,"score_gpt":0.3020599101299465,"score_spread":0.2750619871928844,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}