{"id":"W4385571325","doi":"10.18653/v1/2023.acl-long.230","title":"A fine-grained comparison of pragmatic language understanding in humans and language models","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":47,"is_retracted":false,"has_abstract":true,"ca_institutions":"Carleton University","funders":"","keywords":"Literal (mathematical logic); Computer science; Pragmatics; Heuristic; Set (abstract data type); Language model; Natural language processing; Artificial intelligence; Human language; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002833402,0.00006989916,0.0001657198,0.0001963526,0.00002796484,0.00003562835,0.0002190307,0.00003086018,0.000007712566],"category_scores_gemma":[0.00003289846,0.00006286956,0.00001944665,0.0003744657,0.00001765132,0.0001942793,0.0001675552,0.00007033124,0.000003958648],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002838617,"about_ca_system_score_gemma":0.00001569984,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002123685,"about_ca_topic_score_gemma":0.0008223375,"domain_scores_codex":[0.9992391,0.00003445442,0.0002180282,0.0001916593,0.0001500953,0.000166623],"domain_scores_gemma":[0.9995444,0.0001367484,0.00004644677,0.0002338578,0.00000741862,0.0000310619],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005906233,0.00007991063,0.008192207,0.0002711458,0.00002097472,0.00005831734,0.1940359,0.02317629,0.005177458,0.7608261,0.0003082565,0.007847528],"study_design_scores_gemma":[0.0003054051,0.00001686511,0.0003852358,0.00005594084,0.000001890551,7.734507e-7,0.008737219,0.9796343,0.0002572279,0.01053109,9.73679e-7,0.00007307365],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4564112,0.00008051521,0.5410552,0.0002100333,0.00002230546,0.00007901329,6.643144e-7,0.0001042039,0.00203687],"genre_scores_gemma":[0.9896802,0.000001737362,0.01001152,0.00002569041,0.00000790032,0.000004609954,0.000002235333,0.000004880557,0.0002612696],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.956458,"threshold_uncertainty_score":0.2563746,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07989764519677654,"score_gpt":0.3256151878462655,"score_spread":0.2457175426494889,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}