{"id":"W4389519305","doi":"10.18653/v1/2023.emnlp-main.160","title":"The Skipped Beat: A Study of Sociopragmatic Understanding in LLMs for 64 Languages","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"Alliance de recherche numérique du Canada; Social Sciences and Humanities Research Council of Canada; Natural Sciences and Engineering Research Council of Canada; Canada Research Chairs","keywords":"Sparrow; Computer science; Benchmark (surveying); Meaning (existential); Spoken language; Scripting language; Task (project management); Natural language processing; Artificial intelligence; Psychology; Programming language; Geography","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003493376,0.002400919,0.001101236,0.001639,0.0009409142,0.001907883,0.002861953,0.0020432,0.005166216],"category_scores_gemma":[0.01642372,0.0005949992,0.001430445,0.001218676,0.001064055,0.005573087,0.002936103,0.003687366,0.003418668],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001625745,"about_ca_system_score_gemma":0.001210753,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0134967,"about_ca_topic_score_gemma":0.02132989,"domain_scores_codex":[0.9975228,0.001072373,0.000173087,0.0007868471,0.0002731728,0.0001717143],"domain_scores_gemma":[0.9923306,0.005357261,0.0002404571,0.001116576,0.0006181124,0.0003369751],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.003157479,0.001512063,0.0421534,0.003261711,0.001213938,0.002059674,0.004206073,0.1654101,0.01669823,0.005904991,0.1083582,0.6460642],"study_design_scores_gemma":[0.0003314455,0.001421782,0.02132162,0.0002523564,0.0002621302,0.001050632,0.002912169,0.9148727,0.01452698,0.01198167,0.03087721,0.0001892313],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8718883,0.009259558,0.05361715,0.001915058,0.0007314334,0.0004509769,0.01941139,0.02520641,0.0175198],"genre_scores_gemma":[0.8842978,0.0008218745,0.04088651,0.0009618299,0.0001575377,0.000412901,0.06276456,0.001582486,0.008114493],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0134967,"threshold_uncertainty_score":0.02683628,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08510672602283446,"score_gpt":0.3370588224076921,"score_spread":0.2519520963848576,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}