{"id":"W6947992274","doi":"10.48448/qj49-ww69","title":"A fine-grained comparison of pragmatic language understanding in humans and language models","year":2022,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"Species Distribution and Climate Change","field":"Environmental Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Carleton University","funders":"","keywords":"Pragmatics; Set (abstract data type); Literal (mathematical logic); Language model; Human language; Computational model; Natural language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0004786868,0.0001843267,0.0003070573,0.000286753,0.0001301604,0.00004043983,0.0004053792,0.00007622441,0.08166125],"category_scores_gemma":[0.0000638504,0.00017403,0.0000315856,0.0008060333,0.0008803603,0.0001246325,0.0004624992,0.0002024448,0.00004489393],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009039761,"about_ca_system_score_gemma":0.00003525304,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001565294,"about_ca_topic_score_gemma":0.01982836,"domain_scores_codex":[0.9982877,0.00004505194,0.0002785274,0.0004244666,0.0006292806,0.0003349721],"domain_scores_gemma":[0.9993589,0.00005351828,0.000219997,0.0002756065,0.000003682372,0.00008831167],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"qualitative","study_design_scores_codex":[0.0001404808,0.003850555,0.1646039,0.001573171,0.0001431911,0.0002748628,0.1532156,0.003976455,0.04379001,0.1992282,0.4155469,0.01365669],"study_design_scores_gemma":[0.006280592,0.0008662976,0.01197535,0.0008548206,0.0001924617,0.00003325267,0.8182216,0.1208906,0.0007270218,0.004848842,0.0319328,0.003176365],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.1062193,0.0008956191,0.001233579,0.0002444948,0.0001494542,0.0006246247,0.0003477938,0.000130469,0.8901547],"genre_scores_gemma":[0.9813246,0.00002792205,0.0002585118,0.00007236481,0.00001628813,0.00001961629,0.0001711558,0.00006511438,0.01804439],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8751053,"threshold_uncertainty_score":0.9980572,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05042189979535362,"score_gpt":0.3155308244965832,"score_spread":0.2651089247012296,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}