{"id":"W4385573227","doi":"10.18653/v1/2022.blackboxnlp-1.24","title":"Probing GPT-3’s Linguistic Knowledge on Semantic Tasks","year":2022,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"York University","keywords":"Computer science; Sentence; Natural language processing; Negation; Artificial intelligence; Linguistics; Semantic memory; Semantic role labeling; Psychology; Cognition","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.007151786,0.00145668,0.0008493399,0.0007336991,0.0004967852,0.001870249,0.002609673,0.001578488,0.005707027],"category_scores_gemma":[0.05399452,0.0006972283,0.000957386,0.0006155379,0.0009819083,0.005745971,0.002929611,0.00329019,0.003055653],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001187866,"about_ca_system_score_gemma":0.00130333,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003062686,"about_ca_topic_score_gemma":0.002442031,"domain_scores_codex":[0.9951199,0.001928679,0.0004675297,0.001125639,0.001039592,0.0003186905],"domain_scores_gemma":[0.9570244,0.031488,0.001161773,0.006816366,0.003050749,0.0004587143],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.001877569,0.001312812,0.03694328,0.001441762,0.0002717427,0.001277304,0.01058317,0.05334785,0.06701346,0.01119827,0.03578745,0.7789453],"study_design_scores_gemma":[0.0002935835,0.0009571184,0.02460435,0.0002627786,0.0002314634,0.001352793,0.002295987,0.795462,0.1007624,0.03004297,0.04345882,0.0002757175],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7072444,0.0003812509,0.221417,0.002120738,0.0002298151,0.0009457194,0.003712519,0.03362967,0.03031882],"genre_scores_gemma":[0.8532448,0.0001857975,0.1328276,0.0008985407,0.00004282122,0.0006661128,0.006368586,0.002032756,0.003732982],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.007151786,"threshold_uncertainty_score":0.03782266,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03223378775137108,"score_gpt":0.268688338529988,"score_spread":0.2364545507786169,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}