{"id":"W4385573227","doi":"10.18653/v1/2022.blackboxnlp-1.24","title":"Probing GPT-3’s Linguistic Knowledge on Semantic Tasks","year":2022,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"York University","keywords":"Computer science; Sentence; Natural language processing; Negation; Artificial intelligence; Linguistics; Semantic memory; Semantic role labeling; Psychology; Cognition","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002949643,0.00008943271,0.00009649866,0.0001003579,0.0002739258,0.0000706998,0.000765348,0.0000150118,0.0001261986],"category_scores_gemma":[0.00005789411,0.00008548029,0.00004050209,0.0002448127,0.000008433251,0.00005380172,0.0006676631,0.0002012651,0.0001472161],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008729409,"about_ca_system_score_gemma":0.00007978907,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002966696,"about_ca_topic_score_gemma":0.000007134011,"domain_scores_codex":[0.9989738,0.00006471595,0.0001570901,0.0003621477,0.0002122786,0.000229899],"domain_scores_gemma":[0.9992722,0.00007677811,0.0000352295,0.0005319358,0.00003156393,0.00005227107],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000003677383,0.0002121874,0.0001787395,0.00003648271,0.00001445586,0.00006477378,0.006745107,0.01276714,0.0004151255,0.9309598,0.002515921,0.04608658],"study_design_scores_gemma":[0.0002066379,0.0001133351,0.00007079278,0.00001500823,0.000003497846,0.00002142606,0.00005605848,0.955847,0.0003798801,0.008298537,0.03478503,0.0002028304],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01739666,0.0001229428,0.8299821,0.0008594681,0.001879638,0.0002356468,7.193436e-7,0.0005358627,0.148987],"genre_scores_gemma":[0.9803652,6.187605e-7,0.01378069,0.0004590146,0.0001179976,0.00003336694,7.419429e-7,0.000008061047,0.005234348],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9629685,"threshold_uncertainty_score":0.3485785,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03223378775137108,"score_gpt":0.268688338529988,"score_spread":0.2364545507786169,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}