{"id":"W2971569798","doi":"10.18653/v1/d19-1006","title":"How Contextual are Contextualized Word Representations? Comparing the Geometry of BERT, ELMo, and GPT-2 Embeddings","year":2019,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":60,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Word (group theory); Similarity (geometry); Context (archaeology); Computer science; Embedding; Cosine similarity; Natural language processing; Task (project management); Artificial intelligence; Variance (accounting); Linguistics; Pattern recognition (psychology); Image (mathematics); History; Philosophy","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006308318,0.0003351465,0.0007810699,0.0002222917,0.0001293104,0.0007464007,0.001825991,0.0002182685,0.00001672644],"category_scores_gemma":[0.0003038412,0.0002492753,0.0001785007,0.0002334323,0.0001655999,0.0003583117,0.003733489,0.0006496249,0.000007086249],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004716387,"about_ca_system_score_gemma":0.00009541451,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006473709,"about_ca_topic_score_gemma":0.0001192394,"domain_scores_codex":[0.9974257,0.0001770215,0.000578934,0.000952478,0.0005325007,0.0003334189],"domain_scores_gemma":[0.9963535,0.0006705467,0.0006942761,0.001900207,0.000285432,0.00009600395],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001747597,0.0003636245,0.2335212,0.001916132,0.001569718,0.00004772351,0.02717924,0.02502817,0.001365134,0.6022791,0.02108919,0.08546606],"study_design_scores_gemma":[0.002148799,0.00004574791,0.02802543,0.0007390559,0.00008992337,0.00004572667,0.005236313,0.9484156,0.001267432,0.01010889,0.002929121,0.0009479766],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2798402,0.001093545,0.7096664,0.003609154,0.0007505333,0.0007381329,0.000008710141,0.0001508155,0.004142575],"genre_scores_gemma":[0.9755782,0.00006525419,0.0197771,0.0003500468,0.00008887046,0.00003831064,0.000007778914,0.00001752421,0.004076884],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9233874,"threshold_uncertainty_score":0.9999959,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05486834225772868,"score_gpt":0.2959037959050607,"score_spread":0.241035453647332,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}