{"id":"W3019416653","doi":"10.18653/v1/2021.acl-long.416","title":"StereoSet: Measuring stereotypical bias in pretrained language models","year":2021,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":102,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University; Canadian Institute for Advanced Research","funders":"","keywords":"Stereotype (UML); Race (biology); Set (abstract data type); Scale (ratio); Language model; Computer science; Measure (data warehouse); Natural language processing; Artificial intelligence; Psychology; Cognitive psychology; Social psychology; Data mining; Geography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006444255,0.0003220999,0.0004756853,0.0002534571,0.00003076351,0.0005317605,0.001736609,0.0003358764,0.00005447397],"category_scores_gemma":[0.00007870488,0.0003136744,0.000160435,0.0002216437,0.00001827954,0.0004268079,0.003531727,0.0009173586,0.0000103932],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001563308,"about_ca_system_score_gemma":0.00028668,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004530553,"about_ca_topic_score_gemma":0.0005467496,"domain_scores_codex":[0.9970995,0.0002189646,0.0005699907,0.001122979,0.0005414976,0.0004470445],"domain_scores_gemma":[0.9978216,0.00008958742,0.0001172715,0.001763278,0.00008059257,0.0001276484],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00004336753,0.0007154866,0.004406173,0.001593951,0.0002899267,0.002053395,0.08241525,0.4537512,0.001326288,0.1260057,0.0001758248,0.3272235],"study_design_scores_gemma":[0.0003034885,0.000009282033,0.0005791088,0.0003155115,0.000004798057,0.00001216635,0.000285609,0.9917388,0.0005845854,0.005767619,0.00001106035,0.0003879874],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2665594,0.0004011492,0.7238649,0.0003398242,0.0005001613,0.0002516182,0.000002176851,0.0002448485,0.007836044],"genre_scores_gemma":[0.8286027,0.0000130245,0.1704733,0.0002583738,0.0000928041,0.00003357705,0.000008865091,0.00001930938,0.0004979825],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5620434,"threshold_uncertainty_score":0.9999315,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1300569881173056,"score_gpt":0.2873309265116926,"score_spread":0.157273938394387,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}