{"id":"W4385570189","doi":"10.18653/v1/2023.findings-acl.139","title":"This prompt is measuring <mask>: evaluating bias evaluation in language models","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"University of Edinburgh; UK Research and Innovation","keywords":"Computer science; Scope (computer science); Measure (data warehouse); Taxonomy (biology); Language model; Field (mathematics); Gender bias; Natural language processing; Artificial intelligence; Data science; Psychology; Data mining; Social psychology; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004709222,0.0001388313,0.000161245,0.0003120271,0.00008937361,0.0001485696,0.0006138278,0.00006758753,0.0001789253],"category_scores_gemma":[0.0002906124,0.0001301168,0.00005375099,0.0009119716,0.000009459369,0.000760723,0.00034682,0.0001411362,0.0003062232],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001516712,"about_ca_system_score_gemma":0.0001690158,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000301471,"about_ca_topic_score_gemma":0.0001309546,"domain_scores_codex":[0.9972839,0.0002231718,0.0003675965,0.0005595784,0.001186054,0.0003797252],"domain_scores_gemma":[0.9989297,0.0001178267,0.00007805062,0.0006718513,0.000142356,0.00006026225],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005919014,0.00006413735,0.001198708,0.00008011515,0.00002768563,0.00002898322,0.06268566,0.3080565,0.01961103,0.01620401,0.001382661,0.5906546],"study_design_scores_gemma":[0.0003510657,0.00001766922,0.0002847507,0.00005564444,0.000004822216,0.000002442536,0.0001443078,0.9885219,0.002113466,0.008325529,0.0000213442,0.000157092],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7253722,0.0001742704,0.2456644,0.001298515,0.0002893136,0.0006494634,8.008876e-7,0.0006507508,0.02590032],"genre_scores_gemma":[0.9550249,0.000005892527,0.04260605,0.0002562267,0.00006328961,0.00008718483,0.00000294413,0.00001376792,0.001939729],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6804653,"threshold_uncertainty_score":0.5306008,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2923696840988225,"score_gpt":0.3645737548356549,"score_spread":0.07220407073683238,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}