{"id":"W4385570189","doi":"10.18653/v1/2023.findings-acl.139","title":"This prompt is measuring <mask>: evaluating bias evaluation in language models","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"University of Edinburgh; UK Research and Innovation","keywords":"Computer science; Scope (computer science); Measure (data warehouse); Taxonomy (biology); Language model; Field (mathematics); Gender bias; Natural language processing; Artificial intelligence; Data science; Psychology; Data mining; Social psychology; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.09417532,0.001452366,0.001654834,0.002974532,0.002268611,0.008408043,0.00213869,0.004584852,0.008770748],"category_scores_gemma":[0.4997372,0.0007114463,0.001543285,0.003046667,0.005853253,0.01250436,0.006732881,0.003948894,0.002135459],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003581235,"about_ca_system_score_gemma":0.005732103,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003528373,"about_ca_topic_score_gemma":0.004065266,"domain_scores_codex":[0.8748277,0.09473871,0.006700133,0.005449112,0.01705811,0.001226215],"domain_scores_gemma":[0.4493327,0.4380099,0.02435762,0.05661663,0.02853182,0.003151351],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.004354732,0.001009408,0.08821746,0.004132224,0.000644497,0.000277014,0.02325848,0.02104625,0.01039471,0.170476,0.03092242,0.6452668],"study_design_scores_gemma":[0.001310372,0.00512911,0.05239546,0.003932854,0.001129449,0.0008655431,0.01209274,0.1477908,0.04748525,0.6290632,0.09788457,0.000920649],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2619459,0.002284394,0.6644248,0.01330549,0.001244103,0.004259778,0.004877265,0.00598043,0.04167787],"genre_scores_gemma":[0.6748155,0.000438972,0.3135806,0.002390432,0.0001942606,0.003227121,0.001586356,0.0008565965,0.002910112],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9058247,"threshold_uncertainty_score":0.4980527,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2923696840988225,"score_gpt":0.3645737548356549,"score_spread":0.07220407073683238,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}