{"id":"W6966905796","doi":"10.48448/y8ry-5s89","title":"This prompt is measuring <mask>: evaluating bias evaluation in language models","year":2022,"lang":"en","type":"other","venue":"Open MIND","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Taxonomy (biology); Scope (computer science); Field (mathematics); Language model; Measure (data warehouse); Gender bias","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0883659,0.001191005,0.00126597,0.002546189,0.002290349,0.00852552,0.002171044,0.003394692,0.01232947],"category_scores_gemma":[0.4772062,0.000565415,0.001086881,0.002871187,0.006532011,0.01340022,0.006894417,0.003258785,0.002655714],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003324865,"about_ca_system_score_gemma":0.004749848,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003338062,"about_ca_topic_score_gemma":0.003661029,"domain_scores_codex":[0.8779857,0.09432933,0.005335734,0.004738247,0.01656211,0.001048833],"domain_scores_gemma":[0.4668117,0.428888,0.0227595,0.05403478,0.0249121,0.002593877],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00313515,0.0009681909,0.07055268,0.003438476,0.0003847468,0.0002742702,0.03874143,0.0122244,0.007139915,0.2678561,0.03045407,0.5648305],"study_design_scores_gemma":[0.0007693314,0.002313839,0.03420091,0.004151117,0.00062896,0.0006928372,0.0172465,0.08358079,0.03391266,0.691976,0.1298823,0.0006448088],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2947267,0.002011957,0.5697455,0.01403594,0.001051342,0.003434614,0.004880826,0.005454637,0.1046586],"genre_scores_gemma":[0.7251228,0.000444368,0.2614405,0.002696038,0.0001530016,0.002734865,0.001709004,0.001101484,0.004597939],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.0883659,"threshold_uncertainty_score":0.4673291,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3598726488151682,"score_gpt":0.4272415834876544,"score_spread":0.06736893467248617,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}