{"id":"W4402667124","doi":"10.18653/v1/2024.acl-long.23","title":"Subtle Biases Need Subtler Measures: Dual Metrics for Evaluating Representative and Affinity Bias in Large Language Models","year":2024,"lang":"en","type":"article","venue":"","topic":"Computational and Text Analysis Methods","field":"Social Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"Brock University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Dual (grammatical number); Natural language processing; Language model; Dual language; Artificial intelligence; Machine learning; Psychology; Linguistics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00527971,0.00008528303,0.0001764916,0.0003357577,0.0002072264,0.0001727984,0.00007356496,0.00005309389,0.0001540982],"category_scores_gemma":[0.005303641,0.00007100807,0.00008968987,0.001333989,0.00006143178,0.0002757031,0.0000491357,0.00009028401,0.000004949733],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005679467,"about_ca_system_score_gemma":0.0001742175,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004875006,"about_ca_topic_score_gemma":0.01024294,"domain_scores_codex":[0.9982486,0.0005805302,0.0002169703,0.0002770919,0.0004540503,0.0002227468],"domain_scores_gemma":[0.9939541,0.005715692,0.00003916664,0.00007108031,0.0001517135,0.0000682672],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001449579,0.0004191594,0.02031208,0.0001455191,0.0004450665,0.00006309994,0.2990194,0.01155483,0.0009349073,0.4596163,0.004580875,0.2027638],"study_design_scores_gemma":[0.0006867524,0.0000688078,0.003068023,0.00006227339,0.0001206428,0.000001104669,0.08374356,0.819475,0.0004545611,0.0905199,0.001497314,0.0003020508],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8668649,0.002419994,0.1161691,0.001118833,0.0001869646,0.000436485,0.00004785165,0.0001285946,0.01262738],"genre_scores_gemma":[0.9823619,0.00005489554,0.01483805,0.00007718361,0.0001125407,0.00002862813,0.0000170299,0.000008586099,0.002501162],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8079202,"threshold_uncertainty_score":0.7369583,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4050501291309129,"score_gpt":0.5104728602784459,"score_spread":0.105422731147533,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}