{"id":"W7082273943","doi":"10.48448/rqsr-d504","title":"CulturalBench: A Robust, Diverse and Challenging Benchmark for Measuring LMs' Cultural Knowledge Through Human-AI Red-Teaming","year":2025,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"Geochemistry and Geologic Mapping","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"Overfitting; Set (abstract data type); Benchmark (surveying); Construct (python library); Frontier; Cultural diversity","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007470987,0.0004936092,0.0004777854,0.0003167762,0.00123398,0.0004790276,0.00195527,0.0002877718,0.00009787758],"category_scores_gemma":[0.000376257,0.0004145501,0.0001171823,0.0008698257,0.0006360905,0.0007986421,0.001408891,0.0004166653,0.00001039139],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001447121,"about_ca_system_score_gemma":0.0002946575,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001566942,"about_ca_topic_score_gemma":0.0001657666,"domain_scores_codex":[0.9968723,0.00004301673,0.0003800177,0.001426593,0.0004437017,0.0008343589],"domain_scores_gemma":[0.9982856,0.0001118857,0.0002757244,0.0007185353,0.0004490316,0.0001592341],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00002088867,0.0005564867,0.0004417865,0.003395883,0.0004287021,0.00008732532,0.0159489,0.003464031,0.0121748,0.4662464,0.4157336,0.08150118],"study_design_scores_gemma":[0.001916113,0.000258062,0.0001289328,0.003458962,0.0001566071,0.00009287279,0.003883235,0.1425347,0.003043239,0.02747317,0.8142763,0.002777783],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.0001089354,0.002395091,0.1448313,0.002634309,0.00119099,0.0007980057,0.00001893303,0.0004903272,0.8475321],"genre_scores_gemma":[0.1687322,0.0005343243,0.1273141,0.0004908542,0.001295925,0.0001471296,0.00007998686,0.00007620765,0.7013293],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.4387732,"threshold_uncertainty_score":0.9998307,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06148170898981459,"score_gpt":0.309999045427913,"score_spread":0.2485173364380984,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}