{"id":"W4285199616","doi":"10.18653/v1/2022.bigscience-1.3","title":"You reap what you sow: On the Challenges of Bias Evaluation Under Multilingual Settings","year":2022,"lang":"en","type":"preprint","venue":"","topic":"Interpreting and Communication in Healthcare","field":"Health Professions","cited_by":62,"is_retracted":false,"has_abstract":true,"ca_institutions":"Simon Fraser University","funders":"","keywords":"oskar; STELLA (programming language); Computer science; Sociology; Psychology; Artificial intelligence; Art; Art history","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts","research_integrity","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01157246,0.0003238956,0.0004744956,0.0001822964,0.00134011,0.00003084566,0.001188209,0.0006094807,0.004768098],"category_scores_gemma":[0.002674452,0.0002320827,0.0002120811,0.0001210493,0.0001013537,0.00006764794,0.002060671,0.004198231,0.0001193981],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005897767,"about_ca_system_score_gemma":0.001375891,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001725379,"about_ca_topic_score_gemma":0.0004506999,"domain_scores_codex":[0.9877358,0.008839267,0.001274447,0.000616683,0.00109099,0.0004428298],"domain_scores_gemma":[0.9888461,0.005857917,0.001349239,0.002690613,0.001176748,0.00007936388],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"qualitative","study_design_scores_codex":[0.000645212,0.0007002963,0.003479347,0.006831865,0.0005431694,0.000001825464,0.4418921,0.006327077,0.00004854159,0.3551294,0.02145582,0.1629453],"study_design_scores_gemma":[0.0007108654,0.000269861,0.002922941,0.009045489,0.0001490098,0.000001215857,0.907849,0.02054728,0.0001471883,0.03747346,0.02029916,0.0005845554],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5455664,0.0163002,0.00003200323,0.2375448,0.007934115,0.006229495,0.0001186009,0.0004600327,0.1858144],"genre_scores_gemma":[0.9818328,0.006659982,0.0002598423,0.005502609,0.0003265575,0.002134227,0.0001914422,0.00006843133,0.003024101],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4659569,"threshold_uncertainty_score":0.99996,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3826491857415912,"score_gpt":0.5224920837813289,"score_spread":0.1398428980397377,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}