{"id":"W4403847593","doi":"10.1111/jedm.12420","title":"Algorithmic Bias in BERT for Response Accuracy Prediction: A Case Study for Investigating Population Validity","year":2024,"lang":"en","type":"article","venue":"Journal of Educational Measurement","topic":"Imbalanced Data Classification Techniques","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"Concordia University of Edmonton; University of Alberta","funders":"","keywords":"Item response theory; Population; Psychology; Test validity; Predictive validity; Statistics; Computer science; Econometrics; Psychometrics; Mathematics; Clinical psychology; Demography","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.05274212,0.000623413,0.0006988132,0.001080537,0.0009960097,0.00197478,0.001684462,0.001478726,0.001558737],"category_scores_gemma":[0.1854301,0.0003285185,0.0008627902,0.001666339,0.001864269,0.002004898,0.001741286,0.002739415,0.0004667262],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002296565,"about_ca_system_score_gemma":0.00163816,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008060279,"about_ca_topic_score_gemma":0.008064169,"domain_scores_codex":[0.9670199,0.02681012,0.001095811,0.002762471,0.001809713,0.0005019567],"domain_scores_gemma":[0.7589012,0.2026378,0.008372414,0.01981615,0.009201323,0.001071099],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.001233786,0.0009111991,0.7872154,0.000150769,0.0004441109,0.000671887,0.004194601,0.08539019,0.001101402,0.01234552,0.003997508,0.1023437],"study_design_scores_gemma":[0.0001604948,0.000753551,0.153401,0.0002311953,0.0001980655,0.0006604039,0.002299721,0.7998552,0.007226409,0.02894317,0.006122136,0.0001486947],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9255655,0.0002005982,0.069655,0.001038499,0.00007127145,0.0001933204,0.0003778416,0.0002574976,0.002640413],"genre_scores_gemma":[0.9834839,0.00002326019,0.01569756,0.0001237459,0.00001802472,0.00009938664,0.0002714812,0.00003913281,0.0002434786],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.05274212,"threshold_uncertainty_score":0.2789304,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3090948321123259,"score_gpt":0.4135517798954063,"score_spread":0.1044569477830803,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}