{"id":"W4327644549","doi":"10.1007/978-3-031-28238-6_24","title":"De-biasing Relevance Judgements for Fair Ranking","year":2023,"lang":"en","type":"book-chapter","venue":"Lecture notes in computer science","topic":"Topic Modeling","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":false,"ca_institutions":"Microsoft (Canada); University of Waterloo; Toronto Metropolitan University","funders":"","keywords":"Ranking (information retrieval); Judgement; Relevance (law); Computer science; Set (abstract data type); Artificial intelligence; Biasing; Gender bias; Process (computing); Machine learning; Information retrieval; Psychology; Social psychology; Voltage","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001596673,0.0004573647,0.0004720545,0.0006980423,0.0003723172,0.0007377302,0.003612163,0.0002799903,0.000004581897],"category_scores_gemma":[0.0003398254,0.0004703209,0.0001631863,0.0005273369,0.0002491763,0.0005982023,0.001423225,0.0005976602,0.00003412249],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004630939,"about_ca_system_score_gemma":0.0005334925,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001495804,"about_ca_topic_score_gemma":0.00003481693,"domain_scores_codex":[0.9958169,0.00002492267,0.0005551521,0.001755547,0.0008834256,0.0009639996],"domain_scores_gemma":[0.9969788,0.0009731919,0.0002787319,0.001398025,0.0002285508,0.0001426857],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000003951727,0.000007870072,0.00002726518,0.00007161346,0.000010187,0.00003935022,0.0006856257,0.08856741,0.000132653,0.03988776,0.00003111373,0.8705352],"study_design_scores_gemma":[0.0002224824,0.00004716069,0.00002242373,0.0004543463,0.000005358134,0.00001664818,1.104676e-7,0.6817338,0.0004366086,0.3149882,0.001677138,0.0003956785],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.00005876031,0.0001842618,0.9932772,0.0009174336,0.003207438,0.0005696316,0.000004077825,0.0004089506,0.001372284],"genre_scores_gemma":[0.03281131,0.00003937981,0.9629582,0.001915489,0.0007952558,0.00003105661,0.000003828058,0.00006980662,0.001375657],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.8701395,"threshold_uncertainty_score":0.9997749,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03901985798360569,"score_gpt":0.2754367443815013,"score_spread":0.2364168863978956,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}