{"id":"W3121928352","doi":"","title":"Get Another Label? Improving Data Quality and Data Mining Using Multiple, Noisy Labelers","year":2008,"lang":"en","type":"article","venue":"The Faculty Digital Archive (New York University)","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":98,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; National Science Foundation","keywords":"Computer science; Quality (philosophy); Sequence labeling; Set (abstract data type); Imperfect; Crowdsourcing; Artificial intelligence; Focus (optics); Outsourcing; Machine learning; Data quality; Data mining; Task (project management); Engineering; Operations management","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002636463,0.0001808517,0.0001723223,0.0001208655,0.0005847756,0.0002859009,0.003732926,0.00003799778,0.0000022185],"category_scores_gemma":[0.0002932131,0.0001499315,0.00002342739,0.0004202051,0.0002457277,0.002421164,0.004218226,0.0002097754,0.00001623761],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003584612,"about_ca_system_score_gemma":0.0001669117,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001446121,"about_ca_topic_score_gemma":0.0002014252,"domain_scores_codex":[0.9983451,0.0001425151,0.000170777,0.000796638,0.0002565727,0.0002883871],"domain_scores_gemma":[0.9971384,0.0003386252,0.0001918752,0.002172295,0.00002845185,0.0001303365],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002910709,0.0003715764,0.148908,0.00009284961,0.000339101,0.0001655307,0.01348501,0.0005567333,0.003735542,0.01352492,0.02467356,0.7938561],"study_design_scores_gemma":[0.002059513,0.00007296279,0.04969462,0.00005714081,0.00005083129,0.00009488981,0.001481662,0.7209167,0.00002492806,0.0002465003,0.2245797,0.0007204562],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2427137,0.00006598484,0.7464391,0.00174435,0.00009822247,0.0002650571,0.003349422,0.0003179844,0.005006137],"genre_scores_gemma":[0.8227534,0.0000294682,0.1675714,0.000324776,0.0001346353,2.107381e-7,0.004457643,0.00002940139,0.004699119],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7931357,"threshold_uncertainty_score":0.6936767,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2440212770188994,"score_gpt":0.3130783756742693,"score_spread":0.06905709865536985,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}