{"id":"W2125943921","doi":"10.1145/1401890.1401965","title":"Get another label? improving data quality and data mining using multiple, noisy labelers","year":2008,"lang":"en","type":"article","venue":"","topic":"Machine Learning and Data Classification","field":"Computer Science","cited_by":1114,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada; National Science Foundation","keywords":"Computer science; Quality (philosophy); Sequence labeling; Set (abstract data type); Crowdsourcing; Artificial intelligence; Imperfect; Focus (optics); Machine learning; Outsourcing; Labeled data; Data quality; Data mining; Task (project management); Engineering","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0533477,0.001870645,0.004226172,0.002902847,0.002759279,0.005474885,0.00561529,0.005733274,0.001407121],"category_scores_gemma":[0.2065193,0.001772843,0.002245947,0.004026951,0.006119312,0.01644064,0.007519447,0.005497869,0.00112703],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002807251,"about_ca_system_score_gemma":0.003057901,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003152638,"about_ca_topic_score_gemma":0.004161607,"domain_scores_codex":[0.9527618,0.02493188,0.002514974,0.009207833,0.009316074,0.001267407],"domain_scores_gemma":[0.7619519,0.1498884,0.01834544,0.05373462,0.0140682,0.002011444],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003048622,0.001307522,0.06509171,0.001938647,0.001354966,0.0006406903,0.005913186,0.1618992,0.02504838,0.04951632,0.009231757,0.6750091],"study_design_scores_gemma":[0.0003331303,0.0009017854,0.01203851,0.000379039,0.0004264354,0.0009377563,0.001500675,0.7234598,0.03792428,0.2099879,0.01182162,0.0002890115],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.05885515,0.001062335,0.934046,0.003378188,0.0000778621,0.0001509867,0.0002091013,0.001351374,0.0008690184],"genre_scores_gemma":[0.3994365,0.0004214309,0.5965488,0.001056486,0.0002192036,0.0002825489,0.0005853832,0.0004156631,0.001033847],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.0533477,"threshold_uncertainty_score":0.282133,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3107738534872337,"score_gpt":0.3854225652339457,"score_spread":0.07464871174671206,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}