{"id":"W4206369543","doi":"10.2139/ssrn.3989656","title":"Learning True Labels from Noisy Crowd-Sourced Annotations","year":2021,"lang":"en","type":"article","venue":"SSRN Electronic Journal","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Computer science; Crowdsourcing; Crowd sourcing; Artificial intelligence; Information retrieval; Natural language processing; Data science; World Wide Web","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004478099,0.003626805,0.00270388,0.004603778,0.002279493,0.003794077,0.003478844,0.005523432,0.002801753],"category_scores_gemma":[0.02287888,0.001277508,0.001619073,0.002806697,0.002299779,0.005778939,0.004251229,0.004575815,0.005141062],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00157102,"about_ca_system_score_gemma":0.002987328,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.009800058,"about_ca_topic_score_gemma":0.01683003,"domain_scores_codex":[0.9931657,0.001424181,0.0002528281,0.002626651,0.002042397,0.0004882759],"domain_scores_gemma":[0.9837236,0.009738217,0.0007169915,0.002484484,0.002886287,0.0004503636],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.003870281,0.001507218,0.02011719,0.002267767,0.0007083541,0.002620877,0.002049544,0.115117,0.04801791,0.02040121,0.1130042,0.6703184],"study_design_scores_gemma":[0.0002100937,0.0002891429,0.005010009,0.0003621442,0.0003141272,0.0004787205,0.0009086127,0.864638,0.02224368,0.07641566,0.02899354,0.0001362151],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1327807,0.005205263,0.8152251,0.003770516,0.00274542,0.0005667121,0.01248964,0.01308023,0.01413641],"genre_scores_gemma":[0.6483491,0.001863842,0.2996724,0.001216592,0.002134503,0.000654099,0.02591614,0.001476825,0.01871642],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.009800058,"threshold_uncertainty_score":0.02368271,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.005698329588986957,"score_gpt":0.2147317712613018,"score_spread":0.2090334416723148,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}