{"id":"W3212676313","doi":"10.2196/28749","title":"Crowdsourcing for Machine Learning in Public Health Surveillance: Lessons Learned From Amazon Mechanical Turk","year":2021,"lang":"en","type":"article","venue":"Journal of Medical Internet Research","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":true,"ca_institutions":"Public Health Agency of Canada; University of Calgary","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Machine learning; Convolutional neural network; Inference; Citizen science; Ground truth; Context (archaeology); Data science; Deep learning; Quality (philosophy); Data quality; World Wide Web; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02467144,0.001076877,0.001418641,0.00142939,0.002186304,0.003789478,0.002736583,0.002981422,0.00415402],"category_scores_gemma":[0.09385797,0.000714048,0.0009478344,0.00235521,0.004730529,0.005031003,0.00375099,0.002758737,0.001322583],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003569093,"about_ca_system_score_gemma":0.003817396,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.04601992,"about_ca_topic_score_gemma":0.03285469,"domain_scores_codex":[0.9841957,0.01162702,0.0003547301,0.001465129,0.001952451,0.000405102],"domain_scores_gemma":[0.8874958,0.09579439,0.001892426,0.007545252,0.006055705,0.001216496],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.001304348,0.0007947406,0.05202067,0.001828682,0.0004510912,0.0009914222,0.007874126,0.1536474,0.001941382,0.1320901,0.08202839,0.5650277],"study_design_scores_gemma":[0.0001981492,0.0002542055,0.01682451,0.0008082719,0.00007344748,0.0001852479,0.002746027,0.585153,0.001863999,0.3369605,0.05473181,0.0002007503],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2189082,0.04074183,0.521255,0.1346636,0.002323936,0.001655854,0.003050835,0.002060507,0.07534017],"genre_scores_gemma":[0.8655013,0.00502174,0.1167427,0.005387967,0.001260674,0.0007793609,0.0009412265,0.0003002502,0.00406469],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.04601992,"threshold_uncertainty_score":0.1304767,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1890378337671646,"score_gpt":0.4315017016306562,"score_spread":0.2424638678634916,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}