{"id":"W3212676313","doi":"10.2196/28749","title":"Crowdsourcing for Machine Learning in Public Health Surveillance: Lessons Learned From Amazon Mechanical Turk","year":2021,"lang":"en","type":"article","venue":"Journal of Medical Internet Research","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":11,"is_retracted":false,"has_abstract":true,"ca_institutions":"Public Health Agency of Canada; University of Calgary","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Machine learning; Convolutional neural network; Inference; Citizen science; Ground truth; Context (archaeology); Data science; Deep learning; Quality (philosophy); Data quality; World Wide Web; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","research_integrity"],"consensus_categories":[],"category_scores_codex":[0.02535991,0.0001753148,0.0006528065,0.0004875281,0.0001884033,0.00069678,0.001767451,0.0002694138,0.0001944727],"category_scores_gemma":[0.01224998,0.0001484133,0.0002281881,0.000927393,0.0001289029,0.0002777311,0.0008954672,0.003459489,0.00001250744],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003872397,"about_ca_system_score_gemma":0.002237566,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008744395,"about_ca_topic_score_gemma":0.001040364,"domain_scores_codex":[0.9917803,0.002622987,0.001078696,0.0005588165,0.002942121,0.00101702],"domain_scores_gemma":[0.9939044,0.003530285,0.0003402304,0.0004535843,0.0008938113,0.0008776712],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001650616,0.0007114576,0.005459959,0.0001450713,0.0001847924,0.002020986,0.004922496,0.0003502087,0.00307114,0.01690735,0.01387731,0.9521841],"study_design_scores_gemma":[0.005312411,0.001446067,0.003038062,0.002156356,0.000005772331,0.0013785,0.002032956,0.7916958,0.005350622,0.01185961,0.1752198,0.000503995],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2416781,0.003142007,0.5049837,0.2489615,0.0008404497,0.0001578661,0.000004376937,0.00005699928,0.0001749863],"genre_scores_gemma":[0.9913194,0.0007259455,0.006379189,0.0005756292,0.0004843135,0.000006148472,0.000009348067,0.00002558002,0.0004744891],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9516802,"threshold_uncertainty_score":0.9988396,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1890378337671646,"score_gpt":0.4315017016306562,"score_spread":0.2424638678634916,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}