{"id":"W4308017781","doi":"10.3758/s13428-022-01999-x","title":"Evaluating CloudResearch’s Approved Group as a solution for problematic data quality on MTurk","year":2022,"lang":"en","type":"article","venue":"Behavior Research Methods","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":227,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University","funders":"","keywords":"Vetting; Data quality; Psychology; Quality (philosophy); Sample (material); Applied psychology; Computer science; Computer security; Operations management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.06308426,0.0005612744,0.0005866969,0.001488889,0.00253437,0.003907966,0.001991632,0.001395024,0.00468292],"category_scores_gemma":[0.2851399,0.0004387481,0.0006518327,0.001190392,0.003005466,0.004322717,0.003399157,0.001451848,0.002559501],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002406168,"about_ca_system_score_gemma":0.004162272,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005248482,"about_ca_topic_score_gemma":0.007339237,"domain_scores_codex":[0.9525912,0.02787052,0.003009371,0.00415513,0.01060504,0.001768689],"domain_scores_gemma":[0.6823344,0.1552774,0.05320716,0.04585618,0.05178805,0.01153684],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.006818337,0.004300511,0.6717526,0.0007442727,0.0002003796,0.0004390627,0.04773401,0.001209746,0.008174689,0.006373896,0.02351073,0.2287418],"study_design_scores_gemma":[0.001325369,0.0122567,0.8352779,0.0005192125,0.0003399206,0.0006016122,0.03624173,0.02110735,0.01556551,0.006748512,0.06966621,0.0003500526],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9818307,0.0000906783,0.006999319,0.001230251,0.0001128742,0.001780067,0.0003792909,0.0005931922,0.00698365],"genre_scores_gemma":[0.9756415,0.00005827204,0.01733141,0.000804005,0.00006440888,0.002535358,0.0006200693,0.0002656564,0.002679162],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9369158,"threshold_uncertainty_score":0.3336254,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.8107478869005017,"score_gpt":0.6970883390979823,"score_spread":0.1136595478025194,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}