{"id":"W4308017781","doi":"10.3758/s13428-022-01999-x","title":"Evaluating CloudResearch’s Approved Group as a solution for problematic data quality on MTurk","year":2022,"lang":"en","type":"article","venue":"Behavior Research Methods","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":227,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University","funders":"","keywords":"Vetting; Data quality; Psychology; Quality (philosophy); Sample (material); Applied psychology; Computer science; Computer security; Operations management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":[],"category_scores_codex":[0.1043095,0.0002286181,0.0003895946,0.0004739044,0.00261186,0.0006045522,0.003978262,0.00008561958,0.00006778041],"category_scores_gemma":[0.008200325,0.0002234739,0.0001384751,0.001415976,0.0001948712,0.0004157061,0.005341304,0.001433607,0.00002370676],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004965728,"about_ca_system_score_gemma":0.0005813192,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006246304,"about_ca_topic_score_gemma":0.00001685032,"domain_scores_codex":[0.9773176,0.01601531,0.0006908594,0.001602242,0.003049981,0.001323994],"domain_scores_gemma":[0.9894028,0.005648254,0.0002078066,0.003932821,0.000506647,0.0003016139],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001582901,0.0007319215,0.00006027553,0.0001888793,0.00002173641,0.00001795226,0.001450766,0.0003025257,0.1853862,0.01780391,0.001745165,0.7921324],"study_design_scores_gemma":[0.001837761,0.003254912,0.001396221,0.000110421,0.00003307175,0.00007080341,0.001201634,0.9497103,0.008023578,0.021653,0.01209036,0.0006179208],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.0862129,0.0001487712,0.9060751,0.001973444,0.0004548134,0.004231249,0.00005615246,0.0002899121,0.000557709],"genre_scores_gemma":[0.1469935,0.000003175631,0.8476409,0.0001003188,0.0001329357,0.003915411,0.0001028121,0.00005018856,0.001060804],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.9494078,"threshold_uncertainty_score":0.9986866,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.8107478869005017,"score_gpt":0.6970883390979823,"score_spread":0.1136595478025194,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}