{"id":"W4213112533","doi":"10.3758/s13428-022-01801-y","title":"Machine learning to detect invalid text responses: Validation and comparison to existing detection methods","year":2022,"lang":"en","type":"article","venue":"Behavior Research Methods","topic":"Text and Document Classification Technologies","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Natural language processing; Artificial intelligence; Coding (social sciences); Word (group theory); Machine learning; Code (set theory); Quality (philosophy); Information retrieval; Linguistics; Statistics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.02275861,0.0001851518,0.0002848392,0.001516224,0.001654063,0.0005400796,0.001253708,0.00007980692,0.00007271101],"category_scores_gemma":[0.006502448,0.0001943026,0.00005687236,0.003122701,0.00008473345,0.0003389998,0.002964927,0.001150561,0.00002561133],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003834141,"about_ca_system_score_gemma":0.00008714058,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002259169,"about_ca_topic_score_gemma":0.00001192256,"domain_scores_codex":[0.9866241,0.01052492,0.0004719569,0.0008730487,0.0009085853,0.0005974084],"domain_scores_gemma":[0.9960991,0.002480107,0.0001304137,0.0008130161,0.0002312854,0.0002460471],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.00004685579,0.00003236019,0.001042145,0.000006853205,0.000003423523,0.000003652,0.001096814,0.000124702,0.2702682,0.0007376681,0.00004625541,0.7265911],"study_design_scores_gemma":[0.0003741769,0.001790483,0.01348249,0.00001740714,0.00001500716,0.00005423289,0.003130416,0.02073489,0.8468761,0.002943081,0.1101096,0.0004721547],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.1176269,0.0002218424,0.8791175,0.0011784,0.0001945725,0.0008947225,0.000003232785,0.0005742231,0.0001885895],"genre_scores_gemma":[0.3259832,0.00000867265,0.6718034,0.00003296527,0.00001423394,0.001167773,0.000002727692,0.00001921232,0.0009678506],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.7261189,"threshold_uncertainty_score":0.9996457,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.421437191862916,"score_gpt":0.5856823952146699,"score_spread":0.1642452033517539,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}