{"id":"W3086958902","doi":"10.18148/srm/2020.v14i3.7639","title":"Automatic Coding of Open-ended Questions into Multiple Classes: Whether and How to Use Double Coded Data","year":2019,"lang":"en","type":"article","venue":"DOAJ (DOAJ: Directory of Open Access Journals)","topic":"Survey Methodology and Nonresponse","field":"Social Sciences","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Coding (social sciences); Computer science; Training set; Artificial intelligence; Statistical learning; Data mining; Machine learning; Natural language processing; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02689865,0.0002307317,0.0008470942,0.0005523594,0.0005525147,0.002061769,0.005103479,0.0001907625,0.003939067],"category_scores_gemma":[0.01676029,0.000213838,0.00007108336,0.001060256,0.0003647498,0.006215981,0.002882065,0.0003345169,0.00001943043],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008511781,"about_ca_system_score_gemma":0.0004832686,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01829273,"about_ca_topic_score_gemma":0.009095444,"domain_scores_codex":[0.9901176,0.007745929,0.0005895425,0.0005744102,0.0005771525,0.0003953785],"domain_scores_gemma":[0.9847736,0.01282869,0.0007107344,0.000970465,0.0003853265,0.0003311292],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.002704829,0.0002751426,0.8490056,0.0000960548,0.0002654509,0.00001224768,0.005227023,0.00003328424,0.1102796,0.0009759933,0.02505698,0.00606774],"study_design_scores_gemma":[0.00170601,0.00002654174,0.9588404,0.0006408765,0.0001038316,0.000004927553,0.002420806,0.0005987762,0.007751853,0.002591803,0.02482615,0.0004880222],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9924664,0.001280318,0.001224088,0.001138377,0.0007144645,0.001282798,0.00009863962,0.00004071422,0.001754233],"genre_scores_gemma":[0.9849269,0.001903984,0.01116149,0.000261163,0.00005877758,0.00003529918,0.00001812901,0.00003238093,0.001601866],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1098347,"threshold_uncertainty_score":0.9989742,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.6833831604247071,"score_gpt":0.6409189352124031,"score_spread":0.04246422521230409,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}