{"id":"W2944733665","doi":"10.1177/0894439319846622","title":"Automatic Coding of Text Answers to Open-Ended Questions: Should You Double Code the Training Data?","year":2019,"lang":"en","type":"article","venue":"Social Science Computer Review","topic":"Survey Methodology and Nonresponse","field":"Social Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Coding (social sciences); Computer science; Natural language processing; Artificial intelligence; Training set; Statistical model; Machine learning; Information retrieval; Speech recognition; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":[],"category_scores_codex":[0.07131962,0.0001221591,0.0004611019,0.00007669969,0.001362247,0.0001988282,0.004042029,0.00006868735,0.0002738311],"category_scores_gemma":[0.003244739,0.00009033041,0.00007242716,0.00204257,0.001101142,0.0007521802,0.000927259,0.0001938291,0.00009596956],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00009422661,"about_ca_system_score_gemma":0.001114641,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005512493,"about_ca_topic_score_gemma":0.000320215,"domain_scores_codex":[0.9931723,0.004792702,0.0004097756,0.0004976695,0.0006591227,0.0004684368],"domain_scores_gemma":[0.9963332,0.002526742,0.0002262584,0.0005974428,0.0001813259,0.0001349915],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001540394,0.0001065059,0.001302882,0.0004441902,0.00005044735,0.000003997126,0.07218419,0.00001249189,0.0004243928,0.1481737,0.01350226,0.7636409],"study_design_scores_gemma":[0.001506774,0.0003049472,0.05490307,0.006986161,0.0002277733,0.00001440782,0.01302959,0.002790031,0.0001003574,0.003874813,0.9150113,0.00125073],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.3951757,0.04619084,0.07268525,0.224498,0.02242031,0.03027097,0.0003200781,0.001168441,0.2072705],"genre_scores_gemma":[0.8834462,0.009414963,0.07618269,0.02785668,0.001211307,0.0001926575,0.00002955043,0.00004799514,0.00161792],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9015091,"threshold_uncertainty_score":0.9999378,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5952878803019198,"score_gpt":0.5472651412840432,"score_spread":0.04802273901787657,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}