{"id":"W4400529059","doi":"10.1145/3626772.3661362","title":"Reflections on the Coding Ability of LLMs for Analyzing Market Research Surveys","year":2024,"lang":"en","type":"article","venue":"","topic":"Computational and Text Analysis Methods","field":"Social Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Coding (social sciences); Market research; Data science; Computer science; Business; Marketing; Social science; Sociology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.03129889,0.00003207998,0.00007481174,0.0001348329,0.0006700802,0.0000925636,0.0001327387,0.00002605043,0.001005132],"category_scores_gemma":[0.002486795,0.0000200253,0.0001127586,0.001284241,0.0001857626,0.00004940371,0.00002331997,0.000110341,0.000006862038],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006895825,"about_ca_system_score_gemma":0.0001406592,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002123243,"about_ca_topic_score_gemma":0.004031767,"domain_scores_codex":[0.9967357,0.002509449,0.0001343696,0.0001492805,0.000330827,0.0001404109],"domain_scores_gemma":[0.9854631,0.0141375,0.00001522437,0.0000933731,0.0002625017,0.00002834282],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.000006987738,0.00003017418,0.0009045388,0.00001855629,0.00005995558,1.848984e-7,0.002564448,0.00006654206,0.000467447,0.9497834,0.006829633,0.03926816],"study_design_scores_gemma":[0.00008840825,0.0001115456,0.04599365,0.00007986681,0.00005491954,1.929938e-7,0.01048496,0.03487853,0.0007360221,0.7897886,0.1176147,0.0001685516],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.04304073,0.0001190817,0.2730168,0.01869046,0.0003657618,0.0004286654,0.00001352418,0.00008798489,0.664237],"genre_scores_gemma":[0.9893644,0.00001719565,0.002528702,0.00001831965,0.0001277236,0.00001847878,0.000001245459,0.000002854089,0.007921062],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9463237,"threshold_uncertainty_score":0.9999081,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.449297537108599,"score_gpt":0.5977982595760435,"score_spread":0.1485007224674444,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}