{"id":"W4360978668","doi":"10.1145/3581754.3584136","title":"Supporting Qualitative Analysis with Large Language Models: Combining Codebook with GPT-3 for Deductive Coding","year":2023,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":207,"is_retracted":false,"has_abstract":true,"ca_institutions":"Microsoft (Canada); Research Canada","funders":"","keywords":"Codebook; Generalizability theory; Computer science; Coding (social sciences); Task (project management); Natural language processing; Curiosity; Qualitative analysis; Qualitative research; Task analysis; Artificial intelligence; Qualitative property; Machine learning; Data science; Psychology; Social psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.002018349,0.0004891785,0.0009402714,0.0006037361,0.0002749002,0.000408251,0.001222504,0.0001882028,0.00001204263],"category_scores_gemma":[0.00009501926,0.0003815638,0.0002619812,0.0007663002,0.00006221954,0.0004768706,0.001271393,0.0005746146,0.000006096022],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000172103,"about_ca_system_score_gemma":0.0003980636,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006684395,"about_ca_topic_score_gemma":0.001138542,"domain_scores_codex":[0.996093,0.000214636,0.0006517818,0.001569389,0.0006346354,0.0008365512],"domain_scores_gemma":[0.9967117,0.0007599348,0.0006844622,0.001244308,0.0004408023,0.0001587888],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00007322615,0.00008565257,0.0003449665,0.0003245992,0.003406311,0.0000728419,0.2932501,0.4564806,0.00002694165,0.2448649,0.0001480263,0.0009218192],"study_design_scores_gemma":[0.000627046,0.0001001256,0.00001443684,0.00020717,0.0004386595,0.000003890709,0.03907756,0.939854,0.0002191157,0.01886528,0.000008957141,0.0005837167],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01694644,0.00005165811,0.9787423,0.0004386948,0.0001444796,0.0009462344,0.00007829611,0.0007773293,0.001874556],"genre_scores_gemma":[0.6016538,0.000002630453,0.3956378,0.0001368993,0.00005137406,0.0002952575,0.0001026435,0.0000475241,0.002072118],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.5847073,"threshold_uncertainty_score":0.9998636,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08960805608801989,"score_gpt":0.3901986999772469,"score_spread":0.3005906438892271,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}