{"id":"W4391220979","doi":"10.1287/mksc.2023.0454","title":"Frontiers: Determining the Validity of Large Language Models for Automated Perceptual Analysis","year":2024,"lang":"en","type":"article","venue":"Marketing Science","topic":"Computational and Text Analysis Methods","field":"Social Sciences","cited_by":131,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Perception; Computer science; Data science; Human language; Natural language processing; Language model; Artificial intelligence; Cognitive psychology; Machine learning; Econometrics; Economics; Psychology; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01630735,0.00006390642,0.0001501737,0.0002763127,0.0008360062,0.0002334918,0.0004600353,0.00002654893,0.00005640084],"category_scores_gemma":[0.002301138,0.00004507177,0.0001952221,0.002911232,0.000483462,0.000288195,0.00007410417,0.00005758837,0.000001335922],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006002837,"about_ca_system_score_gemma":0.0002310021,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001471048,"about_ca_topic_score_gemma":0.0001720601,"domain_scores_codex":[0.9982141,0.0004795403,0.0001880457,0.0002735005,0.0005704641,0.0002743751],"domain_scores_gemma":[0.9977688,0.001780049,0.00006709395,0.0001290827,0.0002031853,0.0000518484],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00009320002,0.0003063875,0.1306297,0.0002662795,0.001199715,0.00001141606,0.3754472,0.05558162,0.002863087,0.06064278,0.01003861,0.3629201],"study_design_scores_gemma":[0.00003841783,0.000006407776,0.02677472,0.00001396792,0.0001849045,7.511179e-8,0.007862331,0.9633987,0.00002259775,0.0008911893,0.0007392697,0.00006741362],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7531711,0.0001464237,0.2401451,0.0004182319,0.0003096164,0.0001473324,0.00002233415,0.0002138379,0.005426083],"genre_scores_gemma":[0.972694,0.000004615933,0.02671732,0.00003667946,0.00007638059,0.000007386324,0.00000372378,0.000003674961,0.0004562317],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9078171,"threshold_uncertainty_score":0.6429968,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04878391879767375,"score_gpt":0.4075470857221072,"score_spread":0.3587631669244334,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}