{"id":"W4394867428","doi":"10.1017/pan.2025.10007","title":"Measuring the Quality of Answers in Political Q&As with Large Language Models","year":2025,"lang":"en","type":"preprint","venue":"Political Analysis","topic":"Expert finding and Q&A systems","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Operationalization; Quality (philosophy); Parallels; House of Commons; Context (archaeology); Relevance (law); Computer science; Period (music); Politics; Measure (data warehouse); Parliament; Epistemology; Political science; Data mining; Law; History","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01796016,0.0006251056,0.001086436,0.004092528,0.0008363174,0.003857707,0.001262142,0.001993738,0.001604406],"category_scores_gemma":[0.09132938,0.0004511921,0.0007200701,0.003057404,0.001627279,0.004855588,0.002198362,0.001797516,0.0005509844],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001911024,"about_ca_system_score_gemma":0.001295171,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.009488025,"about_ca_topic_score_gemma":0.01132752,"domain_scores_codex":[0.9861014,0.009559372,0.000566242,0.001405821,0.001922024,0.0004452094],"domain_scores_gemma":[0.8846846,0.09736057,0.006502366,0.005658692,0.004594224,0.001199562],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002128265,0.0008744613,0.1767652,0.0005217938,0.0009471743,0.0003752427,0.003968923,0.57221,0.01440251,0.02197362,0.005400304,0.2004326],"study_design_scores_gemma":[0.00001997255,0.00007360539,0.008712864,0.00001686843,0.00003367067,0.00003478796,0.0002307814,0.9762819,0.002535813,0.01117482,0.0008536222,0.00003118486],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7262837,0.001002324,0.263209,0.002133145,0.00006289796,0.0001578744,0.001374843,0.001638294,0.004137937],"genre_scores_gemma":[0.9770666,0.00005166949,0.02150938,0.00009371736,0.0000357872,0.00003820796,0.0007201462,0.00005741319,0.0004271606],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.01796016,"threshold_uncertainty_score":0.09498352,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07669152858246686,"score_gpt":0.3559642944476354,"score_spread":0.2792727658651686,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}