{"id":"W4385979136","doi":"10.1145/3604570","title":"The End of the Policy Analyst? Testing the Capability of Artificial Intelligence to Generate Plausible, Persuasive, and Useful Policy Analysis","year":2023,"lang":"en","type":"article","venue":"Digital Government Research and Practice","topic":"E-Government and Public Services","field":"Social Sciences","cited_by":29,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Regina; Shared Services Canada","funders":"","keywords":"Rubric; Computer science; Government (linguistics); Artificial intelligence; Task (project management); Heuristic; Public policy; Quality (philosophy); Political science; Psychology; Engineering; Mathematics education; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.08113033,0.0006512704,0.0006199308,0.001386702,0.002050164,0.008325666,0.001851554,0.003453584,0.006626636],"category_scores_gemma":[0.2908729,0.0006222622,0.0005738515,0.001195523,0.004525894,0.01257229,0.003307828,0.003226199,0.001215496],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001978596,"about_ca_system_score_gemma":0.003012395,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001767026,"about_ca_topic_score_gemma":0.001334332,"domain_scores_codex":[0.9172177,0.06598728,0.002963251,0.005171487,0.007676334,0.0009839233],"domain_scores_gemma":[0.5017852,0.4555305,0.01107176,0.01677006,0.01256006,0.002282397],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.01092826,0.01314095,0.1204162,0.004251433,0.0008028942,0.001178223,0.1573127,0.04226543,0.01855792,0.1224412,0.0156001,0.4931047],"study_design_scores_gemma":[0.005762263,0.0245929,0.1097865,0.004523193,0.001308143,0.001223759,0.1211255,0.243326,0.0745816,0.2452593,0.1674506,0.001060299],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8855031,0.0005947517,0.03070636,0.01053545,0.0002940962,0.001901551,0.0003524816,0.0005014905,0.06961074],"genre_scores_gemma":[0.9535346,0.0003153067,0.04116453,0.001781856,0.00006767629,0.0008922598,0.000209327,0.00008887418,0.001945592],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9188697,"threshold_uncertainty_score":0.4290633,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1653987492323391,"score_gpt":0.4347940881314314,"score_spread":0.2693953388990923,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}