{"id":"W4410817084","doi":"10.1001/jamanetworkopen.2025.12621","title":"Evaluating the Quality of Guidelines Using the AGREE II Tool by a Large Language Model vs Human Appraisers","year":2025,"lang":"en","type":"article","venue":"JAMA Network Open","topic":"Clinical practice guidelines implementation","field":"Medicine","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"Quality (philosophy); Computer science; Medical physics; Medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3452227,0.001168052,0.002808737,0.006102216,0.001611837,0.006001823,0.00255885,0.002230586,0.003900247],"category_scores_gemma":[0.6392332,0.000804803,0.00469089,0.005088184,0.003379083,0.006109349,0.005185847,0.003154079,0.0006062782],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006572364,"about_ca_system_score_gemma":0.01411968,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003504972,"about_ca_topic_score_gemma":0.005171319,"domain_scores_codex":[0.5052204,0.4018527,0.03520165,0.007738844,0.04785676,0.002129603],"domain_scores_gemma":[0.1664061,0.6893471,0.06751811,0.0261696,0.04690325,0.003655821],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.05852316,0.01285122,0.3570916,0.01200235,0.01306586,0.0003553614,0.01902138,0.02867946,0.003680875,0.022416,0.01863806,0.4536747],"study_design_scores_gemma":[0.05870384,0.111002,0.4070557,0.01064399,0.02457255,0.0009408383,0.01402313,0.2426268,0.02359915,0.05003578,0.05439258,0.002403636],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7979171,0.003936213,0.1301872,0.01016737,0.0009864628,0.02809396,0.003601582,0.001135132,0.023975],"genre_scores_gemma":[0.873739,0.0005353718,0.1054917,0.001682136,0.0002022036,0.01626591,0.00122137,0.0001167756,0.0007455584],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6547773,"threshold_uncertainty_score":0.807457,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5471494929472598,"score_gpt":0.6434007615904526,"score_spread":0.09625126864319278,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}