{"id":"W4387001850","doi":"10.1001/jamanetworkopen.2023.35377","title":"APPRAISE-AI Tool for Quantitative Evaluation of AI Studies for Clinical Decision Support","year":2023,"lang":"en","type":"review","venue":"JAMA Network Open","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":113,"is_retracted":false,"has_abstract":true,"ca_institutions":"Institute for Clinical Evaluative Sciences; Princess Margaret Cancer Centre; Public Health Ontario; Hospital for Sick Children; SickKids Foundation; University Health Network; University of Toronto","funders":"","keywords":"Inter-rater reliability; Artificial intelligence; Medicine; Reliability (semiconductor); Machine learning; Medical physics; Computer science; Psychology; Rating scale","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3184882,0.003469556,0.00875411,0.03346738,0.001797045,0.009140805,0.00331695,0.003768808,0.0376321],"category_scores_gemma":[0.7080998,0.002178119,0.01609056,0.02880469,0.004485143,0.008678812,0.01191056,0.008155107,0.004399574],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.005634666,"about_ca_system_score_gemma":0.01421089,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000880147,"about_ca_topic_score_gemma":0.00145807,"domain_scores_codex":[0.472467,0.3384145,0.1337301,0.008850915,0.04447487,0.002062581],"domain_scores_gemma":[0.09189831,0.7662993,0.06536405,0.021648,0.05227755,0.002512765],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"not_applicable","study_design_scores_codex":[0.01257889,0.0006715467,0.03257183,0.3461886,0.02559854,0.0004667586,0.006691398,0.004591713,0.003014745,0.03643537,0.2391652,0.2920255],"study_design_scores_gemma":[0.01455784,0.004108634,0.1036546,0.1719294,0.02451902,0.001254003,0.006917567,0.02381509,0.005930136,0.1115907,0.5297406,0.001982403],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"review","genre_scores_codex":[0.03314481,0.05128079,0.3906671,0.0273823,0.01247275,0.2291878,0.1882049,0.01622799,0.05143161],"genre_scores_gemma":[0.06592538,0.003954674,0.5894617,0.002989124,0.0006512899,0.3247997,0.0094963,0.001363559,0.001358051],"genre_candidate":"review","genre_consensus":null,"teacher_disagreement_score":0.6815118,"threshold_uncertainty_score":0.8404254,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.8642618477117467,"score_gpt":0.7294409870398627,"score_spread":0.134820860671884,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}