{"id":"W4384561507","doi":"10.1038/s42256-023-00652-2","title":"Federated benchmarking of medical artificial intelligence with MedPerf","year":2023,"lang":"en","type":"article","venue":"Nature Machine Intelligence","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":149,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; University of Toronto","funders":"National Institutes of Health; Agence Nationale de la Recherche; National Cancer Institute; National Research Foundation Singapore; National Research Foundation","keywords":"Benchmarking; Software deployment; Computer science; Process (computing); Health care; Cloud computing; Knowledge management; Data science; Big data; Process management; Engineering management; Artificial intelligence; Business; Software engineering; Engineering; Data mining","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001482016,0.0003243058,0.000534395,0.0005161698,0.00022864,0.00004991378,0.0003986068,0.0007567265,0.001558194],"category_scores_gemma":[0.001487511,0.0002475106,0.0001355223,0.002550508,0.0003514887,0.000127579,0.0001131814,0.001798179,0.0002775977],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001035342,"about_ca_system_score_gemma":0.0007637155,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0009483836,"about_ca_topic_score_gemma":0.0008580852,"domain_scores_codex":[0.9959005,0.0001024983,0.001060655,0.0006164752,0.001719893,0.0006000057],"domain_scores_gemma":[0.9976476,0.0006806991,0.0002386335,0.0004085702,0.0006062712,0.000418228],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.000668389,0.0003335735,0.01321063,0.0003525523,0.0001015617,0.0002334971,0.001483716,0.0003667332,0.0009500398,0.01523307,0.0006924785,0.9663737],"study_design_scores_gemma":[0.0000712257,0.002613194,0.00526336,0.002336563,0.0002064123,0.0007040483,0.006958475,0.2740083,0.6821298,0.02116386,0.003519696,0.00102509],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.819351,0.002330815,0.1346459,0.03424352,0.003096922,0.001582613,0.00003349864,0.0007670472,0.003948681],"genre_scores_gemma":[0.995986,0.0007446486,0.001216459,0.0009434895,0.0007227432,0.00004031796,0.0001397493,0.00004750545,0.0001590366],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9653487,"threshold_uncertainty_score":0.9999977,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08459942258698776,"score_gpt":0.4346746750443853,"score_spread":0.3500752524573975,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}