{"id":"W4308368314","doi":"10.1177/08465371221135760","title":"Assessment of Radiology Artificial Intelligence Software: A Validation and Evaluation Framework","year":2022,"lang":"en","type":"article","venue":"Canadian Association of Radiologists Journal","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":55,"is_retracted":false,"has_abstract":true,"ca_institutions":"Western University; Centre Intégré de Santé et de Services Sociaux des Laurentides; Vancouver General Hospital; University of British Columbia; University of Toronto; Dalhousie University; Université Laval; Trillium Health Centre; Université de Montréal; Centre intégré de santé et de services sociaux de Chaudière-Appalaches; Centre Hospitalier de l’Université de Montréal","funders":"Fonds de recherche du Québec","keywords":"Workflow; Medicine; Software; Computer science; Standardization; Software development; Software engineering; Verification and validation; Artificial intelligence; Engineering","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004523505,0.00008689758,0.0002960866,0.0004020865,0.0003221415,0.00001534474,0.00009608683,0.0001500099,0.0006085185],"category_scores_gemma":[0.004135969,0.00009201586,0.00007551819,0.0003359009,0.00008305414,0.00007439531,0.00001572844,0.0005522547,0.000001436783],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002727053,"about_ca_system_score_gemma":0.002906841,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003067822,"about_ca_topic_score_gemma":0.001230975,"domain_scores_codex":[0.9977503,0.0006187848,0.0007427463,0.0001618876,0.000477732,0.0002485151],"domain_scores_gemma":[0.997494,0.0005866044,0.0008307272,0.000133645,0.0007252273,0.0002298253],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00007827982,0.0001296889,0.8029274,0.00005335081,0.0001710527,0.000008800642,0.002147798,0.005761339,0.0005883203,0.01305372,0.002762511,0.1723178],"study_design_scores_gemma":[0.0002401013,0.003125544,0.7557557,0.000153916,0.0005454053,0.001653903,0.009043462,0.0188734,0.003459981,0.2043172,0.002381235,0.0004502154],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9741707,0.0004284952,0.01701028,0.006361196,0.001277236,0.0004507734,0.00003624526,0.00001007126,0.0002549726],"genre_scores_gemma":[0.9927533,0.0001820091,0.006392726,0.0002838241,0.0002575514,0.00002942141,0.00005738546,0.000008408115,0.00003541732],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1912634,"threshold_uncertainty_score":0.7131149,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1291509090875911,"score_gpt":0.4291231438017092,"score_spread":0.2999722347141182,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}