{"id":"W4362588244","doi":"10.3390/diagnostics13071315","title":"Error Consistency for Machine Learning Evaluation and Validation with Application to Biomedical Diagnostics","year":2023,"lang":"en","type":"article","venue":"Diagnostics","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto; Nova Scotia Health Authority; St. Francis Xavier University","funders":"Natural Sciences and Engineering Research Council of Canada; Nova Scotia Health Research Foundation; St. Francis Xavier University","keywords":"Machine learning; Generalizability theory; Artificial intelligence; Computer science; Consistency (knowledge bases); Reliability (semiconductor); Software; Implementation; Variety (cybernetics); Software deployment; Sample (material); Data mining; Software engineering; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009938052,0.0001327046,0.0001400431,0.0002107505,0.0002711263,0.0001384368,0.0002659989,0.00007564967,0.000005137443],"category_scores_gemma":[0.006914291,0.000124177,0.00001950201,0.0008972072,0.0000670266,0.0002058915,0.0001299355,0.0001089668,0.000135578],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000568684,"about_ca_system_score_gemma":0.00009904345,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005084309,"about_ca_topic_score_gemma":0.00004656275,"domain_scores_codex":[0.9984952,0.00008314568,0.0002587056,0.0004158824,0.0004511011,0.0002959511],"domain_scores_gemma":[0.996254,0.002780176,0.00009701095,0.0003032723,0.0004073835,0.0001582163],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00008162395,0.0005065281,0.05012515,0.0002458621,0.00009179006,0.00005073758,0.009053499,0.05269108,0.002911464,0.177898,0.01445104,0.6918932],"study_design_scores_gemma":[0.0002934926,0.0005820374,0.004841222,0.00005367719,0.00005196062,0.000006977898,0.0003225323,0.9614357,0.005238577,0.008157091,0.01874705,0.0002697278],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08309042,0.0001330908,0.9112706,0.003594331,0.0002098682,0.001314905,0.00001879471,0.0002767037,0.00009125021],"genre_scores_gemma":[0.9688962,0.0003498844,0.02896268,0.0003661607,0.0001219175,0.0008791103,0.0003412784,0.00002595354,0.00005680493],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9087446,"threshold_uncertainty_score":0.8277552,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04383271295815738,"score_gpt":0.3332272922367434,"score_spread":0.289394579278586,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}