{"id":"W4362588244","doi":"10.3390/diagnostics13071315","title":"Error Consistency for Machine Learning Evaluation and Validation with Application to Biomedical Diagnostics","year":2023,"lang":"en","type":"article","venue":"Diagnostics","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":13,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto; Nova Scotia Health Authority; St. Francis Xavier University","funders":"Natural Sciences and Engineering Research Council of Canada; Nova Scotia Health Research Foundation; St. Francis Xavier University","keywords":"Machine learning; Generalizability theory; Artificial intelligence; Computer science; Consistency (knowledge bases); Reliability (semiconductor); Software; Implementation; Variety (cybernetics); Software deployment; Sample (material); Data mining; Software engineering; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1352005,0.002737039,0.002301112,0.004567343,0.00205058,0.003735228,0.004475213,0.004767707,0.002842168],"category_scores_gemma":[0.386447,0.001312346,0.002931316,0.003184893,0.00490994,0.003690559,0.006390761,0.006551581,0.001238697],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002526361,"about_ca_system_score_gemma":0.005292064,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004942924,"about_ca_topic_score_gemma":0.004519374,"domain_scores_codex":[0.8959652,0.06332596,0.009406195,0.008810999,0.02120527,0.001286496],"domain_scores_gemma":[0.5399551,0.3497973,0.01991303,0.04881001,0.0399085,0.001616125],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001803704,0.000690756,0.03411824,0.001707065,0.001435213,0.000553565,0.001230781,0.5540674,0.01215209,0.07821843,0.01185902,0.3021637],"study_design_scores_gemma":[0.0001372652,0.0004231147,0.004554698,0.0004472032,0.0001083748,0.0002642008,0.0001377744,0.9350306,0.01290685,0.04126648,0.00462494,0.00009843365],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.008260928,0.0005772507,0.9877374,0.000263084,0.0001657006,0.0002579562,0.0001849616,0.001681684,0.0008709916],"genre_scores_gemma":[0.2041242,0.0003132023,0.7896522,0.0005510326,0.0002001751,0.001143739,0.001318801,0.001390668,0.001306006],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8647995,"threshold_uncertainty_score":0.7150171,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04383271295815738,"score_gpt":0.3332272922367434,"score_spread":0.289394579278586,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}