{"id":"W4315646142","doi":"10.1148/ryai.220056","title":"Hurdles to Artificial Intelligence Deployment: Noise in Schemas and “Gold” Labels","year":2023,"lang":"en","type":"article","venue":"Radiology Artificial Intelligence","topic":"Big Data and Business Intelligence","field":"Business, Management and Accounting","cited_by":24,"is_retracted":false,"has_abstract":true,"ca_institutions":"Trillium Health Centre; University of Toronto","funders":"Government of Canada; TD Bank","keywords":"Medicine; Software deployment; Noise (video); Artificial intelligence; Software engineering","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001052601,0.0004816146,0.0005990611,0.001299866,0.0002854831,0.0003022421,0.0008255438,0.0003147347,0.0006416507],"category_scores_gemma":[0.001098623,0.0004799888,0.00009339279,0.003544752,0.0005078247,0.001015137,0.0007066618,0.0004571308,0.00765664],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006407264,"about_ca_system_score_gemma":0.00005175132,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006798696,"about_ca_topic_score_gemma":0.001220014,"domain_scores_codex":[0.9962792,0.00005497688,0.001101862,0.001135043,0.0003538756,0.001075061],"domain_scores_gemma":[0.9985574,0.0003407943,0.0002172096,0.0006131501,0.0001889772,0.00008250761],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0002990505,0.0002408493,0.00611889,0.0001658119,0.00003608877,0.0001754588,0.0004636543,0.003693518,0.01744152,0.577419,0.002588204,0.391358],"study_design_scores_gemma":[0.0001403772,0.0002784453,0.01757362,0.0007874882,0.0001795312,0.0001376383,0.005941968,0.1454612,0.08935238,0.6363311,0.09986108,0.003955242],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9112354,0.0005260556,0.07217726,0.009059221,0.002492098,0.001237653,0.00003386748,0.0008264095,0.002412041],"genre_scores_gemma":[0.9951226,0.000302951,0.001001491,0.00179379,0.001282318,0.000141613,0.00005919591,0.00006414637,0.0002319498],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3874027,"threshold_uncertainty_score":0.9997652,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1293718940637283,"score_gpt":0.3324809187442301,"score_spread":0.2031090246805019,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}