{"id":"W3196387024","doi":"10.1503/cmaj.210036","title":"Evaluation of machine learning solutions in medicine","year":2021,"lang":"en","type":"letter","venue":"Canadian Medical Association Journal","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":49,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"","keywords":"Key (lock); Computer science; Medicine; Computer security","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02446239,0.0005072126,0.0004957329,0.001226863,0.001526317,0.004200698,0.001923653,0.007694745,0.02526688],"category_scores_gemma":[0.1462843,0.0001631455,0.0006083449,0.000900502,0.002392266,0.003750618,0.001916705,0.007027091,0.007657815],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.009635309,"about_ca_system_score_gemma":0.005691882,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004061249,"about_ca_topic_score_gemma":0.007527312,"domain_scores_codex":[0.9690082,0.02022189,0.001307825,0.0009527657,0.007746344,0.0007630315],"domain_scores_gemma":[0.8965222,0.07122267,0.002329115,0.003143084,0.02321558,0.003567479],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0003377065,0.0001876851,0.003041462,0.0004641424,0.00004829404,0.0003330243,0.0001748444,0.001397374,0.0001891628,0.0379088,0.678734,0.2771835],"study_design_scores_gemma":[0.0003657522,0.0009362273,0.004716713,0.004240933,0.00008246121,0.001644804,0.0008790627,0.01048514,0.0009588191,0.07964139,0.8959692,0.00007961781],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"commentary","genre_gemma":"commentary","genre_scores_codex":[0.003856066,0.01343104,0.003275104,0.8760908,0.01020778,0.0001321591,0.000258364,0.00009908302,0.0926496],"genre_scores_gemma":[0.3179965,0.03710886,0.02415304,0.5294459,0.04415728,0.0007469859,0.001590064,0.0002796841,0.0445216],"genre_candidate":"commentary","genre_consensus":"commentary","teacher_disagreement_score":0.9755376,"threshold_uncertainty_score":0.129371,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1972308945399638,"score_gpt":0.4186825425135353,"score_spread":0.2214516479735715,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}