{"id":"W3125451872","doi":"10.15353/jcvis.v6i1.3548","title":"Where do Clinical Language Models Break Down? A Critical Behavioural Exploration of the ClinicalBERT Deep Transformer Model","year":2021,"lang":"en","type":"article","venue":"Journal of Computational Vision and Imaging Systems","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"University of Waterloo","keywords":"Computer science; Transformer; Inference; Artificial intelligence; Language model; Leverage (statistics); Realm; Deep learning; Encoder; Language understanding; Machine learning; Data science; Natural language processing; Engineering","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008818956,0.00008589806,0.00032647,0.00006502865,0.00008965218,0.00005669593,0.00006277711,0.00006783792,0.00001508247],"category_scores_gemma":[0.00035143,0.00005604888,0.0001965657,0.0001217647,0.0001175908,0.0004589263,0.00001249765,0.0003025213,0.000001794963],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003408367,"about_ca_system_score_gemma":0.0004406417,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005598266,"about_ca_topic_score_gemma":0.000009290964,"domain_scores_codex":[0.9978173,0.0002050293,0.001288959,0.0001161858,0.0004642689,0.0001082065],"domain_scores_gemma":[0.9976102,0.0006085603,0.000296438,0.0001102564,0.001234005,0.0001405618],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001573862,0.003706503,0.1849562,0.002008948,0.0003063652,0.0001466358,0.03882864,0.3662407,0.002824801,0.02365324,0.009020085,0.366734],"study_design_scores_gemma":[0.0003435563,0.0002023845,0.007539182,0.001216366,0.0001346686,0.0007608192,0.009135395,0.9706548,0.0001883992,0.009621504,0.000112002,0.00009090163],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6433778,0.005724376,0.3302735,0.01886597,0.001290981,0.0002085344,0.000006662507,0.000007816423,0.0002443286],"genre_scores_gemma":[0.9962407,0.0002229519,0.00284306,0.000353118,0.0002631561,0.000002011957,0.000004826225,0.000009410448,0.00006078158],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6044141,"threshold_uncertainty_score":0.2285607,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1765323993807969,"score_gpt":0.4868758544396559,"score_spread":0.310343455058859,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}