{"id":"W4200445293","doi":"10.2196/27386","title":"Benchmarking Effectiveness and Efficiency of Deep Learning Models for Semantic Textual Similarity in the Clinical Domain: Validation Study","year":2021,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Topic Modeling","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"U.S. National Library of Medicine; National Institutes of Health","keywords":"Benchmarking; Computer science; Natural language processing; Similarity (geometry); Artificial intelligence; Deep learning; Domain (mathematical analysis); Semantic similarity; Information retrieval","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01355927,0.003602884,0.001287483,0.002459366,0.0007607902,0.00151998,0.00295168,0.002867989,0.001800295],"category_scores_gemma":[0.02981839,0.0007456811,0.001590439,0.001797037,0.001231273,0.002483767,0.002297773,0.002848074,0.001188106],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003342024,"about_ca_system_score_gemma":0.002598392,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02896931,"about_ca_topic_score_gemma":0.02363536,"domain_scores_codex":[0.9926031,0.00340779,0.0008930522,0.001651858,0.0008968639,0.0005472776],"domain_scores_gemma":[0.9752363,0.01681933,0.001099382,0.00226846,0.003801278,0.0007752976],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.004464249,0.00292206,0.06129748,0.001720511,0.001846476,0.0003994488,0.0003807099,0.6700041,0.005942602,0.001273036,0.02235105,0.2273982],"study_design_scores_gemma":[0.0002004939,0.0008858732,0.006110363,0.00008311582,0.0001863988,0.00008188954,0.000123463,0.984284,0.006070843,0.0008938035,0.001044323,0.00003540662],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9463187,0.006936477,0.03009433,0.001457583,0.0004655269,0.0004582043,0.004595655,0.005183927,0.004489583],"genre_scores_gemma":[0.9585772,0.000787773,0.02730361,0.0003764259,0.0001038421,0.0002063751,0.0108629,0.0002859054,0.001495967],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02896931,"threshold_uncertainty_score":0.0717091,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04756599648023278,"score_gpt":0.3516154646810429,"score_spread":0.3040494682008101,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}