{"id":"W3035321115","doi":"10.18653/v1/2020.bionlp-1.19","title":"Extensive Error Analysis and a Learning-Based Evaluation of Medical Entity Recognition Systems to Approximate User Experience","year":2020,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Computer science; Classifier (UML); Artificial intelligence; Machine learning; Judgement; Metric (unit); Test set; Medical diagnosis; Natural language processing; Gold standard (test); Information retrieval; Pattern recognition (psychology); Data mining; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001138332,0.00006794283,0.0001731048,0.0001101891,0.00004795359,0.00007195987,0.0002550959,0.0000482135,0.00008104734],"category_scores_gemma":[0.0007400984,0.00006046643,0.0000393706,0.0006565236,0.0000203555,0.0001885006,0.0001243509,0.00008467189,0.000007267845],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002035,"about_ca_system_score_gemma":0.00007378823,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000260118,"about_ca_topic_score_gemma":0.00003042395,"domain_scores_codex":[0.9979763,0.0002634758,0.0002357209,0.000359392,0.001059141,0.0001059742],"domain_scores_gemma":[0.9991136,0.00006436071,0.00008341428,0.0001855119,0.0003940868,0.0001589834],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00007086828,0.0002188015,0.02719988,0.0003971081,0.0004534203,0.00002405132,0.04673172,0.7396612,0.004921218,0.004045006,0.0002332417,0.1760435],"study_design_scores_gemma":[0.000214992,0.00004012361,0.00101161,0.000018695,0.00006490942,8.254377e-7,0.0004813652,0.9971314,0.0009080461,0.00003314847,0.00002262148,0.00007223497],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.404662,0.00002069063,0.5942547,0.0008174211,0.00003495303,0.0001346539,4.195464e-7,0.00003787979,0.00003731937],"genre_scores_gemma":[0.9787431,0.000001422567,0.02076618,0.0004194949,0.00001972674,0.00003643631,0.0000031443,0.000002492914,0.000007971868],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5740812,"threshold_uncertainty_score":0.2465749,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08982900735465392,"score_gpt":0.3426717351197669,"score_spread":0.252842727765113,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}