{"id":"W3030521948","doi":"","title":"HardEval: Focusing on Challenging Tokens to Assess Robustness of NER.","year":2020,"lang":"en","type":"article","venue":"NPARC","topic":"Topic Modeling","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"Institut de Valorisation des Données","keywords":"Robustness (evolution); Computer science; Ambiguity; Exploit; Artificial intelligence; Machine learning; Natural language processing; Programming language; Computer security","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0001538247,0.00009199553,0.0001570605,0.00005451829,0.00005166672,0.0000512856,0.0005875037,0.00003367525,0.00002516921],"category_scores_gemma":[0.00007019058,0.00009145447,0.00004312088,0.0002183721,0.000008137335,0.0001509305,0.0002554943,0.0001095297,0.00001817439],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0000233546,"about_ca_system_score_gemma":0.00003428574,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000002981738,"about_ca_topic_score_gemma":8.76306e-7,"domain_scores_codex":[0.9989814,0.00003397273,0.0001760616,0.000316634,0.0002814335,0.0002105047],"domain_scores_gemma":[0.9993579,0.00005153755,0.00004664182,0.0003575113,0.00005835065,0.0001280278],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001853115,0.00006756205,0.0001559391,0.0001297917,0.0000276482,0.00003388887,0.005100145,0.6564192,0.07195307,0.1024422,0.001550932,0.1621011],"study_design_scores_gemma":[0.0001651768,0.00007061007,0.0001449666,0.00006362921,0.000003290469,0.000002110174,0.00005162113,0.9801069,0.01733158,0.0004555625,0.001459754,0.0001448275],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08561567,0.00001064889,0.8956082,0.01154887,0.0002667299,0.00009345837,6.155855e-7,0.0001149051,0.006740867],"genre_scores_gemma":[0.8586079,0.000001989647,0.1404532,0.0007237562,0.0001765574,0.000003484045,1.917163e-7,0.000008553243,0.00002443566],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7729922,"threshold_uncertainty_score":0.3729405,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1133778731560633,"score_gpt":0.2902726815032783,"score_spread":0.176894808347215,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}