{"id":"W54100278","doi":"10.63317/42vnn94pj3ga","title":"A Trainable Tokenizer, solution for multilingual texts and compound expression tokenization","year":2008,"lang":"en","type":"article","venue":"","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Lexical analysis; Computer science; Task (project management); Natural language processing; Security token; Artificial intelligence; Character (mathematics); Expression (computer science); Alphanumeric; Representation (politics); Regular expression; Lemmatisation; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001410477,0.001532935,0.001383087,0.001342877,0.001419685,0.001862436,0.002827564,0.001864375,0.01754699],"category_scores_gemma":[0.004124482,0.00120216,0.001182176,0.001909669,0.0008966014,0.003792931,0.003238988,0.003152149,0.01122617],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001066539,"about_ca_system_score_gemma":0.003252173,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004045208,"about_ca_topic_score_gemma":0.008532515,"domain_scores_codex":[0.9985265,0.0002621435,0.0001561656,0.0005793708,0.0002947319,0.0001811394],"domain_scores_gemma":[0.9984793,0.0005865878,0.00009053135,0.0003170528,0.000442361,0.00008421166],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0006517787,0.0001499342,0.0006804483,0.000514331,0.0001068728,0.0005385252,0.0003383293,0.02096931,0.0469536,0.02318493,0.0370302,0.8688817],"study_design_scores_gemma":[0.0001649409,0.0001871772,0.0007667266,0.00006812202,0.0001851286,0.0009067098,0.0004505982,0.7318541,0.1582758,0.04731136,0.05971751,0.0001118299],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.004010186,0.0001105929,0.9807265,0.0001395432,0.0001218647,0.0001028672,0.0007699788,0.0129532,0.001065282],"genre_scores_gemma":[0.05554784,0.0001779221,0.9260871,0.0001259691,0.0000878019,0.0002639281,0.004026647,0.002126535,0.01155624],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.01754699,"threshold_uncertainty_score":0.0587005,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02290123785870335,"score_gpt":0.2812507490096366,"score_spread":0.2583495111509332,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}