{"id":"W4301605997","doi":"10.23889/ijpds.v7i1.1757","title":"A scoping review of preprocessing methods for unstructured text data to assess data quality","year":2022,"lang":"en","type":"review","venue":"International Journal for Population Data Science","topic":"Data Quality and Management","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Manitoba; George & Fay Yee Centre for Healthcare Innovation; Manitoba Health","funders":"Canada Research Chairs","keywords":"Computer science; Preprocessor; Punctuation; Data quality; Data pre-processing; Stop words; Information retrieval; Lexical analysis; Natural language processing; Artificial intelligence; Data mining","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3047388,0.004313474,0.01258414,0.06724623,0.005149239,0.01435362,0.007198966,0.006442463,0.008895167],"category_scores_gemma":[0.6163544,0.00394858,0.01559892,0.05063309,0.007084679,0.01285067,0.00893735,0.004504711,0.002226648],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01893029,"about_ca_system_score_gemma":0.07112107,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00921451,"about_ca_topic_score_gemma":0.01570804,"domain_scores_codex":[0.6298958,0.1525816,0.155146,0.01022623,0.05021289,0.001937386],"domain_scores_gemma":[0.2455446,0.5672291,0.07037822,0.01741115,0.09772573,0.001711308],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[0.0002245341,0.00004398415,0.001337383,0.8101359,0.003208777,0.0001957899,0.002838248,0.0003493071,0.000392405,0.002875,0.006883039,0.1715157],"study_design_scores_gemma":[0.00006915846,0.00007684782,0.0009031791,0.964733,0.004128648,0.0001294444,0.0008000334,0.0001521069,0.0003625865,0.001473489,0.02711928,0.00005221091],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.002155249,0.9327207,0.0225668,0.006529752,0.002190696,0.02631473,0.003254149,0.0002257357,0.004042217],"genre_scores_gemma":[0.01936316,0.8126554,0.06903563,0.00352418,0.0007787148,0.09007607,0.003370205,0.0002277749,0.0009689345],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.6952612,"threshold_uncertainty_score":0.8573809,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9239911994930267,"score_gpt":0.7624971008446532,"score_spread":0.1614940986483735,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}