{"id":"W6906393256","doi":"10.17605/osf.io/4k69q","title":"Comparing the National Library of Medicine (NLM)’s Medical Text Indexer (MTI) to Human Indexing: A Pilot Study","year":2022,"lang":"en","type":"article","venue":"OSF Preprints (OSF Preprints)","topic":"Biomedical Text Mining and Ontologies","field":"Biochemistry, Genetics and Molecular Biology","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"National library; Medical library; MEDLINE; Medical journal; Digital library","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"methods","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":[],"domain":null,"study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04836595,0.0006180813,0.001307568,0.00596666,0.001252346,0.003509442,0.001975848,0.001511522,0.007166621],"category_scores_gemma":[0.1711539,0.0003394348,0.001357045,0.006307944,0.001410911,0.007877285,0.003258244,0.001383787,0.002280759],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002553267,"about_ca_system_score_gemma":0.002938712,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0113476,"about_ca_topic_score_gemma":0.007768673,"domain_scores_codex":[0.9739667,0.01527819,0.003459018,0.002575731,0.004144143,0.0005760881],"domain_scores_gemma":[0.7664585,0.1827242,0.007414387,0.01722686,0.0219385,0.004237567],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0701324,0.01698974,0.2789791,0.008648978,0.003812648,0.0006386864,0.01080257,0.00707618,0.02026594,0.006021413,0.03408956,0.5425429],"study_design_scores_gemma":[0.011708,0.04665317,0.68806,0.001270015,0.008387591,0.002890165,0.01385895,0.0800965,0.04257144,0.01059659,0.09320499,0.0007025189],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9516302,0.002125745,0.01366189,0.002687644,0.0003588723,0.002461615,0.01447629,0.002222899,0.01037485],"genre_scores_gemma":[0.9349881,0.000909747,0.03893224,0.0007397937,0.0002307537,0.001389517,0.01958242,0.0006928492,0.002534595],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.951634,"threshold_uncertainty_score":0.2557867,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04820279487776538,"score_gpt":0.3148710831137579,"score_spread":0.2666682882359925,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}