{"id":"W3201812531","doi":"10.2196/26065","title":"Active Annotation in Evaluating the Credibility of Web-Based Medical Information: Guidelines for Creating Training Data Sets for Machine Learning","year":2021,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Misinformation and Its Impacts","field":"Social Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Computer science; Annotation; Credibility; Protocol (science); Information retrieval; Misinformation; Artificial intelligence; Machine learning; World Wide Web; Medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1582165,0.002669845,0.002310544,0.01308882,0.005010489,0.009439165,0.01009862,0.007133905,0.00545554],"category_scores_gemma":[0.3204094,0.002890891,0.002760456,0.008361403,0.009418465,0.01039469,0.009567841,0.008777615,0.004389543],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00451329,"about_ca_system_score_gemma":0.008464411,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004553338,"about_ca_topic_score_gemma":0.008040631,"domain_scores_codex":[0.8057977,0.1407071,0.02118101,0.007813097,0.02321569,0.00128536],"domain_scores_gemma":[0.5077397,0.360146,0.01735332,0.0588275,0.05272793,0.003205616],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001270107,0.002128753,0.01494131,0.00709071,0.0004667575,0.0009984976,0.02017508,0.03976096,0.02292824,0.1544887,0.06644291,0.6693079],"study_design_scores_gemma":[0.0006187948,0.0009259296,0.01151936,0.006818266,0.0002953427,0.0007683261,0.004459036,0.3104055,0.04880339,0.3740658,0.240682,0.0006382831],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.003582662,0.0004264702,0.9818053,0.001295568,0.00008562435,0.005376899,0.002216561,0.001952135,0.003258843],"genre_scores_gemma":[0.01285772,0.000130348,0.9746609,0.0001952844,0.00004232568,0.009089807,0.002329737,0.0002814331,0.0004124909],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.1582165,"threshold_uncertainty_score":0.8367389,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3115417149346357,"score_gpt":0.5227119047136074,"score_spread":0.2111701897789717,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}