{"id":"W4394696642","doi":"10.2196/57001","title":"Assessing and Optimizing Large Language Models on Spondyloarthritis Multi-Choice Question Answering: Protocol for Enhancement and Assessment","year":2024,"lang":"en","type":"article","venue":"JMIR Research Protocols","topic":"Machine Learning in Healthcare","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Preprint; Protocol (science); Computer science; Question answering; Language model; Data science; Artificial intelligence; Natural language processing; Medicine; World Wide Web; Alternative medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02803553,0.003345057,0.001531386,0.001511553,0.00142425,0.001523424,0.003217801,0.003188846,0.04515121],"category_scores_gemma":[0.08195639,0.001638496,0.003430168,0.001061951,0.001433525,0.001347834,0.003220337,0.004055027,0.008913494],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002637546,"about_ca_system_score_gemma":0.009182858,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003398232,"about_ca_topic_score_gemma":0.005369328,"domain_scores_codex":[0.9870041,0.008410084,0.001694362,0.001259398,0.001140728,0.0004914289],"domain_scores_gemma":[0.9510697,0.0278545,0.002480594,0.006249845,0.01135862,0.0009867734],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.03911726,0.0150656,0.008921484,0.04130086,0.002773015,0.001559776,0.003405178,0.1299115,0.02583912,0.01830154,0.1790175,0.5347873],"study_design_scores_gemma":[0.07752705,0.03087707,0.02148015,0.01863235,0.003831798,0.001456163,0.002079889,0.2758853,0.06632479,0.07490065,0.4254215,0.001583306],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"protocol","genre_gemma":"protocol","genre_scores_codex":[0.03770357,0.001778402,0.2207196,0.001963934,0.0008368308,0.669126,0.04768754,0.007584581,0.01259953],"genre_scores_gemma":[0.0217207,0.0003942933,0.160459,0.0005214048,0.00005872264,0.8059387,0.008792521,0.0002813973,0.001833271],"genre_candidate":"protocol","genre_consensus":"protocol","teacher_disagreement_score":0.04515121,"threshold_uncertainty_score":0.1510458,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3059897700266886,"score_gpt":0.628692378855669,"score_spread":0.3227026088289804,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}