{"id":"W4399023868","doi":"10.2196/55799","title":"Evaluating Large Language Models for Automated Reporting and Data Systems Categorization: Cross-Sectional Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Preprint; Categorization; Computer science; Natural language processing; Cross-sectional study; Information retrieval; Data science; Artificial intelligence; World Wide Web; Medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.05500966,0.0009128624,0.0008165851,0.001256935,0.0007759702,0.002297487,0.001180792,0.001346698,0.002014645],"category_scores_gemma":[0.1728831,0.000856566,0.00145298,0.0006588419,0.001387571,0.003715455,0.002114824,0.002476616,0.001253383],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001510373,"about_ca_system_score_gemma":0.001338055,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002170178,"about_ca_topic_score_gemma":0.002839252,"domain_scores_codex":[0.9702195,0.02095857,0.001978463,0.003394321,0.002775153,0.000674],"domain_scores_gemma":[0.6918579,0.2349377,0.02446861,0.01752695,0.02725395,0.003954899],"domain_codex":null,"domain_gemma":"reporting","domain_candidate":"reporting","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.006372452,0.01216215,0.861079,0.001228516,0.001410858,0.0003540766,0.02418369,0.006645429,0.002838002,0.0005573452,0.003679311,0.07948902],"study_design_scores_gemma":[0.001241321,0.03740118,0.8166995,0.001003647,0.002387448,0.001977996,0.0197177,0.09225934,0.01246756,0.002893863,0.01145484,0.0004956814],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9928151,0.0001821325,0.005505879,0.0001043578,0.00002763971,0.0005818604,0.0002792742,0.0001099006,0.0003936971],"genre_scores_gemma":[0.9890599,0.0001164089,0.007911271,0.0002488885,0.00003553549,0.00128276,0.0008160802,0.00008808052,0.0004410514],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9449903,"threshold_uncertainty_score":0.2909224,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.378983326224231,"score_gpt":0.5856021836091797,"score_spread":0.2066188573849487,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}