{"id":"W4399023868","doi":"10.2196/55799","title":"Evaluating Large Language Models for Automated Reporting and Data Systems Categorization: Cross-Sectional Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Preprint; Categorization; Computer science; Natural language processing; Cross-sectional study; Information retrieval; Data science; Artificial intelligence; World Wide Web; Medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004392454,0.0001073936,0.0002227678,0.0001077769,0.0001969667,0.0002325379,0.0001259663,0.0001672822,0.00004918645],"category_scores_gemma":[0.002409591,0.00008520904,0.0000279477,0.0002309132,0.0000433496,0.0005590628,0.0001064158,0.0002636408,0.00001796567],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008107264,"about_ca_system_score_gemma":0.0006748792,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001482551,"about_ca_topic_score_gemma":0.00001770026,"domain_scores_codex":[0.9968968,0.00003355487,0.001879073,0.0001936535,0.0007525792,0.0002443399],"domain_scores_gemma":[0.9984789,0.000323673,0.0003435419,0.0003681819,0.0002753689,0.0002102976],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003774561,0.001892742,0.421842,0.03183109,0.000809765,0.0002590481,0.3271959,0.004503065,0.0001058889,0.009403431,0.02263064,0.179149],"study_design_scores_gemma":[0.0001229837,0.0002181795,0.002706364,0.0002640457,0.00003696671,0.0001633461,0.02007961,0.9758179,0.00001070172,0.000110576,0.0003879952,0.00008133684],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9681999,0.0004702616,0.0272949,0.000234778,0.001223942,0.001736705,0.00004709191,0.0005126102,0.0002797956],"genre_scores_gemma":[0.9964538,0.0000162223,0.001330715,0.0001967042,0.0007423346,0.0002060303,0.0008390319,0.00001698615,0.0001981539],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9713148,"threshold_uncertainty_score":0.3474724,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.378983326224231,"score_gpt":0.5856021836091797,"score_spread":0.2066188573849487,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}