{"id":"W4400356709","doi":"10.2196/59258","title":"Evaluating the Capabilities of Generative AI Tools in Understanding Medical Papers: Qualitative Study","year":2024,"lang":"en","type":"article","venue":"JMIR Medical Informatics","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Preprint; Computer science; Generative grammar; Data science; Artificial intelligence; World Wide Web","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.005154601,0.0001306737,0.0003200296,0.0001565149,0.00009328057,0.00004949835,0.0001607596,0.000186187,0.001028104],"category_scores_gemma":[0.004365517,0.00007851874,0.00007112348,0.0005024832,0.0004025181,0.0002590083,0.00005699047,0.0008062318,0.00003036926],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003479749,"about_ca_system_score_gemma":0.001817774,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002357823,"about_ca_topic_score_gemma":0.0003038251,"domain_scores_codex":[0.9961048,0.0003660001,0.001345349,0.0001185438,0.001805715,0.0002595255],"domain_scores_gemma":[0.9961163,0.003189372,0.0001059972,0.0002210538,0.0001479789,0.0002193251],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"qualitative","study_design_scores_codex":[0.00005835092,0.0002338235,0.000741949,0.0007235151,0.00006409556,0.00001443157,0.9338416,0.00001178719,0.00001027011,0.009601177,0.0007615068,0.05393742],"study_design_scores_gemma":[0.0001284362,0.001036154,0.0001762881,0.001150696,0.00003093199,0.0000209752,0.9378994,0.05375884,0.0001191377,0.00539758,0.0001957532,0.00008578351],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9797936,0.000206243,0.001546548,0.01449809,0.0004698168,0.001157658,0.000004444284,0.00004631983,0.002277259],"genre_scores_gemma":[0.9970374,0.00005531495,0.000206562,0.002169702,0.0002516856,0.0001813188,0.00001586134,0.00001019497,0.000071924],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.05385164,"threshold_uncertainty_score":0.9998851,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4930343013499484,"score_gpt":0.5937801521859366,"score_spread":0.1007458508359882,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}