{"id":"W4293918660","doi":"10.1101/2022.08.30.22279318","title":"Evaluating Progress in Automatic Chest X-Ray Radiology Report Generation","year":2022,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Topic Modeling","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"National Institute of Biomedical Imaging and Bioengineering; National Institutes of Health","keywords":"Workflow; Computer science; Metric (unit); Rank (graph theory); Quality (philosophy); Artificial intelligence; Medical imaging; Identification (biology); Contrast (vision); Medical physics; Information retrieval; Machine learning; Natural language processing; Data science; Radiology; Medicine; Database","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003635125,0.0002164781,0.0003748315,0.0002327789,0.0001010987,0.0001283024,0.001297371,0.0001671212,0.00008126759],"category_scores_gemma":[0.0003186191,0.0002245492,0.00007956027,0.0002406793,0.00003345765,0.0001205981,0.001904048,0.0007149052,0.000008034086],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000200867,"about_ca_system_score_gemma":0.000418835,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0000303295,"about_ca_topic_score_gemma":0.00001542409,"domain_scores_codex":[0.9967536,0.0005766192,0.000794661,0.001031045,0.0005287717,0.0003153066],"domain_scores_gemma":[0.9977877,0.00007702718,0.0004594614,0.001556036,0.00006517887,0.00005456691],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002432626,0.0001655064,0.4184892,0.0004329994,0.00006370823,0.001910717,0.005393311,0.3145282,0.0008879742,0.003530166,0.0001753563,0.2544204],"study_design_scores_gemma":[0.0001266787,0.00002802601,0.04329374,0.00004072076,0.000007138043,0.0001072667,0.00001061397,0.9549049,0.00004973209,0.001062937,0.0001619286,0.000206339],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8086158,0.000592645,0.186666,0.0009550212,0.002254586,0.0004889995,6.167206e-7,0.0002293708,0.0001969067],"genre_scores_gemma":[0.7656132,0.00000710224,0.2333398,0.00007542624,0.0002845152,0.0005177878,0.00003151912,0.00001618186,0.0001144937],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6403767,"threshold_uncertainty_score":0.9156851,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1107970950486012,"score_gpt":0.3653045299124789,"score_spread":0.2545074348638777,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}