{"id":"W4405064339","doi":"10.1115/1.4067333","title":"DesignQA: A Multimodal Benchmark for Evaluating Large Language Models’ Understanding of Engineering Documentation","year":2024,"lang":"en","type":"article","venue":"Journal of Computing and Information Science in Engineering","topic":"Design Education and Practice","field":"Engineering","cited_by":29,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"Directorate for Engineering","keywords":"Documentation; Benchmark (surveying); Computer science; Software engineering; Natural language processing; Engineering drawing; Systems engineering; Artificial intelligence; Engineering; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002592893,0.00008411191,0.0001280168,0.0008264807,0.00004429141,0.0002013013,0.0001004529,0.00002992234,0.000003329502],"category_scores_gemma":[0.0003481557,0.00008348202,0.00003323301,0.0006010001,0.00001312144,0.005276864,0.0000141865,0.0001629139,4.099554e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001927096,"about_ca_system_score_gemma":0.00008200497,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000001766292,"about_ca_topic_score_gemma":1.22838e-7,"domain_scores_codex":[0.9989799,0.000007690373,0.0005171794,0.00005251927,0.0002664578,0.0001762358],"domain_scores_gemma":[0.9992958,0.0003990751,0.0001006526,0.00004656089,0.0001045219,0.00005335102],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000003919706,0.000003478194,0.00001995312,0.000335039,0.000008019561,4.5384e-7,0.009759651,0.9738992,0.007114694,0.004276854,0.00001528311,0.00456338],"study_design_scores_gemma":[0.0002563187,0.00003807791,0.00028279,0.0003897026,0.000008964951,0.00002886258,0.00240466,0.9950718,0.001239697,0.00007679948,0.0001188669,0.00008345041],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2546398,0.0002502545,0.7441515,0.00001909937,0.0006678206,0.00008003484,0.000001470063,0.0000349895,0.0001549642],"genre_scores_gemma":[0.9616142,0.0000379294,0.03827966,0.00000805733,0.00004896947,0.000001641337,0.000001333858,0.00000724873,9.388253e-7],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7069744,"threshold_uncertainty_score":0.3825599,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03288544794624475,"score_gpt":0.3343424033057408,"score_spread":0.301456955359496,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}