{"id":"W4402716477","doi":"10.1109/cvpr52733.2024.00913","title":"MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","year":2024,"lang":"en","type":"article","venue":"","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":245,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Victoria; University of Waterloo","funders":"","keywords":"Benchmark (surveying); Computer science; Artificial intelligence; Cognitive science; Psychology; Geology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002941309,0.002822069,0.000859192,0.002817094,0.0009989219,0.002621144,0.003718298,0.003443238,0.01226431],"category_scores_gemma":[0.01686873,0.0004631427,0.001663685,0.00201437,0.0008699822,0.003952968,0.004265556,0.002789638,0.006168539],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002775596,"about_ca_system_score_gemma":0.002557525,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02164984,"about_ca_topic_score_gemma":0.02662444,"domain_scores_codex":[0.996995,0.001061933,0.0002333604,0.0007736659,0.0006394805,0.000296606],"domain_scores_gemma":[0.9946449,0.002851316,0.0002679452,0.0008580398,0.0008578173,0.000519914],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.002235776,0.002763939,0.01561159,0.003358727,0.0008044827,0.0007285161,0.001338545,0.1728493,0.008460232,0.01322595,0.325524,0.4530988],"study_design_scores_gemma":[0.0006006227,0.001227517,0.01429746,0.0006033094,0.000234905,0.0005195384,0.001540394,0.7825888,0.01654877,0.03085206,0.1508304,0.0001562419],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"methods","genre_scores_codex":[0.4126042,0.01200299,0.2367111,0.005452842,0.002084343,0.003651148,0.1035858,0.09968363,0.1242239],"genre_scores_gemma":[0.5216844,0.001554238,0.2589757,0.001931075,0.0002845746,0.002311501,0.1917499,0.002599522,0.01890909],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.02164984,"threshold_uncertainty_score":0.04304767,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04329207881041666,"score_gpt":0.3393487419959848,"score_spread":0.2960566631855682,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}