{"id":"W4402716477","doi":"10.1109/cvpr52733.2024.00913","title":"MMMU: A Massive Multi-Discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","year":2024,"lang":"en","type":"article","venue":"","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":245,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Victoria; University of Waterloo","funders":"","keywords":"Benchmark (surveying); Computer science; Artificial intelligence; Cognitive science; Psychology; Geology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003138123,0.0001933609,0.0001659252,0.0001663144,0.0003612673,0.0005087583,0.0004141594,0.00007712817,0.00002577766],"category_scores_gemma":[0.0001356761,0.0001657222,0.00007664437,0.0003329183,0.00006452441,0.0004072683,0.0003228272,0.0002072311,0.00001982744],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001452827,"about_ca_system_score_gemma":0.00005914657,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003425523,"about_ca_topic_score_gemma":0.00004901454,"domain_scores_codex":[0.9985488,0.00003763948,0.0002209618,0.0006926687,0.0001670203,0.0003328894],"domain_scores_gemma":[0.9988267,0.0005391802,0.00004711848,0.0004030674,0.00004444684,0.000139417],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0000133776,0.0001013912,0.001025481,0.0001204586,0.00009322617,0.00001633035,0.007511381,0.001918972,0.008016533,0.9368691,0.002436196,0.04187749],"study_design_scores_gemma":[0.0004130539,0.00004208295,0.001721348,0.00008872141,0.000007339887,0.00001283891,0.0002952841,0.9903998,0.0002446013,0.005010615,0.001534922,0.000229365],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.005412483,0.0004617868,0.9851009,0.006652568,0.0002281198,0.0004777848,0.000009739605,0.0006145946,0.001042037],"genre_scores_gemma":[0.5672937,0.00001137191,0.4319476,0.0001117008,0.00006791369,0.0001308077,0.000007979491,0.00001615856,0.00041281],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9884809,"threshold_uncertainty_score":0.6757956,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04329207881041666,"score_gpt":0.3393487419959848,"score_spread":0.2960566631855682,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}