{"id":"W4387389875","doi":"10.48550/arxiv.2310.02567","title":"Improving Automatic VQA Evaluation Using Large Language Models","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Canadian Institute for Advanced Research; Samsung; Nvidia","keywords":"Leverage (statistics); Computer science; Metric (unit); Machine learning; Proxy (statistics); Question answering; Task (project management); Artificial intelligence; Set (abstract data type); Context (archaeology); Data mining; Information retrieval","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009868939,0.0002776934,0.0002656297,0.0003843358,0.0002750587,0.0001836683,0.001653265,0.00023444,0.00002726082],"category_scores_gemma":[0.0001251676,0.0003429442,0.0001589834,0.0007692224,0.0000339446,0.000470549,0.002483519,0.0006379568,0.0001853309],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004685287,"about_ca_system_score_gemma":0.0003512229,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002631533,"about_ca_topic_score_gemma":0.00008576987,"domain_scores_codex":[0.9977291,0.0002923814,0.000232124,0.001133045,0.0002174271,0.0003959234],"domain_scores_gemma":[0.9975131,0.0001383919,0.0003732724,0.00164976,0.0002005151,0.0001249361],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000001278762,0.00004427717,0.0004101477,0.00007773249,0.00003694676,0.00003112311,0.000688935,0.9548672,0.000211952,0.04007309,0.00001686787,0.00354041],"study_design_scores_gemma":[0.0003248446,0.000008319071,0.001465011,0.00007222067,0.00009604236,0.000002101592,0.0000952235,0.9642697,0.00002761439,0.03331173,0.000003728963,0.0003234389],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4083998,0.00002300951,0.5899733,0.00004968529,0.0001987087,0.0003981435,0.00001204377,0.0006999017,0.0002454061],"genre_scores_gemma":[0.9724457,0.000006024175,0.02708229,0.00003915644,0.00006483164,0.000006115744,0.00004449295,0.0000364632,0.0002748929],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5640459,"threshold_uncertainty_score":0.9999022,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1362128373710949,"score_gpt":0.2611552107093333,"score_spread":0.1249423733382385,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}