{"id":"W7126429335","doi":"10.18653/v1/2024.findings-eacl.14","title":"An Examination of the Robustness of Reference-Free Image Captioning Evaluation Metrics","year":2024,"lang":"","type":"article","venue":"","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Canadian Institute for Advanced Research; Nvidia","keywords":"Closed captioning; Robustness (evolution); Image (mathematics); Image processing; Pattern recognition (psychology)","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02723199,0.004774928,0.001805173,0.009712998,0.00179254,0.005192486,0.00357521,0.003048148,0.003623007],"category_scores_gemma":[0.1170272,0.0005915578,0.001419941,0.004918128,0.001857677,0.005526208,0.004789618,0.003138187,0.002472574],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003030004,"about_ca_system_score_gemma":0.001442686,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.009654256,"about_ca_topic_score_gemma":0.009569305,"domain_scores_codex":[0.970767,0.01214743,0.003250158,0.005244668,0.007576356,0.001014473],"domain_scores_gemma":[0.9093574,0.04900191,0.005286279,0.01530731,0.01927314,0.001773991],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.003748488,0.0009312241,0.03931122,0.005986211,0.002556568,0.0005172752,0.001782943,0.07600989,0.03075253,0.007138333,0.1490701,0.6821952],"study_design_scores_gemma":[0.0005705359,0.003880799,0.0803005,0.001652207,0.001012718,0.002538264,0.002419646,0.6781394,0.1134356,0.02158218,0.09359042,0.0008777478],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5080059,0.03802408,0.291558,0.002699162,0.00529411,0.003651125,0.04073592,0.06797133,0.04206041],"genre_scores_gemma":[0.7024797,0.001892472,0.213121,0.00111176,0.0006249058,0.0013676,0.06821676,0.005655779,0.005530045],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.972768,"threshold_uncertainty_score":0.1440182,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05109654786901989,"score_gpt":0.3427500987957064,"score_spread":0.2916535509266865,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}