{"id":"W4378465251","doi":"10.48550/arxiv.2305.14998","title":"An Examination of the Robustness of Reference-Free Image Captioning Evaluation Metrics","year":2023,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Multimodal Machine Learning Applications","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Canadian Institute for Advanced Research; Nvidia","keywords":"Closed captioning; Robustness (evolution); Computer science; Artificial intelligence; Computer vision; Image (mathematics); Natural language processing","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001390422,0.0001578461,0.0002113111,0.0004489599,0.0001348076,0.00004741155,0.00297205,0.0001818031,0.00001230283],"category_scores_gemma":[0.0004990558,0.0001578855,0.0001005654,0.001825798,0.0001126726,0.0003264849,0.00161392,0.0004356514,0.000007076157],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000192365,"about_ca_system_score_gemma":0.0002284152,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0009187682,"about_ca_topic_score_gemma":0.00009544587,"domain_scores_codex":[0.998081,0.0005811433,0.0002617455,0.0006393694,0.0002943309,0.0001423735],"domain_scores_gemma":[0.9958702,0.000222562,0.000699465,0.002272315,0.0008856916,0.00004971112],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002364853,0.00006327035,0.002485079,0.00005693933,0.00002198658,6.924083e-7,0.0002473427,0.944097,0.0003985353,0.04979198,0.00001252686,0.002822245],"study_design_scores_gemma":[0.0001928425,0.0000178027,0.2173714,0.00004849481,0.00006377821,3.191358e-7,0.00006014256,0.7726909,0.0002800934,0.009161095,0.000001988105,0.0001111808],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5039629,0.000005475089,0.4949347,0.00005481012,0.000151379,0.0002884423,0.00001425427,0.00008293379,0.0005051259],"genre_scores_gemma":[0.9890393,0.00001491538,0.01074247,0.000003129156,0.00002396987,0.000004346515,0.0000369062,0.00001345049,0.0001214883],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4850764,"threshold_uncertainty_score":0.6438383,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1578930036878109,"score_gpt":0.2572585749079035,"score_spread":0.09936557122009265,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}