{"id":"W4404783705","doi":"10.18653/v1/2024.emnlp-main.211","title":"Decompose and Compare Consistency: Measuring VLMs’ Answer Reliability via Task-Decomposition Consistency Comparison","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Canadian Institute for Advanced Research","keywords":"Consistency (knowledge bases); Reliability (semiconductor); Computer science; Task (project management); Decomposition; Reliability engineering; Data mining; Artificial intelligence; Engineering","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02266552,0.00134545,0.001015843,0.003857194,0.0006367365,0.002796105,0.001242328,0.001510528,0.003463318],"category_scores_gemma":[0.1950502,0.0005096719,0.001074348,0.001879498,0.0008299609,0.003688816,0.003776863,0.001680405,0.001508938],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008012131,"about_ca_system_score_gemma":0.001257996,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00152587,"about_ca_topic_score_gemma":0.001559391,"domain_scores_codex":[0.9807798,0.009038879,0.00187831,0.003194189,0.00458762,0.0005211378],"domain_scores_gemma":[0.8411556,0.1111115,0.01614725,0.01306094,0.01655152,0.001973092],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.005222668,0.001152183,0.2523559,0.002047274,0.001144236,0.0002231046,0.00787474,0.01909174,0.03229488,0.00957304,0.01498544,0.6540349],"study_design_scores_gemma":[0.001153283,0.003316145,0.4004427,0.000616847,0.0009970483,0.0009729548,0.003734901,0.440912,0.05575345,0.07055472,0.02084956,0.0006963777],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4835125,0.001578962,0.4881622,0.0007477864,0.0002294938,0.002266731,0.00362643,0.008680138,0.0111958],"genre_scores_gemma":[0.8231834,0.0001385372,0.1686363,0.0003443992,0.0001262318,0.001743893,0.003563544,0.0008646897,0.001399062],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.02266552,"threshold_uncertainty_score":0.1198682,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03196609638306724,"score_gpt":0.2864189849729321,"score_spread":0.2544528885898649,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}