{"id":"W4398757454","doi":"10.1162/tacl_a_00667","title":"Evaluating Correctness and Faithfulness of Instruction-Following Models for Question Answering","year":2024,"lang":"en","type":"article","venue":"Transactions of the Association for Computational Linguistics","topic":"Topic Modeling","field":"Computer Science","cited_by":69,"is_retracted":false,"has_abstract":true,"ca_institutions":"Canadian Institute for Advanced Research; McGill University; Mila - Quebec Artificial Intelligence Institute; Minnow Environmental (Canada); Research Canada","funders":"","keywords":"Correctness; Computer science; Question answering; Natural language processing; Artificial intelligence; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0007699236,0.00008434834,0.00015215,0.0001057828,0.0002134809,0.00006618185,0.0001916228,0.00006318949,3.573119e-7],"category_scores_gemma":[0.0007533106,0.00008108369,0.0001544718,0.0002447588,0.00001686121,0.0001499504,0.00001421584,0.00008107107,9.965335e-8],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001267515,"about_ca_system_score_gemma":0.0001532528,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002465592,"about_ca_topic_score_gemma":0.000005163105,"domain_scores_codex":[0.9989496,0.00004338321,0.0003851098,0.0001987476,0.0003147357,0.000108447],"domain_scores_gemma":[0.9977589,0.001168447,0.0002052235,0.0001217265,0.0007253863,0.00002026614],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000004891626,0.00001269882,0.00007464747,0.0001369978,0.00006244984,2.58636e-8,0.0004174364,0.8073781,0.00005528296,0.1815955,0.000004424018,0.01025765],"study_design_scores_gemma":[0.0002811209,0.00003283201,0.0001393257,0.0001131828,0.00007535487,0.000001020297,0.00003366423,0.856438,0.000475256,0.1422618,0.000080665,0.00006773806],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01619233,0.00008257401,0.9801859,0.0001559232,0.002853577,0.0003383785,0.00006211163,0.00006568449,0.00006357394],"genre_scores_gemma":[0.7754974,0.000001383816,0.2242914,0.000007629467,0.00008232595,0.00003225439,0.000007264649,0.000008556008,0.00007180391],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7593051,"threshold_uncertainty_score":0.3306497,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03778902820660011,"score_gpt":0.3285510699647791,"score_spread":0.290762041758179,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}