{"id":"W4404780849","doi":"10.18653/v1/2024.findings-emnlp.730","title":"“Knowing When You Don’t Know”: A Multilingual Relevance Assessment Dataset for Robust Retrieval-Augmented Generation","year":2024,"lang":"en","type":"article","venue":"","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Relevance (law); Computer science; Information retrieval; Relevance feedback; Artificial intelligence; Natural language processing; Image retrieval; Image (mathematics); Political science","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002259395,0.001483085,0.000959651,0.003028274,0.001522955,0.001520374,0.002891379,0.002614076,0.008990653],"category_scores_gemma":[0.007211625,0.0003871883,0.001306878,0.002140886,0.0007013962,0.001932482,0.003427198,0.00162041,0.01079025],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001130329,"about_ca_system_score_gemma":0.001834827,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02303111,"about_ca_topic_score_gemma":0.06104676,"domain_scores_codex":[0.997938,0.0007281901,0.0002226064,0.0004922393,0.0004567746,0.0001622132],"domain_scores_gemma":[0.9963469,0.001261767,0.0001445753,0.001072001,0.000810632,0.0003642148],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.002098491,0.0013609,0.01244288,0.002311827,0.0003380432,0.001033106,0.0009143917,0.00557552,0.01129931,0.00324418,0.7636592,0.1957223],"study_design_scores_gemma":[0.001819791,0.001246936,0.05512293,0.0007293297,0.0004962853,0.002639598,0.003704269,0.08521054,0.03014239,0.009699314,0.808658,0.0005306057],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"dataset","genre_gemma":"dataset","genre_scores_codex":[0.1775456,0.006327914,0.03171016,0.003181537,0.001386239,0.001891675,0.7306902,0.0225996,0.02466705],"genre_scores_gemma":[0.07440202,0.0003792901,0.03192094,0.0004152814,0.00008903268,0.0006277326,0.885666,0.0004908646,0.006008856],"genre_candidate":"dataset","genre_consensus":"dataset","teacher_disagreement_score":0.02303111,"threshold_uncertainty_score":0.04579413,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03810460359996788,"score_gpt":0.3433172520810044,"score_spread":0.3052126484810366,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}