{"id":"W4389523693","doi":"10.18653/v1/2023.findings-emnlp.549","title":"Open Domain Multi-document Summarization: A Comprehensive Study of Model Brittleness under Retrieval","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; University of Toronto","funders":"Alliance de recherche numérique du Canada","keywords":"Automatic summarization; Computer science; Information retrieval; Domain (mathematical analysis); Multi-document summarization; Task (project management); Set (abstract data type); Natural language processing","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01134073,0.001552163,0.002979797,0.002636527,0.0009731255,0.002856114,0.003377906,0.002288523,0.002456598],"category_scores_gemma":[0.04476495,0.0009957602,0.001741787,0.002630674,0.002469259,0.007392485,0.002239048,0.003457668,0.0006681415],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002740834,"about_ca_system_score_gemma":0.001469899,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007681186,"about_ca_topic_score_gemma":0.004443608,"domain_scores_codex":[0.9959599,0.002046121,0.0002629089,0.001066937,0.0004571102,0.0002070187],"domain_scores_gemma":[0.9464096,0.0423432,0.003687815,0.004390661,0.002504072,0.0006645972],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003185808,0.0001678716,0.00238612,0.0007021613,0.0003602448,0.0002053506,0.000566949,0.8825654,0.001850069,0.0341318,0.00356289,0.0731825],"study_design_scores_gemma":[0.00001744287,0.00008208469,0.000477454,0.00003280879,0.0000413703,0.00005822947,0.00005199579,0.9645892,0.0004757097,0.03313562,0.001015535,0.00002252862],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05566109,0.005778699,0.9322712,0.001828458,0.0001139681,0.0002144795,0.0007622967,0.0009787817,0.002391078],"genre_scores_gemma":[0.7984326,0.00364218,0.1871875,0.0007630275,0.0009382853,0.0005091786,0.002925914,0.0005960769,0.00500526],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.01134073,"threshold_uncertainty_score":0.05997616,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1222880486129483,"score_gpt":0.3515873804412257,"score_spread":0.2292993318282775,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}