{"id":"W4389523693","doi":"10.18653/v1/2023.findings-emnlp.549","title":"Open Domain Multi-document Summarization: A Comprehensive Study of Model Brittleness under Retrieval","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"Vector Institute; University of Toronto","funders":"Alliance de recherche numérique du Canada","keywords":"Automatic summarization; Computer science; Information retrieval; Domain (mathematical analysis); Multi-document summarization; Task (project management); Set (abstract data type); Natural language processing","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0002107531,0.0001125338,0.0002148497,0.0001092899,0.00009895643,0.0001839924,0.001546547,0.00003899963,0.00001395203],"category_scores_gemma":[0.00001230404,0.0001048213,0.00003019733,0.0006986063,0.00001858918,0.0004704827,0.002220849,0.00006862123,0.00002448414],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004186743,"about_ca_system_score_gemma":0.00009864676,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003094733,"about_ca_topic_score_gemma":0.00009481849,"domain_scores_codex":[0.9985361,0.00009427233,0.0003269832,0.000449745,0.000389403,0.0002034949],"domain_scores_gemma":[0.9988769,0.00007646928,0.00008020225,0.0007337845,0.0001711292,0.00006155254],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003007522,0.0005101397,0.0009160992,0.00002445521,0.00007781478,0.00002521904,0.005871663,0.8615257,0.002578223,0.1246563,0.0003842075,0.003400022],"study_design_scores_gemma":[0.001459499,0.00007405687,0.002546988,0.00001204967,0.00000449769,0.000001461087,0.001506567,0.9852597,0.0003910627,0.008564209,0.00004353247,0.0001363376],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3077251,0.000006819781,0.6907598,0.0003103853,0.0001199355,0.0006375226,9.853958e-7,0.0001119264,0.0003274547],"genre_scores_gemma":[0.8884655,0.000004636176,0.1095151,0.0002043508,0.00001373338,0.00002664018,0.000003444821,0.000009692628,0.001756927],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.5812448,"threshold_uncertainty_score":0.4274487,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1222880486129483,"score_gpt":0.3515873804412257,"score_spread":0.2292993318282775,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}