{"id":"W4312443713","doi":"10.1109/re54965.2022.00011","title":"Automated Question Answering for Improved Understanding of Compliance Requirements: A Multi-Document Study","year":2022,"lang":"en","type":"article","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":33,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa","funders":"","keywords":"Jargon; Computer science; Compliance (psychology); Question answering; Natural language; Requirements engineering; Software requirements; Subject (documents); Information retrieval; Software; World Wide Web; Artificial intelligence; Software development; Software design; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008728719,0.00008875163,0.0001298181,0.0001433752,0.0001885795,0.00005841958,0.0005957345,0.00001115059,0.00001422649],"category_scores_gemma":[0.0001603613,0.00009405457,0.00003330281,0.0004213941,0.00001318569,0.0002473381,0.0004915359,0.00008432792,9.408192e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005842047,"about_ca_system_score_gemma":0.00004661748,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009169439,"about_ca_topic_score_gemma":0.000008806866,"domain_scores_codex":[0.9988503,0.00006260951,0.0002297727,0.000299138,0.0003097485,0.0002484045],"domain_scores_gemma":[0.9992189,0.0002356034,0.00006237711,0.0003695976,0.00006852433,0.00004501657],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0006280307,0.01088974,0.1381751,0.002177247,0.001719395,0.0001037158,0.03242352,0.2917482,0.3071022,0.1753768,0.004797307,0.03485877],"study_design_scores_gemma":[0.001234083,0.0006453826,0.005048165,0.00002216305,0.000003551419,0.000002113989,0.0006100984,0.9909697,0.001021012,0.0002772859,0.00004280692,0.000123657],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02889454,0.00002004272,0.9688705,0.0001147704,0.0002893424,0.0009437333,0.000003331254,0.0008524468,0.00001124855],"genre_scores_gemma":[0.8970816,5.571752e-7,0.1025353,0.00001614539,0.000005784136,0.0002258229,0.000002169649,0.0000104017,0.000122291],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.868187,"threshold_uncertainty_score":0.3835434,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1345397905093531,"score_gpt":0.380177912239094,"score_spread":0.2456381217297409,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}