{"id":"W2929767294","doi":"10.18653/v1/n19-1381","title":"Evaluating Coherence in Dialogue Systems using Entailment","year":2019,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Computer science; Coherence (philosophical gambling strategy); Textual entailment; Sentence; Natural language processing; Artificial intelligence; Logical consequence; Scalability; Quality (philosophy); Meaning (existential); Machine learning; Mathematics; Statistics; Psychology; Epistemology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001215633,0.0002311201,0.0003746411,0.0001651532,0.00003457045,0.0003703717,0.001425895,0.0001972364,0.00001279523],"category_scores_gemma":[0.00004957573,0.000221908,0.00006458811,0.0001284208,0.00001132659,0.0001745867,0.002835947,0.0004687851,0.00005052573],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004196835,"about_ca_system_score_gemma":0.0004355004,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004031644,"about_ca_topic_score_gemma":0.00005038544,"domain_scores_codex":[0.9974504,0.0002256068,0.0005593566,0.0008812323,0.0005416551,0.0003417335],"domain_scores_gemma":[0.9982101,0.0001041641,0.0002389913,0.001307401,0.00008241559,0.00005696428],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[8.158975e-7,0.00001928698,0.003226516,0.0001710013,0.000009455342,0.000007273631,0.0005431761,0.9841937,0.0004260752,0.006457297,0.00001247754,0.004932871],"study_design_scores_gemma":[0.0001675959,0.00001466308,0.0002213185,0.0004994092,0.000004524707,0.00000486912,0.00004323853,0.9973357,0.00009519528,0.001349884,0.0000168076,0.0002467708],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.198553,0.0003052838,0.7952601,0.00005946718,0.003144467,0.0007851669,0.000001533163,0.0001036572,0.001787261],"genre_scores_gemma":[0.8506708,0.000005408888,0.1488321,0.00006504199,0.0001082938,0.00004740851,0.000003735354,0.00001035326,0.00025685],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6521177,"threshold_uncertainty_score":0.9049147,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2056351445495016,"score_gpt":0.3773597918694956,"score_spread":0.1717246473199941,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}