{"id":"W2584220694","doi":"10.1609/aaai.v32i1.11321","title":"RUBER: An Unsupervised Method for Automatic Evaluation of Open-Domain Dialog Systems","year":2018,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Artificial Intelligence","topic":"Topic Modeling","field":"Computer Science","cited_by":162,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"National Key Research and Development Program of China; National Natural Science Foundation of China; Tencent; National Science Foundation","keywords":"Dialog box; Computer science; Annotation; Metric (unit); Open domain; Artificial intelligence; Utterance; Domain (mathematical analysis); Natural language processing; Dialog system; Transferability; Conversation; Information retrieval; Machine learning; World Wide Web; Linguistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004853962,0.0001928799,0.0003671267,0.0001349922,0.0001998547,0.0004013134,0.003760353,0.00009393309,0.00003701712],"category_scores_gemma":[0.0007927412,0.0001483833,0.00008696393,0.0004986056,0.0001814213,0.0007952574,0.0005035775,0.0001160081,0.0000134811],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006903306,"about_ca_system_score_gemma":0.0002784203,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002016459,"about_ca_topic_score_gemma":0.00002857745,"domain_scores_codex":[0.9975374,0.000112053,0.0007909015,0.0005666017,0.0006975083,0.0002955762],"domain_scores_gemma":[0.9960194,0.0001723426,0.0005766864,0.0005644841,0.002584134,0.00008300849],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002846247,0.0001097413,0.00003207493,0.00006903071,0.00001553846,1.992726e-8,0.002498206,0.0002483038,0.04483236,0.8040802,0.00003167563,0.1480544],"study_design_scores_gemma":[0.00005149189,0.0002321942,0.00002879049,0.0001515216,0.00002063177,0.000001329215,0.0003998284,0.6006531,0.1354056,0.2629367,0.00001581043,0.0001031351],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1849867,0.00001930255,0.8061517,0.0009264852,0.000779648,0.001973266,0.000007182832,0.00006658245,0.005089107],"genre_scores_gemma":[0.8815126,0.000002216087,0.1181141,0.00006981583,0.0001087352,0.000145932,6.199734e-7,0.00001172968,0.00003424446],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6965259,"threshold_uncertainty_score":0.6987734,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2435706272612481,"score_gpt":0.407469443211161,"score_spread":0.1638988159499128,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}