{"id":"W3022592851","doi":"10.18653/v1/2020.acl-main.220","title":"Learning an Unreferenced Metric for Online Dialogue Evaluation","year":2020,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"Mila - Quebec Artificial Intelligence Institute; McGill University","funders":"McGill University; Canadian Institute for Advanced Research","keywords":"Inference; Computer science; Metric (unit); Task (project management); Artificial intelligence; Domain (mathematical analysis); Natural language processing; Quality (philosophy); Machine learning; Open domain; Question answering; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009667872,0.0001927332,0.000271699,0.0002200032,0.00007908073,0.0002343935,0.001233353,0.0001915228,0.00002542206],"category_scores_gemma":[0.0008406505,0.000182391,0.00009819269,0.0002925991,0.000007781296,0.0002097227,0.001037594,0.0004902059,0.00001123804],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001175104,"about_ca_system_score_gemma":0.0004288054,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001407987,"about_ca_topic_score_gemma":0.00007868866,"domain_scores_codex":[0.9978437,0.0002061714,0.0003326176,0.0008953882,0.0004831199,0.0002389326],"domain_scores_gemma":[0.9985329,0.0001582724,0.0001806636,0.0006734064,0.0003265951,0.0001281955],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000004537537,0.00005083983,0.00007497332,0.0000663176,0.00002789884,7.605554e-7,0.0007391648,0.2637673,0.0001370103,0.01614538,0.00007385042,0.7189119],"study_design_scores_gemma":[0.0002481561,0.000106924,0.0004372929,0.00001791116,0.00002654383,4.484742e-7,0.00002685676,0.9577046,0.0001156873,0.0407052,0.0004023677,0.000207989],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01679795,0.00006515826,0.9792957,0.00112203,0.0009156932,0.000739244,0.000007611523,0.0003935434,0.0006630382],"genre_scores_gemma":[0.5312889,0.000009137696,0.4677016,0.000185267,0.0003266309,0.00008322224,0.0002939822,0.00001044792,0.0001008307],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.718704,"threshold_uncertainty_score":0.7437689,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2226464616326511,"score_gpt":0.3771446736430029,"score_spread":0.1544982120103518,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}