{"id":"W4401042661","doi":"10.18653/v1/2024.naacl-long.208","title":"Long-form evaluation of model editing","year":2024,"lang":"en","type":"article","venue":"","topic":"Model-Driven Software Engineering Techniques","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Dalhousie University","funders":"Alliance de recherche numérique du Canada; Killam Trusts","keywords":"Computer science","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000749986,0.00006017907,0.00006333043,0.0001003569,0.00001280268,0.00004898943,0.0002993779,0.00003350187,0.00001337705],"category_scores_gemma":[0.000008284349,0.00005333783,0.0000351358,0.0001922775,0.000008174102,0.0004196007,0.0001101171,0.00006288278,0.000008582039],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005599556,"about_ca_system_score_gemma":0.00007405481,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000006629825,"about_ca_topic_score_gemma":0.000001721895,"domain_scores_codex":[0.9991716,0.00001090658,0.000135423,0.00017009,0.0004144094,0.00009763895],"domain_scores_gemma":[0.99953,0.00001960755,0.00001670961,0.0002795596,0.0001310438,0.0000231181],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[1.324462e-7,0.000007038309,0.00002450747,0.00003837028,0.000007089114,0.000001077756,0.000307357,0.09090319,0.0003195954,0.5622038,0.001270107,0.3449178],"study_design_scores_gemma":[0.00002437261,0.000009727702,0.00004256846,0.00004905612,0.000005822695,0.000002870373,4.107801e-7,0.9755433,0.006194889,0.0177792,0.0002929587,0.00005483432],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.003183424,0.0001102726,0.9928767,0.0001138409,0.0001907794,0.0001010247,7.120551e-7,0.001171082,0.002252128],"genre_scores_gemma":[0.514662,0.000002062679,0.4851971,0.000009955592,0.00002690844,0.00001319376,7.016106e-7,0.000004436314,0.00008361948],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8846401,"threshold_uncertainty_score":0.2175054,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04670331372071215,"score_gpt":0.3104295284407657,"score_spread":0.2637262147200536,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}