{"id":"W4403334020","doi":"10.1145/3654777.3676450","title":"Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences","year":2024,"lang":"en","type":"article","venue":"","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":131,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"","keywords":"Computer science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002414408,0.0001908476,0.000196947,0.0001705827,0.0002653303,0.0007339276,0.001124373,0.00006123649,0.0001495506],"category_scores_gemma":[0.00008509173,0.0001109956,0.00006933932,0.0009812253,0.00014131,0.0009803624,0.000217585,0.0001429649,0.00008668765],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00007011749,"about_ca_system_score_gemma":0.0002319931,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003794447,"about_ca_topic_score_gemma":0.0003169961,"domain_scores_codex":[0.9973073,0.0003559139,0.0004060156,0.000522015,0.001099968,0.0003087994],"domain_scores_gemma":[0.9982741,0.0003070982,0.0001210622,0.0007383279,0.0004961723,0.00006320511],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.00001049066,0.0002644316,0.001872475,0.000135329,0.0002440605,0.00001593837,0.01628629,0.002223358,0.01739517,0.8652802,0.003851483,0.09242076],"study_design_scores_gemma":[0.0001480645,0.0006203249,0.002189409,0.000442483,0.0001549717,0.00001952678,0.002853529,0.2013077,0.7298962,0.060121,0.001804742,0.0004420408],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7104168,0.0007533851,0.2503306,0.002016837,0.000520216,0.0008307959,0.000003960851,0.0005702379,0.03455719],"genre_scores_gemma":[0.9958168,0.000007286147,0.003187867,0.00009534427,0.00005931704,0.00005689347,0.000004842114,0.00001376444,0.0007578763],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8051592,"threshold_uncertainty_score":0.7077281,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1043031927296003,"score_gpt":0.3456309742213749,"score_spread":0.2413277814917746,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}