{"id":"W4403334020","doi":"10.1145/3654777.3676450","title":"Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences","year":2024,"lang":"en","type":"article","venue":"","topic":"Explainable Artificial Intelligence (XAI)","field":"Computer Science","cited_by":131,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"","keywords":"Computer science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.08069227,0.001898957,0.001253103,0.002822484,0.00149481,0.007230446,0.003061451,0.002580537,0.01087377],"category_scores_gemma":[0.2378781,0.0011794,0.001211808,0.001139663,0.003163745,0.009716406,0.006491593,0.002775383,0.005410732],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002894879,"about_ca_system_score_gemma":0.004137042,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002595937,"about_ca_topic_score_gemma":0.003598084,"domain_scores_codex":[0.9121781,0.06606486,0.004001225,0.006848701,0.009332247,0.001574783],"domain_scores_gemma":[0.803749,0.1213938,0.0112773,0.03134229,0.02923187,0.003005815],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001570073,0.0009179696,0.03934165,0.002062815,0.0002492057,0.0008605415,0.05543586,0.008251722,0.03491231,0.06295838,0.04429901,0.7491404],"study_design_scores_gemma":[0.00072959,0.001535586,0.0221967,0.003442684,0.0003866989,0.001322413,0.02216824,0.3571322,0.1292093,0.13547,0.3254575,0.0009491891],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.06849205,0.0003200733,0.8719776,0.004587342,0.0001990893,0.001221224,0.0004343064,0.0303861,0.02238238],"genre_scores_gemma":[0.3563363,0.0001440995,0.6283394,0.001690917,0.00005712709,0.001143522,0.0007029919,0.004887674,0.006697992],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9193077,"threshold_uncertainty_score":0.4267466,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1043031927296003,"score_gpt":0.3456309742213749,"score_spread":0.2413277814917746,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}