{"id":"W4404689550","doi":"10.1109/beliv64461.2024.00018","title":"We Don't Know How to Assess LLM Contributions in VIS/HCI","year":2024,"lang":"en","type":"article","venue":"","topic":"Artificial Intelligence in Law","field":"Social Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Human–computer interaction","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch","research_integrity"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.5161391,0.001303836,0.003427602,0.03054508,0.007773435,0.02692139,0.006554962,0.007006625,0.01016268],"category_scores_gemma":[0.8992592,0.001698133,0.003813601,0.01585568,0.0105436,0.02828848,0.01236739,0.00706105,0.007808723],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.009602892,"about_ca_system_score_gemma":0.02646152,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004738327,"about_ca_topic_score_gemma":0.007680655,"domain_scores_codex":[0.4816185,0.3049607,0.09429245,0.01608763,0.09938645,0.003654299],"domain_scores_gemma":[0.06578219,0.4299781,0.06393978,0.05545585,0.3776814,0.007162737],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0004535631,0.0001522152,0.02007227,0.03942436,0.003085268,0.0003160034,0.02533176,0.0006888714,0.00309146,0.03412899,0.262668,0.6105872],"study_design_scores_gemma":[0.0003405811,0.0008721719,0.02126881,0.0703103,0.002918286,0.0008140082,0.0288578,0.002983668,0.005202206,0.067239,0.7984232,0.0007699511],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"commentary","genre_gemma":"commentary","genre_scores_codex":[0.02941377,0.1481489,0.1454087,0.4769519,0.1416697,0.007153745,0.004012757,0.002565049,0.0446755],"genre_scores_gemma":[0.3536007,0.08278012,0.3467973,0.1117239,0.04959444,0.02301536,0.00373061,0.005433117,0.02332453],"genre_candidate":"commentary","genre_consensus":"commentary","teacher_disagreement_score":0.9929934,"threshold_uncertainty_score":0.5966866,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.09195651535672372,"score_gpt":0.4302569987492309,"score_spread":0.3383004833925072,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}