{"id":"W4404689550","doi":"10.1109/beliv64461.2024.00018","title":"We Don't Know How to Assess LLM Contributions in VIS/HCI","year":2024,"lang":"en","type":"article","venue":"","topic":"Artificial Intelligence in Law","field":"Social Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Human–computer interaction","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0008934745,0.00009074068,0.0001287811,0.0001339314,0.0002695394,0.0004584379,0.000292989,0.0001157976,0.002031296],"category_scores_gemma":[0.0007923332,0.00008277801,0.0000624596,0.001032443,0.0001984253,0.0003725612,0.00006174063,0.0001745516,0.001439207],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003003447,"about_ca_system_score_gemma":0.0002862198,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004990018,"about_ca_topic_score_gemma":0.0674101,"domain_scores_codex":[0.9986448,0.0001548661,0.0001748199,0.0002745722,0.00031769,0.0004332843],"domain_scores_gemma":[0.9990596,0.0004883081,0.00001503839,0.0001479226,0.0001132377,0.0001758811],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000002810829,0.00004703752,0.000435929,0.000005043846,0.000007049607,0.00002120877,0.006075527,0.0000329883,0.0006561559,0.9442103,0.03094861,0.01755733],"study_design_scores_gemma":[0.00001593137,0.00002876546,0.0000774764,0.00006543406,0.000005336988,4.320214e-7,0.01153832,0.0005614041,0.003838561,0.01737252,0.9663471,0.0001487325],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.0511295,0.001118619,0.06426818,0.3779293,0.004248129,0.001222003,0.00006633955,0.001007015,0.4990109],"genre_scores_gemma":[0.9731277,0.000137529,0.0004338426,0.0003666826,0.0004193363,0.00004488366,0.000002051973,0.000009522736,0.02545851],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9353985,"threshold_uncertainty_score":0.9993383,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.09195651535672372,"score_gpt":0.4302569987492309,"score_spread":0.3383004833925072,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}