{"id":"W6929033617","doi":"10.48448/pcmq-qp41","title":"Automatic Evaluation of Generative Models with Instruction Tuning","year":2023,"lang":"en","type":"other","venue":"Open MIND","topic":"Human Motion and Animation","field":"Engineering","cited_by":0,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Generative model; Matching (statistics); Feature (linguistics); Generative grammar; Stability (learning theory); Process (computing)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0001936336,0.00009376875,0.000136065,0.0001367102,0.00001577965,0.00003638237,0.00007885233,0.00007434334,0.006448664],"category_scores_gemma":[0.000005554513,0.00008623338,0.00001326141,0.0001041287,0.00001171179,0.0001295295,0.00001301812,0.00005586179,0.0001779756],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005918801,"about_ca_system_score_gemma":0.00003279334,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002026786,"about_ca_topic_score_gemma":0.0004315971,"domain_scores_codex":[0.9994755,0.0000304192,0.0001241083,0.0001020689,0.000211689,0.00005624727],"domain_scores_gemma":[0.9997573,0.000004564146,0.0000764435,0.0001055008,0.0000398442,0.00001633414],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002772667,0.00001589148,0.000004799351,0.0001086836,0.0002049449,0.000001052156,0.002186508,0.26736,0.0009639445,0.000205636,0.01427399,0.7146718],"study_design_scores_gemma":[0.0004265773,0.00001632272,0.00004046438,0.0004328685,0.00006526885,0.000001161845,0.0001353276,0.9955391,0.0006414941,0.00009777088,0.00248385,0.000119845],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.021925,0.00005266923,0.01178659,0.000005504332,0.0002053724,0.0007410419,0.00002416281,0.00005986949,0.9651998],"genre_scores_gemma":[0.5645842,0.0001046184,0.1001048,0.000008305223,0.0005392422,0.0002813741,0.0004639179,0.002302609,0.3316109],"genre_candidate":"other","genre_consensus":null,"teacher_disagreement_score":0.728179,"threshold_uncertainty_score":0.9944596,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.09370928327921453,"score_gpt":0.309072205950342,"score_spread":0.2153629226711274,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}