{"id":"W4401043137","doi":"10.18653/v1/2024.naacl-long.161","title":"Evaluating In-Context Learning of Libraries for Code Generation","year":2024,"lang":"en","type":"article","venue":"","topic":"Digital Accessibility for Disabilities","field":"Social Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Allen Institute for Artificial Intelligence; Natural Sciences and Engineering Research Council of Canada; McGill University","keywords":"Computer science; Context (archaeology); Code (set theory); Code generation; Programming language; Operating system","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009934796,0.00004166245,0.00008321317,0.00004709085,0.00009337594,0.0002855077,0.00008860945,0.00003387782,0.0001913773],"category_scores_gemma":[0.001502968,0.00003666409,0.0000499402,0.0001700793,0.0001755385,0.001062564,0.00002371799,0.00004162824,0.00000603435],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00004734363,"about_ca_system_score_gemma":0.0002027984,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000683863,"about_ca_topic_score_gemma":0.007418794,"domain_scores_codex":[0.9993018,0.00006631617,0.0001988645,0.0001424348,0.0001683547,0.000122223],"domain_scores_gemma":[0.999367,0.0004907424,0.00002340319,0.0000490882,0.0000506166,0.00001912278],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.000006597607,0.00002007254,0.006718716,0.000098502,0.000006019453,8.516038e-8,0.03077798,0.0001857821,0.0004490767,0.8207129,0.0002273084,0.1407969],"study_design_scores_gemma":[0.0004910404,0.0004498853,0.001721727,0.000269456,0.00002903742,2.310117e-7,0.2313548,0.1792916,0.01329209,0.5203713,0.05229398,0.0004349667],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9135516,0.000371468,0.0008240855,0.0006200817,0.0001864577,0.000268895,0.00000589874,0.00009193761,0.08407953],"genre_scores_gemma":[0.9907885,0.000003811325,0.001181077,0.00002326242,0.0001045429,0.00003322004,0.000006974958,0.000005721352,0.007852855],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3003417,"threshold_uncertainty_score":0.4139861,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1684341334377057,"score_gpt":0.4342591159206821,"score_spread":0.2658249824829764,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}