{"id":"W4401043137","doi":"10.18653/v1/2024.naacl-long.161","title":"Evaluating In-Context Learning of Libraries for Code Generation","year":2024,"lang":"en","type":"article","venue":"","topic":"Digital Accessibility for Disabilities","field":"Social Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Allen Institute for Artificial Intelligence; Natural Sciences and Engineering Research Council of Canada; McGill University","keywords":"Computer science; Context (archaeology); Code (set theory); Code generation; Programming language; Operating system","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002749189,0.001350568,0.0008607538,0.002043055,0.0008040679,0.001837139,0.001769705,0.001738788,0.003190555],"category_scores_gemma":[0.02480662,0.0003893169,0.0007573485,0.0009782886,0.0004035012,0.00423898,0.002396054,0.001830924,0.002346047],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001067878,"about_ca_system_score_gemma":0.001915256,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006010956,"about_ca_topic_score_gemma":0.01461253,"domain_scores_codex":[0.9964424,0.001533434,0.0002174442,0.0008319026,0.0007907157,0.0001840874],"domain_scores_gemma":[0.9824896,0.01174059,0.000742199,0.001806969,0.002354023,0.0008665731],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.002299818,0.001727815,0.02183771,0.0005891659,0.0002551936,0.0002397492,0.0004693389,0.03909385,0.009712643,0.001455612,0.01597137,0.9063478],"study_design_scores_gemma":[0.0005240809,0.002351081,0.008768241,0.0002032509,0.0003391851,0.0003104384,0.001046929,0.9113513,0.05299418,0.007927892,0.01410362,0.00007972526],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"methods","genre_scores_codex":[0.802761,0.008837556,0.1437863,0.001538425,0.0006824931,0.0005305186,0.001944858,0.02698011,0.01293872],"genre_scores_gemma":[0.8848845,0.0007274186,0.1039107,0.0003022716,0.000128645,0.0001529732,0.005127443,0.0006594957,0.004106515],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.006010956,"threshold_uncertainty_score":0.0145393,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1684341334377057,"score_gpt":0.4342591159206821,"score_spread":0.2658249824829764,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}