{"id":"W4223598146","doi":"10.1145/3524610.3527886","title":"On the effectiveness of pretrained models for API learning","year":2022,"lang":"en","type":"preprint","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia, Okanagan Campus; Kelowna General Hospital; University of British Columbia","funders":"","keywords":"Computer science; Natural language processing; Artificial intelligence; Natural language; Language model; Lexical analysis; Task (project management); Information retrieval; Context (archaeology); Encoder; Question answering; Transformer; Automatic summarization; Parsing; ENCODE; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.007169214,0.00443526,0.001707529,0.001427548,0.0009874471,0.002400411,0.002953497,0.004121108,0.006136402],"category_scores_gemma":[0.03627221,0.001172516,0.001242204,0.001110995,0.001210923,0.007436298,0.002381454,0.005986129,0.002429833],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002170254,"about_ca_system_score_gemma":0.002359336,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02298011,"about_ca_topic_score_gemma":0.02081067,"domain_scores_codex":[0.9973736,0.001052712,0.0001974895,0.0007846513,0.0003571278,0.0002344149],"domain_scores_gemma":[0.9746303,0.02051522,0.0005108496,0.002180802,0.001754215,0.0004086923],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001412815,0.0007654211,0.006704802,0.0004417406,0.0004905607,0.0002146023,0.0001243365,0.6533466,0.002641661,0.009065745,0.01773753,0.3070542],"study_design_scores_gemma":[0.00003568917,0.0001116907,0.0003731568,0.0000461038,0.00005327766,0.00003155023,0.00002403289,0.9920762,0.001199146,0.00546763,0.0005681756,0.00001333843],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3361261,0.02427858,0.5756951,0.009450257,0.001658394,0.0005137004,0.003317169,0.01542182,0.03353898],"genre_scores_gemma":[0.8812171,0.003521533,0.09480514,0.002219866,0.0004618559,0.000362562,0.005370255,0.0008522285,0.01118946],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.02298011,"threshold_uncertainty_score":0.04569268,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03599107396813926,"score_gpt":0.293690002250762,"score_spread":0.2576989282826227,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}