{"id":"W4223598146","doi":"10.1145/3524610.3527886","title":"On the effectiveness of pretrained models for API learning","year":2022,"lang":"en","type":"preprint","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia, Okanagan Campus; Kelowna General Hospital; University of British Columbia","funders":"","keywords":"Computer science; Natural language processing; Artificial intelligence; Natural language; Language model; Lexical analysis; Task (project management); Information retrieval; Context (archaeology); Encoder; Question answering; Transformer; Automatic summarization; Parsing; ENCODE; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003186566,0.0001464527,0.0002154071,0.0001356059,0.00009120392,0.00007250804,0.001838914,0.0000813279,0.0000417332],"category_scores_gemma":[0.002297545,0.0001058208,0.0001434285,0.0002066506,0.00002467781,0.00005429841,0.002013451,0.0007956672,0.00000185966],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001071706,"about_ca_system_score_gemma":0.0001450176,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002556912,"about_ca_topic_score_gemma":2.045406e-7,"domain_scores_codex":[0.9982938,0.0004199323,0.0001422134,0.0004448266,0.0004758744,0.00022333],"domain_scores_gemma":[0.9823207,0.01664547,0.00005651653,0.0008389014,0.000105352,0.0000330159],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002458092,0.00001617186,0.00005381554,0.000321805,0.00003558019,7.645316e-7,0.0002054643,0.7567834,0.00007099094,0.2416955,0.00007983635,0.000712071],"study_design_scores_gemma":[0.0001534128,0.0001805229,0.0006724309,0.00009901732,0.00000304317,5.559401e-7,0.00001052378,0.9010897,0.001734261,0.09585389,0.00006815444,0.0001345112],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.02518828,0.00004232454,0.9715753,0.00009758239,0.000349409,0.0010532,0.000005082932,0.0002711153,0.001417728],"genre_scores_gemma":[0.9873056,0.000003374704,0.01132107,0.00001335564,0.00001877728,0.0008778595,0.00000647967,0.00002188378,0.0004315328],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9621174,"threshold_uncertainty_score":0.4315247,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03599107396813926,"score_gpt":0.293690002250762,"score_spread":0.2576989282826227,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}