{"id":"W3111275476","doi":"10.1109/tse.2021.3078384","title":"A Comparison of Natural Language Understanding Platforms for Chatbots in Software Engineering","year":2021,"lang":"en","type":"article","venue":"IEEE Transactions on Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":106,"is_retracted":false,"has_abstract":true,"ca_institutions":"Concordia University","funders":"","keywords":"Chatbot; Computer science; IBM; Natural language understanding; Natural language; Task (project management); Software; Data science; Artificial intelligence; Software engineering; World Wide Web; Natural language processing; Programming language; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00027223,0.000331683,0.0004944089,0.0006905418,0.00008296878,0.00008984734,0.0005173066,0.0001647354,0.00001189353],"category_scores_gemma":[0.000766188,0.000377216,0.0002133429,0.001256635,0.00001903179,0.0004311006,0.00001493554,0.0006546523,0.000005072938],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004506119,"about_ca_system_score_gemma":0.0001169696,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001091441,"about_ca_topic_score_gemma":0.00001765384,"domain_scores_codex":[0.9978154,0.00001104409,0.0005032845,0.0005299294,0.0004797815,0.0006605635],"domain_scores_gemma":[0.9968122,0.002347107,0.00006313984,0.00053561,0.00009654368,0.0001454262],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002903061,0.0002000243,0.003053425,0.0006776905,0.0001185257,0.00006960763,0.002405948,0.9817206,0.005136516,0.0005774833,0.0000243041,0.005986894],"study_design_scores_gemma":[0.001802031,0.0001788176,0.003272298,0.001018967,0.00002655729,0.00005965224,0.0004011731,0.7721497,0.2201372,0.0000308308,0.00007803629,0.0008446582],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1258185,0.0006317081,0.8712961,0.00001555115,0.001169361,0.0002812821,0.00002398965,0.0007625606,9.746337e-7],"genre_scores_gemma":[0.844659,0.000009598366,0.1550711,0.00001064918,0.00003873936,0.00009895856,0.000007323848,0.00006194464,0.00004265101],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.7188405,"threshold_uncertainty_score":0.999868,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0374386258312366,"score_gpt":0.2990128889211052,"score_spread":0.2615742630898686,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}