{"id":"W4403036283","doi":"10.1007/s10664-024-10548-3","title":"An empirical study on the effectiveness of large language models for SATD identification and classification","year":2024,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":22,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Manitoba; Queen's University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Identification (biology); Computer science; Empirical research; Natural language processing; Artificial intelligence; Data science; Statistics; Mathematics; Biology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001309393,0.0001290622,0.0001411885,0.0001168871,0.00006698204,0.0001798554,0.0004001559,0.00006937946,6.919034e-7],"category_scores_gemma":[0.000469164,0.00008937769,0.00004429621,0.0003815466,0.00001526874,0.0003997906,0.00007634996,0.0001798364,0.000001488111],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006231861,"about_ca_system_score_gemma":0.00002603913,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000002742039,"about_ca_topic_score_gemma":6.034356e-7,"domain_scores_codex":[0.9989317,0.0001256364,0.0001906809,0.0003813851,0.0002089054,0.0001616811],"domain_scores_gemma":[0.9979991,0.001462456,0.00003263625,0.0003995827,0.00005980528,0.00004644186],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0004466712,0.004246131,0.07825341,0.006784437,0.0006479669,0.0001369919,0.07012993,0.01302269,0.2829572,0.3980004,0.002662091,0.1427121],"study_design_scores_gemma":[0.0002420221,0.0003961704,0.07359385,0.0002488804,0.00002779864,0.000006732423,0.0001563421,0.8916962,0.02080678,0.0123955,0.0001353881,0.0002943374],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2642289,0.0005783385,0.7338858,0.0001621737,0.00008749258,0.000411389,0.000006774099,0.0006370893,0.000002040754],"genre_scores_gemma":[0.9673632,0.000001409018,0.03236688,0.00004117642,0.00003960314,0.0001561051,0.000007054527,0.00001877846,0.000005791449],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8786735,"threshold_uncertainty_score":0.3644716,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02754798223888654,"score_gpt":0.3451586090035,"score_spread":0.3176106267646134,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}