{"id":"W3152918650","doi":"10.1007/s10664-023-10314-x","title":"Evaluating pre-trained models for user feedback analysis in software engineering: a study on classification of app-reviews","year":2023,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":36,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of British Columbia, Okanagan Campus; University of British Columbia","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Machine learning; Artificial intelligence; F1 score; Macro; Set (abstract data type); Task (project management); Binary classification; Support vector machine; Data mining; Engineering","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01112455,0.00159958,0.00145356,0.002506997,0.0008464092,0.00234399,0.00210419,0.002373301,0.0009961292],"category_scores_gemma":[0.0546048,0.0005244246,0.001135974,0.001439088,0.0005410224,0.002830767,0.001160274,0.00244104,0.001379473],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00222015,"about_ca_system_score_gemma":0.002011131,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01221319,"about_ca_topic_score_gemma":0.01339018,"domain_scores_codex":[0.9910953,0.004404254,0.000808308,0.001873245,0.001417745,0.0004009863],"domain_scores_gemma":[0.8748576,0.1037999,0.003044737,0.005029435,0.01185186,0.001416433],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.005560372,0.007036302,0.2485826,0.001476001,0.001468492,0.0004431819,0.003064018,0.08501057,0.01128884,0.0008432296,0.01816367,0.6170627],"study_design_scores_gemma":[0.0001318034,0.001833635,0.05812186,0.0001795796,0.0004365423,0.0003125734,0.0007043941,0.9259763,0.008908194,0.0008217448,0.002482913,0.00009057463],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9693755,0.00173374,0.02402757,0.0002863336,0.0001826134,0.0002661787,0.0008844003,0.001551331,0.001692343],"genre_scores_gemma":[0.9757111,0.0002931204,0.01840079,0.0001234232,0.00006643726,0.0001506769,0.00322014,0.0001387328,0.001895698],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9888754,"threshold_uncertainty_score":0.05883294,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1510793216345061,"score_gpt":0.4013715572634857,"score_spread":0.2502922356289796,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}