{"id":"W4235970556","doi":"10.1007/s10664-021-10009-1","title":"Conclusion stability for natural language based mining of design discussions","year":2021,"lang":"en","type":"article","venue":"Empirical Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":8,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Artifact (error); Code refactoring; Computer science; Relevance (law); Task (project management); Context (archaeology); Documentation; Stability (learning theory); Software; Machine learning; Artificial intelligence; Data mining; Software engineering; Engineering; Programming language; Systems engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.009759915,0.0008006552,0.001223106,0.006181894,0.002081123,0.004542124,0.002269743,0.001385105,0.01072909],"category_scores_gemma":[0.09417272,0.0006392752,0.00210784,0.00307486,0.001924499,0.007263926,0.003387846,0.002233748,0.003211234],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002320975,"about_ca_system_score_gemma":0.002666763,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004450572,"about_ca_topic_score_gemma":0.003429539,"domain_scores_codex":[0.988539,0.003144238,0.001026429,0.00349572,0.003116169,0.0006784639],"domain_scores_gemma":[0.9044801,0.06735961,0.004227835,0.01006219,0.01259136,0.001278827],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.003389885,0.0008010621,0.08377755,0.001714167,0.0005323887,0.0007561265,0.004859746,0.08771267,0.0347743,0.24976,0.02968717,0.5022349],"study_design_scores_gemma":[0.00009342159,0.0002438832,0.009610051,0.0001787125,0.0001502574,0.0003690852,0.001155171,0.7075464,0.02306689,0.2484002,0.00910558,0.00008050787],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1142282,0.0006064289,0.8683203,0.001534403,0.0001368652,0.0003135786,0.00489998,0.004361906,0.005598328],"genre_scores_gemma":[0.8117827,0.000249039,0.1740303,0.0004074845,0.0001821954,0.0004687211,0.007869638,0.0007573651,0.004252582],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9902401,"threshold_uncertainty_score":0.05161601,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04044770805115119,"score_gpt":0.3104229520862787,"score_spread":0.2699752440351275,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}