{"id":"W2125759561","doi":"10.1145/1368088.1368115","title":"On the difficulty of replicating human subjects studies in software engineering","year":2008,"lang":"en","type":"article","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":67,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Replication (statistics); Comparability; Computer science; Context (archaeology); Literal (mathematical logic); Empirical research; Software; Test (biology); Artificial intelligence; Software engineering; Cognitive psychology; Programming language; Psychology; Mathematics; Statistics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003133231,0.00009263608,0.0001358131,0.0001162715,0.00007671813,0.00001105904,0.0006343283,0.00002475029,0.000003197041],"category_scores_gemma":[0.003447009,0.00006106873,0.00003325455,0.0005160997,0.00004687076,0.00008050084,0.0002481095,0.0001757019,0.000006803044],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005628275,"about_ca_system_score_gemma":0.00001902694,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003232548,"about_ca_topic_score_gemma":0.000006522251,"domain_scores_codex":[0.9990743,0.00002183191,0.0001775437,0.0002301021,0.0002785289,0.0002176625],"domain_scores_gemma":[0.9968998,0.002356271,0.00002937746,0.000619816,0.00006619524,0.00002848664],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00001733343,0.0007659083,0.5335919,0.0008059095,0.0003132418,0.0004980062,0.04829545,0.1072459,0.05428502,0.2369514,0.01159381,0.005636172],"study_design_scores_gemma":[0.0005968635,0.0002637942,0.9058578,0.0003920185,0.00000232054,0.00006406644,0.0003050294,0.02171306,0.0695592,0.000672797,0.0000774138,0.0004956541],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9636606,0.000103664,0.03566987,0.00009441796,0.00005401225,0.0001193666,2.289962e-7,0.00025424,0.00004359489],"genre_scores_gemma":[0.9881768,0.000007797162,0.01162264,0.00002529318,0.00001451472,0.00002982705,2.003227e-7,0.000008048416,0.000114861],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3722658,"threshold_uncertainty_score":0.4126641,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07825651788581889,"score_gpt":0.3125074568354242,"score_spread":0.2342509389496053,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}