{"id":"W3176225318","doi":"10.1145/3463274.3463359","title":"Towards a More Structured Peer Review Process with Empirical Standards","year":2021,"lang":"en","type":"article","venue":"Evaluation and Assessment in Software Engineering","topic":"scientometrics and bibliometrics research","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; Dalhousie University","funders":"","keywords":"Process (computing); Computer science; Process management; Empirical research; Peer review; Business; Programming language; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.664901,0.00245129,0.00392592,0.01978119,0.008485331,0.03077512,0.01259858,0.01060988,0.004877345],"category_scores_gemma":[0.7217215,0.002671184,0.0029821,0.01117731,0.02160466,0.02350517,0.02471172,0.01871015,0.00604097],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01410726,"about_ca_system_score_gemma":0.08482743,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002075764,"about_ca_topic_score_gemma":0.002868325,"domain_scores_codex":[0.1823423,0.6122468,0.04761062,0.02812204,0.1272415,0.002436935],"domain_scores_gemma":[0.08444589,0.3926679,0.08072937,0.156885,0.2679001,0.01737165],"domain_codex":"methods","domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0003669366,0.001972328,0.008583817,0.01153228,0.0009439542,0.0006739433,0.03965186,0.006159274,0.01167246,0.2005524,0.0498413,0.6680495],"study_design_scores_gemma":[0.001177796,0.002251221,0.01616481,0.01925677,0.0003526408,0.001445106,0.0159833,0.01635328,0.007570147,0.441592,0.4767262,0.001126702],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.008466537,0.003943673,0.9072648,0.04853059,0.003470315,0.01256702,0.0001669617,0.001877745,0.01371236],"genre_scores_gemma":[0.03794423,0.001414977,0.9422894,0.005114207,0.001875862,0.00845051,0.0002850799,0.0003019941,0.002323715],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.335099,"threshold_uncertainty_score":0.4132368,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4179059071787115,"score_gpt":0.6254396821821306,"score_spread":0.2075337750034191,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}