{"id":"W3176225318","doi":"10.1145/3463274.3463359","title":"Towards a More Structured Peer Review Process with Empirical Standards","year":2021,"lang":"en","type":"article","venue":"Evaluation and Assessment in Software Engineering","topic":"scientometrics and bibliometrics research","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; Dalhousie University","funders":"","keywords":"Process (computing); Computer science; Process management; Empirical research; Peer review; Business; Programming language; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","bibliometrics"],"consensus_categories":[],"category_scores_codex":[0.0232108,0.0001473521,0.0003170516,0.004494478,0.0000740394,0.0007816319,0.0003483125,0.00006357041,0.0008647828],"category_scores_gemma":[0.05911246,0.0001009953,0.00004509603,0.03838604,0.00003046041,0.0003903114,0.0001535098,0.000354619,0.000002339537],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002747027,"about_ca_system_score_gemma":0.001716274,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000003812974,"about_ca_topic_score_gemma":0.000008198741,"domain_scores_codex":[0.9817023,0.0001635877,0.000472286,0.0005478231,0.01681363,0.0003003174],"domain_scores_gemma":[0.9883053,0.000780272,0.00009743478,0.0003433322,0.01027482,0.0001988219],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00001595856,0.0001415257,0.4304905,0.0004696297,0.00004350088,0.00008055152,0.0006330712,0.01254423,0.00006737345,0.0002317378,0.009643277,0.5456386],"study_design_scores_gemma":[0.001235052,0.00009650244,0.8553563,0.0005226606,0.00003074629,0.0000378759,0.0005582723,0.1033787,0.0001965514,0.0008840855,0.0373697,0.0003335371],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7250865,0.02374579,0.2299976,0.01715855,0.0006369853,0.00140927,0.0001474696,0.0001291923,0.00168861],"genre_scores_gemma":[0.9611326,0.00116073,0.03657568,0.0004982896,0.00005735666,0.000109631,0.0000598864,0.00002118967,0.0003845837],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5453051,"threshold_uncertainty_score":0.9820535,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4179059071787115,"score_gpt":0.6254396821821306,"score_spread":0.2075337750034191,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}