{"id":"W4205513494","doi":"10.1109/ase51524.2021.9678640","title":"Is Historical Data an Appropriate Benchmark for Reviewer Recommendation Systems? : A Case Study of the Gerrit Community","year":2021,"lang":"en","type":"article","venue":"2021 36th IEEE/ACM International Conference on Automated Software Engineering (ASE)","topic":"Software Engineering Research","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo; Polytechnique Montréal; McGill University","funders":"","keywords":"Context (archaeology); Pessimism; Computer science; Benchmark (surveying); Replicate; Task (project management); Data science; History; Epistemology; Management","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001458419,0.0003407481,0.000452716,0.0002462915,0.0002544318,0.0003437579,0.003865959,0.0001454117,0.0001231521],"category_scores_gemma":[0.008897702,0.0003089131,0.0001067238,0.0007087103,0.00002401712,0.0006367081,0.001285211,0.0007623667,0.00001093637],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006325644,"about_ca_system_score_gemma":0.0003487794,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007942355,"about_ca_topic_score_gemma":0.00006435008,"domain_scores_codex":[0.9968402,0.0004645654,0.0007486186,0.0007575616,0.0008117369,0.0003773567],"domain_scores_gemma":[0.9928439,0.001590019,0.0002539728,0.003510125,0.001628298,0.0001737414],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0005111056,0.03086802,0.05423665,0.01053627,0.008833637,0.006679065,0.05537367,0.3209176,0.01040358,0.01683459,0.2687843,0.2160215],"study_design_scores_gemma":[0.0009132074,0.0002920013,0.001954201,0.0004834124,0.00004472603,0.000312612,0.0004582244,0.989701,0.0003712407,0.00005450015,0.004971264,0.0004435856],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4243603,0.0002227622,0.5577116,0.00142473,0.01186795,0.001952438,0.0007935853,0.001625899,0.00004072053],"genre_scores_gemma":[0.9731131,0.00003585086,0.02569008,0.00008641899,0.0002015348,0.0002132171,0.0003601008,0.00005427002,0.0002453751],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6687834,"threshold_uncertainty_score":0.9999363,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1459378586426111,"score_gpt":0.3631536798642139,"score_spread":0.2172158212216028,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}