{"id":"W4393902987","doi":"10.3138/cjpe-2024-0005","title":"Learning From Evaluation Data: Discoveries From the Inaugural Evaluation Capacity Case Challenge","year":2024,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa; University of Alberta; McGill University; Queen's University","funders":"","keywords":"Data science; Computer science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2014431,0.0005201411,0.001017504,0.003114197,0.009807399,0.0223036,0.004169613,0.005740942,0.003224634],"category_scores_gemma":[0.3129793,0.0006214331,0.0006969718,0.003724898,0.02469859,0.02947956,0.01732977,0.01416162,0.0005492344],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.012624,"about_ca_system_score_gemma":0.02317364,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007332087,"about_ca_topic_score_gemma":0.02094515,"domain_scores_codex":[0.8447177,0.1231408,0.003866969,0.003887018,0.02062017,0.003767298],"domain_scores_gemma":[0.5326445,0.3884751,0.01029332,0.02686284,0.03316042,0.008563727],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"qualitative","study_design_scores_codex":[0.0000991452,0.0002231101,0.0107967,0.000701779,0.00003753297,0.001325524,0.1113618,0.001160498,0.0004546769,0.6306852,0.04424883,0.198905],"study_design_scores_gemma":[0.00005105589,0.0001613109,0.005246313,0.003126117,0.00002831587,0.001507426,0.1025878,0.005989801,0.001871893,0.4299713,0.4492547,0.0002039583],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"commentary","genre_gemma":"empirical","genre_scores_codex":[0.118403,0.0157972,0.1359059,0.6135002,0.002334021,0.0006363472,0.0004606179,0.0001828459,0.1127798],"genre_scores_gemma":[0.8451725,0.008659849,0.1110264,0.01746667,0.001334093,0.0009254405,0.0003937746,0.0002211419,0.01480017],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7985569,"threshold_uncertainty_score":0.9847628,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.7100870622061499,"score_gpt":0.544183741665453,"score_spread":0.1659033205406969,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}