{"id":"W4405366177","doi":"10.3138/cjpe-2023-0012","title":"Examining Youth Sector Stakeholders’ Experiences in an Online Program Evaluation Certificate","year":2023,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University; Youth Research and Evaluation eXchange; Brock University","funders":"","keywords":"Certificate; Business; Public relations; Political science; Computer science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":true,"confidence":"high","status":"direct model label, unvalidated"},{"model":"grok","categories":[],"domain":null,"study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":true,"confidence":"high","status":"direct model label, unvalidated"},{"model":"opus","categories":["metaresearch"],"domain":"incentives","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":true,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.03396134,0.0002106657,0.00034876,0.002077955,0.00026852,0.0008795586,0.0007440196,0.0001321123,0.002267102],"category_scores_gemma":[0.003783699,0.0001715516,0.0001036003,0.003289009,0.0001455966,0.001650894,0.0000193615,0.0003199647,0.00008056897],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006563222,"about_ca_system_score_gemma":0.006035719,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005627401,"about_ca_topic_score_gemma":0.05345097,"domain_scores_codex":[0.9911465,0.001645934,0.001618322,0.0004717838,0.004553515,0.0005639161],"domain_scores_gemma":[0.994144,0.0002350727,0.000937588,0.0004729576,0.003571895,0.0006384538],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001985322,0.0001272653,0.04696091,0.000003484373,0.00001158832,0.000007904368,0.03678437,0.009406613,0.00006583357,0.00001208567,0.0003108905,0.9062892],"study_design_scores_gemma":[0.00133834,0.00128698,0.3406734,0.00007755812,0.00007493971,0.00001002685,0.1698465,0.48259,0.00007376269,0.001209322,0.002584442,0.0002347529],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9943652,0.0001851612,0.00007188006,0.0007213355,0.001234296,0.002924538,0.00001848903,0.00005008925,0.0004290623],"genre_scores_gemma":[0.9968215,0.000009124609,0.001801955,0.00008396574,0.0002582383,0.0007197795,0.000225695,0.00002060145,0.00005913871],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9060544,"threshold_uncertainty_score":0.9995992,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9339822613619181,"score_gpt":0.581308825436659,"score_spread":0.3526734359252591,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}