{"meta":{"page":1,"per_page":50,"max_per_page":100,"total":892,"total_is_capped":false,"direct_labels_cover":52,"predictions_cover":892,"direct_label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline (scores rank; they never assert a category)","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"query_hash":"466ddf5883b4","filters":{"venue":"Canadian Journal of Program Evaluation"}},"results":[{"id":"W1663638353","doi":"10.3138/cjpe.30.1.108","title":"Robert K. Yin. (2014). <i>Case Study Research Design and Methods (5th ed.).</i>","year":2015,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Qualitative Comparative Analysis Research","field":"Social Sciences","cited_by":406,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Psychology; Philosophy; Sociology","retraction":null,"screen_n_in":null,"score":{"opus":0.6829255352418476,"gpt":0.6779845172067633,"spread":0.004941018035084288,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1300038,0.0001114162,0.000288692,0.0009188388,0.0009018808,0.0005513845,0.0003479343,0.00007868379,0.0004392636],"category_scores_gemma":[0.01210423,0.00009938134,0.00006347804,0.001260096,0.000714916,0.0005516117,0.00002686566,0.0005122131,0.00004489878],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001076443,"about_ca_system_score_gemma":0.01127178,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.06006487,"about_ca_topic_score_gemma":0.1763618,"domain_scores_codex":[0.9637802,0.03294355,0.0004820823,0.0002356051,0.001997596,0.000561027],"domain_scores_gemma":[0.9879221,0.001462212,0.0002246231,0.0001947182,0.008571646,0.001624719],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"qualitative","study_design_scores_codex":[0.00006260302,0.0002923244,0.01024329,0.000006716605,0.0002581963,0.0006884043,0.1762019,0.0005726541,0.00004626918,0.001247984,0.06313013,0.7472495],"study_design_scores_gemma":[0.003167323,0.005915598,0.002739306,0.00006782213,0.0004023678,0.0003603138,0.7070971,0.02120946,0.0001125372,0.03387226,0.2245779,0.0004780347],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8589314,0.01689059,0.09131406,0.006767602,0.000768924,0.01203107,0.000004476901,0.00003890074,0.01325304],"genre_scores_gemma":[0.9453751,0.00003810703,0.05374508,0.00003258929,0.0002644554,0.0001603434,0.000001520072,0.00001409556,0.0003687113],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7467715,"threshold_uncertainty_score":0.9962173,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"not_applicable","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":[],"domain":null,"study_design":"not_applicable","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"low"},{"model":"opus","categories":[],"domain":null,"study_design":"not_applicable","genre":"commentary","about_ca_system":false,"about_ca_topic":false,"confidence":"low"}],"label_agreement":"agree"},{"id":"W2113515522","doi":"10.3138/cjpe.016.001","title":"Addressing Attribution through Contribution Analysis: Using Performance Measures Sensibly","year":2001,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":342,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"Government of Canada","funders":"","keywords":"Attribution; Accountability; Authorship attribution; Public relations; Administration (probate law); Psychology; Business; Political science; Social psychology; Computer science; Law; Artificial intelligence","retraction":null,"screen_n_in":null,"score":{"opus":0.6796845234554371,"gpt":0.5702891541640307,"spread":0.1093953692914064,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01855108,0.0001717153,0.0004181861,0.001231228,0.0006017481,0.000654636,0.0003546966,0.0001321038,0.001019635],"category_scores_gemma":[0.00228243,0.0001382279,0.0002745675,0.003573975,0.0001165234,0.001845184,0.0000119094,0.0002630332,0.00004416694],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009571845,"about_ca_system_score_gemma":0.004131797,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005635114,"about_ca_topic_score_gemma":0.0101921,"domain_scores_codex":[0.993841,0.0008973504,0.001319936,0.0002705077,0.003234984,0.0004362063],"domain_scores_gemma":[0.9895816,0.0001514445,0.001232262,0.0003376986,0.008321461,0.000375514],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005729624,0.0000558953,0.2139138,0.000003362315,0.0002584292,0.00001239033,0.0008444975,0.1098887,0.0003892263,0.0002682935,0.0003693116,0.6739388],"study_design_scores_gemma":[0.001325101,0.0004245279,0.2091643,0.00009822765,0.001331502,0.0001780687,0.0008806009,0.7357028,0.0007337685,0.00230982,0.04758309,0.0002682058],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8888697,0.0008263701,0.1075659,0.0007287594,0.0006935711,0.0006666715,0.000008013981,0.00001363579,0.0006273757],"genre_scores_gemma":[0.9956023,0.00004838652,0.003818422,0.0001350753,0.0002826333,0.00001613176,0.00004216899,0.000008724095,0.00004621566],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6736705,"threshold_uncertainty_score":0.9998935,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4230488502","doi":"10.3138/cjpe.230","title":"Useful Theory of Change Models","year":2015,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":213,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Causality (physics); Theory of change; Development theory; Intervention (counseling); Epistemology; Computer science; Psychological intervention; Management science; Psychology; Sociology; Economics; Philosophy; Physics","retraction":null,"screen_n_in":null,"score":{"opus":0.8815743086593234,"gpt":0.5830366120430426,"spread":0.2985376966162808,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02951679,0.00008291962,0.0002148965,0.000645357,0.00006086782,0.0001321304,0.0004306173,0.00006207573,0.0009688999],"category_scores_gemma":[0.002523195,0.00006067476,0.0001000823,0.0006161088,0.00008981459,0.001054459,0.00001074054,0.0001199315,0.00004725626],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002436918,"about_ca_system_score_gemma":0.005279141,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003148608,"about_ca_topic_score_gemma":0.005280161,"domain_scores_codex":[0.995709,0.000712624,0.0008087651,0.0001212273,0.002461384,0.0001870183],"domain_scores_gemma":[0.9930302,0.0001720576,0.0007556617,0.0002630102,0.005218913,0.0005601838],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00003011145,0.0000372332,0.01456073,0.000002721423,0.00002123891,0.000002753057,0.00536727,0.004864279,0.000006562896,0.005610535,0.002937779,0.9665588],"study_design_scores_gemma":[0.002441526,0.001567708,0.06029489,0.00009861121,0.0001692313,0.0000638531,0.008755816,0.2816721,0.0002045228,0.5905778,0.05390577,0.0002480593],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9596446,0.002650479,0.00972136,0.003488117,0.002291091,0.001916573,0.00001772277,0.00001230612,0.02025782],"genre_scores_gemma":[0.9969894,0.000007206828,0.002442915,0.0001788082,0.0001888478,0.00003806376,0.0000045625,0.000006667958,0.0001434966],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9663107,"threshold_uncertainty_score":0.9999443,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1594257894","doi":"10.3138/cjpe.23.003","title":"Reframing Evaluation: Defining an Indigenous Evaluation Framework","year":2008,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":159,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Cognitive reframing; Indigenous; Traditional knowledge; Focus group; Common ground; Sociology; Public relations; Political science; Engineering ethics; Psychology; Engineering; Social psychology; Anthropology; Ecology","retraction":null,"screen_n_in":null,"score":{"opus":0.396733003676957,"gpt":0.552227878522765,"spread":0.155494874845808,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.07245219,0.0002736312,0.0004493441,0.001522558,0.00118894,0.0006292202,0.0009150127,0.0002739923,0.008595575],"category_scores_gemma":[0.01642953,0.0002358241,0.0002122362,0.001882623,0.0001953106,0.002185793,0.00002159183,0.0005971259,0.0002806355],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001730375,"about_ca_system_score_gemma":0.02561126,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005223277,"about_ca_topic_score_gemma":0.008125919,"domain_scores_codex":[0.9832129,0.0031676,0.001969507,0.0005177734,0.0105287,0.0006034719],"domain_scores_gemma":[0.9810528,0.0006045217,0.00170871,0.0007943672,0.01490828,0.0009313123],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002261393,0.00008495262,0.03465482,0.00000290023,0.00004113437,0.00001091034,0.01349066,0.02055896,0.00001956082,0.0003557827,0.00076893,0.9299888],"study_design_scores_gemma":[0.002924512,0.001940261,0.2955871,0.0001803921,0.0006329354,0.0005999489,0.007993684,0.6049691,0.0001406582,0.06572765,0.0186782,0.0006255984],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9838338,0.002488669,0.003372459,0.0007311968,0.002198379,0.002861754,0.000009603757,0.00002705073,0.004477085],"genre_scores_gemma":[0.9794262,0.00002905847,0.01921723,0.0003187683,0.0005447072,0.0002830508,0.00009365455,0.00002850955,0.00005883276],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9293632,"threshold_uncertainty_score":0.9923107,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"qualitative","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":[],"domain":null,"study_design":"qualitative","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"opus","categories":["sts"],"domain":null,"study_design":"qualitative","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"medium"}],"label_agreement":"split"},{"id":"W2775174218","doi":"10.3138/cjpe.31122","title":"Theory of Change Analysis: Building Robust Theories of Change","year":2017,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":145,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Theory of change; Computer science; Management science; Risk analysis (engineering); Positive economics; Econometrics; Economics; Business; Management","retraction":null,"screen_n_in":null,"score":{"opus":0.7318312016170765,"gpt":0.5642255157815168,"spread":0.1676056858355597,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02695549,0.0001136358,0.0004325363,0.001245814,0.00031207,0.0002771632,0.0009126268,0.0000778667,0.002079111],"category_scores_gemma":[0.004102443,0.00008389902,0.0002730514,0.0007235947,0.0002827659,0.001225016,0.00002846376,0.0001323784,0.000008144712],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001332133,"about_ca_system_score_gemma":0.001414063,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00131182,"about_ca_topic_score_gemma":0.01937543,"domain_scores_codex":[0.9960204,0.0006297944,0.001003098,0.0001674196,0.001959151,0.00022014],"domain_scores_gemma":[0.9925872,0.0002683639,0.002643926,0.0006208509,0.003619072,0.0002606036],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00003053009,0.00003606013,0.1773194,0.000009459533,0.0002196487,0.000002095182,0.004115089,0.0006708325,0.00004289546,0.01083199,0.00009081062,0.8066311],"study_design_scores_gemma":[0.0008044912,0.0005412624,0.9022572,0.0001362424,0.00105172,0.000007164045,0.003570409,0.03193347,0.0005632939,0.05624328,0.002730956,0.0001605033],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9858212,0.001821543,0.004568031,0.002264469,0.001038813,0.001255007,0.00003408555,0.000005312036,0.003191527],"genre_scores_gemma":[0.9968542,0.00002688133,0.002668766,0.00006407727,0.0002457231,0.00006105687,0.000005214231,0.000006776988,0.00006730475],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8064706,"threshold_uncertainty_score":0.9988331,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W47101223","doi":"10.3138/cjpe.0015.003","title":"Predictors of Educator’s Valuing of Systematic Inquiry in Schools","year":2001,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Educational Assessment and Improvement","field":"Decision Sciences","cited_by":116,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Toronto; University of Ottawa","funders":"","keywords":"Psychology; Variance (accounting); Relevance (law); Exploratory research; Medical education; Value (mathematics); Medicine; Sociology; Social science","retraction":null,"screen_n_in":null,"score":{"opus":0.3838212522965732,"gpt":0.506814129178217,"spread":0.1229928768816438,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01226135,0.00008603389,0.0003629735,0.00114575,0.00003830417,0.000092401,0.0004479513,0.00005095997,0.0002875268],"category_scores_gemma":[0.005816371,0.00006633748,0.0001119014,0.001033022,0.00006732433,0.0004697507,0.000009429882,0.000136373,0.000006611624],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004018026,"about_ca_system_score_gemma":0.00583431,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007502328,"about_ca_topic_score_gemma":0.003509972,"domain_scores_codex":[0.9957548,0.0004723785,0.001693326,0.0001279275,0.001767239,0.0001842955],"domain_scores_gemma":[0.9955229,0.0003093847,0.001399272,0.0002383448,0.002264174,0.0002659544],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00001390649,0.000208385,0.9505356,0.0003700196,0.00006379027,0.000002927648,0.004715934,0.0007688257,0.0003358633,0.0004640937,0.001809667,0.04071101],"study_design_scores_gemma":[0.001421502,0.001175078,0.9218672,0.004734701,0.0002620705,0.00004883312,0.04621554,0.004375058,0.0006013688,0.01825941,0.0008147054,0.0002245636],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9955839,0.0008773927,0.0001405759,0.0006167907,0.001340484,0.001086473,0.000002169075,0.000001520005,0.0003507077],"genre_scores_gemma":[0.9985282,0.000008369994,0.001156202,0.00002356178,0.0001630163,0.00005298762,0.000002932559,0.000005643511,0.00005910895],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0414996,"threshold_uncertainty_score":0.9998017,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366450241","doi":"10.3138/cjpe.024.007","title":"Richard A. Krueger and Mary Anne Casey. (2009). <i>Focus Groups: A Practical Guide for Applied Research</i> (4th ed.). Thousand Oaks, CA: Sage. 205 pages, plus preface, acknowledgements, references, and indices.","year":2009,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Focus Groups and Qualitative Methods","field":"Social Sciences","cited_by":114,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"SAGE; Psychology; Focus (optics); Sociology; Library science; Gerontology; Computer science; Medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.1941600388254136,"gpt":0.4936611830970598,"spread":0.2995011442716462,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02578051,0.0002042431,0.0003853199,0.0004486959,0.001077095,0.0004945752,0.0002194527,0.0002146992,0.0002840165],"category_scores_gemma":[0.005268101,0.0001799043,0.00006363603,0.0005861354,0.0006147248,0.0005728286,0.00002279487,0.0005168231,0.000007874624],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006421467,"about_ca_system_score_gemma":0.003958883,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003523844,"about_ca_topic_score_gemma":0.09069684,"domain_scores_codex":[0.9949156,0.002246515,0.0006404837,0.0003764506,0.001046808,0.0007741559],"domain_scores_gemma":[0.995626,0.0007833024,0.0004334686,0.0001638594,0.001983189,0.001010182],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000128405,0.00008053198,0.0003396072,0.00003816886,0.0000692561,0.00002603345,0.008677612,0.00000443073,0.00003935711,0.003395675,0.1530487,0.8341522],"study_design_scores_gemma":[0.002340312,0.001412144,0.009409009,0.0001669804,0.0002386535,0.00002984167,0.01131582,0.0003709398,0.00003271008,0.02310292,0.9512413,0.0003394423],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4127546,0.329577,0.0339631,0.06405917,0.005958704,0.04025992,0.000715394,0.0002339972,0.1124781],"genre_scores_gemma":[0.7794095,0.006320129,0.2056273,0.0007771636,0.003240994,0.0007553368,0.000126207,0.00008474042,0.003658557],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8338128,"threshold_uncertainty_score":0.9258956,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366725272","doi":"10.3138/cjpe.19.006","title":"Integrating Evaluative Inquiry into the Organizational Culture: A Review and Synthesis of the Knowledge Base","year":2004,"lang":"en","type":"review","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":112,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Knowledge management; Organizational culture; Organizational learning; Knowledge base; Conceptual framework; Empirical research; Sociology; Work (physics); Management science; Epistemology; Political science; Computer science; Public relations; Social science; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.3839810064574307,"gpt":0.5592321866859272,"spread":0.1752511802284965,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03196302,0.000325034,0.001195544,0.0004720632,0.0004879673,0.0002913973,0.001278892,0.0001844529,0.001114717],"category_scores_gemma":[0.03397734,0.0001442282,0.0004637331,0.00271828,0.0004494941,0.0003524668,0.00007426611,0.0006250832,0.00003371994],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001036621,"about_ca_system_score_gemma":0.03164942,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001483265,"about_ca_topic_score_gemma":0.006192862,"domain_scores_codex":[0.9907445,0.003953284,0.00226149,0.0003208791,0.002473714,0.0002461489],"domain_scores_gemma":[0.9872314,0.001441374,0.003409076,0.0005855007,0.007030153,0.0003024605],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[8.683301e-7,0.00002965434,0.00004999515,0.001807372,0.0001026794,4.922361e-7,0.004615124,0.00001890381,1.044579e-7,0.0005796123,0.001590246,0.991205],"study_design_scores_gemma":[0.000413716,0.0002550049,0.0003591805,0.0577091,0.006331761,0.0001734913,0.002671518,0.0008318374,0.00000427559,0.007302665,0.9236063,0.0003411495],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.00002965983,0.9939319,0.0001150299,0.001382245,0.0006667533,0.003200454,0.00001524958,0.000003573414,0.0006551265],"genre_scores_gemma":[0.001939169,0.9954885,0.001600335,0.0002084179,0.0002584776,0.0003421315,0.00002112166,0.00002847112,0.0001134126],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.9908638,"threshold_uncertainty_score":0.9997984,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1826075361","doi":"10.3138/cjpe.22.007","title":"Participatory Impact Pathways Analysis: A Practical Application of Program Theory in Research-for-Development","year":2007,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":104,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Toronto","funders":"","keywords":"Logic model; Citizen journalism; Food security; Computer science; Poverty; Narrative; Theory of change; Management science; Knowledge management; Process management; Sociology; Business; Engineering; Economics; Economic growth; Social science","retraction":null,"screen_n_in":null,"score":{"opus":0.815960147008348,"gpt":0.6979548155552535,"spread":0.1180053314530944,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1473003,0.0001223299,0.0003643957,0.002815181,0.0001761275,0.0001823291,0.0003874307,0.0001198564,0.0003676154],"category_scores_gemma":[0.0082124,0.00009187088,0.0002180485,0.003494817,0.000189221,0.0004923323,0.00001522796,0.0003272312,0.00001777695],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009958664,"about_ca_system_score_gemma":0.01092426,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002999508,"about_ca_topic_score_gemma":0.03537372,"domain_scores_codex":[0.9926903,0.001381967,0.001843712,0.0002733814,0.003256735,0.0005538514],"domain_scores_gemma":[0.9909407,0.001536389,0.0009826467,0.0003597396,0.005615707,0.0005647854],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.000138129,0.0002518608,0.1080769,0.000007347388,0.0001130297,0.000003501054,0.002925762,0.002346568,0.00007749243,0.00242063,0.0001938791,0.8834448],"study_design_scores_gemma":[0.001252024,0.001360007,0.8425184,0.00004692863,0.0002727489,0.000009805493,0.004816143,0.1063667,0.00062225,0.02439897,0.01815967,0.0001763707],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9548617,0.0002846832,0.04028691,0.0005058593,0.0001422591,0.003130554,0.000006028909,0.000008852585,0.0007731308],"genre_scores_gemma":[0.9805804,0.000002974489,0.0188414,0.00002765196,0.00007662437,0.00042456,0.00002188559,0.000008829913,0.00001568293],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8832685,"threshold_uncertainty_score":0.9946829,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4255004666","doi":"10.3138/cjpe.30.3.05","title":"Decolonizing and Indigenizing Evaluation Practice in Africa: Toward African Relational Evaluation Approaches","year":2016,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":103,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Indigenization; Decolonization; Sociology; African culture; Metaphor; Epistemology; African philosophy; Political science; Gender studies; Anthropology; Linguistics; Philosophy","retraction":null,"screen_n_in":null,"score":{"opus":0.6556148384806202,"gpt":0.5116189627666728,"spread":0.1439958757139473,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.08925153,0.0001997701,0.0003175055,0.001617502,0.0002873926,0.0004182764,0.0003562937,0.0001636897,0.001603082],"category_scores_gemma":[0.03800416,0.0001420932,0.00009661925,0.001403072,0.0001489104,0.002789038,0.00002771176,0.0002675442,0.00004579013],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00200363,"about_ca_system_score_gemma":0.01060369,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002238024,"about_ca_topic_score_gemma":0.01027848,"domain_scores_codex":[0.9886178,0.003111959,0.001552354,0.0004511544,0.00580743,0.0004593302],"domain_scores_gemma":[0.9895741,0.00190395,0.00147564,0.0003076291,0.006295128,0.0004435435],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00004277796,0.00005646494,0.02891705,0.000003127751,0.00003373402,0.00000280444,0.007473428,0.002845065,0.00004892925,0.0008893684,0.0003373591,0.9593499],"study_design_scores_gemma":[0.00613897,0.0008221081,0.5347691,0.0002531704,0.0006291003,0.0001623701,0.01630327,0.3467783,0.0001046841,0.05241629,0.0411096,0.0005130569],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9357244,0.005259812,0.004710685,0.02307783,0.001349726,0.005721228,0.00001438391,0.00002133593,0.02412054],"genre_scores_gemma":[0.9902479,0.00003587262,0.00904714,0.00007809587,0.0001829267,0.0003275304,0.00001366233,0.00001578684,0.00005107632],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9588369,"threshold_uncertainty_score":0.9993096,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366448482","doi":"10.3138/cjpe.020.005","title":"The Delphi Technique as a Method for Increasing Inclusion in the Evaluation Process","year":2005,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":80,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Delphi method; Delphi; Compromise; Inclusion (mineral); Process (computing); Set (abstract data type); Evaluation methods; Psychology; Economic Justice; Representation (politics); Public relations; Management science; Sociology; Engineering ethics; Computer science; Social psychology; Political science; Engineering; Social science; Artificial intelligence; Law","retraction":null,"screen_n_in":null,"score":{"opus":0.3689523270921288,"gpt":0.6178187315297932,"spread":0.2488664044376644,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1891226,0.0001308472,0.0001877721,0.0005886743,0.001220381,0.0006664667,0.001067328,0.00009939401,0.0002623583],"category_scores_gemma":[0.01875096,0.00007047594,0.0001257412,0.001180838,0.00007494464,0.0007575926,0.00004232506,0.0002864145,0.00001338475],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007672923,"about_ca_system_score_gemma":0.007992775,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006280525,"about_ca_topic_score_gemma":0.05213422,"domain_scores_codex":[0.9898994,0.003581483,0.001287675,0.0002357102,0.004642139,0.0003535981],"domain_scores_gemma":[0.991171,0.00218325,0.0009108325,0.0003463913,0.005207561,0.0001809668],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005281939,0.00003220536,0.0008700857,0.000002726985,0.000007951707,5.379457e-7,0.005155796,0.00598011,0.00008232221,0.0005739133,0.0005906152,0.9866509],"study_design_scores_gemma":[0.001548652,0.0005961659,0.01440502,0.00009347479,0.0001404915,0.0001616433,0.007122014,0.7008614,0.0004952725,0.1824598,0.09194803,0.0001680684],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7846106,0.003387119,0.07454904,0.1030093,0.001137514,0.02417315,0.00001156022,0.00002648515,0.009095257],"genre_scores_gemma":[0.9740414,0.00001186715,0.02354275,0.000714045,0.0002856642,0.001363523,0.000009716074,0.00001002617,0.00002097967],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9864829,"threshold_uncertainty_score":0.997631,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2289811664","doi":"10.3138/cjpe.20.001","title":"Evaluability Assessment of a Survivors of Torture Program: Lessons Learned","year":2005,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Interpreting and Communication in Healthcare","field":"Health Professions","cited_by":71,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Calgary","funders":"","keywords":"Torture; Accountability; Logic model; Program evaluation; Process (computing); Political science; Psychology; Public relations; Public administration; Computer science; Human rights; Law","retraction":null,"screen_n_in":null,"score":{"opus":0.3722469645419353,"gpt":0.6063679190213934,"spread":0.2341209544794581,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01066644,0.0001320234,0.0003991422,0.0002931013,0.000348081,0.00001288745,0.0004234586,0.0002409467,0.0008035655],"category_scores_gemma":[0.001739353,0.000118769,0.0001692078,0.0003821289,0.0001615888,0.0001459136,0.00002584369,0.0009091306,0.000004810353],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001008517,"about_ca_system_score_gemma":0.01149326,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.008565924,"about_ca_topic_score_gemma":0.09196035,"domain_scores_codex":[0.9940667,0.003098071,0.001542595,0.0001561395,0.0007449475,0.0003915204],"domain_scores_gemma":[0.992027,0.0004455213,0.001659676,0.0005436489,0.004927088,0.0003970161],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00004391436,0.0002397094,0.1573232,0.000367704,0.00007332508,6.187921e-7,0.007449365,0.0007369284,0.00006412651,0.001877639,0.0008663143,0.8309572],"study_design_scores_gemma":[0.003813395,0.004313612,0.7426845,0.003908748,0.0006604582,0.00002018205,0.02409644,0.06933713,0.0001981154,0.006193337,0.1442198,0.0005542721],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9688594,0.001245069,0.0001940516,0.01710219,0.0008496771,0.004106349,0.00004489462,0.00003414515,0.007564251],"genre_scores_gemma":[0.9838941,0.00004478628,0.01529023,0.0001052308,0.0001844609,0.0003551849,0.00003069184,0.00001800318,0.00007729165],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8304029,"threshold_uncertainty_score":0.9980361,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W79165327","doi":"10.3138/cjpe.018.001","title":"They’re Happy, but Did They Make a Difference? Applying Kirkpatrick’s Framework to the Evaluation of a National Leadership Program","year":2003,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Human Resource Development and Performance Evaluation","field":"Psychology","cited_by":59,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Saskatchewan","funders":"","keywords":"Formative assessment; Process (computing); Program evaluation; Leadership development; Psychology; Investment (military); Management science; Process management; Medical education; Political science; Business; Computer science; Public relations; Pedagogy; Engineering; Public administration; Medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.3557958721915049,"gpt":0.4343449716571408,"spread":0.07854909946563587,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01729048,0.0002724634,0.0003236921,0.0007773985,0.0004025715,0.0002011975,0.0004885135,0.0002567418,0.002426811],"category_scores_gemma":[0.003170078,0.0002104796,0.0001878109,0.000676733,0.000101871,0.0001452944,0.00001179159,0.0005570619,0.0001010096],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007465408,"about_ca_system_score_gemma":0.004604795,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005035535,"about_ca_topic_score_gemma":0.01497319,"domain_scores_codex":[0.9934617,0.001789255,0.0009820595,0.0003183894,0.002850034,0.0005985175],"domain_scores_gemma":[0.9946517,0.0002996478,0.0008212996,0.0003444206,0.003492742,0.0003901616],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00007034007,0.0001955555,0.02233886,0.00002805866,0.0002112037,0.000002566621,0.03171688,0.002982261,0.00002875877,0.003260536,0.00111201,0.938053],"study_design_scores_gemma":[0.005249077,0.002803743,0.8248347,0.001020506,0.001519233,0.0001948185,0.03899578,0.02296877,0.0001684835,0.03928679,0.06173581,0.001222284],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9746876,0.001399587,0.00097919,0.001805389,0.001503412,0.00821586,0.000019146,0.00003625392,0.01135351],"genre_scores_gemma":[0.993634,0.00000474322,0.003191314,0.0002668592,0.0004331513,0.002180799,0.00004857781,0.0000343708,0.0002062285],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9368307,"threshold_uncertainty_score":0.9984851,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2188387579","doi":"10.3138/cjpe.019.002","title":"Reporting on Outcomes: Setting Performance Expectations and Telling Performance Stories","year":2004,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":57,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"Government of Canada","funders":"","keywords":"Outcome (game theory); Chart; Public relations; Actuarial science; Business; Psychology; Political science; Economics; Microeconomics; Statistics","retraction":null,"screen_n_in":null,"score":{"opus":0.4090562215533984,"gpt":0.5347074822731473,"spread":0.125651260719749,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0141548,0.0001526749,0.0002869089,0.0006951425,0.0006547028,0.0004552597,0.0002574006,0.00006630614,0.0001733925],"category_scores_gemma":[0.006707187,0.0001153633,0.00008894204,0.0006148996,0.00009660298,0.001083278,0.00001093261,0.000283894,0.0000390632],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004796234,"about_ca_system_score_gemma":0.003421588,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001209069,"about_ca_topic_score_gemma":0.001979388,"domain_scores_codex":[0.9953899,0.0001596489,0.001926901,0.0002377942,0.001978777,0.0003069764],"domain_scores_gemma":[0.9946496,0.0002519746,0.002481395,0.000257917,0.002009096,0.000349948],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00001076868,0.00002609826,0.2800477,0.000009756869,0.00001991084,0.000004961333,0.006424821,0.03151759,0.00001985446,0.000220507,0.00006768226,0.6816303],"study_design_scores_gemma":[0.001840081,0.001491561,0.8909363,0.0003684623,0.0001026799,0.0001795554,0.01601203,0.08072186,0.000848031,0.002707003,0.004422542,0.0003698901],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9946532,0.0002018398,0.0005174054,0.001668336,0.0006848122,0.0005423995,0.000001664828,0.00001376931,0.00171654],"genre_scores_gemma":[0.991978,0.00001600867,0.007561838,0.0001535012,0.0001439078,0.00004358854,0.000005142505,0.00001136837,0.00008667346],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6812605,"threshold_uncertainty_score":0.8029613,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4362681378","doi":"10.3138/cjpe.18.001","title":"Evaluation and Research: Differences and Similarities","year":2003,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":57,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Generalization; Similarity (geometry); Relevance (law); Causality (physics); Psychology; Management science; Computer science; Knowledge management; Epistemology; Artificial intelligence; Political science","retraction":null,"screen_n_in":null,"score":{"opus":0.7713106731453503,"gpt":0.617815743068949,"spread":0.1534949300764012,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.08505214,0.0000954722,0.0001893079,0.0009065759,0.0004349087,0.0008093224,0.0001902908,0.00007599516,0.001613657],"category_scores_gemma":[0.01742486,0.00007102668,0.00003748609,0.0006272737,0.0002954662,0.0006845726,0.00001092887,0.0002482963,0.00001161657],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002338544,"about_ca_system_score_gemma":0.005247738,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000255523,"about_ca_topic_score_gemma":0.01173654,"domain_scores_codex":[0.9910994,0.003073377,0.0006434766,0.0002176959,0.004689834,0.0002762164],"domain_scores_gemma":[0.9909535,0.0006852574,0.0003042629,0.000184083,0.007424066,0.0004488312],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00000538315,0.00001839489,0.07548368,0.000003946727,0.00001920544,0.000001576365,0.002120622,0.00008591494,0.00001783085,0.003550024,0.001054728,0.9176387],"study_design_scores_gemma":[0.001673118,0.001028292,0.5774608,0.00008341578,0.0001759767,0.0001101379,0.0123404,0.07123373,0.00009436619,0.2831823,0.05240655,0.0002109959],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9844016,0.00381466,0.0001452874,0.002057424,0.000425475,0.001016421,0.000003009512,0.000003857947,0.008132269],"genre_scores_gemma":[0.99806,0.00008386681,0.001533728,0.00007177858,0.00006955137,0.00006099391,0.000001629731,0.000005473414,0.0001129416],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9174277,"threshold_uncertainty_score":0.999299,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2996731041","doi":"10.3138/cjpe.68004","title":"Revisiting Contribution Analysis","year":2019,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":48,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Operationalization; Set (abstract data type); Epistemology; Positive economics; Causal analysis; Narrative; Key (lock); Sociology; Economics; Computer science; Econometrics; Philosophy","retraction":null,"screen_n_in":null,"score":{"opus":0.2744377398669307,"gpt":0.5383881020311411,"spread":0.2639503621642104,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02371973,0.00008365711,0.0002938214,0.001226589,0.0001299931,0.0004330582,0.0003347346,0.00006114939,0.009356517],"category_scores_gemma":[0.003447909,0.00006387504,0.0002472199,0.002173973,0.0000301729,0.0006346132,0.000007212774,0.0001533913,0.0003742027],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003828183,"about_ca_system_score_gemma":0.002694418,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002237128,"about_ca_topic_score_gemma":0.003713876,"domain_scores_codex":[0.9959131,0.000559216,0.0009378265,0.0001706604,0.002192221,0.0002269396],"domain_scores_gemma":[0.9931853,0.0002544405,0.0009064205,0.0002825798,0.005038084,0.0003331445],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.000007729902,0.000008769211,0.394148,0.000001608286,0.0001186157,0.00000178111,0.0002341725,0.009463499,0.00004557036,0.0009116623,0.0003959333,0.5946627],"study_design_scores_gemma":[0.0009944829,0.0002739023,0.6328114,0.00003517746,0.0006561247,0.0000168485,0.000809462,0.2913534,0.00008634327,0.004749384,0.0680642,0.0001491934],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9866833,0.0004662942,0.003709601,0.002203968,0.0007378972,0.000837289,0.000005675303,0.00000716785,0.005348801],"genre_scores_gemma":[0.9981985,0.000004428255,0.001212919,0.0001518297,0.0001678385,0.00001310447,0.00002285508,0.000004100151,0.0002244126],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5945134,"threshold_uncertainty_score":0.9915491,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2163998785","doi":"10.3138/cjpe.0023.002","title":"Organizational Capacity To Do and Use Evaluation: Results of a Pan-Canadian Survey of Evaluators","year":2009,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":48,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"National Research Council Canada; Atlantic Canada Opportunities Agency; University of Ottawa","funders":"","keywords":"Respondent; Psychology; Stakeholder; Knowledge management; Organizational learning; Context (archaeology); Descriptive statistics; Exploratory research; Social psychology; Applied psychology; Public relations; Sociology; Political science; Computer science; Social science; Geography","retraction":null,"screen_n_in":null,"score":{"opus":0.4439444040034558,"gpt":0.5021568546325215,"spread":0.05821245062906572,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04944271,0.0001788613,0.0004387946,0.001723777,0.000191066,0.000309523,0.0004785089,0.0001339367,0.001185625],"category_scores_gemma":[0.04300472,0.0001528033,0.00008777263,0.00252158,0.000137533,0.0007612117,0.00001240437,0.0001802325,0.00001949797],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006370758,"about_ca_system_score_gemma":0.01531216,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.05135973,"about_ca_topic_score_gemma":0.57705,"domain_scores_codex":[0.9912189,0.001847089,0.001912246,0.0003302943,0.00435993,0.000331548],"domain_scores_gemma":[0.9762391,0.0006689486,0.001582139,0.0004464706,0.02005524,0.001008027],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001306522,0.00009618478,0.3509576,0.000006304948,0.0000622109,0.000001955232,0.003994016,0.005419582,0.0001805605,0.0009057941,0.003923823,0.6343213],"study_design_scores_gemma":[0.001514511,0.0009327134,0.9673308,0.00007171756,0.0001132669,0.00001919332,0.0003370427,0.02085061,0.000192373,0.006691575,0.001789646,0.0001566059],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9942273,0.0002608137,0.0002009356,0.001962055,0.0004326339,0.001634576,0.0002229308,0.00000354357,0.001055184],"genre_scores_gemma":[0.9956908,0.000009099865,0.0038756,0.000229132,0.00006957332,0.00001564959,0.00006818087,0.000009861198,0.00003209427],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6341648,"threshold_uncertainty_score":0.9997274,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2161163161","doi":"","title":"Knowlton, L.W. & Phillips, C.C. (2013). The Logic Model Guidebook: Better Strategies for Great Results (2nd Edition), Thousand Oaks, CA: Sage.","year":2012,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Intelligent Tutoring Systems and Adaptive Learning","field":"Computer Science","cited_by":47,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"SAGE; Physics; Nuclear physics","retraction":null,"screen_n_in":null,"score":{"opus":0.1031034195254785,"gpt":0.3427688452820291,"spread":0.2396654257565506,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.004583641,0.0001662193,0.0001880611,0.0001824026,0.0004669089,0.000536442,0.0005166723,0.00008937458,0.00003393007],"category_scores_gemma":[0.000287819,0.0001142379,0.0001366108,0.0001602752,0.00006506876,0.001138498,0.00001821636,0.0002444228,0.00001717414],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002888759,"about_ca_system_score_gemma":0.001396788,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007449661,"about_ca_topic_score_gemma":0.003227555,"domain_scores_codex":[0.9979963,0.0002566762,0.000567208,0.0001825446,0.0004850267,0.0005121913],"domain_scores_gemma":[0.9976229,0.000122976,0.0004887536,0.000286125,0.001160835,0.0003184325],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005069364,0.00008788355,0.0007724313,0.00006178429,0.0002355044,0.00001602523,0.01081953,0.1016266,0.0001473197,0.1737107,0.115348,0.5971236],"study_design_scores_gemma":[0.0008545709,0.0003807559,0.001420338,0.0002025353,0.000125711,0.0001272904,0.0005401986,0.3021923,0.0001142694,0.02338983,0.6703419,0.0003103362],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.008109172,0.003212719,0.9758968,0.006049774,0.003343354,0.001696348,0.00002289598,0.00003737074,0.001631536],"genre_scores_gemma":[0.9619273,0.00002094071,0.03260931,0.0004177001,0.002782732,0.0001625587,0.00001985473,0.00002342359,0.002036172],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9538181,"threshold_uncertainty_score":0.5172923,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1490602178","doi":"10.3138/cjpe.0028.005","title":"A Professional Grounding and History of the Development and Formal Use of Evaluator Competencies","year":2014,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":44,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Professionalization; Viewpoints; Credentialing; Engineering ethics; Value (mathematics); Professional association; Field (mathematics); Foundation (evidence); Professional development; Psychology; Political science; Sociology; Pedagogy; Public relations; Engineering; Computer science; Social science; Law","retraction":null,"screen_n_in":null,"score":{"opus":0.3985154956302613,"gpt":0.4680725057001939,"spread":0.06955701006993253,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01470362,0.0000784705,0.0002021007,0.0003169925,0.0001427595,0.00006089974,0.0002012313,0.00004514754,0.0001836011],"category_scores_gemma":[0.002625795,0.00004784211,0.00004944337,0.0002018564,0.0002224433,0.0006453085,0.00003093566,0.0001130683,9.573965e-7],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003434881,"about_ca_system_score_gemma":0.006019714,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001416638,"about_ca_topic_score_gemma":0.005262525,"domain_scores_codex":[0.9966415,0.0005461826,0.0008401165,0.0001062796,0.001718764,0.0001471422],"domain_scores_gemma":[0.9966117,0.000375993,0.000935033,0.0001488298,0.00174384,0.0001845875],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00002121186,0.00002787876,0.1750433,0.00002352564,0.00002338894,1.596445e-7,0.007321842,0.0001617902,0.0002799205,0.001693369,0.0009099821,0.8144937],"study_design_scores_gemma":[0.0007983416,0.0002741465,0.8656971,0.0001703079,0.00006490148,0.00002209347,0.00158223,0.04331083,0.000214032,0.001763403,0.08601237,0.00009024481],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9972078,0.0004734724,0.0004124052,0.0003720891,0.0007777945,0.0004634596,0.000001552318,0.000001390271,0.0002900378],"genre_scores_gemma":[0.9914935,0.000003557874,0.008165202,0.00006982547,0.00003314269,0.00001559378,0.000001293286,0.00000389175,0.0002139731],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8144034,"threshold_uncertainty_score":0.9996153,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W166307397","doi":"10.3138/cjpe.24.001","title":"Building an Evaluative Culture: The Key to Effective Evaluation and Results Management","year":2009,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Human Resource Development and Performance Evaluation","field":"Psychology","cited_by":44,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"Rogers Communications (Canada)","funders":"","keywords":"Key (lock); Organizational culture; Knowledge management; Order (exchange); Empirical research; Business; Sociology; Computer science; Public relations; Psychology; Process management; Epistemology; Political science; Computer security","retraction":null,"screen_n_in":null,"score":{"opus":0.0790348188226643,"gpt":0.4286523090145266,"spread":0.3496174901918623,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01734217,0.0002008066,0.0001963737,0.0005274426,0.0004385415,0.0002299364,0.0002774915,0.0001066003,0.0003447336],"category_scores_gemma":[0.0005776681,0.0001499788,0.00006248871,0.0005386153,0.00006673222,0.0003428853,0.000009698892,0.0002688305,0.00004288964],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000620087,"about_ca_system_score_gemma":0.0004880439,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001355062,"about_ca_topic_score_gemma":0.001743754,"domain_scores_codex":[0.9960943,0.001354937,0.0006663783,0.0003157827,0.001187458,0.0003811275],"domain_scores_gemma":[0.9971735,0.0000712301,0.0004630483,0.0002810767,0.001583581,0.0004275785],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001878121,0.00004748555,0.0004031201,0.000003791084,0.0001015052,0.000005082723,0.0373991,0.001239509,0.00005455128,0.001197721,0.00255836,0.956802],"study_design_scores_gemma":[0.005367289,0.004389231,0.8904018,0.0001982258,0.0009307844,0.00008451681,0.005254802,0.02195859,0.0001047658,0.009766259,0.06112072,0.0004230278],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9760506,0.0009707171,0.0006482035,0.002052095,0.0007832614,0.00876655,0.000006080875,0.00002084734,0.01070163],"genre_scores_gemma":[0.9925449,0.000007231068,0.005767976,0.0004661506,0.0003682389,0.0006242919,0.00004642076,0.00001414347,0.000160624],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9563789,"threshold_uncertainty_score":0.6115959,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2732819684","doi":"10.3138/cjpe.20.006","title":"Evaluator Competencies in University-Based Evaluation Training Programs","year":2005,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":44,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Credentialing; Accreditation; Licensure; Certificate; Taxonomy (biology); Competence (human resources); Computer science; Medical education; Psychology; Medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.4101080033795569,"gpt":0.5034090358417598,"spread":0.09330103246220289,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.04209973,0.0001766822,0.0003380694,0.001771872,0.0002399719,0.0003651587,0.0006436126,0.000119027,0.00300078],"category_scores_gemma":[0.00302336,0.0001564063,0.0001762416,0.001800314,0.0001472594,0.001134907,0.00001118357,0.0003088309,0.0001028926],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001835343,"about_ca_system_score_gemma":0.01827317,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004243129,"about_ca_topic_score_gemma":0.08864555,"domain_scores_codex":[0.9917924,0.001756535,0.001146272,0.0003110222,0.004549505,0.0004442845],"domain_scores_gemma":[0.9929327,0.0003085411,0.0008512482,0.0003317755,0.005067927,0.000507794],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0000261083,0.00007310702,0.02386614,0.000002671812,0.00001339781,0.000004927019,0.002537213,0.05224651,0.00002220088,0.0003653379,0.0003167801,0.9205256],"study_design_scores_gemma":[0.003135606,0.0005278931,0.05854617,0.000102562,0.0001259656,0.00001912814,0.007297549,0.8207593,0.00002846036,0.002559721,0.1066883,0.00020934],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9816959,0.0006237194,0.002328594,0.006192556,0.0008146368,0.003089844,0.000007188582,0.00002389479,0.00522367],"genre_scores_gemma":[0.9832349,0.000004146198,0.01618359,0.0002010888,0.0001979389,0.00004278359,0.00003061808,0.00001194685,0.00009296449],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9203163,"threshold_uncertainty_score":0.9979106,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2274776964","doi":"10.3138/cjpe.022.003","title":"Developing a Tool to Measure Knowledge Exchange Outcomes","year":2007,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Health Policy Implementation Science","field":"Health Professions","cited_by":41,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"","keywords":"USable; Knowledge management; Measure (data warehouse); Process (computing); Computer science; Best practice; Reliability (semiconductor); Point (geometry); Process management; Business; World Wide Web; Data mining","retraction":null,"screen_n_in":null,"score":{"opus":0.8485447477899826,"gpt":0.719432757191686,"spread":0.1291119905982966,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02655754,0.0001196899,0.0002342772,0.000860391,0.0007149095,0.00003470849,0.0002763862,0.000112064,0.0007703861],"category_scores_gemma":[0.008443861,0.0001057765,0.00005710261,0.0009722848,0.00003716687,0.0002806553,0.00001904951,0.0003637438,0.0003244039],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002694302,"about_ca_system_score_gemma":0.01927166,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001916389,"about_ca_topic_score_gemma":0.2093629,"domain_scores_codex":[0.9961933,0.0008144001,0.001213853,0.0001510637,0.0007309387,0.0008964284],"domain_scores_gemma":[0.9945447,0.0005904788,0.0005289098,0.0001716751,0.002976059,0.001188127],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00001368227,0.0000100185,0.09310091,0.00008121575,0.0000179494,0.00000757934,0.03153177,0.000004786666,0.000030096,0.001078498,0.01330706,0.8608164],"study_design_scores_gemma":[0.001061915,0.0001757434,0.4549415,0.000217912,0.00002836891,0.000009077087,0.003359677,0.00005874809,0.00002589035,0.0004555667,0.5395155,0.000150045],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9091344,0.0002666506,0.02950277,0.04193629,0.003987953,0.007415172,0.00002139481,0.00004946939,0.00768589],"genre_scores_gemma":[0.9536825,0.0000027925,0.03159853,0.01333523,0.0006807729,0.0002944459,0.000004502996,0.00002412367,0.0003770617],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8606664,"threshold_uncertainty_score":0.9999084,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2994626447","doi":"10.3138/cjpe.68444","title":"Increasing Cultural Competence in Support of Indigenous-Led Evaluation: A Necessary Step toward Indigenous-Led Evaluation","year":2019,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":41,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Indigenous; Aotearoa; Competence (human resources); Traditional knowledge; Sociology; Political science; Psychology; Gender studies; Social psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.2345971341053808,"gpt":0.4965004891018339,"spread":0.2619033549964531,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.07576697,0.0003400173,0.0008069795,0.001982501,0.0002648956,0.0005184945,0.001068337,0.0002469052,0.00821133],"category_scores_gemma":[0.005116344,0.0002833641,0.0002720531,0.002353419,0.0001667331,0.002127646,0.00004446702,0.0005204178,0.0001879124],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002202654,"about_ca_system_score_gemma":0.02456376,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00302346,"about_ca_topic_score_gemma":0.03024615,"domain_scores_codex":[0.9829466,0.004004569,0.002888987,0.0005613554,0.008903108,0.0006953171],"domain_scores_gemma":[0.9835783,0.0005971664,0.002608703,0.0006444506,0.01201843,0.0005529434],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001470977,0.0001752166,0.3895294,0.00003430238,0.00008108084,0.000007458065,0.01946143,0.01589076,0.0004954046,0.00006010243,0.0004416647,0.573676],"study_design_scores_gemma":[0.007124279,0.001952826,0.7230923,0.0002852674,0.0004784246,0.0001970459,0.01378916,0.2438193,0.0003495763,0.003905556,0.004450884,0.0005553263],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9883426,0.0008667771,0.0001647643,0.0008144276,0.001359407,0.005491401,0.00001974863,0.00001260298,0.002928286],"genre_scores_gemma":[0.995545,0.00002824983,0.003547724,0.0002438941,0.0001732315,0.0002209582,0.0001511568,0.00002447739,0.00006532084],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5731208,"threshold_uncertainty_score":0.9999619,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366721381","doi":"10.3138/cjpe.15.006","title":"Logic Models in Primary Care Reform: Navigating the Evaluation","year":2000,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":39,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"Dalhousie University","funders":"","keywords":"Primary care; Logic model; Nova scotia; Strengths and weaknesses; Quality (philosophy); Political science; Public administration; Process management; Business; Economic growth; Medicine; Psychology; Sociology; Economics; Family medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.4326891670577493,"gpt":0.5423916433560159,"spread":0.1097024762982666,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02946173,0.0001379505,0.0002321677,0.000336243,0.0003194109,0.0004184403,0.0006356841,0.00009531771,0.003803845],"category_scores_gemma":[0.0007618512,0.00008566247,0.0001241698,0.001113248,0.00009601763,0.001172638,0.00001029734,0.000476058,0.00008853933],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002557547,"about_ca_system_score_gemma":0.007856986,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001753441,"about_ca_topic_score_gemma":0.03236518,"domain_scores_codex":[0.9930535,0.001273551,0.001223916,0.0002335852,0.003887868,0.000327595],"domain_scores_gemma":[0.9951437,0.000170815,0.0005827518,0.0003877822,0.003466948,0.0002480689],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001304674,0.00001484982,0.002671239,0.00000345215,0.000006322055,0.000002122456,0.003910898,0.09258626,0.000005176884,0.0001759437,0.0001411176,0.9004695],"study_design_scores_gemma":[0.001960941,0.0004483057,0.1474077,0.0002370274,0.0001123484,0.00006022556,0.01089581,0.7681948,0.00002430668,0.06321129,0.007217966,0.0002293219],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9403664,0.002559396,0.0001770309,0.003848096,0.0004720423,0.002092118,0.000005801141,0.000008605857,0.0504705],"genre_scores_gemma":[0.9973456,0.00003424065,0.001665738,0.0005179776,0.0001526189,0.0001475973,0.00003981331,0.00000964805,0.00008680016],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9002402,"threshold_uncertainty_score":0.9993734,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366452622","doi":"10.3138/cjpe.0026.005","title":"L’approche <i>Realist</i> à l’épreuve du réel de l’évaluation des programmes","year":2012,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":37,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"Centre Hospitalier de l’Université de Montréal; Université de Montréal","funders":"","keywords":"Epistemology; Context (archaeology); Valuation (finance); Psychological intervention; Outcome (game theory); Sociology; Psychology; GRASP; Mechanism (biology); Philosophy; Computer science; Economics; History","retraction":null,"screen_n_in":null,"score":{"opus":0.4946108818232891,"gpt":0.5147343735641988,"spread":0.02012349174090972,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.04735767,0.000192258,0.0002899536,0.0006935721,0.0004353957,0.000626008,0.0005697161,0.0001677701,0.00138836],"category_scores_gemma":[0.007978791,0.0001502922,0.0002026394,0.001095933,0.0002130403,0.001689407,0.00001892493,0.0003204223,0.0001180517],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000876735,"about_ca_system_score_gemma":0.005596429,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008721581,"about_ca_topic_score_gemma":0.01022194,"domain_scores_codex":[0.9937447,0.001371264,0.001139178,0.0002105826,0.002817339,0.0007169672],"domain_scores_gemma":[0.9933913,0.0003907806,0.0008818717,0.0003443107,0.00391221,0.001079564],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0000140474,0.00009640321,0.231807,0.000006305317,0.00002807842,0.000001649661,0.008050774,0.0007763054,0.00006052359,0.000417196,0.006651802,0.7520899],"study_design_scores_gemma":[0.001989281,0.0008937028,0.6332878,0.0001009487,0.0003826694,0.0003779312,0.008991606,0.05536036,0.0004882059,0.04106676,0.2566084,0.0004523437],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9705555,0.001446435,0.01191076,0.003403249,0.001283298,0.001758391,0.000008951802,0.00003226987,0.009601187],"genre_scores_gemma":[0.9728618,0.00003099288,0.02552081,0.0003938174,0.0007195025,0.0001769871,0.00002572527,0.00001896087,0.0002514563],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7516375,"threshold_uncertainty_score":0.9995245,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366384242","doi":"10.3138/cjpe.0023.008","title":"Insights into Evaluation Capacity Building: Motivations, Strategies, Outcomes, and Lessons Learned","year":2009,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":37,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Variety (cybernetics); Capacity building; Accountability; Face (sociological concept); Public relations; Business; Psychology; Process management; Knowledge management; Political science; Sociology; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.536890060585536,"gpt":0.5401212105282374,"spread":0.00323114994270135,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01517437,0.0001898501,0.0003326865,0.001053863,0.0004900606,0.0009908496,0.0004113778,0.0001325607,0.0005911456],"category_scores_gemma":[0.005554219,0.000146242,0.0001116417,0.0009687603,0.0001205945,0.001805975,0.0000108296,0.000313654,0.00002053607],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006728097,"about_ca_system_score_gemma":0.007194589,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007362855,"about_ca_topic_score_gemma":0.016562,"domain_scores_codex":[0.9938912,0.001041999,0.001134726,0.0003255178,0.003329569,0.0002770174],"domain_scores_gemma":[0.993187,0.0002812643,0.0008867626,0.0003600008,0.004739135,0.0005458742],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.000009682146,0.00005965206,0.01495359,0.000002761883,0.00002833068,0.000001800151,0.002211576,0.004433325,0.0003634506,0.01581573,0.0007583259,0.9613618],"study_design_scores_gemma":[0.001308582,0.0004503526,0.5793656,0.00004037739,0.0001310448,0.00001201436,0.001792088,0.0814666,0.00009676786,0.3186343,0.01651054,0.0001917443],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9771731,0.0008353729,0.003600928,0.01420573,0.0005273416,0.001219933,0.000002760713,0.00001539383,0.002419369],"genre_scores_gemma":[0.9917321,0.00002670229,0.00765213,0.0003511661,0.000116594,0.00004164846,0.00001762846,0.000008058163,0.00005402716],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.96117,"threshold_uncertainty_score":0.9984337,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W193071","doi":"10.3138/cjpe.17.006","title":"Developing an Evaluation Habit of Mind","year":2002,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":36,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"University of Toronto","funders":"","keywords":"Habit; Accountability; Sustainability; Psychology; Political science; Sociology; Social psychology; Law","retraction":null,"screen_n_in":null,"score":{"opus":0.6840143153944287,"gpt":0.5727226976961833,"spread":0.1112916176982454,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02704721,0.0001220696,0.0002646083,0.0009649515,0.0001767037,0.0002582843,0.0005041179,0.00009143713,0.01233043],"category_scores_gemma":[0.004134873,0.00009931815,0.0001145847,0.001006828,0.00009670808,0.001162082,0.000008937411,0.0001544143,0.0001118253],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005813457,"about_ca_system_score_gemma":0.00451257,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002050906,"about_ca_topic_score_gemma":0.01263693,"domain_scores_codex":[0.9931573,0.0009862486,0.001342561,0.000215445,0.004043758,0.0002546617],"domain_scores_gemma":[0.9902801,0.0001923993,0.001167115,0.0003438836,0.007615906,0.0004005892],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005835037,0.00004048847,0.01410033,0.000002892387,0.00001776647,0.000001243876,0.001804494,0.002914849,0.00005338065,0.0003139141,0.0009203903,0.9798244],"study_design_scores_gemma":[0.002248259,0.001048947,0.132314,0.0001039123,0.0002124083,0.00005958135,0.002547444,0.8046436,0.0008518737,0.01603496,0.0396677,0.0002672684],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9869411,0.0008244836,0.002334263,0.002452586,0.0009430344,0.001236961,0.000005767903,0.000004637263,0.005257194],"genre_scores_gemma":[0.9843367,0.00001041494,0.01523116,0.0001015424,0.0001581125,0.00003865218,0.00001195881,0.000008789544,0.0001026157],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9795572,"threshold_uncertainty_score":0.9885724,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3099471317","doi":"10.3138/cjpe.70004","title":"Sustainability Analysis of Intervention Benefits: A Theory of Change Approach","year":2020,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":34,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Sustainability; Intervention (counseling); Spell; Sustainability organizations; Work (physics); Environmental economics; Business; Environmental resource management; Economics; Psychology; Engineering; Sociology; Ecology","retraction":null,"screen_n_in":null,"score":{"opus":0.6214875283452821,"gpt":0.5413536083546764,"spread":0.0801339199906057,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01928089,0.00008898947,0.0004167877,0.001074238,0.00005517175,0.0000713291,0.0004207345,0.00006246421,0.001401033],"category_scores_gemma":[0.00526972,0.00006900889,0.000399922,0.002886235,0.0001155159,0.0005020531,0.00001691862,0.0001220873,0.000002286451],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002372274,"about_ca_system_score_gemma":0.001733893,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004851808,"about_ca_topic_score_gemma":0.003241553,"domain_scores_codex":[0.995625,0.001002869,0.001298815,0.0001783713,0.001739188,0.0001557717],"domain_scores_gemma":[0.9918914,0.0002018465,0.001433619,0.0002465546,0.005914357,0.000312178],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00005865338,0.00009999217,0.06936284,0.00003405463,0.0002711926,5.039697e-7,0.006607831,0.00864966,0.000007777219,0.003007723,0.0001025678,0.9117972],"study_design_scores_gemma":[0.001008329,0.001253745,0.6038186,0.00004169832,0.001715338,0.000002748413,0.01683423,0.3611896,0.0001294047,0.01224175,0.001635786,0.0001287778],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9825515,0.001092265,0.01180876,0.002313226,0.0001653063,0.001273743,0.00003411217,0.000004786139,0.0007563237],"genre_scores_gemma":[0.99839,0.000005921827,0.001370071,0.00009029479,0.00005976651,0.00004341816,0.00002471951,0.00000443682,0.00001132339],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9116684,"threshold_uncertainty_score":0.9995118,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366449629","doi":"10.3138/cjpe.0021.006","title":"Studies Are Not Enough: The Necessary Transformation of Evaluation","year":2007,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":33,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"GRASP; Public sector; Business; Knowledge management; Profit (economics); Public relations; Not for profit; Political science; Economics; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.55640313541793,"gpt":0.5856812041578379,"spread":0.0292780687399079,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.07960297,0.0001369451,0.0003114922,0.0007803059,0.0003314114,0.0001714244,0.0004454855,0.00008525326,0.0007678888],"category_scores_gemma":[0.006037403,0.00008464888,0.0001728983,0.001162417,0.0001931782,0.001029379,0.000009335649,0.000227835,0.00002801048],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005279353,"about_ca_system_score_gemma":0.003415695,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001845018,"about_ca_topic_score_gemma":0.02437774,"domain_scores_codex":[0.9918206,0.0009551927,0.001796158,0.000170025,0.004964783,0.0002932744],"domain_scores_gemma":[0.9865968,0.0008069305,0.001796986,0.0003287498,0.01024012,0.0002304428],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00004888551,0.0000439499,0.006318354,0.00001262062,0.00007524605,0.000001893069,0.009665096,0.005878947,0.0001267158,0.0007180395,0.001822944,0.9752873],"study_design_scores_gemma":[0.004272938,0.001498361,0.6412517,0.0003516647,0.0009702812,0.0001272313,0.09883677,0.1498227,0.006340445,0.03036771,0.06570961,0.0004506217],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9739197,0.003103955,0.003092745,0.0111721,0.001776417,0.002412696,0.00001104049,0.000008873436,0.004502467],"genre_scores_gemma":[0.9985262,0.00004516835,0.0008227323,0.0002820731,0.0002094443,0.00004655704,0.00001027814,0.000007774967,0.00004972646],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9748367,"threshold_uncertainty_score":0.9934248,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2484173420","doi":"10.3138/cjpe.0026.006","title":"Logic Analysis: Testing Program Theory to Better Evaluate Complex Interventions","year":2012,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"School Health and Nursing Education","field":"Health Professions","cited_by":33,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":true,"about_ca":false},"ca_institutions":"Université de Sherbrooke; Université de Montréal","funders":"Canadian Institutes of Health Research; U.S. Public Health Service","keywords":"Psychological intervention; Computer science; Logic model; Risk analysis (engineering); Action (physics); Management science; Psychology; Medicine; Sociology; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.5926332520068205,"gpt":0.6085805738558242,"spread":0.01594732184900371,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01501158,0.0001633475,0.0003386909,0.001236443,0.0008687048,0.00005855925,0.0002393092,0.0001513312,0.00189968],"category_scores_gemma":[0.00418575,0.0001436081,0.0002438413,0.001870212,0.00005830695,0.000345691,0.00001298839,0.0006216301,0.0002385083],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001260567,"about_ca_system_score_gemma":0.003323157,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001221599,"about_ca_topic_score_gemma":0.01098647,"domain_scores_codex":[0.9946679,0.002505703,0.001138854,0.0001737437,0.0005644687,0.000949319],"domain_scores_gemma":[0.9943152,0.0004046459,0.0007999697,0.0002765278,0.002492212,0.00171143],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00003130009,0.0001893083,0.2918167,0.00009955731,0.0001724756,8.662133e-7,0.003239302,0.0001164855,0.00001539875,0.0002337285,0.008701886,0.695383],"study_design_scores_gemma":[0.0006164454,0.0007397264,0.9686468,0.0005156004,0.001886223,0.0000110361,0.002731233,0.002062046,0.000002395717,0.003216462,0.0193943,0.0001776806],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9658114,0.00172473,0.00507942,0.01096972,0.005019388,0.007703552,0.00002000236,0.0001130487,0.003558708],"genre_scores_gemma":[0.9439402,0.000001593888,0.05200109,0.001627902,0.001268245,0.0009273237,0.00009043375,0.00002347037,0.0001197125],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6952053,"threshold_uncertainty_score":0.9990127,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2919316438","doi":"10.3138/cjpe.52946","title":"Using Actor-Based Theories of Change to Conduct Robust Evaluation in Complex Settings","year":2019,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":33,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Theory of change; Democracy; Management science; Computer science; Psychological intervention; Process management; Sociology; Operations research; Business; Risk analysis (engineering); Political science; Economics; Psychology; Management; Mathematics; Law","retraction":null,"screen_n_in":null,"score":{"opus":0.8242816801062659,"gpt":0.5913763997790368,"spread":0.2329052803272291,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02935395,0.0001365608,0.0003542662,0.001370027,0.00008731398,0.0001956837,0.0004205964,0.00008156753,0.00516839],"category_scores_gemma":[0.003054036,0.0001117952,0.0001050651,0.001384441,0.00007024145,0.0007741345,0.00001349553,0.0001642238,0.00004565926],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007504324,"about_ca_system_score_gemma":0.005517053,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001092077,"about_ca_topic_score_gemma":0.01382545,"domain_scores_codex":[0.9941851,0.001073314,0.00113593,0.0002334254,0.003087193,0.0002850565],"domain_scores_gemma":[0.9933407,0.0003024253,0.0009710562,0.0003228837,0.004764558,0.0002983191],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00008632374,0.00009187051,0.2329136,0.00002940337,0.00002811089,0.000002101505,0.005677781,0.1646518,0.003230715,0.0007943336,0.0006542801,0.5918397],"study_design_scores_gemma":[0.001971305,0.0006721533,0.2688283,0.0001897558,0.00008196938,0.00001016714,0.003402985,0.7147289,0.0005200219,0.004031066,0.005370786,0.0001926291],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9924901,0.0001423871,0.0009085916,0.002223197,0.0006352767,0.002691471,0.00001326365,0.000004061136,0.0008917177],"genre_scores_gemma":[0.9892964,9.105789e-7,0.0100545,0.000432919,0.0001041399,0.00006089969,0.00002460729,0.00001076939,0.00001487729],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.591647,"threshold_uncertainty_score":0.9994844,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1594443358","doi":"10.3138/cjpe.027.002","title":"Research on Evaluation: A Needs Assessment","year":2012,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":31,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Context (archaeology); Psychology; Survey research; Engineering ethics; Management science; Sociology; Applied psychology; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.7833440326357193,"gpt":0.6849524617018541,"spread":0.09839157093386519,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.1574821,0.0001518222,0.0002654825,0.002936783,0.0005387983,0.0006491278,0.0006902911,0.0001232373,0.01286504],"category_scores_gemma":[0.005142995,0.0001135166,0.000156873,0.002851841,0.0001505863,0.001235866,0.00002188766,0.0006262976,0.0005924988],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001875963,"about_ca_system_score_gemma":0.01332643,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002222662,"about_ca_topic_score_gemma":0.002242581,"domain_scores_codex":[0.9827187,0.003858387,0.001176559,0.000205916,0.01133779,0.0007026419],"domain_scores_gemma":[0.9852979,0.0009106096,0.0006316192,0.0005442426,0.0115287,0.001086911],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00001717527,0.0001518961,0.04477359,0.000002554974,0.00003200587,0.000001427511,0.001778282,0.003215401,0.00001935718,0.006179741,0.02045452,0.9233741],"study_design_scores_gemma":[0.002224075,0.00188131,0.5218678,0.0001042619,0.000170378,0.00006762116,0.01343927,0.09685926,0.0001289739,0.01783021,0.3451289,0.0002979787],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8563039,0.001330534,0.0007642173,0.008681882,0.004203503,0.003315702,0.000007656716,0.00001555178,0.1253771],"genre_scores_gemma":[0.9957566,0.00000809221,0.00238996,0.0002798121,0.0009232257,0.0002489675,0.00001436859,0.00001411229,0.0003648336],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9230761,"threshold_uncertainty_score":0.9922671,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W242703065","doi":"10.3138/cjpe.0014.005","title":"Organizational Development and Evaluation","year":2000,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Organization development; Organizational learning; Organizational effectiveness; Organizational culture; Organizational behavior and human resources; Knowledge management; Facilitation; Organizational commitment; Organizational performance; Organizational studies; Organizational engineering; Process (computing); Psychology; Process management; Business; Computer science; Public relations; Political science; Social psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.3470470019193243,"gpt":0.5167326550286309,"spread":0.1696856531093066,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01888613,0.0001025171,0.0001592197,0.0004929701,0.0002921759,0.0003924748,0.0002472586,0.00006402005,0.03004304],"category_scores_gemma":[0.001684612,0.00008153411,0.00003917536,0.0007273301,0.0000671318,0.0005667342,0.000005872479,0.0001169412,0.0002247069],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003954455,"about_ca_system_score_gemma":0.008964811,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006852469,"about_ca_topic_score_gemma":0.004759162,"domain_scores_codex":[0.995114,0.000479311,0.0008657491,0.0001870194,0.003160817,0.0001931353],"domain_scores_gemma":[0.994701,0.0001314747,0.0003607743,0.0001647828,0.004245061,0.0003968598],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.000006679298,0.00001940677,0.01674352,0.000001219435,0.00001476917,0.000001073759,0.001295335,0.001542666,0.000008297396,0.0002880532,0.000943133,0.9791359],"study_design_scores_gemma":[0.002336901,0.0003609886,0.4542094,0.00005719761,0.0001584683,0.0001409227,0.00118837,0.1808004,0.000297725,0.02987434,0.3302712,0.0003040879],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9848315,0.0005987391,0.0002094372,0.001591989,0.0003473457,0.000852512,0.000002194987,0.000006861273,0.01155938],"genre_scores_gemma":[0.9920371,0.0000105149,0.007120703,0.0001934676,0.0001393051,0.00004427169,0.00002504624,0.000008064774,0.0004215544],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9788318,"threshold_uncertainty_score":0.9966534,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1720965121","doi":"10.3138/cjpe.28.001","title":"Measuring Organizational Evaluation Capacity in the Canadian Federal Government","year":2013,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"Saint Paul University; Carleton University; École Nationale d'Administration Publique","funders":"","keywords":"Government (linguistics); Identification (biology); Organization development; Business; Evaluation methods; Organizational effectiveness; Organizational learning; Organizational identification; Public relations; Process management; Organizational commitment; Knowledge management; Political science; Computer science; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.5879919298116747,"gpt":0.4514966114809471,"spread":0.1364953183307276,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.03407819,0.0001405196,0.0001881899,0.0005392281,0.0005719237,0.001377531,0.0006910252,0.0000983563,0.008415329],"category_scores_gemma":[0.006274463,0.00009659027,0.00008399943,0.001166024,0.00008285926,0.001036074,0.000008824198,0.0003200953,0.0002365503],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003510752,"about_ca_system_score_gemma":0.01313434,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.1323244,"about_ca_topic_score_gemma":0.948618,"domain_scores_codex":[0.9899738,0.001466077,0.0009977651,0.0002152928,0.006955659,0.00039137],"domain_scores_gemma":[0.9931259,0.0002471033,0.0005484644,0.0003083284,0.005239357,0.0005309196],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.000005206911,0.00006601281,0.3680124,0.000003666029,0.00003054914,0.000004280207,0.005184403,0.01159351,0.0000467446,0.001499963,0.008538914,0.6050144],"study_design_scores_gemma":[0.0008491836,0.0001426798,0.8455033,0.00002865771,0.00004164374,0.00004552945,0.002684727,0.1223806,0.00003949021,0.01958001,0.008557946,0.0001462025],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9693754,0.0001634765,0.0001782323,0.01269564,0.0008297865,0.002748273,0.000009060739,0.000004392744,0.0139957],"genre_scores_gemma":[0.9976972,0.000002050821,0.001044751,0.0007699055,0.0002008578,0.0002069006,0.00001590529,0.000009373804,0.00005308049],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8162935,"threshold_uncertainty_score":0.9996591,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2946489075","doi":"10.3138/cjpe.43050","title":"Principles, Approaches, and Methods for Evaluation in Indigenous Contexts: A Grey Literature Scoping Review","year":2019,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"University of Toronto; Public Health Ontario","funders":"","keywords":"Grey literature; Indigenous; Interpretation (philosophy); Management science; Engineering ethics; Sociology; Psychology; Computer science; MEDLINE; Political science; Engineering; Ecology","retraction":null,"screen_n_in":null,"score":{"opus":0.4738564796823058,"gpt":0.5813766564612814,"spread":0.1075201767789757,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.07604852,0.0001513961,0.000458946,0.0007774321,0.0001237684,0.0005091424,0.0003260751,0.0001158156,0.0004871044],"category_scores_gemma":[0.008845795,0.0001171251,0.0001123156,0.0009619015,0.00004228988,0.0009457357,0.00001425945,0.0002404068,0.00001093857],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005066055,"about_ca_system_score_gemma":0.007721358,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0000841212,"about_ca_topic_score_gemma":0.008692618,"domain_scores_codex":[0.9945547,0.001885023,0.001396876,0.0003221797,0.001541022,0.0003001796],"domain_scores_gemma":[0.9939412,0.0005861751,0.001015734,0.0003123124,0.003849279,0.0002952343],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001655477,0.00002246222,0.01280837,0.0003632248,0.00001501781,6.613218e-7,0.001959103,0.0006451087,0.000007874664,0.0002658199,0.00008643279,0.9838094],"study_design_scores_gemma":[0.01474622,0.003293051,0.1725699,0.07188507,0.0008411455,0.0003377644,0.002212439,0.6199259,0.0001506713,0.02879181,0.08408332,0.001162786],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"review","genre_gemma":"empirical","genre_scores_codex":[0.410897,0.509248,0.01196458,0.007109176,0.002560317,0.05472733,0.0000317711,0.0000224758,0.003439381],"genre_scores_gemma":[0.8659421,0.002779839,0.1268103,0.001415565,0.0003061517,0.002182031,0.0001757212,0.00004563891,0.000342714],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9826466,"threshold_uncertainty_score":0.9995031,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W18830643","doi":"10.3138/cjpe.026.001","title":"Advancing Empirical Scholarship to Further Develop Evaluation Theory and Practice","year":2011,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":29,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Scholarship; Grounded theory; Context (archaeology); Development theory; Field (mathematics); Empirical research; Contingency; Epistemology; Sociology; Intersection (aeronautics); Contingency theory; Management science; Engineering ethics; Qualitative research; Knowledge management; Computer science; Social science; Political science; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.580695406400756,"gpt":0.5966934622915074,"spread":0.01599805589075143,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.123081,0.00013875,0.0002115006,0.0008215165,0.0002405246,0.000378912,0.0003749638,0.00009107417,0.003545433],"category_scores_gemma":[0.07867451,0.000108101,0.00005783772,0.001140743,0.00007516366,0.001823277,0.00002333024,0.000260603,0.0001565621],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005323553,"about_ca_system_score_gemma":0.007967063,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001158139,"about_ca_topic_score_gemma":0.007066055,"domain_scores_codex":[0.9910715,0.004155572,0.0009729883,0.0002907674,0.003171006,0.0003382128],"domain_scores_gemma":[0.987357,0.001231334,0.0006682547,0.0002901028,0.009655432,0.0007979153],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0001218474,0.0000388359,0.02729316,0.000001902025,0.00002783669,0.000004534122,0.01140147,0.0002413089,0.00004289647,0.0006582591,0.0008976988,0.9592702],"study_design_scores_gemma":[0.001834932,0.001408475,0.7638695,0.0001189442,0.0003783461,0.0003487947,0.01839562,0.01073665,0.0002738029,0.1019099,0.100315,0.0004101049],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9583588,0.001152224,0.01218419,0.009083507,0.001217044,0.002492861,0.00000358477,0.00001649766,0.01549134],"genre_scores_gemma":[0.9545121,0.000008134364,0.04362605,0.00155642,0.0001305912,0.00008400523,0.000003412745,0.00001293575,0.00006631498],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9588602,"threshold_uncertainty_score":0.9976568,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4367029949","doi":"10.3138/cjpe.25.002","title":"Contribution Analysis Applied: Reflections on Scope and Methodology","year":2010,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":29,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Scope (computer science); Perspective (graphical); Relation (database); Elaboration; Management science; Epistemology; Computer science; Engineering ethics; Sociology; Engineering; Data mining; Philosophy","retraction":null,"screen_n_in":null,"score":{"opus":0.6370922382666173,"gpt":0.6304102886950795,"spread":0.006681949571537782,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0240344,0.00009312559,0.0002705803,0.001543479,0.000301154,0.0003059904,0.0002223,0.0001113841,0.002028733],"category_scores_gemma":[0.005607335,0.00007051111,0.0001122583,0.001601133,0.0001160831,0.0002522332,0.000007314232,0.0003513648,0.00004533047],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001356015,"about_ca_system_score_gemma":0.001822214,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003017261,"about_ca_topic_score_gemma":0.08202588,"domain_scores_codex":[0.9971011,0.0006299663,0.000673239,0.0002041449,0.001188276,0.0002032806],"domain_scores_gemma":[0.9955599,0.0005912374,0.0005623714,0.0002540403,0.002631719,0.0004006875],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00003710946,0.00003831101,0.01679327,0.000001075587,0.0001712828,0.000001864717,0.0006567735,0.004007944,0.001291518,0.01930451,0.0008308703,0.9568655],"study_design_scores_gemma":[0.002545694,0.001840806,0.6997486,0.00001820608,0.001940667,0.0001298343,0.001809527,0.0534916,0.00149986,0.08617953,0.150395,0.0004006796],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9580563,0.0001956865,0.02084571,0.004537449,0.001748715,0.001133468,0.00001190302,0.00001434534,0.01345647],"genre_scores_gemma":[0.9870341,0.000007095737,0.01240402,0.000271026,0.0001634345,0.00004366703,0.00001387867,0.000004390042,0.00005832124],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9564648,"threshold_uncertainty_score":0.9988835,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366451089","doi":"10.3138/cjpe.022.006","title":"Youth Voices: Evaluation of Participatory Action Research","year":2007,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Participatory Visual Research Methods","field":"Social Sciences","cited_by":28,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"York University; University of Toronto","funders":"","keywords":"Participatory action research; Participatory evaluation; Citizen journalism; Reflexivity; Psychosocial; Action research; Psychology; Sociology; Action (physics); Process (computing); Applied psychology; Community-based participatory research; Public relations; Medical education; Pedagogy; Political science; Social science; Computer science; Medicine; Psychotherapist","retraction":null,"screen_n_in":null,"score":{"opus":0.9730869743906994,"gpt":0.7857522584583132,"spread":0.1873347159323862,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2612474,0.00008875116,0.0001967859,0.001060296,0.000591389,0.0001163579,0.0003484261,0.0001663386,0.0005892872],"category_scores_gemma":[0.02888243,0.00008932872,0.0001005764,0.001396319,0.0007991311,0.0005854733,0.0000107512,0.0004659141,0.0000174677],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002437609,"about_ca_system_score_gemma":0.01922766,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.01596213,"about_ca_topic_score_gemma":0.2456339,"domain_scores_codex":[0.9809486,0.0110556,0.0008135965,0.0001891054,0.006049624,0.0009434698],"domain_scores_gemma":[0.9865939,0.0006493418,0.0004609487,0.0002039694,0.01096776,0.001124059],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00006382753,0.0001160836,0.01848199,0.00001854748,0.00005200145,0.000005056826,0.04726173,0.0002725096,0.0007194399,0.001299309,0.0002593147,0.9314502],"study_design_scores_gemma":[0.005656881,0.003948134,0.4410921,0.0007852058,0.001800808,0.00001145186,0.3798693,0.02725414,0.02042712,0.03941339,0.07891896,0.0008224823],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9884681,0.0007924435,0.0005543883,0.0002413001,0.0006465947,0.001656988,0.000003699607,0.00001022234,0.007626316],"genre_scores_gemma":[0.9981159,0.00001500865,0.001254022,0.00001221412,0.0004851432,0.0000656144,0.000004152228,0.00001294205,0.00003500158],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9306277,"threshold_uncertainty_score":0.9905907,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2184444384","doi":"10.3138/cjpe.19.007","title":"The Analysis of Focus Groups in Published Research Articles","year":2004,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Focus Groups and Qualitative Methods","field":"Social Sciences","cited_by":28,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Focus group; sort; Inter-rater reliability; Coding (social sciences); Focus (optics); Psychology; Computer science; Sociology; Applied psychology; Information retrieval; Social science; Developmental psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.3988284955321382,"gpt":0.5595611463391841,"spread":0.1607326508070459,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.05281096,0.0000471366,0.0001711792,0.001021797,0.000447151,0.0002384103,0.0003024811,0.00005834795,0.0001118088],"category_scores_gemma":[0.009379963,0.00003416979,0.0001178673,0.004121621,0.0006043029,0.0003212023,0.000006195813,0.0002482761,0.00000144689],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007341662,"about_ca_system_score_gemma":0.003453801,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.08972934,"about_ca_topic_score_gemma":0.8353028,"domain_scores_codex":[0.9948072,0.003209973,0.0004757986,0.00009172693,0.001058282,0.0003570085],"domain_scores_gemma":[0.9966793,0.0006464465,0.0002392736,0.000123468,0.002035583,0.0002759416],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00001779868,0.00005748451,0.01387128,0.000002516374,0.0001553781,0.000005552159,0.04779629,0.0008837333,0.00003958284,0.1010611,0.0001102913,0.835999],"study_design_scores_gemma":[0.001282336,0.0006269204,0.3592228,0.00007534496,0.0004437488,0.00000141617,0.08757886,0.001636942,0.0002620613,0.5331162,0.01559303,0.0001603496],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9834052,0.001674825,0.0005444842,0.01087621,0.000181542,0.0006005813,0.000002297841,0.00000307473,0.002711744],"genre_scores_gemma":[0.9968807,0.00002447903,0.002954634,0.0000100151,0.00007656041,0.0000275536,0.000001184911,0.000003780135,0.00002110242],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8358386,"threshold_uncertainty_score":0.9989644,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2185731738","doi":"10.3138/cjpe.19.001","title":"The Role of the Evaluator in a Political World","year":2004,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":28,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Politics; Power (physics); Simplicity; Argument (complex analysis); Elite; Ambiguity; Servant; Sociology; Independence (probability theory); Fundamentalism; Environmental ethics; Law; Political science; Political economy; Epistemology; Philosophy","retraction":null,"screen_n_in":null,"score":{"opus":0.2293825789903462,"gpt":0.5204448598505118,"spread":0.2910622808601656,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0183713,0.00007869399,0.0001562358,0.0004047749,0.0002161198,0.0002021653,0.0007169961,0.00004093286,0.0004017472],"category_scores_gemma":[0.004548977,0.00003987999,0.0001356708,0.001257482,0.0002038213,0.0002547996,0.00001703895,0.0002375935,0.00002736458],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000662716,"about_ca_system_score_gemma":0.009871162,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001185784,"about_ca_topic_score_gemma":0.1566141,"domain_scores_codex":[0.9954082,0.0006664001,0.00095686,0.0001129631,0.002541552,0.0003140066],"domain_scores_gemma":[0.9966816,0.0003650018,0.0005285559,0.0003360446,0.001812259,0.0002765894],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00002295112,0.00006305381,0.1148359,0.000001749122,0.0000185368,0.000001506913,0.00118086,0.005478553,0.00007306213,0.08459651,0.0004013288,0.793326],"study_design_scores_gemma":[0.001647529,0.0002648496,0.448495,0.0001067888,0.00006424385,0.00003033235,0.004152455,0.02197911,0.0009453879,0.4478423,0.07435546,0.0001165775],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9626817,0.001232826,0.00007808899,0.02158749,0.001017611,0.001386307,0.000004697785,0.000003281482,0.01200805],"genre_scores_gemma":[0.9991939,0.000002950341,0.0003911972,0.0001807177,0.00009499727,0.00003628516,5.784734e-7,0.000004892448,0.00009449571],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7932094,"threshold_uncertainty_score":0.995742,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2150927259","doi":"10.3138/cjpe.27.003","title":"The Art of the Nudge: Five Practices for Developmental Evaluators","year":2012,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":27,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"Concordia University","funders":"","keywords":"Servant; Psychology; Adaptation (eye); Servant leadership; Energy (signal processing); Pedagogy; Applied psychology; Public relations; Social psychology; Political science; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.5020626007095538,"gpt":0.5787691116820812,"spread":0.07670651097252745,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03657033,0.00009411802,0.0001539886,0.0001759944,0.0005625016,0.0002518278,0.0006672526,0.00005405636,0.0005739893],"category_scores_gemma":[0.01717263,0.00004700099,0.0001605275,0.0005574756,0.0001548321,0.000815033,0.00001898251,0.0001526935,0.00004404363],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003638521,"about_ca_system_score_gemma":0.0072142,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001226727,"about_ca_topic_score_gemma":0.008814751,"domain_scores_codex":[0.9955459,0.0007332911,0.0008957553,0.0001033782,0.002420191,0.0003014311],"domain_scores_gemma":[0.9926672,0.001213038,0.002297419,0.0002398772,0.003310638,0.0002718268],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00003684526,0.00005077911,0.0847872,0.000004416725,0.00006525969,1.108431e-7,0.003151422,0.0002907005,0.00006077068,0.0011041,0.05290604,0.8575423],"study_design_scores_gemma":[0.0008657894,0.0002682006,0.1873099,0.00002742228,0.0001695954,0.00003942937,0.007618861,0.0113286,0.0006366092,0.004598858,0.7870238,0.0001129441],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9608999,0.001873575,0.0006508701,0.0192021,0.004954397,0.003883451,0.00001855507,0.00000582638,0.008511353],"genre_scores_gemma":[0.9947085,0.00001106961,0.003695438,0.0002620782,0.0002142186,0.0001267584,0.000003496292,0.000007437577,0.0009709466],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8574294,"threshold_uncertainty_score":0.998414,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1493255413","doi":"10.3138/cjpe.027.003","title":"Capturing the Complexity of Evaluations of Health Promotion Interventions: A Scoping Review","year":2012,"lang":"en","type":"review","venue":"Canadian Journal of Program Evaluation","topic":"Health Policy Implementation Science","field":"Health Professions","cited_by":27,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"Lloydminster Hospital; University of Calgary","funders":"","keywords":"Psychological intervention; Perspective (graphical); Promotion (chess); Grey literature; Health promotion; Management science; Psychology; Applied psychology; Computer science; MEDLINE; Medicine; Public health; Nursing; Political science; Engineering; Artificial intelligence","retraction":null,"screen_n_in":null,"score":{"opus":0.9727255468434597,"gpt":0.7916901593614977,"spread":0.181035387481962,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.05778972,0.0002563429,0.001905095,0.0008203111,0.0007890927,0.00001817693,0.0006260421,0.0001715465,0.001456275],"category_scores_gemma":[0.007360358,0.0001830568,0.0006522698,0.001355092,0.0004335377,0.000346164,0.00004400105,0.0008879217,0.00002905734],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001874632,"about_ca_system_score_gemma":0.05331322,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003432831,"about_ca_topic_score_gemma":0.03339038,"domain_scores_codex":[0.9770094,0.01407347,0.006692659,0.0002061886,0.001298877,0.0007193867],"domain_scores_gemma":[0.9814976,0.0009931073,0.01306587,0.0005133277,0.003359902,0.0005702277],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"systematic_review","study_design_scores_codex":[5.398449e-7,0.00003356773,0.00005131781,0.1963604,0.00004881236,1.114732e-7,0.001636756,0.000001380677,3.130721e-8,0.0002814798,0.000559432,0.8010262],"study_design_scores_gemma":[0.0003874985,0.0003872594,0.0004066368,0.8429852,0.0017809,0.00002775882,0.000550012,0.00003291992,1.85081e-7,0.0005085205,0.1527765,0.0001566971],"study_design_candidate":"systematic_review","study_design_consensus":null,"genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.00001861632,0.9732137,0.0006892736,0.004612012,0.00094421,0.02026239,0.000117751,0.000008370353,0.0001336607],"genre_scores_gemma":[0.000963637,0.9940994,0.002209587,0.0003914526,0.000281397,0.001890252,0.0001105071,0.00003442926,0.00001929721],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.8008695,"threshold_uncertainty_score":0.9994565,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":["metaepi_broad"],"domain":null,"study_design":"design_other","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"opus","categories":["metaresearch","metaepi_broad"],"domain":"methods","study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"},{"id":"W2043777716","doi":"10.3138/cjpe.29.1.36","title":"Supporting Knowledge Translation Through Evaluation: Evaluator as Knowledge Broker","year":2014,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"McMaster University; Queen's University","funders":"","keywords":"Knowledge translation; Context (archaeology); Knowledge management; Computer science; History","retraction":null,"screen_n_in":null,"score":{"opus":0.4946854783565933,"gpt":0.5959329823406669,"spread":0.1012475039840736,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","insufficient_payload"],"consensus_categories":["metaresearch","insufficient_payload"],"category_scores_codex":[0.07480533,0.0003111415,0.0005323099,0.0009844174,0.0005857131,0.0007604281,0.0008366985,0.0002239572,0.01259638],"category_scores_gemma":[0.01430965,0.0002569336,0.0003552363,0.001689271,0.0001757274,0.001847452,0.00002174334,0.0004215061,0.001268188],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009279299,"about_ca_system_score_gemma":0.01524301,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002630216,"about_ca_topic_score_gemma":0.01449681,"domain_scores_codex":[0.9879287,0.003570183,0.002433014,0.0005339553,0.004865519,0.0006685862],"domain_scores_gemma":[0.9837446,0.001166799,0.001668269,0.0006473886,0.01200564,0.0007672675],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002283637,0.0001016486,0.00405141,0.00001140184,0.00004587492,0.000001130871,0.007287828,0.00201529,0.00008327828,0.001259857,0.007003614,0.9781159],"study_design_scores_gemma":[0.003028983,0.001147647,0.01314982,0.0001174775,0.000453902,0.00008029847,0.002161451,0.4814937,0.0003197775,0.05012747,0.447538,0.0003815079],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6728983,0.01030731,0.0193339,0.007066739,0.008343509,0.007601594,0.00001829282,0.00009115905,0.2743392],"genre_scores_gemma":[0.9921967,0.00001502023,0.005625112,0.0002702803,0.0009463542,0.0002165563,0.00004819002,0.00003316428,0.0006486035],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9777343,"threshold_uncertainty_score":0.9999883,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366450544","doi":"10.3138/cjpe.021.002","title":"Audit and Evaluation in Public Management: Challenges, Reforms, and Different Roles","year":2006,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Public Policy and Administration Research","field":"Social Sciences","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Audit; Public sector; Confusion; Business; New public management; Public relations; Public management; Face (sociological concept); Relation (database); Accounting; Political science; Sociology; Psychology; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.1978141145687296,"gpt":0.4281585864049609,"spread":0.2303444718362313,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006150761,0.00006689182,0.00009893884,0.000503425,0.0002536151,0.0003127891,0.0001125887,0.00007020515,0.0001111669],"category_scores_gemma":[0.0003569538,0.00005938685,0.00002022502,0.0002295012,0.0001632235,0.0004715213,0.000008286876,0.0001214608,0.000001091446],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004708153,"about_ca_system_score_gemma":0.001600339,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005357585,"about_ca_topic_score_gemma":0.495417,"domain_scores_codex":[0.9980501,0.0004966287,0.0002707399,0.0001169524,0.0007711551,0.0002944057],"domain_scores_gemma":[0.9990117,0.0000365983,0.0001421215,0.00006543597,0.0004041986,0.0003399473],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.000003964762,0.00003875209,0.004460057,0.00001420838,0.00001071032,0.000005116509,0.001117075,0.000007072168,0.00000238168,0.09855928,0.0002395269,0.8955418],"study_design_scores_gemma":[0.001804189,0.0002348676,0.756873,0.0001036385,0.0000645277,0.00001685599,0.006099225,0.004074114,0.000006875241,0.1028632,0.1276747,0.0001847161],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8744881,0.005338577,0.0000113366,0.03196076,0.0001295255,0.001447454,0.000004369161,0.000009672451,0.08661022],"genre_scores_gemma":[0.9988686,0.0006083311,0.0001187065,0.00002771895,0.0001767751,0.00007469889,0.00001185235,0.000004121168,0.000109169],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8953571,"threshold_uncertainty_score":0.8099102,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3105492687","doi":"10.3138/cjpe.69535","title":"Toward Learning from Change Pathways: Reviewing Theory of Change and Its Discontents","year":2020,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"University of Guelph","funders":"","keywords":"Underpinning; Vagueness; Theory of change; Context (archaeology); Stakeholder; Process (computing); Stakeholder engagement; Epistemology; Management science; Knowledge management; Sociology; Computer science; Political science; Public relations; Economics; Engineering; Artificial intelligence","retraction":null,"screen_n_in":null,"score":{"opus":0.918735116591738,"gpt":0.5122009043033375,"spread":0.4065342122884005,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.008744089,0.0001039163,0.0003041797,0.0002271541,0.000107031,0.0001526376,0.0002821319,0.00005303327,0.001059017],"category_scores_gemma":[0.004376398,0.00007762983,0.00009439585,0.000418962,0.000042016,0.0009068071,0.00002240047,0.0002005993,0.00002590121],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008091484,"about_ca_system_score_gemma":0.00066467,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002543539,"about_ca_topic_score_gemma":0.0007692763,"domain_scores_codex":[0.9967198,0.0008509593,0.0007557599,0.0001820044,0.001316597,0.0001749047],"domain_scores_gemma":[0.9970682,0.0002208189,0.0008225329,0.0001043938,0.001332355,0.0004517425],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00001748847,0.000009742632,0.02397019,0.00001694268,0.00002328758,0.000004139696,0.0171205,0.000030459,0.0001031884,0.0003070726,0.00006208722,0.9583349],"study_design_scores_gemma":[0.004172196,0.003367363,0.6562167,0.001560946,0.0006414417,0.00003428644,0.03217653,0.223127,0.0007680103,0.0199878,0.05726856,0.0006791361],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9723896,0.01800383,0.0004567557,0.006803091,0.0004864843,0.001474452,0.0000162697,0.000008239178,0.0003612186],"genre_scores_gemma":[0.9982341,0.0002624346,0.0003770197,0.0006695898,0.0003632235,0.00006248595,0.00001196362,0.000008143978,0.000011055],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9576558,"threshold_uncertainty_score":0.9998541,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":[],"domain":null,"study_design":"design_other","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"opus","categories":["metaresearch"],"domain":"methods","study_design":"design_other","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"medium"}],"label_agreement":"split"},{"id":"W281362470","doi":"10.3138/cjpe.23.004","title":"Challenges in Applying Indigenous Evaluation Practices in Mainstream Grant Programs to Indigenous Communities","year":2008,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Indigenous; Mainstream; Government (linguistics); Substance abuse prevention; Competence (human resources); Cultural competence; Traditional knowledge; Political science; Culturally appropriate; Public relations; Substance use; Sociology; Psychology; Medicine; Pedagogy; Gerontology; Social psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.6836246640574786,"gpt":0.5359148392352847,"spread":0.1477098248221939,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.04188688,0.000228603,0.0004564505,0.002872157,0.0004256106,0.0003632879,0.0008517355,0.0001554553,0.0003092811],"category_scores_gemma":[0.003468703,0.0001982018,0.0001053074,0.001895216,0.0001231659,0.001300074,0.00002997437,0.0005348659,0.00006201759],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001556455,"about_ca_system_score_gemma":0.01032931,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.0118608,"about_ca_topic_score_gemma":0.6369312,"domain_scores_codex":[0.990861,0.00234355,0.001672224,0.0003155377,0.004174825,0.0006328358],"domain_scores_gemma":[0.993859,0.0004530336,0.001652407,0.0005076333,0.003033222,0.0004946901],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00002964456,0.0001865994,0.2017064,0.000007537806,0.00001372635,0.00002698135,0.06978314,0.006109251,0.000005864498,0.00002521095,0.00002355505,0.722082],"study_design_scores_gemma":[0.002889778,0.001935491,0.866802,0.0003429452,0.0000798074,0.0004050004,0.08073063,0.02902875,0.00003772603,0.004880816,0.01242668,0.000440351],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9874831,0.003014898,0.000053148,0.001199602,0.000467846,0.005391425,0.000003227307,0.00001200969,0.002374748],"genre_scores_gemma":[0.9956695,0.0003918479,0.002439309,0.0001246831,0.0001163575,0.001186971,0.00003243436,0.00001886631,0.00001998828],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7216417,"threshold_uncertainty_score":0.9952812,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2945587103","doi":"10.3138/cjpe.42190","title":"Evaluation Literacy: Perspectives of Internal Evaluators in Non-Government Organizations","year":2019,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Government (linguistics); Context (archaeology); Conversation; Narrative; Literacy; Public relations; Psychology; Knowledge management; Sociology; Political science; Pedagogy; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.1121734083055392,"gpt":0.4916620196187995,"spread":0.3794886113132602,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.03273753,0.0001561126,0.0003470908,0.0009645036,0.00007138366,0.0002020236,0.0005638409,0.00009066147,0.01383499],"category_scores_gemma":[0.006326363,0.0001297231,0.0001269458,0.001875706,0.00007851945,0.001103741,0.00002421308,0.0002389864,0.0001132538],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001686054,"about_ca_system_score_gemma":0.007404455,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005865979,"about_ca_topic_score_gemma":0.006461604,"domain_scores_codex":[0.9906327,0.001088445,0.001513011,0.0002904479,0.006213699,0.0002617654],"domain_scores_gemma":[0.9866698,0.0002277279,0.001289473,0.0003903839,0.01114837,0.0002742887],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00004263855,0.0001976794,0.354305,0.000008992573,0.00007144702,0.000002043431,0.02465465,0.014929,0.0004200722,0.0004715461,0.0004549385,0.604442],"study_design_scores_gemma":[0.002378138,0.0008505872,0.6150318,0.0001550162,0.0001872203,0.00001861289,0.01551965,0.356939,0.0004990225,0.006571633,0.001661348,0.0001880129],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.988338,0.0005250182,0.0009673468,0.0003672165,0.0007274098,0.00190316,0.00001009254,0.000003711532,0.007158021],"genre_scores_gemma":[0.997655,0.00001188824,0.001982821,0.00005511622,0.00008194984,0.00005663615,0.000008067348,0.00001428551,0.0001342282],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.604254,"threshold_uncertainty_score":0.9982226,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4234946968","doi":"10.3138/cjpe.30.3.04","title":"Lessons on Decolonizing Evaluation from Kaupapa Māori Evaluation","year":2016,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false},"ca_institutions":"","funders":"","keywords":"Aotearoa; Indigenous; Sociology; Traditional knowledge; Equity (law); Christian ministry; Community development; Cultural competence; Pedagogy; Public relations; Political science; Gender studies","retraction":null,"screen_n_in":null,"score":{"opus":0.6218029580620157,"gpt":0.5966281862836175,"spread":0.02517477177839822,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch","insufficient_payload"],"category_scores_codex":[0.05812495,0.0002644896,0.0004000673,0.001277898,0.0004523172,0.0005933336,0.0007773972,0.0001967921,0.01966214],"category_scores_gemma":[0.0193947,0.0001738481,0.0002429436,0.001030944,0.0001327782,0.001412795,0.00002087021,0.0002628762,0.0009154424],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002719115,"about_ca_system_score_gemma":0.01322162,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005779843,"about_ca_topic_score_gemma":0.02840033,"domain_scores_codex":[0.9858571,0.002900598,0.00166839,0.0005500306,0.008518367,0.0005054679],"domain_scores_gemma":[0.9845241,0.001371984,0.001603414,0.0007146759,0.0110867,0.0006991334],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.00004313811,0.00006247364,0.01136966,0.00000111281,0.00005207137,0.000002357933,0.0006137678,0.002794091,0.0003966936,0.0004391378,0.005214055,0.9790114],"study_design_scores_gemma":[0.008954713,0.001850902,0.4635614,0.000508078,0.00105898,0.00003315095,0.00184622,0.2979965,0.001880581,0.1382044,0.08340207,0.0007030366],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9594777,0.0008938044,0.002926406,0.01931495,0.0040838,0.003681087,0.00005540767,0.00002733525,0.0095395],"genre_scores_gemma":[0.9964114,0.00002285169,0.001998685,0.0003469608,0.0006463031,0.0003330836,0.00004879966,0.00002425476,0.0001676021],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9783084,"threshold_uncertainty_score":0.9998624,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4366721395","doi":"10.3138/cjpe.0017.007","title":"Introducing Program Teams to Logic Models: Facilitating the Learning Process","year":2003,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":22,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":true,"about_ca":false},"ca_institutions":"Health Canada","funders":"University of Ottawa","keywords":"Logic model; Process (computing); Key (lock); Computer science; Articulation (sociology); Point (geometry); Facilitation; Knowledge management; Process management; Management science; Psychology; Business; Engineering; Sociology; Political science; Mathematics","retraction":null,"screen_n_in":null,"score":{"opus":0.367515472877912,"gpt":0.5381391714975737,"spread":0.1706236986196618,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03103167,0.0001535057,0.0002377015,0.000528041,0.0006339829,0.0008261352,0.0005700234,0.00006613876,0.000854969],"category_scores_gemma":[0.01641264,0.0000957489,0.0001168403,0.001485612,0.00008308391,0.0007759884,0.00001104028,0.0004470427,0.00008554665],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004017035,"about_ca_system_score_gemma":0.004779187,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001921838,"about_ca_topic_score_gemma":0.005602709,"domain_scores_codex":[0.994343,0.001338899,0.001022898,0.0002991863,0.002563711,0.0004322509],"domain_scores_gemma":[0.9940469,0.0004295634,0.0006449866,0.0003052311,0.004079579,0.0004937344],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000004734057,0.00002184563,0.003547528,0.000003108916,0.00001049707,0.000001114633,0.004287993,0.3951189,0.000008428144,0.000631068,0.000321046,0.5960438],"study_design_scores_gemma":[0.0007495616,0.001550775,0.004239885,0.00008811879,0.00008331992,0.00007682385,0.03507449,0.8327271,0.00008107216,0.05442232,0.07062607,0.0002805128],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9289944,0.0008535786,0.0388575,0.007368288,0.001347848,0.005143493,0.000003245076,0.00005396156,0.01737775],"genre_scores_gemma":[0.9877298,0.000002648484,0.01135397,0.0002733226,0.0001407472,0.0002684847,0.000003719869,0.00001145197,0.0002157881],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5957632,"threshold_uncertainty_score":0.9977568,"prediction_status":"machine_predicted_unvalidated"},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":[],"domain":null,"study_design":"design_other","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"opus","categories":[],"domain":null,"study_design":"not_applicable","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"},{"id":"W4366382532","doi":"10.3138/cjpe.31132","title":"Moving Beyond the Buzzword: A Framework for Teaching Culturally Responsive Approaches to Evaluation","year":2017,"lang":"en","type":"article","venue":"Canadian Journal of Program Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":22,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Conceptual framework; Competence (human resources); Cultural competence; Sociology; Domain (mathematical analysis); Knowledge management; Computer science; Engineering ethics; Psychology; Pedagogy; Social psychology; Social science; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.6407724104552023,"gpt":0.5597165071488638,"spread":0.0810559033063385,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","scholarly_communication"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.06413532,0.0001721144,0.0002715307,0.0004758911,0.002505651,0.002436947,0.001461816,0.0001208257,0.0004466583],"category_scores_gemma":[0.05975384,0.0001066961,0.000210111,0.0002718735,0.0001537707,0.001201083,0.00003365618,0.0004203364,0.00004305618],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006113987,"about_ca_system_score_gemma":0.005132222,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002937876,"about_ca_topic_score_gemma":0.01284876,"domain_scores_codex":[0.99375,0.001299871,0.0009640276,0.0003207131,0.003283185,0.0003822013],"domain_scores_gemma":[0.9916731,0.001200062,0.001548161,0.0008505245,0.004258902,0.0004692256],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0000634708,0.0000213101,0.00302003,0.000003174102,0.00004120647,8.601991e-7,0.007699091,0.006430849,0.00003430091,0.01218198,0.004391362,0.9661124],"study_design_scores_gemma":[0.001733697,0.001117979,0.1500711,0.0002102308,0.0004454044,0.0000452784,0.01645248,0.3153688,0.0001664003,0.4219718,0.09201159,0.0004052935],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7964171,0.0006996965,0.06172205,0.1167933,0.003923082,0.0104971,0.00002938925,0.00001937502,0.009898834],"genre_scores_gemma":[0.9442305,0.000002086705,0.05366974,0.0006807805,0.0006299297,0.0005111801,0.00001082973,0.00001451437,0.0002504212],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9657071,"threshold_uncertainty_score":0.9987929,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null}]}