{"meta":{"page":1,"per_page":50,"max_per_page":100,"total":73,"total_is_capped":false,"direct_labels_cover":1,"predictions_cover":73,"direct_label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline (scores rank; they never assert a category)","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"query_hash":"3e0a82fc70b4","filters":{"venue":"New Directions for Evaluation"}},"results":[{"id":"W2101260272","doi":"10.1002/ev.313","title":"Policy implementation: Implications for evaluation","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McGill University; Douglas Mental Health University Institute","funders":"","keywords":"Accountability; Toolbox; Corporate governance; Context (archaeology); Process (computing); Government (linguistics); Sustainability; Political science; Public relations; Process management; Public administration; Sociology; Computer science; Business; Management; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.3635462965884845,"gpt":0.6388161775626578,"spread":0.2752698809741734,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01358166,0.0002077073,0.0002487538,0.0008773612,0.0009758741,0.0003909091,0.0003774592,0.0001076764,0.001836226],"category_scores_gemma":[0.005147405,0.0001904094,0.0002693822,0.001772456,0.00002751593,0.001002281,0.00001792112,0.00007509626,0.000110293],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007423776,"about_ca_system_score_gemma":0.003105188,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006686457,"about_ca_topic_score_gemma":0.0005281173,"domain_scores_codex":[0.9954602,0.0003487246,0.001061865,0.0006729682,0.002091924,0.0003642849],"domain_scores_gemma":[0.992761,0.0009590209,0.0005403029,0.0007275004,0.004836425,0.0001757241],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00004016024,0.00007711609,0.0007934293,0.000001744592,0.00002009965,2.913662e-9,0.0003842374,0.002747438,0.0007377552,0.0318734,0.0545567,0.9087679],"study_design_scores_gemma":[0.005524888,0.0004321373,0.1852218,0.000007237064,0.0002644456,0.000003640356,0.0005999928,0.09917181,0.001033,0.5684986,0.1389862,0.0002563681],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.07374496,0.0006967795,0.6075945,0.2374683,0.003738799,0.02575078,0.0005198896,0.0005055534,0.04998042],"genre_scores_gemma":[0.9738463,0.00004564255,0.01323451,0.001846266,0.001714816,0.003894322,0.001139169,0.00002624064,0.004252798],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9085115,"threshold_uncertainty_score":0.9990762,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1543074450","doi":"10.1002/ev.20076","title":"Framing the Capacity to Do and Use Evaluation","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Framing (construction); Conceptualization; Situated; Construct (python library); Capacity building; Organizational change; Sociology; Management science; Knowledge management; Public relations; Computer science; Political science; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.3372561403528904,"gpt":0.5217974321371192,"spread":0.1845412917842288,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.026411,0.0001261476,0.0001504556,0.0002956879,0.0007741525,0.0007006883,0.000213896,0.00006843606,0.0007899149],"category_scores_gemma":[0.01906536,0.00008517937,0.00007428456,0.0008077761,0.00004072328,0.0007644431,0.00004259715,0.00009733968,0.0001531388],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001775034,"about_ca_system_score_gemma":0.0002765738,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001651167,"about_ca_topic_score_gemma":0.001132772,"domain_scores_codex":[0.9953135,0.0008927742,0.0004902242,0.0004521127,0.002664241,0.000187123],"domain_scores_gemma":[0.9946947,0.002410647,0.0002094448,0.0005585527,0.0019984,0.0001282675],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002238382,0.0000249226,0.005238763,0.000001899219,0.00001367058,3.481585e-9,0.002191893,0.0105815,0.0004526603,0.004710428,0.01214551,0.9646164],"study_design_scores_gemma":[0.0008699119,0.0001158958,0.2141308,0.00001193432,0.0001158859,0.000002062037,0.000406897,0.551722,0.000344243,0.05130178,0.1808522,0.0001263272],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8407602,0.0001050474,0.1420498,0.007279453,0.001531165,0.00273256,0.00001010445,0.00005555324,0.005476167],"genre_scores_gemma":[0.9918239,0.00000993597,0.005234292,0.0007115088,0.0003485385,0.0005133222,0.00001426716,0.00001129584,0.00133296],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9644901,"threshold_uncertainty_score":0.9891975,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2140465263","doi":"10.1002/ev.240","title":"Going through the process: An examination of the operationalization of process use in empirical research on evaluation","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":74,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Operationalization; Process (computing); Empirical research; Empirical examination; Process management; Research methodology; Psychology; Management science; Computer science; Sociology; Epistemology; Business; Actuarial science; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.6748434947289444,"gpt":0.6733453144103931,"spread":0.001498180318551312,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04806614,0.0001254999,0.0001825362,0.0006113081,0.000539225,0.0001416873,0.0004646991,0.0001206341,0.0003232053],"category_scores_gemma":[0.01756558,0.00007662128,0.00008133943,0.003995198,0.0001387037,0.001598563,0.00002836105,0.0002077275,0.000009105584],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004021439,"about_ca_system_score_gemma":0.001639733,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009816256,"about_ca_topic_score_gemma":0.003767385,"domain_scores_codex":[0.9887614,0.002412768,0.00122602,0.0004791107,0.006886432,0.0002342625],"domain_scores_gemma":[0.9872273,0.003313587,0.0005687432,0.0006070317,0.008242385,0.00004099082],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","study_design_scores_codex":[0.0003842263,0.0009781404,0.08523186,0.00004376277,0.00002950153,4.031249e-8,0.03042653,0.556344,0.001438039,0.01084527,0.001484945,0.3127937],"study_design_scores_gemma":[0.001286146,0.0002666049,0.5076444,0.00007751615,0.00004625197,9.898677e-7,0.007140429,0.441682,0.00993726,0.03086284,0.0009655764,0.00009003848],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9813769,0.00005153265,0.009700155,0.001179931,0.0004178843,0.003330042,0.000008975244,0.00001405882,0.003920532],"genre_scores_gemma":[0.9985067,0.000007810736,0.0003947002,0.0001100081,0.0001354068,0.0003781367,0.00006702243,0.00001301391,0.0003871774],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4224125,"threshold_uncertainty_score":0.9907099,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2101892546","doi":"10.1002/ev.316","title":"Knowledge theories can inform evaluation practice: What can a complexity lens add?","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":60,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Calgary","funders":"","keywords":"Competence (human resources); Fidelity; Knowledge management; Computer science; Process (computing); Context (archaeology); Complex adaptive system; Paradigm shift; Psychology; Epistemology; Artificial intelligence; Social psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.3755486202484554,"gpt":0.565178999495196,"spread":0.1896303792467406,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02362859,0.0002844073,0.0003585937,0.0006657516,0.001089137,0.001187649,0.0004138593,0.0001574518,0.00324071],"category_scores_gemma":[0.01489184,0.0002423748,0.000216194,0.001738381,0.0001067181,0.004270114,0.00004068721,0.000217217,0.000233913],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001051516,"about_ca_system_score_gemma":0.003564172,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001493456,"about_ca_topic_score_gemma":0.007542036,"domain_scores_codex":[0.9927024,0.001035541,0.001093825,0.00065177,0.004112427,0.0004040805],"domain_scores_gemma":[0.9891194,0.001797782,0.0007161945,0.0008224178,0.007352861,0.0001913952],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0001186897,0.000209852,0.0002076425,0.000003638281,0.00004033529,6.504539e-8,0.007464712,0.002322532,0.00009852554,0.02369707,0.02615753,0.9396794],"study_design_scores_gemma":[0.004205012,0.0007317927,0.03972557,0.00007056997,0.0005773504,0.0000177249,0.01363226,0.2909358,0.0007459727,0.3332121,0.3156413,0.0005046349],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.2022611,0.006921707,0.04561784,0.1889031,0.01984962,0.02303287,0.0003040281,0.001112827,0.5119969],"genre_scores_gemma":[0.9877691,0.0002737078,0.003843514,0.002313378,0.0007249766,0.0005837576,0.0004694851,0.00002148346,0.004000544],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9391748,"threshold_uncertainty_score":0.9998492,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2066945648","doi":"10.1002/ev.315","title":"Knowledge translation: Implications for evaluation","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":49,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Canadian Institutes of Health Research","funders":"","keywords":"Knowledge translation; Computer science; Knowledge management; Field (mathematics); Translation (biology); Sustainability","retraction":null,"screen_n_in":null,"score":{"opus":0.5370852562015647,"gpt":0.6053385358954619,"spread":0.06825327969389727,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01536231,0.0002110028,0.0002687965,0.0006156423,0.0009147095,0.0003205815,0.0003619714,0.0001466648,0.001384666],"category_scores_gemma":[0.004748076,0.0001914615,0.0003083455,0.001474273,0.00003113193,0.0009602904,0.000009574584,0.00009097812,0.00016939],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004359178,"about_ca_system_score_gemma":0.001680196,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008872319,"about_ca_topic_score_gemma":0.0003081708,"domain_scores_codex":[0.9959677,0.0004023062,0.001009322,0.0006938581,0.001616521,0.0003102649],"domain_scores_gemma":[0.9918218,0.001959858,0.0003961982,0.000698105,0.004960346,0.0001636656],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005609732,0.0001400621,0.0002280579,0.000002710488,0.0000177811,2.254718e-9,0.0007071762,0.004064792,0.0007406283,0.008322448,0.04093262,0.9447876],"study_design_scores_gemma":[0.004305592,0.0003140101,0.05412227,0.00001214034,0.0002998819,0.000002486509,0.0002109389,0.3860098,0.0005475296,0.2717831,0.2821639,0.0002283469],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.0247402,0.002971145,0.7936661,0.05993576,0.003883346,0.01612306,0.0001902594,0.0003958752,0.09809429],"genre_scores_gemma":[0.9679669,0.00005481833,0.02018528,0.0005375909,0.001185848,0.003194266,0.0007362723,0.00002775202,0.006111262],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9445593,"threshold_uncertainty_score":0.9995282,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2948770661","doi":"10.1002/ev.20365","title":"Sustainability‐Ready Evaluation: A Call to Action","year":2019,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Canadian Evaluation Society","funders":"","keywords":"Sustainability; Transformative learning; Sustainability science; Sustainability organizations; Relevance (law); Process management; Sustainable development; Social sustainability; Environmental resource management; Checklist; Natural resource; Action (physics); Business; Engineering ethics; Management science; Environmental planning; Computer science; Risk analysis (engineering); Political science; Sociology; Engineering; Psychology; Economics; Ecology","retraction":null,"screen_n_in":null,"score":{"opus":0.3058992048136394,"gpt":0.5825784932667877,"spread":0.2766792884531483,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.02122766,0.0002038142,0.000285122,0.0007400702,0.0003894655,0.0003530841,0.0003538267,0.0001286374,0.009956214],"category_scores_gemma":[0.009593405,0.0001786159,0.0001756137,0.001756021,0.00002236294,0.001006432,0.00005309839,0.0001310464,0.002598305],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001652014,"about_ca_system_score_gemma":0.002891407,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002087144,"about_ca_topic_score_gemma":0.001072809,"domain_scores_codex":[0.992732,0.0007197569,0.0008573946,0.0008293165,0.004516198,0.000345329],"domain_scores_gemma":[0.9897026,0.001016447,0.0003140695,0.0009505406,0.007784057,0.0002322165],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001363748,0.0001149064,0.004589782,0.00001024963,0.00002824535,3.190196e-8,0.001018908,0.04899589,0.0006839047,0.001877138,0.0689676,0.8735769],"study_design_scores_gemma":[0.002839396,0.0004582398,0.06353687,0.00001423548,0.0001716157,0.000003912855,0.001960321,0.3624991,0.0008967879,0.04306342,0.524255,0.0003010579],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8355998,0.0001986683,0.04872486,0.02798719,0.007995368,0.01676195,0.0000414325,0.0002826285,0.06240806],"genre_scores_gemma":[0.9581212,0.000006835649,0.002048132,0.0005750735,0.000492778,0.001592451,0.0001473956,0.00002450317,0.03699163],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8732759,"threshold_uncertainty_score":0.9987492,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1986224130","doi":"10.1002/ev.133","title":"Background and history of the Joint Committee's Program Evaluation Standards","year":2004,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":27,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"Queen's University","funders":"","keywords":"Joint (building); Context (archaeology); Political science; Library science; Engineering ethics; Computer science; Engineering; History; Archaeology; Civil engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.3548191126107928,"gpt":0.5240969201650518,"spread":0.169277807554259,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01762353,0.0001271088,0.0002012131,0.000237916,0.0002751182,0.00009502291,0.0002082025,0.00007783092,0.001156578],"category_scores_gemma":[0.003021478,0.00008801436,0.0001446085,0.0005635328,0.0001428297,0.0004717207,0.00004642344,0.0001040393,0.00001289038],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002188554,"about_ca_system_score_gemma":0.004614126,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001835624,"about_ca_topic_score_gemma":0.001072916,"domain_scores_codex":[0.99417,0.0004632364,0.0006914855,0.0003459838,0.004174207,0.000155136],"domain_scores_gemma":[0.9952222,0.0002799276,0.0004795327,0.0005274784,0.003416728,0.00007418441],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00004260381,0.0001734349,0.001050797,0.00001069643,0.00003149766,1.315229e-8,0.00177708,0.007806424,0.0006823186,0.003472627,0.03258309,0.9523694],"study_design_scores_gemma":[0.006239746,0.0005079239,0.1398679,0.00007369381,0.0004004885,0.000006804413,0.00137479,0.1302083,0.001559522,0.1020565,0.617464,0.0002404066],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8635142,0.00509544,0.03123543,0.01374022,0.009536527,0.01705874,0.00009849425,0.0001950927,0.05952581],"genre_scores_gemma":[0.9904878,0.0000488538,0.004656688,0.0001779834,0.0001726649,0.001021408,0.0000296342,0.00001538944,0.003389539],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.952129,"threshold_uncertainty_score":0.9997565,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3203797720","doi":"10.1002/ev.20478","title":"An introduction to policy advocacy evaluation: The concepts, history, and literature of the field","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Impact","funders":"","keywords":"Field (mathematics); Adaptation (eye); Policy development; Engineering ethics; Management science; Evaluation methods; Computer science; Political science; Public administration; Psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.12003077569205,"gpt":0.5220412336486495,"spread":0.4020104579565995,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.007650383,0.0001072322,0.000143171,0.0002573703,0.0003844317,0.0001631276,0.0002743523,0.00007397093,0.001967794],"category_scores_gemma":[0.01362795,0.00006515134,0.000100028,0.001704842,0.00005296363,0.0004649663,0.00004697687,0.0001195453,0.00001133711],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003630283,"about_ca_system_score_gemma":0.002796269,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007740122,"about_ca_topic_score_gemma":0.0004808959,"domain_scores_codex":[0.9961274,0.0009714849,0.0004700996,0.0004101978,0.001894446,0.0001263833],"domain_scores_gemma":[0.9946122,0.0008083424,0.0002511702,0.0008498271,0.003399324,0.00007914507],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005500391,0.00007916832,0.001041202,0.000006278407,0.00002551978,3.381087e-8,0.009098953,0.006305797,0.003293351,0.01130394,0.2710662,0.6977246],"study_design_scores_gemma":[0.001157222,0.0002286396,0.03931433,0.00002521979,0.0001666805,0.00001433253,0.00226073,0.06779349,0.006148704,0.02224011,0.8605142,0.0001364043],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"commentary","genre_gemma":"empirical","genre_scores_codex":[0.3865713,0.005755718,0.01649167,0.5611899,0.01127386,0.005626065,0.00003841384,0.00008037855,0.0129726],"genre_scores_gemma":[0.9823692,0.00006747121,0.00159819,0.003252346,0.002527639,0.0003723172,0.00006203856,0.00001270231,0.00973811],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6975882,"threshold_uncertainty_score":0.9989445,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1929107339","doi":"10.1002/ev.20078","title":"Cross‐Case Analysis and Implications for Research, Theory, and Practice","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Leverage (statistics); Theory of change; Organizational change; Affect (linguistics); Thematic analysis; Context (archaeology); Order (exchange); Evaluation methods; Organization development; Knowledge management; Management science; Public relations; Sociology; Computer science; Political science; Management; Business; Qualitative research; Social science; Engineering; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.4689937918488332,"gpt":0.6700118587679306,"spread":0.2010180669190974,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.06620824,0.00009258839,0.0001710689,0.0007911834,0.001371108,0.0006310081,0.0001248861,0.00007162867,0.0002570345],"category_scores_gemma":[0.05830085,0.00007662498,0.00009255832,0.001637292,0.0001235872,0.0007831592,0.00004306033,0.00008333861,0.000014931],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008131051,"about_ca_system_score_gemma":0.0002454568,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001035356,"about_ca_topic_score_gemma":0.0006853878,"domain_scores_codex":[0.9963924,0.00156916,0.0004602101,0.0005148989,0.0008598513,0.0002034436],"domain_scores_gemma":[0.9725251,0.02304235,0.0002207124,0.000468556,0.003600519,0.0001427526],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0001884608,0.00005961956,0.007614968,0.000005240965,0.0001312969,4.249007e-8,0.0008667086,0.000865255,0.0002264932,0.08111557,0.00452521,0.9044011],"study_design_scores_gemma":[0.001901635,0.0002807207,0.159562,0.000004644648,0.0009821125,0.00006834728,0.001335303,0.1873945,0.0001487982,0.4061846,0.2419771,0.0001602581],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.168217,0.000371608,0.8029016,0.01494262,0.000306054,0.002483449,0.00006344786,0.00005691079,0.01065733],"genre_scores_gemma":[0.9745942,0.0000751635,0.01866915,0.0002709866,0.0002274983,0.001036491,0.00005721204,0.00001207785,0.005057216],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9042409,"threshold_uncertainty_score":0.999929,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1992053221","doi":"10.1002/ev.325","title":"Enhancing disaster and emergency preparedness, response, and recovery through evaluation","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Preparedness; Disaster response; Emergency management; Disaster preparedness; Emergency response; Process management; Evaluation methods; Disaster recovery; Disaster planning; Computer science; Business; Risk analysis (engineering); Computer security; Public relations; Political science; Medical emergency; Poison control; Human factors and ergonomics; Engineering; Medicine; Reliability engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.1637329626934722,"gpt":0.5114703559569701,"spread":0.3477373932634978,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01923286,0.0001683252,0.0001994032,0.0002752845,0.0005882491,0.0002575804,0.0001401196,0.0001276721,0.003677361],"category_scores_gemma":[0.01058643,0.0001436997,0.00007725303,0.0005825302,0.00005707318,0.001529189,0.00005406828,0.000143287,0.00005409091],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000117101,"about_ca_system_score_gemma":0.0007617656,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007801387,"about_ca_topic_score_gemma":0.003041922,"domain_scores_codex":[0.9958462,0.0006345945,0.0007423634,0.0006284172,0.001939101,0.000209371],"domain_scores_gemma":[0.9964079,0.00110678,0.0003384211,0.0004812588,0.00155514,0.0001104634],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0006838833,0.0001007595,0.007807197,0.00001428266,0.0000497492,6.299228e-8,0.01073652,0.0005569189,0.03856595,0.001059799,0.01633322,0.9240916],"study_design_scores_gemma":[0.005034,0.0006324694,0.3066417,0.00005026221,0.0005587229,0.00002951039,0.005520883,0.2630357,0.005985218,0.2529367,0.1588935,0.000681449],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9752097,0.0003805103,0.01267523,0.001711366,0.003324509,0.001658257,0.00001630038,0.00005184856,0.004972261],"genre_scores_gemma":[0.9870321,0.0001431135,0.006553541,0.00009989898,0.000386143,0.0005523154,0.0000492588,0.00001979408,0.005163873],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9234102,"threshold_uncertainty_score":0.9977478,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2097486283","doi":"10.1002/ev.20045","title":"Rethinking the Relationship Among Monitoring, Evaluation, and Results‐Based Management: Observations From Canada","year":2013,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Public Policy and Administration Research","field":"Social Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Accountability; Government (linguistics); Monitoring and evaluation; Dual (grammatical number); Performance management; Process management; Political science; Public administration; Business; Marketing","retraction":null,"screen_n_in":null,"score":{"opus":0.2437606918516431,"gpt":0.4279213200316784,"spread":0.1841606281800354,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.00461092,0.00009442323,0.00007597301,0.0001132477,0.002654754,0.0004121889,0.0001933034,0.00008620907,0.0002461306],"category_scores_gemma":[0.007940806,0.00008622287,0.00003714042,0.0006512862,0.0001263044,0.0005952404,0.0000187503,0.0001328787,0.000005621489],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005385614,"about_ca_system_score_gemma":0.002649392,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.7622146,"about_ca_topic_score_gemma":0.7844575,"domain_scores_codex":[0.9971409,0.000683565,0.0003566166,0.0002756471,0.001293481,0.0002497954],"domain_scores_gemma":[0.9965351,0.001920546,0.0001845301,0.0002725837,0.0009356948,0.0001516121],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00006711613,0.0001502549,0.3769011,0.00002958827,0.000266086,3.859907e-7,0.02375833,0.00439118,0.00008181138,0.1051739,0.2825956,0.2065847],"study_design_scores_gemma":[0.0009438709,0.00001142865,0.8559817,0.00003602444,0.0001069895,6.639898e-8,0.002209753,0.02283179,0.00003714984,0.07689896,0.04083137,0.0001108672],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.860548,0.0003120295,0.001613235,0.07823433,0.001278153,0.005756032,0.00009611248,0.0001519057,0.05201024],"genre_scores_gemma":[0.9912386,0.00002582625,0.001838183,0.0001694771,0.0005342852,0.001320036,0.0002169307,0.000009943825,0.004646721],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4790806,"threshold_uncertainty_score":0.9986436,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1999716079","doi":"10.1002/ev.243","title":"Infusing evaluative thinking as process use: The case of the International Development Research Centre (IDRC)","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":21,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Accountability; Process (computing); Political science; Sociology; Process management; Management; Business; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.453763683231604,"gpt":0.6131986765418254,"spread":0.1594349933102213,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.06376707,0.0001225646,0.0001364053,0.000442126,0.001443328,0.0003875556,0.0006892894,0.00007581469,0.0008620387],"category_scores_gemma":[0.02071734,0.00007020439,0.00009717375,0.001987329,0.0001233171,0.0007225436,0.0001515559,0.0002533552,0.00005913584],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007199033,"about_ca_system_score_gemma":0.003579946,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003278787,"about_ca_topic_score_gemma":0.006386649,"domain_scores_codex":[0.9918494,0.0007995476,0.0008792299,0.0003866346,0.005803255,0.0002819634],"domain_scores_gemma":[0.9859741,0.003521078,0.0004773521,0.0004891024,0.00945406,0.00008435587],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002156689,0.0001992829,0.01487742,0.000009401295,0.0001178238,0.000002117505,0.06079681,0.01693803,0.0005323885,0.006124012,0.01629481,0.8838922],"study_design_scores_gemma":[0.004591064,0.0002065242,0.1069155,0.0002443235,0.000234732,0.0002466364,0.1009031,0.2864179,0.03494455,0.2538088,0.2110081,0.0004788191],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9707329,0.00008861873,0.005401148,0.005605602,0.001859189,0.002124463,0.000007640155,0.00002701482,0.01415345],"genre_scores_gemma":[0.9925522,0.000003837635,0.002175755,0.0001886945,0.0002249148,0.00009399388,0.00001606293,0.00001182934,0.004732683],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8834134,"threshold_uncertainty_score":0.9998567,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2070292114","doi":"10.1002/ev.393","title":"Internal evaluation, historically speaking","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":20,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Centre for Advancing Health Outcomes; University of British Columbia","funders":"","keywords":"Context (archaeology); Conservatism; Speculation; Government (linguistics); Function (biology); Evaluation methods; Internal validity; Political science; Public administration; Sociology; Public relations; Law; Business; History","retraction":null,"screen_n_in":null,"score":{"opus":0.4596669989734262,"gpt":0.5336734648069541,"spread":0.07400646583352793,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01812725,0.0001636143,0.0002083475,0.0005522726,0.0004234069,0.0001627216,0.0004275002,0.00008961283,0.02066295],"category_scores_gemma":[0.007109472,0.0001378697,0.0001906123,0.0009179246,0.000037804,0.0008693263,0.00004272782,0.0001170788,0.000673414],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000803622,"about_ca_system_score_gemma":0.001248788,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003649111,"about_ca_topic_score_gemma":0.0009106458,"domain_scores_codex":[0.9941237,0.0005208685,0.0008935625,0.0005509414,0.003665056,0.000245871],"domain_scores_gemma":[0.9943635,0.0005012636,0.0004678326,0.0005714491,0.003943295,0.0001526457],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00006994035,0.0001064948,0.004718515,0.000001770001,0.00003186737,8.036219e-8,0.002874009,0.0006423387,0.0001989345,0.002345803,0.03861834,0.9503919],"study_design_scores_gemma":[0.003079146,0.0003135103,0.1162775,0.00002492564,0.0003393954,0.00001053144,0.0007121035,0.3699005,0.001260747,0.1659406,0.3417998,0.0003412171],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.08032717,0.0007554456,0.3456487,0.002010304,0.01121147,0.004128161,0.00001711348,0.0002695734,0.5556321],"genre_scores_gemma":[0.9690315,0.00001807375,0.01840582,0.0002803786,0.0006846088,0.0005473531,0.00004555079,0.00002392617,0.01096284],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9500507,"threshold_uncertainty_score":0.9802323,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3011548789","doi":"10.1002/ev.20398","title":"Finding the Impact: Methods for Assessing the Contribution of Collective Impact to Systems and Population Change in a Multi‐Site Study","year":2020,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":17,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Impact","funders":"","keywords":"Scale (ratio); Context (archaeology); Interim; Population; Work (physics); SPARK (programming language); Management science; Computer science; Political science; Sociology; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.5624400587745525,"gpt":0.6718887697007279,"spread":0.1094487109261754,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01809795,0.0001265875,0.0002775141,0.0002914034,0.0005374275,0.0003434438,0.0001467087,0.00005224661,0.00003597342],"category_scores_gemma":[0.009708745,0.0000690774,0.0001153246,0.001729711,0.00001635683,0.0005876988,0.00003489725,0.00008546314,0.0000022493],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005032026,"about_ca_system_score_gemma":0.000365719,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002346691,"about_ca_topic_score_gemma":0.001272085,"domain_scores_codex":[0.9967269,0.001415418,0.0006408828,0.0003223559,0.0007215775,0.0001728892],"domain_scores_gemma":[0.9949687,0.003157871,0.00043921,0.0002139906,0.001143334,0.00007689281],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003400712,0.0001981289,0.5428273,0.00000989335,0.0001075488,3.055341e-8,0.04792937,0.07795132,0.002071462,0.000160786,0.0007597859,0.3276443],"study_design_scores_gemma":[0.001155353,0.0002800741,0.487921,0.00001224446,0.00005761697,5.09522e-7,0.002619958,0.5073546,0.00005268321,0.0003668967,0.0001332614,0.00004579678],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7796181,0.0002325078,0.2094104,0.00176398,0.0003291203,0.008569719,0.00003090798,0.00001561651,0.00002959904],"genre_scores_gemma":[0.995693,0.000004095486,0.002425569,0.00007293177,0.0001339983,0.001577101,0.00003713378,0.000009356584,0.00004676538],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4294033,"threshold_uncertainty_score":0.9986329,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1781948974","doi":"10.1002/ev.20111","title":"How Does Accreditation Fit Into the Picture?","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Accounting Education and Careers","field":"Business, Management and Accounting","cited_by":17,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Victoria","funders":"","keywords":"Accreditation; Blank; Computer science; Public relations; Political science; Law; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.06017099692544828,"gpt":0.3194472065074327,"spread":0.2592762095819845,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001086486,0.000101706,0.00007247279,0.0001612696,0.0003669901,0.0006871027,0.0001430748,0.00005497132,0.0000936232],"category_scores_gemma":[0.001797046,0.00006644284,0.00006520136,0.0004524521,0.00002482486,0.001212296,0.0000235261,0.00006614211,0.00007939389],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001253909,"about_ca_system_score_gemma":0.0001761877,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003587237,"about_ca_topic_score_gemma":0.0009091328,"domain_scores_codex":[0.9990979,0.00001793932,0.0001302251,0.0001861324,0.0004416143,0.0001261766],"domain_scores_gemma":[0.9985556,0.00006755922,0.0001647974,0.0001968208,0.0009993584,0.00001583758],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00004872615,0.00007155378,0.003364061,0.00003977052,0.00004956599,4.899269e-8,0.003206604,0.0009326217,0.0001779436,0.01142665,0.7180243,0.2626581],"study_design_scores_gemma":[0.0008415886,0.000008802595,0.004979547,0.000009937213,0.0001912917,4.37711e-7,0.008812946,0.01761545,0.00008623333,0.03979125,0.9275196,0.0001429671],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6506878,0.0007916495,0.03714958,0.2192923,0.03355635,0.005199547,0.000007463145,0.001203581,0.05211169],"genre_scores_gemma":[0.9837978,0.000002993641,0.0006875604,0.001757642,0.006268106,0.0003402694,0.0001702523,0.00002333528,0.006952059],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.33311,"threshold_uncertainty_score":0.6625747,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2013250096","doi":"10.1002/ev.20007","title":"When one must go: The Canadian experience with strategic review and judging program value","year":2012,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Valuation (finance); Function (biology); Government (linguistics); Value (mathematics); Citizen journalism; Public relations; Process (computing); Value for money; Public administration; Business; Political science; Economics; Public economics; Computer science; Accounting; Law","retraction":null,"screen_n_in":null,"score":{"opus":0.38211926976493,"gpt":0.524848788433648,"spread":0.142729518668718,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.007510602,0.0001196516,0.0001641573,0.0001629735,0.0008972551,0.0003402054,0.0001986835,0.00004593869,0.001256927],"category_scores_gemma":[0.0005795008,0.00007346598,0.0000439995,0.0006748976,0.00007698065,0.0007578798,0.00001677845,0.0001051742,0.00007023394],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002340365,"about_ca_system_score_gemma":0.001059537,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.008264406,"about_ca_topic_score_gemma":0.1060416,"domain_scores_codex":[0.9974876,0.0002844725,0.0003629938,0.0002735514,0.001291249,0.0003001053],"domain_scores_gemma":[0.9982114,0.0002894883,0.0001867416,0.0003959119,0.0006698557,0.0002466147],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00001444448,0.0001126382,0.01461808,0.00004245267,0.00004323125,5.189962e-8,0.005087813,0.0003933167,0.00002756513,0.008344883,0.01094541,0.9603701],"study_design_scores_gemma":[0.002063702,0.0006359054,0.1959909,0.0004230351,0.001104438,0.00004625676,0.00545183,0.08325389,0.000196274,0.02260666,0.6875635,0.0006635707],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5310917,0.04930536,0.008846051,0.1046422,0.003832904,0.0303958,0.00006451092,0.0005104629,0.271311],"genre_scores_gemma":[0.9895501,0.000372417,0.005079953,0.001238898,0.0002197462,0.001451938,0.00002744831,0.00001260548,0.002046907],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9597065,"threshold_uncertainty_score":0.9996561,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4388306673","doi":"10.1002/ev.20564","title":"Artificial intelligence and the future of evaluation education: Possibilities and prototypes","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Fraser Health","funders":"","keywords":"Chatbot; Evaluation methods; Engineering ethics; Literacy; Computer science; Program evaluation; Management science; Psychology; Pedagogy; Artificial intelligence; Political science; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.2331504610129478,"gpt":0.5285796286386468,"spread":0.295429167625699,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01728557,0.00009295601,0.0001587205,0.0003258435,0.0003833795,0.0001679127,0.0001099248,0.00005694804,0.0004731944],"category_scores_gemma":[0.003221796,0.00006000196,0.00005839018,0.001034678,0.0001632111,0.0003777951,0.00003314192,0.00006323629,0.00002652376],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005495895,"about_ca_system_score_gemma":0.001091945,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006165556,"about_ca_topic_score_gemma":0.0003979991,"domain_scores_codex":[0.9969491,0.000518244,0.0005869541,0.0003046232,0.001532963,0.0001080899],"domain_scores_gemma":[0.99639,0.001250022,0.0002604376,0.0002989531,0.001753137,0.00004739401],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00009152033,0.00002359094,0.0005221862,0.000009302708,0.0000109287,3.423711e-9,0.002727089,0.0007941752,0.00003930739,0.03605647,0.001988458,0.957737],"study_design_scores_gemma":[0.0004153337,0.00008834241,0.02025618,0.0000169729,0.0001054219,0.000002574249,0.00967973,0.3223081,0.0005779981,0.6381781,0.008294486,0.00007680805],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9249324,0.005009631,0.008771664,0.0379023,0.004190451,0.01217176,0.0000299394,0.0001242143,0.006867656],"genre_scores_gemma":[0.9944193,0.0002849851,0.001026437,0.00009483676,0.0005414897,0.001588974,0.0000429196,0.000008171827,0.001992887],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9576601,"threshold_uncertainty_score":0.5990867,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2033749959","doi":"10.1002/ev.1190","title":"Evaluative inquiry in university‐school professional learning partnerships","year":2000,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":12,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Queen's University","funders":"","keywords":"Professional learning community; Professional development; Pedagogy; Psychology; Sociology","retraction":null,"screen_n_in":null,"score":{"opus":0.3781556627691831,"gpt":0.5350127287443462,"spread":0.1568570659751632,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.01174245,0.0001423421,0.000201503,0.000600313,0.000504633,0.0001064397,0.0002648488,0.0001130554,0.03080477],"category_scores_gemma":[0.00369462,0.0001292158,0.0001168165,0.001591727,0.0000516474,0.0009103186,0.00002498482,0.0002596322,0.00116282],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000529168,"about_ca_system_score_gemma":0.001562808,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004950674,"about_ca_topic_score_gemma":0.0007052243,"domain_scores_codex":[0.9950977,0.001463309,0.0005340239,0.0005073165,0.002138424,0.0002592354],"domain_scores_gemma":[0.9973262,0.001176817,0.0002025267,0.0002845038,0.0008540255,0.0001559027],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0003624262,0.0001720332,0.01966922,0.000003647788,0.00002330942,6.210902e-7,0.004460967,0.06219227,0.00012365,0.001126557,0.03115151,0.8807138],"study_design_scores_gemma":[0.004220424,0.0002143043,0.1445049,0.00004687511,0.0000694911,0.000003484694,0.007197182,0.3116674,0.0001206169,0.01479451,0.5169045,0.0002562752],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9267965,0.0001498448,0.002481498,0.004913901,0.001502991,0.002197325,0.000007968982,0.0001175054,0.06183248],"genre_scores_gemma":[0.8720748,0.00005159846,0.001334831,0.0001586301,0.0003224039,0.0001090688,0.00007994012,0.00001349786,0.1258552],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8804575,"threshold_uncertainty_score":0.9996149,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4224927296","doi":"10.1002/ev.20494","title":"Evaluation policy and organizational evaluation capacity building: A study of international aid agency evaluation policies","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":11,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Agency (philosophy); Pace; Evaluation methods; Program evaluation; Policy analysis; Capacity building; Public relations; Management science; Knowledge management; Process management; Business; Political science; Computer science; Public administration; Economics; Sociology; Economic growth","retraction":null,"screen_n_in":null,"score":{"opus":0.3393749749893569,"gpt":0.529793832087028,"spread":0.1904188570976711,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow","sts","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.0702406,0.0003319473,0.0004262672,0.002328449,0.001524308,0.000310774,0.0006565615,0.0001099304,0.01129295],"category_scores_gemma":[0.0273062,0.0003355342,0.0001647515,0.003545425,0.0000897113,0.001202335,0.0002698326,0.0002525375,0.00002023639],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003124144,"about_ca_system_score_gemma":0.005966497,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001098213,"about_ca_topic_score_gemma":0.001091184,"domain_scores_codex":[0.9721263,0.004871704,0.001861967,0.001122017,0.01964198,0.0003760186],"domain_scores_gemma":[0.9817331,0.00113395,0.001406007,0.000785499,0.01477206,0.0001693687],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002194581,0.001678603,0.02878462,0.00001036467,0.0003688436,7.767655e-8,0.0177446,0.2687704,0.003533458,0.005806412,0.004765143,0.668318],"study_design_scores_gemma":[0.00819758,0.0007128904,0.1400173,0.00001078236,0.001098284,0.00001291398,0.005615712,0.7586678,0.0004194326,0.07785241,0.007091615,0.0003032608],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9749468,0.0003991393,0.003908445,0.002609985,0.002569824,0.008554827,0.0001283582,0.00007449843,0.006808142],"genre_scores_gemma":[0.9930107,0.00002960577,0.001449687,0.0001524099,0.000669078,0.003607129,0.0004806008,0.00003867874,0.0005620984],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6680148,"threshold_uncertainty_score":0.9999096,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2008070236","doi":"10.1002/ev.299","title":"Reflections on the dilemmas of conducting environmental evaluations","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":10,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Impact","funders":"","keywords":"Cognitive reframing; Multidisciplinary approach; Asset (computer security); Context (archaeology); Set (abstract data type); Face (sociological concept); Computer science; Management science; Theory of change; Work (physics); Sociology; Engineering ethics; Psychology; Social science; Social psychology; Economics; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.5676539202947379,"gpt":0.5954441216399692,"spread":0.02779020134523125,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.009315129,0.0001380301,0.0001783721,0.0003602433,0.0008502192,0.0001117338,0.0002908323,0.00006704957,0.004187814],"category_scores_gemma":[0.00451452,0.00009410539,0.0001895821,0.0008748849,0.00006232294,0.0003853635,0.00001603735,0.0001378194,0.0001898197],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002366712,"about_ca_system_score_gemma":0.0004274402,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001898199,"about_ca_topic_score_gemma":0.00007948813,"domain_scores_codex":[0.9958905,0.0005305228,0.000741035,0.0003684286,0.002286591,0.0001828954],"domain_scores_gemma":[0.9964492,0.001923575,0.0004442231,0.0006017363,0.000509823,0.00007146307],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00009788146,0.0004723879,0.0005505753,0.00000177892,0.00006305515,5.579351e-8,0.003091196,0.02428401,0.03114713,0.05045737,0.03608853,0.8537461],"study_design_scores_gemma":[0.004504933,0.002190218,0.1032492,0.00006797702,0.0005718948,0.00001797289,0.009199553,0.2383419,0.04886747,0.388825,0.2035493,0.0006146566],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7503828,0.0002998798,0.03773119,0.04359316,0.003101198,0.005631652,0.0001116804,0.0001662996,0.1589821],"genre_scores_gemma":[0.9935433,0.00001929259,0.001553653,0.000536803,0.000249532,0.0001967013,0.00004855883,0.000008932285,0.003843227],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8531314,"threshold_uncertainty_score":0.9967225,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2891656379","doi":"10.1002/ev.20346","title":"The Evolving Market for Systematic Evaluation in Canada","year":2018,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":10,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"University of Ottawa","funders":"","keywords":"Accreditation; Professionalization; Context (archaeology); Supply and demand; Government (linguistics); Supply side; Business; Exploratory research; Affect (linguistics); Public relations; Marketing; Economics; Political science; Economic growth; Sociology; Social science","retraction":null,"screen_n_in":null,"score":{"opus":0.1834456181021273,"gpt":0.4959395639226441,"spread":0.3124939458205168,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03830241,0.0001357571,0.0002330049,0.0002540105,0.0009370116,0.0003191387,0.0003637202,0.00005014844,0.001459473],"category_scores_gemma":[0.02540108,0.00009128386,0.00009584875,0.0009488863,0.00003445672,0.0004608813,0.00002361854,0.00006540828,0.00004059229],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002197262,"about_ca_system_score_gemma":0.0076341,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.07362279,"about_ca_topic_score_gemma":0.9436849,"domain_scores_codex":[0.9941405,0.0009443959,0.001044904,0.0004001142,0.003174943,0.0002951584],"domain_scores_gemma":[0.9890132,0.005511147,0.0004750648,0.0005685717,0.004353782,0.00007822949],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002174891,0.00006305654,0.007075967,0.0002650531,0.00008390297,4.72551e-8,0.001481421,0.005686263,0.000134621,0.001710881,0.4264057,0.5568756],"study_design_scores_gemma":[0.001457052,0.00008495279,0.02742544,0.0002014257,0.000107867,0.000001343121,0.001751982,0.9301785,0.0001425885,0.02058451,0.01794586,0.0001184341],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4183674,0.00615795,0.2231886,0.03805948,0.04139125,0.1027075,0.000216339,0.0002981038,0.1696134],"genre_scores_gemma":[0.9877036,0.00001204603,0.001280432,0.0001833876,0.0004426396,0.003982652,0.00002753326,0.00001573564,0.006351939],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9244923,"threshold_uncertainty_score":0.9994533,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1557717928","doi":"10.1002/ev.20082","title":"The Value in Validity","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"123 Certification (Canada)","funders":"","keywords":"Beauty; Prioritization; Economic Justice; Value (mathematics); Sociology; Balance (ability); Management science; Psychology; Social psychology; Epistemology; Computer science; Political science; Law; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.3622601772662026,"gpt":0.5450118219863205,"spread":0.1827516447201178,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02828205,0.00008381571,0.0001184255,0.0002003183,0.0005597453,0.0002651981,0.0002929553,0.0000515813,0.0004453527],"category_scores_gemma":[0.01400155,0.00005413893,0.00009171481,0.0007903934,0.00003311063,0.0003258729,0.00002427962,0.00008651624,0.0002515431],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001737388,"about_ca_system_score_gemma":0.0003940238,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009400039,"about_ca_topic_score_gemma":0.001564777,"domain_scores_codex":[0.9963982,0.0008966677,0.0005428375,0.0002901654,0.001685257,0.0001868754],"domain_scores_gemma":[0.994442,0.004180217,0.0001902834,0.0004646098,0.0006610256,0.00006189734],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003208042,0.0000354521,0.006722787,0.000001101709,0.000005441278,9.353223e-9,0.0004275869,0.01422823,0.00009164034,0.01937688,0.03017574,0.928903],"study_design_scores_gemma":[0.0007418859,0.00006068935,0.05916897,0.000003837933,0.00001399135,5.561759e-7,0.0001307301,0.4134066,0.0002153973,0.1348695,0.3913291,0.00005873568],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5854608,0.0003399578,0.2047027,0.03869582,0.009478688,0.004196056,0.00001243521,0.0001903395,0.1569231],"genre_scores_gemma":[0.9894294,0.00003332132,0.001398457,0.0002727378,0.0003654762,0.0003094481,0.00001088535,0.000008599464,0.008171696],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9288443,"threshold_uncertainty_score":0.9943039,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4214553715","doi":"10.1002/ev.20486","title":"How does teaching with cases support the development of evaluation competencies?","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Adaptability; Craft; Evaluation methods; Professional development; Medical education; Psychology; Computer science; Knowledge management; Pedagogy; Engineering; Medicine; Management","retraction":null,"screen_n_in":null,"score":{"opus":0.2469212988547501,"gpt":0.4932914012784128,"spread":0.2463701024236627,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01723243,0.0001460801,0.0002178717,0.0002160946,0.0008875165,0.0003504437,0.0002065299,0.00005288366,0.001587626],"category_scores_gemma":[0.006729027,0.00007824304,0.0001025402,0.0007163964,0.00005734323,0.0006435508,0.00004308176,0.000120076,0.00002027352],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002612741,"about_ca_system_score_gemma":0.004420108,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0000261368,"about_ca_topic_score_gemma":0.004298388,"domain_scores_codex":[0.9940189,0.0007884622,0.0006669287,0.0004177259,0.003918371,0.0001896321],"domain_scores_gemma":[0.9931706,0.001588731,0.0005124198,0.0005450607,0.004113745,0.00006948125],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005756029,0.0001432844,0.004928084,0.00001157134,0.00009228633,3.213059e-7,0.00588848,0.003547771,0.002413282,0.00158314,0.005159792,0.9761744],"study_design_scores_gemma":[0.005898815,0.0005534716,0.1361867,0.000122028,0.000957343,0.00009666706,0.04295234,0.2288122,0.0636058,0.01931284,0.500869,0.0006327886],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8607902,0.0002918822,0.1043453,0.01326082,0.001844451,0.002941698,0.00003107076,0.00007475683,0.01641985],"genre_scores_gemma":[0.9648371,0.0000063739,0.02707852,0.0001146897,0.000154394,0.0004217245,0.0001461932,0.00001224891,0.0072288],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9755417,"threshold_uncertainty_score":0.999325,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2278381810","doi":"10.1002/ev.20158","title":"Assessing the Practice Impact of Research on Evaluation","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Evaluation methods; Program evaluation; Psychology; Applied psychology; Computer science; Political science; Public administration; Reliability engineering; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.8085447410014557,"gpt":0.7537715169629834,"spread":0.05477322403847229,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1363261,0.0001246237,0.0001843619,0.0007253175,0.0005902221,0.0005644747,0.0004022713,0.00008864773,0.0009687738],"category_scores_gemma":[0.0782761,0.00007504276,0.0001816336,0.002562855,0.00009757536,0.001777695,0.00004383408,0.0002418452,0.0002397557],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001147903,"about_ca_system_score_gemma":0.005921238,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004554048,"about_ca_topic_score_gemma":0.0001362,"domain_scores_codex":[0.9844223,0.004374872,0.0007865654,0.0004409667,0.009705986,0.0002693584],"domain_scores_gemma":[0.9732386,0.008652723,0.0005587357,0.0008078654,0.0165972,0.000144912],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002084419,0.0002652725,0.002055082,0.000002455305,0.00006822788,5.364864e-8,0.002552198,0.1836199,0.0002455624,0.002425811,0.1086735,0.6998835],"study_design_scores_gemma":[0.002833281,0.001043572,0.06293412,0.00003792742,0.0002043318,0.000006957615,0.01094069,0.8076981,0.0006344976,0.07064318,0.04288464,0.0001387279],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7026923,0.0007861179,0.02127291,0.0186942,0.003574813,0.00707513,0.00002096266,0.00009542397,0.2457881],"genre_scores_gemma":[0.9958755,0.00001262826,0.00168673,0.00009992368,0.0004209589,0.0003986408,0.00003755301,0.00001354178,0.001454461],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6997447,"threshold_uncertainty_score":0.9999444,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2067449005","doi":"10.1002/ev.326","title":"Real‐time evaluation in humanitarian emergencies","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Formative assessment; Credibility; Evaluation methods; Program evaluation; Field (mathematics); Emergency management; Process management; Computer science; Monitoring and evaluation; Risk analysis (engineering); Political science; Business; Psychology; Engineering; Public administration","retraction":null,"screen_n_in":null,"score":{"opus":0.2369325683844146,"gpt":0.526100458986193,"spread":0.2891678906017783,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.02473062,0.000166799,0.0002240592,0.0008356414,0.0004360841,0.000225463,0.0003461498,0.0001440369,0.02111136],"category_scores_gemma":[0.007601687,0.0001493982,0.0001354214,0.001349853,0.00004347226,0.000930701,0.00003354747,0.0002025159,0.0009041436],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002848825,"about_ca_system_score_gemma":0.001409882,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005338709,"about_ca_topic_score_gemma":0.01547348,"domain_scores_codex":[0.9945185,0.0004904731,0.0009112529,0.0005576141,0.003247,0.00027519],"domain_scores_gemma":[0.9958813,0.0007165552,0.0003233731,0.0006398227,0.002326706,0.000112252],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00006143743,0.0002419479,0.00970449,0.000003804331,0.00002548199,1.009647e-7,0.002576572,0.006711751,0.03248901,0.006500219,0.03778033,0.9039049],"study_design_scores_gemma":[0.002290877,0.0001454365,0.2089537,0.00001012086,0.0001158524,0.00000263069,0.0006367078,0.6159611,0.001640717,0.09887509,0.071104,0.0002637062],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9112027,0.00005001839,0.001530063,0.002535789,0.003859328,0.002819557,0.00001626215,0.0001038417,0.07788242],"genre_scores_gemma":[0.9831802,0.0000303897,0.004547333,0.00008297061,0.0006410053,0.0009268737,0.0001874923,0.0000236074,0.01038009],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9036412,"threshold_uncertainty_score":0.9998738,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4293180673","doi":"10.1002/ev.20490","title":"The importance of implementation: Putting evaluation policy to work","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McGill University","funders":"","keywords":"Work (physics); Government (linguistics); White paper; Public administration; Public relations; Early adopter; Program evaluation; Evaluation methods; Policy analysis; Business; Political science; Marketing","retraction":null,"screen_n_in":null,"score":{"opus":0.2980319978182459,"gpt":0.5961758186242162,"spread":0.2981438208059703,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.03211819,0.0001170788,0.0001602029,0.000489687,0.00185349,0.0001744437,0.0004579157,0.00002514043,0.004750548],"category_scores_gemma":[0.005320163,0.00009303616,0.0001403991,0.003396682,0.00002638577,0.0003234785,0.0001232704,0.0001000461,0.00004121472],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008516531,"about_ca_system_score_gemma":0.002430512,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001021378,"about_ca_topic_score_gemma":0.0008873572,"domain_scores_codex":[0.9924928,0.0009251672,0.001063622,0.0004202659,0.0048392,0.0002589448],"domain_scores_gemma":[0.9951912,0.001343301,0.0006796181,0.0005956315,0.002096661,0.00009358202],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005482755,0.00003213319,0.01388251,0.000001167026,0.00002573282,1.246013e-8,0.001369331,0.03815974,0.0001446965,0.00326675,0.05568473,0.8873783],"study_design_scores_gemma":[0.003172344,0.000496369,0.185131,0.000009426855,0.0002312033,0.000004388635,0.01039038,0.1072074,0.00114497,0.1185198,0.5734286,0.0002641207],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7828459,0.001607929,0.03320837,0.1342232,0.00710684,0.01612485,0.0001682483,0.0001750146,0.02453962],"genre_scores_gemma":[0.9893297,0.00001105247,0.003197593,0.0007444617,0.0004466729,0.003292273,0.0001131433,0.00001727349,0.002847841],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8871142,"threshold_uncertainty_score":0.999446,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1534140708","doi":"10.1002/ev.20088","title":"House With a View: Validity and Evaluative Argument","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"123 Certification (Canada)","funders":"","keywords":"CLARITY; Argument (complex analysis); Style (visual arts); Epistemology; Narrative; Evaluation methods; Psychology; Computer science; Sociology; Linguistics; Philosophy; History","retraction":null,"screen_n_in":null,"score":{"opus":0.240140513715607,"gpt":0.5046303264375429,"spread":0.2644898127219358,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01470984,0.0001630329,0.0002359726,0.0002837742,0.0004739558,0.0002613241,0.0001637112,0.00006029147,0.000967174],"category_scores_gemma":[0.003387637,0.0001144784,0.00006992483,0.0006268728,0.00006107021,0.0005457156,0.00003137769,0.00008689269,0.0001660828],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000166844,"about_ca_system_score_gemma":0.0004715519,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005176118,"about_ca_topic_score_gemma":0.0005579006,"domain_scores_codex":[0.9956893,0.0007325164,0.0004859223,0.0005260939,0.002364073,0.0002020958],"domain_scores_gemma":[0.9962776,0.001326908,0.0003010812,0.0004531097,0.001491778,0.0001494988],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001423616,0.0001200493,0.008983283,0.000007454547,0.00003833132,4.086496e-8,0.001670273,0.003960565,0.0002208531,0.002459656,0.01871305,0.9636841],"study_design_scores_gemma":[0.004800286,0.001216396,0.1073725,0.00003609455,0.0003053381,0.000008932052,0.0006307045,0.4839411,0.001335116,0.06402384,0.3359998,0.0003299913],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6927229,0.0002786646,0.2771509,0.006789342,0.001234768,0.00379451,0.0000172652,0.0002592919,0.01775236],"genre_scores_gemma":[0.9857093,0.00005422368,0.009424304,0.0004004726,0.0002791543,0.0005754763,0.00002101126,0.00002483789,0.003511203],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9633541,"threshold_uncertainty_score":0.9999461,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2038633574","doi":"10.1002/ev.1171","title":"Translating evaluation findings into “policy language”","year":2000,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"Université du Québec à Montréal","funders":"","keywords":"Affect (linguistics); Welfare; Order (exchange); Policy learning; Policy analysis; Program evaluation; Language policy; Computer science; Management science; Political science; Public administration; Sociology; Pedagogy; Economics; Machine learning; Law; Finance","retraction":null,"screen_n_in":null,"score":{"opus":0.1976021849984889,"gpt":0.5590826968278899,"spread":0.3614805118294011,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.01548644,0.0001839433,0.0002229187,0.0006725255,0.0007088773,0.0003290183,0.0003058249,0.0001166315,0.02525264],"category_scores_gemma":[0.004425323,0.0001622508,0.0001905848,0.001804895,0.00003870415,0.0009614254,0.00001251555,0.0001251103,0.0009265425],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006330734,"about_ca_system_score_gemma":0.001744748,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005207291,"about_ca_topic_score_gemma":0.001314081,"domain_scores_codex":[0.9942243,0.0005624556,0.0008258471,0.0005654122,0.003533534,0.0002884412],"domain_scores_gemma":[0.997288,0.0006400135,0.0001839078,0.0005170975,0.001229255,0.0001417045],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003721376,0.00004250967,0.0005101329,0.000002962589,0.00001671524,3.288837e-8,0.006538921,0.01210415,0.0009759983,0.0004426057,0.00496491,0.9743639],"study_design_scores_gemma":[0.003685717,0.0001731392,0.01708404,0.00002835201,0.0002131928,0.000005516265,0.00175896,0.783249,0.001858256,0.05703496,0.1345949,0.0003139448],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8476119,0.0007453475,0.01511153,0.01480731,0.001152936,0.003829965,0.00002342629,0.0002435239,0.1164741],"genre_scores_gemma":[0.9772639,0.00003327219,0.005804547,0.000393117,0.0008135595,0.0006148553,0.0001628282,0.00002466972,0.01488924],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9740499,"threshold_uncertainty_score":0.9998513,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3196514161","doi":"10.1002/ev.20464","title":"Supporting systems transformation through design‐driven evaluation","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Complex Systems and Decision Making","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Canadian Economics Association","funders":"","keywords":"Cognitive reframing; Computer science; Process (computing); Process management; Service (business); New product development; Knowledge management; Product (mathematics); Value (mathematics); Management science; Engineering management; Systems engineering; Engineering; Business","retraction":null,"screen_n_in":null,"score":{"opus":0.4219147858864458,"gpt":0.5201704823327796,"spread":0.09825569644633386,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01734815,0.0002037253,0.0003903597,0.0003646695,0.0007646474,0.0009120423,0.0002827346,0.0001418251,0.001447475],"category_scores_gemma":[0.01319594,0.0001815955,0.0002818685,0.001712518,0.00002209655,0.001387336,0.00003649429,0.0001119708,0.0002451797],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004387379,"about_ca_system_score_gemma":0.001239359,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001409008,"about_ca_topic_score_gemma":0.0002379121,"domain_scores_codex":[0.9913511,0.001594084,0.001690936,0.0007548037,0.004254531,0.0003545892],"domain_scores_gemma":[0.9906854,0.002388693,0.0007348344,0.0007190817,0.00535545,0.0001165143],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00007972401,0.000109094,0.0002383825,0.00002007083,0.00007523369,0.00000107534,0.004907487,0.3818917,0.003830248,0.01635457,0.08946788,0.5030245],"study_design_scores_gemma":[0.001627975,0.00005986518,0.0008915386,0.00005080715,0.0001657649,0.0000413396,0.002642056,0.8009515,0.001091371,0.1012829,0.09100044,0.000194365],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01023346,0.002218681,0.9662533,0.0006207823,0.003739641,0.002948854,0.00002226989,0.0001240497,0.01383894],"genre_scores_gemma":[0.9768347,0.00001279722,0.01685558,0.00007917305,0.0006668266,0.0006781117,0.0001014694,0.00003009282,0.00474123],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9666013,"threshold_uncertainty_score":0.9994653,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2055999462","doi":"10.1002/ev.392","title":"Internal evaluation a quarter‐century later: A conversation with Arnold J. Love","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Conversation; Evaluation methods; Perspective (graphical); Association (psychology); Quarter (Canadian coin); Sociology; Psychology; Computer science; Artificial intelligence; History; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.2229234698309984,"gpt":0.4641575431803136,"spread":0.2412340733493152,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.009713752,0.0002194726,0.0002404111,0.0006054447,0.0002157167,0.0002361033,0.0003020742,0.0001150959,0.01145379],"category_scores_gemma":[0.001223773,0.0001690873,0.0001462317,0.0008711343,0.00005668517,0.001267031,0.00002631969,0.0001389492,0.0009926495],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004672486,"about_ca_system_score_gemma":0.001016102,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002381308,"about_ca_topic_score_gemma":0.0007713031,"domain_scores_codex":[0.9944335,0.0005540317,0.0007827567,0.000625641,0.003327966,0.0002761534],"domain_scores_gemma":[0.9953578,0.0003737025,0.0005801295,0.0005659725,0.002970226,0.0001521699],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001285794,0.0004887917,0.02512361,0.00001552443,0.0002417346,5.324846e-7,0.03601621,0.003045821,0.0008227548,0.001805335,0.03802973,0.8931242],"study_design_scores_gemma":[0.009654631,0.001365003,0.1733542,0.00008060967,0.0007345598,0.00002568238,0.007355947,0.7264556,0.003707619,0.03146711,0.04524055,0.0005585041],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6133327,0.0005666818,0.2070441,0.005391363,0.006261089,0.009201546,0.00006719652,0.0003026188,0.1578327],"genre_scores_gemma":[0.9925314,0.00002764217,0.004075015,0.0002134255,0.0002760134,0.0003221997,0.0001203602,0.000022049,0.002411908],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8925657,"threshold_uncertainty_score":0.9997852,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1994122017","doi":"10.1002/ev.333","title":"Evaluation of disaster and emergency management: Do no harm, but do better","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Disaster Response and Management","field":"Health Professions","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Emergency management; Harm; Medical emergency; Business; Medicine; Psychology; Political science; Social psychology","retraction":null,"screen_n_in":null,"score":{"opus":0.08876767161420009,"gpt":0.4590029695750237,"spread":0.3702352979608236,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00542736,0.0001550341,0.0001700373,0.0002866621,0.0004495994,0.00002112575,0.0001100333,0.0001184709,0.006138894],"category_scores_gemma":[0.0004785154,0.0001467376,0.00008186319,0.0002390739,0.00003004353,0.0002948141,0.00008746587,0.000185042,0.0002464186],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001033331,"about_ca_system_score_gemma":0.0001420362,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007568279,"about_ca_topic_score_gemma":0.0002336088,"domain_scores_codex":[0.9973167,0.0005691018,0.0005484735,0.0003632601,0.0009332186,0.0002692234],"domain_scores_gemma":[0.9979997,0.0001173664,0.0002669398,0.0004327316,0.001090308,0.00009294393],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0002405544,0.0002213903,0.004223055,0.0003412839,0.0003180024,1.52936e-7,0.006827462,0.00004014367,0.004197447,0.006487779,0.07542427,0.9016784],"study_design_scores_gemma":[0.01390747,0.0003276941,0.1451233,0.0003390713,0.005508251,0.000001798709,0.009567867,0.02369628,0.0003704341,0.07108074,0.729369,0.000708043],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8304724,0.0002272196,0.002887579,0.0006661951,0.004514499,0.00516342,0.00003291498,0.00007008088,0.1559657],"genre_scores_gemma":[0.9587472,0.0001561197,0.002897068,0.0002158929,0.0008591238,0.00328265,0.0001249099,0.000053416,0.0336636],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9009704,"threshold_uncertainty_score":0.9947696,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2139159602","doi":"10.1002/ev.92","title":"PIE à la Mode: Mainstreaming Evaluation and Accountability in Each Program in Every County of a Statewide School Readiness Initiative","year":2003,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of British Columbia","funders":"","keywords":"Accountability; Mandate; Mainstreaming; Legislature; Program evaluation; Plan (archaeology); Mainstream; Benchmarking; Quality (philosophy); Public administration; Political science; Public relations; Medical education; Psychology; Business; Pedagogy; Special education; Medicine","retraction":null,"screen_n_in":null,"score":{"opus":0.2119641125322008,"gpt":0.5393907107995318,"spread":0.327426598267331,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02727976,0.0001656699,0.0003093257,0.0007689873,0.0001309035,0.0001445582,0.0001321472,0.0001261081,0.0004491635],"category_scores_gemma":[0.01350881,0.0001506323,0.00006181662,0.001714985,0.00007781114,0.001254649,0.00002220656,0.0001723772,0.000006877861],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007648396,"about_ca_system_score_gemma":0.002174522,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0005654849,"about_ca_topic_score_gemma":0.006039717,"domain_scores_codex":[0.9943261,0.001667312,0.001070939,0.0005440626,0.002153022,0.0002385515],"domain_scores_gemma":[0.9955288,0.001736531,0.0004163414,0.0003542086,0.001878742,0.00008536489],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0002101608,0.0005914194,0.2649134,0.00002782792,0.00002262746,1.572709e-7,0.003182675,0.0242478,0.0003186727,0.0008954399,0.0004752357,0.7051146],"study_design_scores_gemma":[0.004210941,0.0001491338,0.5022712,0.00007911863,0.00007120272,0.000002677038,0.003822383,0.4356048,0.0006198055,0.05085579,0.002130345,0.0001826665],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9821094,0.0001883176,0.002773273,0.000192576,0.0002747822,0.003444088,0.00003292804,0.0000276957,0.01095698],"genre_scores_gemma":[0.9945275,0.0000551865,0.003779954,0.00003887956,0.00002675565,0.001329425,0.00007134672,0.00001165238,0.0001593403],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7049319,"threshold_uncertainty_score":0.9948008,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1541185587","doi":"10.1002/ev.20112","title":"Credentialed Evaluator Designation Program, the Canadian Experience","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Value (mathematics); Professional development; Service (business); Administration (probate law); Political science; Public relations; Management; Engineering ethics; Sociology; Pedagogy; Computer science; Engineering; Law; Business; Marketing; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.4863368872817085,"gpt":0.5792195495990002,"spread":0.09288266231729175,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01647855,0.0001439102,0.0001486316,0.0004373063,0.001097156,0.0008145981,0.0004925731,0.00009679403,0.001629223],"category_scores_gemma":[0.008709146,0.00009891103,0.0001160744,0.001386405,0.00007822403,0.0007963037,0.00002554133,0.0001044037,0.0003694113],"about_ca_system_candidate":true,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009569103,"about_ca_system_score_gemma":0.006002241,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.01427033,"about_ca_topic_score_gemma":0.2733112,"domain_scores_codex":[0.994406,0.0006213262,0.0005987458,0.0004311518,0.003638185,0.0003046287],"domain_scores_gemma":[0.9944095,0.0004991141,0.0002689944,0.0005721606,0.003864764,0.0003854627],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00004647927,0.00007537816,0.002614932,0.000001168706,0.00002327973,1.081123e-7,0.005058243,0.006543837,0.000103402,0.002467509,0.1036166,0.8794491],"study_design_scores_gemma":[0.001690913,0.0002075841,0.01678341,0.000006808045,0.00009383341,0.000005388142,0.00221719,0.4237547,0.0006125888,0.02705079,0.5273989,0.0001778751],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5349118,0.001890456,0.1297499,0.06076821,0.02291181,0.0323635,0.0000839934,0.0009676042,0.2163527],"genre_scores_gemma":[0.9850383,0.000005933656,0.004685005,0.0003735166,0.0004518623,0.00226479,0.00008753935,0.00001534644,0.007077704],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8792712,"threshold_uncertainty_score":0.9996409,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1505207325","doi":"10.1002/ev.20084","title":"Credible Judgment: Combining Truth, Beauty, and Justice","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"Université du Québec à Montréal","funders":"","keywords":"Beauty; Argumentation theory; Economic Justice; Process (computing); Variety (cybernetics); Psychology; Social psychology; Public relations; Computer science; Law; Epistemology; Political science","retraction":null,"screen_n_in":null,"score":{"opus":0.260603245733291,"gpt":0.5132600924299058,"spread":0.2526568466966148,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01094448,0.0001180313,0.0001786342,0.0003215816,0.0006095197,0.0003062333,0.0001708434,0.0000681336,0.001058154],"category_scores_gemma":[0.00657607,0.0001013275,0.00006794571,0.000564161,0.0000374132,0.0005508513,0.0000387243,0.00008280047,0.0001587956],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001323175,"about_ca_system_score_gemma":0.0002699592,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004399345,"about_ca_topic_score_gemma":0.0001139706,"domain_scores_codex":[0.9970622,0.0002961498,0.0004940866,0.0004051151,0.001534101,0.0002083661],"domain_scores_gemma":[0.9965985,0.001933775,0.0002322284,0.0003443847,0.0007485913,0.0001424851],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00004769461,0.00007613646,0.002128429,0.0000138024,0.00002460813,2.421048e-8,0.001036736,0.005211268,0.0004672191,0.04055332,0.02721679,0.923224],"study_design_scores_gemma":[0.002476879,0.0002445968,0.01857428,0.00001865377,0.0002529254,0.000004026927,0.0006525713,0.6494474,0.0006561152,0.08997813,0.2375283,0.0001661949],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4710881,0.0007191148,0.3404806,0.01009882,0.01175056,0.002778663,0.00002755868,0.000332174,0.1627245],"genre_scores_gemma":[0.988413,0.00004368636,0.00638005,0.0003714717,0.0005321868,0.0001790639,0.00003972977,0.00001387057,0.00402695],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9230578,"threshold_uncertainty_score":0.999855,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2512463988","doi":"10.1002/ev.20199","title":"Building Evaluation Capacity Through CLIPs: Communities of Learning, Inquiry, and Practice","year":2016,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"University of British Columbia","funders":"","keywords":"CLIPS; Context (archaeology); Work (physics); Community of practice; Medical education; Professional development; Higher education; Pedagogy; Knowledge management; Computer science; Sociology; Public relations; Psychology; Political science; Medicine; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.5953867694134086,"gpt":0.5760182223031797,"spread":0.01936854711022895,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02790357,0.0001392817,0.000231896,0.000301101,0.0005167918,0.0001304024,0.0002078889,0.0000971628,0.001088128],"category_scores_gemma":[0.0269649,0.0000980987,0.00008998557,0.0005492824,0.0001645857,0.002025291,0.00005698586,0.0001096805,0.00003115267],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002667743,"about_ca_system_score_gemma":0.0005723466,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003949165,"about_ca_topic_score_gemma":0.0006101837,"domain_scores_codex":[0.9939329,0.00223033,0.0007634783,0.000311107,0.002580754,0.000181426],"domain_scores_gemma":[0.988035,0.006809541,0.0007450168,0.0004336888,0.003912688,0.00006411213],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0001788885,0.0001123621,0.008914937,0.00001189592,0.00006578926,1.67323e-8,0.0106744,0.003148324,0.004252719,0.01105528,0.005279138,0.9563062],"study_design_scores_gemma":[0.009569176,0.001118329,0.06909404,0.000244745,0.0009758353,0.00003200299,0.02792457,0.194545,0.006115075,0.2463345,0.4434799,0.0005668887],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7530027,0.0004714587,0.2232267,0.006355595,0.001464907,0.001885313,0.00001505368,0.00007808533,0.01350021],"genre_scores_gemma":[0.9759446,0.0002613874,0.0222553,0.0001108557,0.0002174322,0.0002595844,0.00001503242,0.00001347229,0.0009223027],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9557394,"threshold_uncertainty_score":0.999825,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3204738611","doi":"10.1002/ev.20471","title":"Contribution analysis: A promising method for assessing advocacy's impact","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Impact","funders":"","keywords":"Context (archaeology); Policy analysis; Outcome (game theory); Political science; Public relations; Management science; Public administration; Economics","retraction":null,"screen_n_in":null,"score":{"opus":0.2401609389579082,"gpt":0.6095421407491777,"spread":0.3693812017912694,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0174533,0.0001727705,0.0004162315,0.0006952653,0.0007474457,0.0009706353,0.000171802,0.0001159728,0.001517164],"category_scores_gemma":[0.01572433,0.0001439643,0.0006780688,0.003289677,0.00001666257,0.001112353,0.00002693436,0.00009355635,0.00002741341],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005745921,"about_ca_system_score_gemma":0.003078288,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007071478,"about_ca_topic_score_gemma":0.0004935118,"domain_scores_codex":[0.9956559,0.0008056699,0.0008594065,0.0006459705,0.00172926,0.0003037764],"domain_scores_gemma":[0.9898345,0.002592309,0.0005245784,0.0005156584,0.00639034,0.0001425436],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001198517,0.0001849386,0.008311507,0.000008261439,0.0007409541,1.854337e-7,0.0007097952,0.08231808,0.009030892,0.001163617,0.01245991,0.884952],"study_design_scores_gemma":[0.002491776,0.00009078541,0.06795547,0.00001686739,0.001837892,0.000005528679,0.0006834483,0.8682188,0.006704398,0.02503485,0.02676984,0.000190327],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05936646,0.0003417032,0.9338968,0.002591342,0.000775826,0.001557196,0.0000421319,0.00007009049,0.001358479],"genre_scores_gemma":[0.8494359,0.00001644749,0.1457291,0.0002562421,0.0004845112,0.0007995724,0.0008543399,0.00002122819,0.002402591],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8847617,"threshold_uncertainty_score":0.9993956,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4242601229","doi":"10.1002/ev.20107","title":"Editors' Notes","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Accreditation; Charter; Certification; Context (archaeology); Credentialing; Conversation; Public relations; Professional association; Scholarship; Task (project management); Computer science; Political science; Engineering ethics; Management; Psychology; Law; History","retraction":null,"screen_n_in":null,"score":{"opus":0.4199570485042934,"gpt":0.5640301890940794,"spread":0.144073140589786,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.01075822,0.00009687734,0.0001368121,0.0003028605,0.0002245322,0.0002173573,0.0002159577,0.00006840214,0.001503748],"category_scores_gemma":[0.01340727,0.00007723775,0.00009613941,0.0007739752,0.00002372901,0.0006379562,0.00002478534,0.00006319691,0.0009479864],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002180906,"about_ca_system_score_gemma":0.001232757,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00008553909,"about_ca_topic_score_gemma":0.0002431278,"domain_scores_codex":[0.9962952,0.000224965,0.0004551762,0.0003171236,0.002547299,0.0001602249],"domain_scores_gemma":[0.9958901,0.0009675914,0.0001898042,0.0003831131,0.002379351,0.000190016],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00003273582,0.0000497017,0.002943861,8.349187e-7,0.00001100587,2.660406e-8,0.0007136293,0.005496763,0.0001060223,0.001043885,0.5306134,0.4589882],"study_design_scores_gemma":[0.001610138,0.0001451287,0.00703516,0.000004504304,0.00004858347,0.000001906027,0.0004888114,0.1392865,0.0009171578,0.07170197,0.7786421,0.0001180062],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1768294,0.001038187,0.4710911,0.05104867,0.08142015,0.005859904,0.00006818633,0.000688639,0.2119557],"genre_scores_gemma":[0.9568493,0.00001170937,0.01292377,0.0004691032,0.009742195,0.0004684441,0.0001036275,0.00002372671,0.01940806],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7800199,"threshold_uncertainty_score":0.9998299,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4403250772","doi":"10.1002/ev.20615","title":"Using contribution analysis to assess evaluation capacity outcomes in community organizations","year":2024,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Université de Montréal; University of Ottawa","funders":"","keywords":"Evaluation methods; Capacity building; Program evaluation; Process management; Business; Political science; Public administration; Economic growth; Economics; Engineering","retraction":null,"screen_n_in":null,"score":{"opus":0.538010031454944,"gpt":0.5850516251312451,"spread":0.04704159367630112,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.03135104,0.0002006722,0.0004017927,0.002616893,0.0007694029,0.0007013812,0.0003223823,0.000133221,0.002142202],"category_scores_gemma":[0.01838579,0.0001777741,0.0002203951,0.01170647,0.00003059056,0.001116545,0.00005686581,0.0002654117,0.0001573973],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001742847,"about_ca_system_score_gemma":0.00158961,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001332677,"about_ca_topic_score_gemma":0.01498995,"domain_scores_codex":[0.9929721,0.002189526,0.001041582,0.0005216671,0.002995895,0.000279171],"domain_scores_gemma":[0.9918265,0.002260649,0.0002093682,0.0006749226,0.004878069,0.0001504596],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003914364,0.0004004596,0.1195667,0.00001375354,0.0006275058,1.648663e-7,0.005168595,0.735638,0.003727194,0.005949491,0.002740212,0.1261288],"study_design_scores_gemma":[0.0006976324,0.0000531409,0.2391568,0.00001952055,0.001082934,0.000001079018,0.0006029268,0.7393069,0.0005010961,0.01564433,0.00277178,0.0001619168],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6357742,0.00007385741,0.3583024,0.001597394,0.001198804,0.00195158,0.00007559378,0.0001011463,0.0009250299],"genre_scores_gemma":[0.9932098,0.000006105072,0.005123807,0.0001587548,0.0001183004,0.0004079335,0.0003806132,0.00001832878,0.0005763966],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3574356,"threshold_uncertainty_score":0.99877,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2017680079","doi":"10.1002/ev.344","title":"Strategy evaluation: Experience at the International Development Research Centre","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"International Development Research Centre","funders":"","keywords":"Corporation; Program evaluation; Strategic planning; Modalities; International development; Sociology; Management; Business; Public relations; Political science; Economics; Public administration; Social science","retraction":null,"screen_n_in":null,"score":{"opus":0.5427282247833871,"gpt":0.6215806160734704,"spread":0.07885239129008326,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","insufficient_payload"],"consensus_categories":["metaresearch","insufficient_payload"],"category_scores_codex":[0.03725204,0.0001469221,0.000130602,0.000405758,0.001744187,0.0006219206,0.0008644863,0.0001062432,0.03820798],"category_scores_gemma":[0.008411651,0.0001001275,0.00009906555,0.001099615,0.0001393518,0.0007680265,0.0001624214,0.0003134174,0.001214487],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006695354,"about_ca_system_score_gemma":0.002814644,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007421249,"about_ca_topic_score_gemma":0.007487623,"domain_scores_codex":[0.9894697,0.0006429789,0.000760878,0.0006443333,0.008142579,0.0003395351],"domain_scores_gemma":[0.99081,0.001681205,0.0002563196,0.0007585415,0.006350917,0.0001430011],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005336565,0.00008940639,0.004995087,0.000001164378,0.00002703769,6.855896e-8,0.004043818,0.002855758,0.003933436,0.002279039,0.08839645,0.8933254],"study_design_scores_gemma":[0.001271554,0.00003829072,0.05281446,0.000007286716,0.00002872871,0.000006111708,0.003419299,0.1531149,0.008417612,0.01147851,0.7692514,0.0001518304],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9103907,0.0001409466,0.0014902,0.01693047,0.006020308,0.002757522,0.00001296052,0.00006168388,0.0621952],"genre_scores_gemma":[0.9515751,0.00001151016,0.001890551,0.0002176332,0.0007345419,0.001125504,0.0001405671,0.00001488827,0.04428971],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8931735,"threshold_uncertainty_score":0.9999409,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W3011218257","doi":"10.1002/ev.20394","title":"Advancing the Measurement of Collective Community Capacity and the Evaluation of Community Capacity‐Building Models","year":2020,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Community Health and Development","field":"Health Professions","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Capacity building; Work (physics); Collective efficacy; Community resilience; Adaptation (eye); Public relations; Psychological resilience; Political science; Sociology; Psychology; Computer science; Engineering; Social psychology; Social science","retraction":null,"screen_n_in":null,"score":{"opus":0.4752172519940911,"gpt":0.4749833008706206,"spread":0.0002339511234704661,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04564258,0.0001581857,0.0003795667,0.0001061942,0.005662041,0.00001051449,0.0002953949,0.0001151722,0.0000438011],"category_scores_gemma":[0.01226295,0.0001119981,0.00009594806,0.0006313116,0.0002268626,0.0001981842,0.000173959,0.001345211,0.000001021969],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001228438,"about_ca_system_score_gemma":0.002724619,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01633341,"about_ca_topic_score_gemma":0.01772843,"domain_scores_codex":[0.9809496,0.01650068,0.0008573019,0.0001184586,0.001294898,0.0002790875],"domain_scores_gemma":[0.9892172,0.005267253,0.0006910146,0.0005896988,0.004102645,0.000132205],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002213753,0.0005155972,0.001526651,0.001460562,0.0004586845,7.567798e-9,0.8345047,0.03341395,0.003459034,0.01837504,0.009104199,0.09496778],"study_design_scores_gemma":[0.01438963,0.0003835417,0.03026976,0.0004841202,0.001112435,0.000001043093,0.08262347,0.5918018,0.001908723,0.2757021,0.001084473,0.0002389348],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9599781,0.0004865828,0.01654246,0.007754753,0.000391798,0.008376143,0.00004109299,0.00005146342,0.006377591],"genre_scores_gemma":[0.9963354,0.00004742828,0.001589321,0.0007917849,0.00006926483,0.001123621,0.00001496258,0.00001575344,0.00001239935],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7518812,"threshold_uncertainty_score":0.9960572,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4378223570","doi":"10.1002/ev.20540","title":"What we can learn from the international program for development evaluation training (IPDET)","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"Program evaluation; Training (meteorology); Monitoring and evaluation; Diversity (politics); Medical education; Corporate governance; Political science; Psychology; Business; Medicine; Geography; Public administration; Finance","retraction":null,"screen_n_in":null,"score":{"opus":0.5304740430548962,"gpt":0.569693978952799,"spread":0.03921993589790285,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.02027583,0.000180733,0.0001945055,0.0003814693,0.0009562689,0.001183605,0.0005673315,0.00009727187,0.001742619],"category_scores_gemma":[0.006198145,0.0001325587,0.0001773384,0.00117908,0.00003853541,0.000978679,0.00006170417,0.0001149685,0.0002619385],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005514317,"about_ca_system_score_gemma":0.002484871,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006608532,"about_ca_topic_score_gemma":0.003626251,"domain_scores_codex":[0.9940389,0.0003564642,0.0008220404,0.0006102448,0.003863023,0.0003093217],"domain_scores_gemma":[0.9939265,0.002695922,0.0003912886,0.0004291877,0.002459302,0.00009782404],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000037923,0.00003987788,0.0003375596,0.000001554949,0.00006255478,2.013879e-8,0.008020009,0.007267925,0.0001297757,0.000216869,0.05327784,0.9306081],"study_design_scores_gemma":[0.001480625,0.00006568262,0.01336402,0.00003419938,0.0000887798,5.442189e-7,0.007277131,0.4473603,0.0003893526,0.02246811,0.5073485,0.0001228005],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6015567,0.001505582,0.05158221,0.262092,0.04319316,0.02989428,0.0002483742,0.001384491,0.008543247],"genre_scores_gemma":[0.8953398,0.0006019179,0.03261326,0.001715562,0.003905639,0.02677898,0.005914544,0.0001050608,0.03302525],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9304853,"threshold_uncertainty_score":0.9998533,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2626410734","doi":"10.1002/ev.20240","title":"Conceptual Indicators Framework for Strengthening the Chinese Health System","year":2017,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Healthcare Systems and Reforms","field":"Economics, Econometrics and Finance","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"","keywords":"China; Government (linguistics); Plan (archaeology); Conceptual framework; Process (computing); Political science; Equity (law); Monitoring and evaluation; Process management; Public relations; Public administration; Economic growth; Business; Sociology; Computer science; Economics; Geography","retraction":null,"screen_n_in":null,"score":{"opus":0.08227829248233864,"gpt":0.3703221274091979,"spread":0.2880438349268593,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.002962941,0.0001149132,0.0002874711,0.0001432658,0.002121999,0.0001523445,0.0002262185,0.0001276734,0.00002766779],"category_scores_gemma":[0.001078897,0.00008184707,0.0001544978,0.0001347953,0.00005117663,0.0001900168,0.00001915976,0.0001203817,0.00003121334],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003903596,"about_ca_system_score_gemma":0.0002055159,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003054057,"about_ca_topic_score_gemma":0.0003860593,"domain_scores_codex":[0.9987464,0.00003446844,0.0006055153,0.0002865982,0.00007408699,0.0002529464],"domain_scores_gemma":[0.9981787,0.0001195272,0.0009975175,0.0005143558,0.00008494388,0.0001049583],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00001811421,0.00002568547,0.01414703,0.00009780756,0.00007761869,2.413396e-8,0.00325156,0.0001223045,5.912313e-7,0.7355856,0.001374945,0.2452988],"study_design_scores_gemma":[0.004181733,0.0005494954,0.3093689,0.0003623088,0.00003425469,0.000008634839,0.00453159,0.03009602,0.00002730999,0.1322492,0.5179996,0.0005909136],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3201105,0.009922449,0.5962879,0.02621294,0.0190218,0.01151643,0.001065965,0.0003431613,0.01551886],"genre_scores_gemma":[0.9940448,0.00004472688,0.003039462,0.000101697,0.001169386,0.0007332201,0.00004297708,0.00002579877,0.000797978],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6739342,"threshold_uncertainty_score":0.9991771,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4249389693","doi":"10.1002/ev.20282","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":true},"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","retraction":null,"screen_n_in":null,"score":{"opus":0.0711548467400742,"gpt":0.2947114174565125,"spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.001152775,0.0001940134,0.0003781772,0.0005338805,0.000355196,0.0002244991,0.0001748384,0.0002272763,0.1461259],"category_scores_gemma":[0.0003479774,0.0002423908,0.000193508,0.0002466833,0.0000540795,0.0007033957,0.00004382425,0.00009012481,0.5442282],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004358186,"about_ca_system_score_gemma":0.0001418389,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003819858,"about_ca_topic_score_gemma":0.00003390296,"domain_scores_codex":[0.9986079,0.00001220689,0.0006784026,0.0003962657,0.00006746652,0.0002378011],"domain_scores_gemma":[0.9986594,0.00004275919,0.0006441558,0.0003298316,0.0002569793,0.00006690674],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.000009166512,0.00001362391,0.000005492795,0.00002571599,0.00009306354,3.455948e-9,0.0004720695,0.000139271,1.721042e-7,0.001982213,0.9877389,0.009520303],"study_design_scores_gemma":[0.0007716732,0.00004498267,0.00004522394,0.00001922779,0.00004998751,4.177001e-7,0.0001374821,0.003712759,0.00001594533,0.003756323,0.9911927,0.0002532647],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.00003435367,0.002113493,0.004931094,0.0003169737,0.05299304,0.001031726,0.002309195,0.00004083091,0.9362293],"genre_scores_gemma":[0.0001284502,0.0005284328,0.0008068433,0.0001820641,0.002089511,0.0003178392,0.001724722,0.00002087916,0.9942012],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.3981023,"threshold_uncertainty_score":0.9884408,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4378223319","doi":"10.1002/ev.20539","title":"Meeting the challenges of educating internal evaluators","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Institute for Clinical Evaluative Sciences; Queen's University","funders":"","keywords":"Front line; Public relations; Geopolitics; Coronavirus disease 2019 (COVID-19); Pandemic; Program evaluation; Political science; Business; Medicine; Public administration","retraction":null,"screen_n_in":null,"score":{"opus":0.3888663697506948,"gpt":0.5659171151344419,"spread":0.1770507453837471,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0243713,0.0001005506,0.0001610651,0.0004154885,0.0004779745,0.00009592403,0.0003741297,0.00004715356,0.0007444051],"category_scores_gemma":[0.01048717,0.00006631506,0.0001503054,0.001265866,0.0000421683,0.0002871208,0.0000567223,0.00008789517,0.0002198569],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001020664,"about_ca_system_score_gemma":0.0005498398,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009024751,"about_ca_topic_score_gemma":0.0004390223,"domain_scores_codex":[0.9960158,0.0005099875,0.0006999962,0.0003111025,0.002276029,0.0001870935],"domain_scores_gemma":[0.9951709,0.002535522,0.0004472402,0.0004326838,0.001357785,0.00005587844],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001666749,0.00004034591,0.00194449,0.000007415414,0.00003356827,1.683456e-8,0.004373027,0.01080351,0.0004864877,0.006708819,0.02214313,0.9534425],"study_design_scores_gemma":[0.001354817,0.0001968208,0.1086238,0.00008520552,0.0001590046,0.000004377574,0.01843545,0.7021917,0.003418212,0.09723347,0.06809587,0.0002012833],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8167965,0.002155344,0.00535571,0.03241059,0.006582715,0.002995843,0.00001947544,0.0002715464,0.1334123],"genre_scores_gemma":[0.9955015,0.0001431534,0.0007614358,0.00005847383,0.0004195982,0.0002911571,0.00001314216,0.00001277766,0.002798749],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9532412,"threshold_uncertainty_score":0.9978479,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4307053258","doi":"10.1002/ev.20514","title":"Identity as a compass when navigating uncharted equitable spaces: Our queer evaluation practices","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Saskatchewan; Saskatchewan Health","funders":"","keywords":"Queer; Dialogic; Situated; Sociology; Normative; Identity (music); Epistemology; Norm (philosophy); Social psychology; Psychology; Computer science; Aesthetics; Gender studies; Pedagogy","retraction":null,"screen_n_in":null,"score":{"opus":0.3794672207651364,"gpt":0.5847148136120114,"spread":0.2052475928468749,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","scholarly_communication","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.05101389,0.000233402,0.0003240644,0.0004527921,0.002600692,0.001041504,0.0006326374,0.00008054481,0.01986374],"category_scores_gemma":[0.01541828,0.0002261905,0.0002137725,0.001870664,0.00002430932,0.003501622,0.0002374707,0.0004018565,0.0005076299],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001290115,"about_ca_system_score_gemma":0.002711433,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001649919,"about_ca_topic_score_gemma":0.002260426,"domain_scores_codex":[0.9861952,0.002465486,0.00110173,0.0008573209,0.008921117,0.0004591495],"domain_scores_gemma":[0.9923459,0.001168767,0.001994279,0.0007475846,0.003572385,0.0001711227],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003123172,0.0005642431,0.00774694,0.00002205486,0.0002155839,6.091701e-7,0.0125858,0.1733419,0.002359428,0.006684666,0.1816537,0.6145128],"study_design_scores_gemma":[0.003025203,0.0003516237,0.004598951,0.00002438069,0.0004123051,0.00001631837,0.02225355,0.6849001,0.0002623454,0.1077003,0.1761509,0.0003040762],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8058267,0.001933348,0.06489756,0.05557442,0.01258568,0.01201977,0.0001775599,0.0005451508,0.04643983],"genre_scores_gemma":[0.9796724,0.00001673025,0.004318316,0.0006521001,0.0005272313,0.002972892,0.0004022313,0.00003389493,0.0114042],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6142088,"threshold_uncertainty_score":0.9999955,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4236772578","doi":"10.1002/ev.239","title":"Editor's notes","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Ottawa","funders":"","keywords":"Citation; Computer science; Library science; Sociology","retraction":null,"screen_n_in":null,"score":{"opus":0.2516694502734112,"gpt":0.5602664481446863,"spread":0.3085969978712751,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01794069,0.00010065,0.0001306202,0.0004196514,0.0003810621,0.0001716485,0.0002106891,0.00008040529,0.002860931],"category_scores_gemma":[0.00755059,0.00008242649,0.0001207346,0.000900167,0.00002611451,0.0005144787,0.00001989154,0.00007535439,0.0005750006],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001884202,"about_ca_system_score_gemma":0.0004874713,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005940183,"about_ca_topic_score_gemma":0.000854121,"domain_scores_codex":[0.9964433,0.000113136,0.0005797027,0.000336796,0.002305403,0.0002216212],"domain_scores_gemma":[0.9955299,0.00220527,0.0002176946,0.0003746425,0.001548107,0.0001243182],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00003785329,0.00004694937,0.003639381,0.000001102956,0.0000108917,3.572027e-8,0.0003374634,0.001107617,0.000579137,0.001414526,0.1545108,0.8383142],"study_design_scores_gemma":[0.00127118,0.0001176209,0.06442342,0.000006218946,0.0000543384,0.00000203408,0.0003828021,0.04293986,0.003902538,0.04095733,0.8457961,0.0001465709],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.09408283,0.0003643102,0.7803776,0.008008094,0.03957959,0.002388824,0.00001944374,0.000261955,0.0749174],"genre_scores_gemma":[0.9634696,0.00001638496,0.01254403,0.0004402238,0.0112263,0.0001391542,0.00006168312,0.00001894787,0.01208367],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8693868,"threshold_uncertainty_score":0.9980506,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4307053074","doi":"10.1002/ev.20511","title":"Empowering the search for pleasure, health and well‐being outside heteronormative definitions: The role of evaluation in shaping structurally sensitive programming for 2SGBTQ+ men who <i>Party and Play</i> in Ontario","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"HIV, Drug Use, Sexual Risk","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"Casey House","funders":"","keywords":"Phenomenon; Agency (philosophy); Transformative learning; Pleasure; Sociology; Public health; Public relations; Health care; Qualitative research; Gender studies; Social psychology; Psychology; Medicine; Nursing; Political science; Epistemology; Social science","retraction":null,"screen_n_in":null,"score":{"opus":0.104289379332773,"gpt":0.3925731318662367,"spread":0.2882837525334636,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005406683,0.0001421672,0.0002729998,0.0002483363,0.0007284509,0.0000436931,0.00005048854,0.00004278232,0.000009913434],"category_scores_gemma":[0.0006598015,0.0001147009,0.00005826625,0.0003080044,0.00005577112,0.0002095154,0.00005025297,0.0002794654,1.656016e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001047909,"about_ca_system_score_gemma":0.001151514,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.0110082,"about_ca_topic_score_gemma":0.08380765,"domain_scores_codex":[0.9979066,0.0004808707,0.0004911429,0.0003062371,0.0005372961,0.0002778191],"domain_scores_gemma":[0.9981733,0.0009901825,0.0002133127,0.0001633061,0.0003956131,0.00006426329],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001428721,0.0001398131,0.01555961,0.0002262716,0.0001772124,2.228403e-7,0.4059172,0.01693419,0.001384932,0.001158153,0.0002833802,0.5567902],"study_design_scores_gemma":[0.01187238,0.001994712,0.1034527,0.0005066273,0.0004863581,0.00005665022,0.2951391,0.5668893,0.002764637,0.0118885,0.004625668,0.0003233524],"study_design_candidate":"qualitative","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9802921,0.0008700623,0.002691385,0.002253121,0.0001270887,0.01289903,0.00006527697,0.00002697255,0.0007749417],"genre_scores_gemma":[0.9930106,0.0000367558,0.003235117,0.0001472149,0.00004885039,0.003114425,0.0003096231,0.00002507338,0.00007229992],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5564669,"threshold_uncertainty_score":0.9955776,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1978303279","doi":"10.1002/ev.382","title":"Integrating a new evaluation unit with an old institution: See no evil; hear no evil; speak no evil","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Guelph","funders":"","keywords":"Unit (ring theory); CLARITY; Institution; Evaluation methods; Association (psychology); Psychology; Sociology; Computer science; Mathematics education; Social science; Engineering; Psychotherapist","retraction":null,"screen_n_in":null,"score":{"opus":0.3909805053050648,"gpt":0.5034779046174458,"spread":0.112497399312381,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.01736029,0.0004849081,0.0005230155,0.0008294718,0.001149549,0.0005766048,0.0006473174,0.0002470767,0.02210397],"category_scores_gemma":[0.008323997,0.0003815557,0.0002383757,0.002161334,0.0001301108,0.003536137,0.00007265158,0.0003837197,0.004990346],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0007844121,"about_ca_system_score_gemma":0.004785973,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001702086,"about_ca_topic_score_gemma":0.00627782,"domain_scores_codex":[0.9904591,0.001241245,0.001472267,0.001188561,0.005096246,0.0005425385],"domain_scores_gemma":[0.9869133,0.0006028721,0.0008953673,0.00129666,0.009825579,0.000466192],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001556903,0.000812517,0.01233173,0.0000509591,0.0002271055,0.000001092588,0.007395059,0.01233461,0.001220521,0.00660834,0.05166478,0.9057964],"study_design_scores_gemma":[0.006232469,0.002845415,0.03890539,0.000318862,0.00068989,0.00002763184,0.001579132,0.7306747,0.0007128062,0.009590889,0.207605,0.0008177863],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.3594787,0.0008401606,0.1223121,0.002521836,0.01261698,0.01383066,0.00009228828,0.0009065377,0.4874008],"genre_scores_gemma":[0.8374768,0.0001774813,0.09508414,0.001319829,0.004451561,0.001798999,0.0008273068,0.0001443221,0.05871948],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9049786,"threshold_uncertainty_score":0.9998636,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4378223534","doi":"10.1002/ev.20536","title":"Learning by linking the Canadian Evaluation Society's student case competition within a graduate evaluation course","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true},"ca_institutions":"Queen's University","funders":"","keywords":"Experiential learning; Dialogic; Pedagogy; Psychology; Nature versus nurture; Situated; Competition (biology); Professional development; Assessment for learning; Sociology; Formative assessment; Computer science","retraction":null,"screen_n_in":null,"score":{"opus":0.342939436925835,"gpt":0.5475533572533501,"spread":0.2046139203275151,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","sts","scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.08201619,0.0002943852,0.0002813505,0.00055217,0.0041688,0.001107166,0.0004136903,0.0001952985,0.001570499],"category_scores_gemma":[0.004627426,0.0002343985,0.000293823,0.002989394,0.000101132,0.0008964866,0.00005741287,0.0004527121,0.0005705869],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002082793,"about_ca_system_score_gemma":0.004821155,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_topic_score_codex":0.01289517,"about_ca_topic_score_gemma":0.2523465,"domain_scores_codex":[0.9863285,0.002559059,0.001168215,0.0008428421,0.008563664,0.0005376657],"domain_scores_gemma":[0.9904892,0.00152772,0.0007585991,0.0006627764,0.006287098,0.0002745857],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00002459207,0.00009808716,0.003322271,0.000008447749,0.0001558221,0.0000014189,0.01602391,0.3554457,0.0002858549,0.0008382401,0.08079413,0.5430016],"study_design_scores_gemma":[0.002326858,0.0001666578,0.01473988,0.00003401322,0.0007020396,0.00004309726,0.01309792,0.9360473,0.0001460785,0.01179326,0.02062775,0.0002752029],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9514962,0.0005276616,0.00617166,0.01902125,0.005344915,0.009802668,0.00008894522,0.0003878107,0.007158844],"genre_scores_gemma":[0.9921185,0.00004963144,0.0004196869,0.0003964445,0.000512786,0.002359136,0.001503502,0.00004005697,0.002600291],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5806016,"threshold_uncertainty_score":0.9999298,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4252169279","doi":"10.1002/ev.309","title":"Editors' notes","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Community Health and Development","field":"Health Professions","cited_by":1,"is_retracted":false,"has_abstract":false,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Calgary","funders":"","keywords":"Citation; Library science; Population; Intervention (counseling); State (computer science); Sociology; Medicine; Computer science; Nursing","retraction":null,"screen_n_in":null,"score":{"opus":0.1565093687785537,"gpt":0.5236589003624142,"spread":0.3671495315838605,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.001635695,0.00008423744,0.0001158271,0.0001405188,0.001497819,0.000007938289,0.00007453472,0.0001210791,0.0007357299],"category_scores_gemma":[0.002053854,0.00008242873,0.00004702525,0.0002596314,0.000007002488,0.0001139661,0.00001311952,0.0002434251,0.0002621702],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003993582,"about_ca_system_score_gemma":0.00135586,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003842217,"about_ca_topic_score_gemma":0.0006363915,"domain_scores_codex":[0.9986604,0.0002832273,0.0003378828,0.0001430751,0.0002848285,0.0002906218],"domain_scores_gemma":[0.9981738,0.0008667855,0.0001112378,0.0002358351,0.0004587686,0.0001535156],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00006515082,0.00007328443,0.001226875,0.0000256806,0.0000077781,2.896923e-8,0.002546301,0.00003420283,0.0001213835,0.001409752,0.4504891,0.5440004],"study_design_scores_gemma":[0.001888332,0.000146664,0.08288344,0.00006369031,0.00004213896,4.231332e-7,0.000428246,0.001860536,0.00008687768,0.02600721,0.8864725,0.0001199445],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.2564925,0.002470484,0.08416079,0.2374264,0.08452646,0.02901904,0.00007716432,0.002649529,0.3031777],"genre_scores_gemma":[0.9000586,0.0002330495,0.03185079,0.01360627,0.02901332,0.002610184,0.0007098879,0.00005752857,0.02186036],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6435661,"threshold_uncertainty_score":0.9998021,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null}]}