{"meta":{"query_hash":"3e0a82fc70b4","filters":{"venue":"New Directions for Evaluation"},"cohort_total":73,"direct_labels_cover":1,"predictions_cover":73,"exported":73,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/3e0a82fc70b4","api":"https://metacan.xera.ac/api/v1/cohort?venue=New+Directions+for+Evaluation"},"results":[{"id":"W1505207325","doi":"10.1002/ev.20084","title":"Credible Judgment: Combining Truth, Beauty, and Justice","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Université du Québec à Montréal","funders":"","keywords":"Beauty; Argumentation theory; Economic Justice; Process (computing); Variety (cybernetics); Psychology; Social psychology; Public relations; Computer science; Law; Epistemology; Political science","score_opus":0.26060324573329097,"score_gpt":0.5132600924299058,"score_spread":0.2526568466966148,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1505207325","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.47108805,0.00071911485,0.3404806,0.010098822,0.011750565,0.0027786626,0.000027558683,0.000332174,0.16272445],"genre_scores_gemma":[0.988413,0.000043686363,0.0063800495,0.00037147166,0.0005321868,0.00017906392,0.000039729774,0.000013870568,0.00402695],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9970622,0.00029614975,0.00049408665,0.00040511513,0.0015341006,0.00020836607],"domain_scores_gemma":[0.99659854,0.0019337749,0.0002322284,0.00034438472,0.0007485913,0.00014248508],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.010944479,0.00011803131,0.00017863416,0.00032158155,0.00060951966,0.00030623333,0.0001708434,0.000068133595,0.0010581542],"category_scores_gemma":[0.00657607,0.0001013275,0.00006794571,0.00056416105,0.000037413196,0.0005508513,0.000038724298,0.00008280047,0.00015879556],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000047694608,0.00007613646,0.0021284295,0.000013802403,0.000024608134,2.421048e-8,0.0010367356,0.005211268,0.00046721907,0.040553324,0.027216794,0.923224],"study_design_scores_gemma":[0.002476879,0.0002445968,0.018574277,0.000018653769,0.00025292544,0.0000040269274,0.0006525713,0.6494474,0.00065611524,0.08997813,0.23752828,0.0001661949],"about_ca_topic_score_codex":0.000043993452,"about_ca_topic_score_gemma":0.00011397061,"teacher_disagreement_score":0.9230578,"about_ca_system_score_codex":0.00013231748,"about_ca_system_score_gemma":0.0002699592,"threshold_uncertainty_score":0.99985504},"labels":[],"label_agreement":null},{"id":"W1534140708","doi":"10.1002/ev.20088","title":"House With a View: Validity and Evaluative Argument","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"123 Certification (Canada)","funders":"","keywords":"CLARITY; Argument (complex analysis); Style (visual arts); Epistemology; Narrative; Evaluation methods; Psychology; Computer science; Sociology; Linguistics; Philosophy; History","score_opus":0.24014051371560705,"score_gpt":0.5046303264375429,"score_spread":0.26448981272193584,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1534140708","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6927229,0.00027866464,0.27715087,0.006789342,0.0012347682,0.0037945097,0.000017265203,0.00025929188,0.017752364],"genre_scores_gemma":[0.9857093,0.000054223678,0.009424304,0.0004004726,0.0002791543,0.00057547627,0.000021011261,0.000024837887,0.0035112035],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9956893,0.0007325164,0.00048592233,0.0005260939,0.0023640727,0.00020209582],"domain_scores_gemma":[0.99627763,0.0013269075,0.00030108116,0.00045310974,0.0014917784,0.00014949881],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0147098405,0.00016303294,0.00023597265,0.0002837742,0.00047395576,0.00026132414,0.0001637112,0.000060291466,0.00096717395],"category_scores_gemma":[0.0033876372,0.00011447844,0.00006992483,0.0006268728,0.00006107021,0.00054571556,0.000031377695,0.00008689269,0.00016608277],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00014236165,0.00012004933,0.008983283,0.0000074545474,0.000038331324,4.086496e-8,0.0016702731,0.003960565,0.00022085306,0.0024596564,0.018713046,0.9636841],"study_design_scores_gemma":[0.0048002857,0.0012163959,0.107372455,0.000036094545,0.00030533806,0.000008932052,0.00063070445,0.4839411,0.0013351155,0.064023845,0.33599976,0.00032999134],"about_ca_topic_score_codex":0.00005176118,"about_ca_topic_score_gemma":0.0005579006,"teacher_disagreement_score":0.9633541,"about_ca_system_score_codex":0.00016684395,"about_ca_system_score_gemma":0.00047155187,"threshold_uncertainty_score":0.99994606},"labels":[],"label_agreement":null},{"id":"W1541185587","doi":"10.1002/ev.20112","title":"Credentialed Evaluator Designation Program, the Canadian Experience","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Value (mathematics); Professional development; Service (business); Administration (probate law); Political science; Public relations; Management; Engineering ethics; Sociology; Pedagogy; Computer science; Engineering; Law; Business; Marketing; Economics","score_opus":0.4863368872817085,"score_gpt":0.5792195495990002,"score_spread":0.09288266231729175,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1541185587","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5349118,0.0018904561,0.12974988,0.060768206,0.022911808,0.0323635,0.0000839934,0.0009676042,0.21635273],"genre_scores_gemma":[0.9850383,0.0000059336558,0.004685005,0.00037351664,0.0004518623,0.0022647902,0.000087539345,0.000015346437,0.007077704],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.994406,0.0006213262,0.00059874577,0.00043115177,0.0036381846,0.00030462866],"domain_scores_gemma":[0.9944095,0.0004991141,0.00026899442,0.00057216064,0.0038647642,0.00038546266],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.016478546,0.00014391018,0.00014863165,0.00043730627,0.001097156,0.00081459805,0.0004925731,0.00009679403,0.0016292229],"category_scores_gemma":[0.008709146,0.000098911034,0.000116074436,0.0013864046,0.00007822403,0.00079630374,0.000025541329,0.00010440366,0.00036941134],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":true,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000046479272,0.00007537816,0.002614932,0.0000011687065,0.000023279734,1.0811226e-7,0.0050582434,0.0065438366,0.00010340196,0.0024675087,0.10361656,0.8794491],"study_design_scores_gemma":[0.0016909133,0.0002075841,0.016783409,0.0000068080453,0.00009383341,0.0000053881417,0.00221719,0.4237547,0.00061258883,0.027050788,0.52739894,0.0001778751],"about_ca_topic_score_codex":0.014270325,"about_ca_topic_score_gemma":0.27331123,"teacher_disagreement_score":0.8792712,"about_ca_system_score_codex":0.0009569103,"about_ca_system_score_gemma":0.0060022413,"threshold_uncertainty_score":0.99964094},"labels":[],"label_agreement":null},{"id":"W1543074450","doi":"10.1002/ev.20076","title":"Framing the Capacity to Do and Use Evaluation","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Framing (construction); Conceptualization; Situated; Construct (python library); Capacity building; Organizational change; Sociology; Management science; Knowledge management; Public relations; Computer science; Political science; Economics","score_opus":0.33725614035289037,"score_gpt":0.5217974321371192,"score_spread":0.1845412917842288,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1543074450","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8407602,0.000105047366,0.14204979,0.0072794533,0.001531165,0.0027325596,0.000010104448,0.000055553242,0.0054761674],"genre_scores_gemma":[0.99182385,0.00000993597,0.005234292,0.0007115088,0.00034853845,0.00051332224,0.0000142671615,0.000011295844,0.0013329595],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9953135,0.0008927742,0.00049022416,0.00045211267,0.0026642405,0.00018712298],"domain_scores_gemma":[0.9946947,0.0024106472,0.00020944483,0.0005585527,0.0019984003,0.00012826748],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.026410995,0.00012614764,0.00015045555,0.00029568793,0.0007741525,0.00070068834,0.00021389604,0.00006843606,0.00078991486],"category_scores_gemma":[0.019065356,0.00008517937,0.00007428456,0.00080777606,0.00004072328,0.0007644431,0.000042597152,0.00009733968,0.0001531388],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002238382,0.000024922598,0.005238763,0.0000018992191,0.000013670584,3.4815855e-9,0.002191893,0.010581496,0.0004526603,0.004710428,0.012145513,0.96461636],"study_design_scores_gemma":[0.00086991186,0.0001158958,0.21413079,0.00001193432,0.00011588586,0.0000020620369,0.00040689702,0.55172205,0.00034424296,0.051301785,0.18085222,0.00012632723],"about_ca_topic_score_codex":0.00016511674,"about_ca_topic_score_gemma":0.0011327721,"teacher_disagreement_score":0.96449006,"about_ca_system_score_codex":0.00017750343,"about_ca_system_score_gemma":0.0002765738,"threshold_uncertainty_score":0.9891975},"labels":[],"label_agreement":null},{"id":"W1557717928","doi":"10.1002/ev.20082","title":"The Value in Validity","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"123 Certification (Canada)","funders":"","keywords":"Beauty; Prioritization; Economic Justice; Value (mathematics); Sociology; Balance (ability); Management science; Psychology; Social psychology; Epistemology; Computer science; Political science; Law; Economics","score_opus":0.36226017726620263,"score_gpt":0.5450118219863205,"score_spread":0.18275164472011785,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1557717928","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.58546084,0.0003399578,0.20470272,0.03869582,0.009478688,0.0041960557,0.0000124352055,0.00019033955,0.15692312],"genre_scores_gemma":[0.98942935,0.000033321325,0.0013984573,0.00027273785,0.00036547624,0.0003094481,0.000010885353,0.000008599464,0.008171696],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9963982,0.0008966677,0.00054283754,0.0002901654,0.001685257,0.00018687545],"domain_scores_gemma":[0.994442,0.004180217,0.00019028345,0.00046460977,0.00066102564,0.000061897335],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02828205,0.00008381571,0.00011842549,0.00020031833,0.0005597453,0.0002651981,0.00029295534,0.000051581304,0.00044535272],"category_scores_gemma":[0.014001549,0.000054138935,0.000091714806,0.0007903934,0.00003311063,0.0003258729,0.000024279625,0.000086516244,0.00025154307],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000032080417,0.000035452103,0.006722787,0.0000011017095,0.0000054412776,9.353223e-9,0.0004275869,0.014228228,0.00009164034,0.019376878,0.030175742,0.92890304],"study_design_scores_gemma":[0.0007418859,0.00006068935,0.05916897,0.0000038379326,0.000013991351,5.561759e-7,0.00013073008,0.41340658,0.00021539727,0.13486955,0.39132908,0.00005873568],"about_ca_topic_score_codex":0.00009400039,"about_ca_topic_score_gemma":0.0015647775,"teacher_disagreement_score":0.92884433,"about_ca_system_score_codex":0.0001737388,"about_ca_system_score_gemma":0.0003940238,"threshold_uncertainty_score":0.99430394},"labels":[],"label_agreement":null},{"id":"W1569252761","doi":"10.1002/ev.20079","title":"Postscript: That Was Then, This Is Now","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec; Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Sustainability; Public relations; Organizational communication; Knowledge management; Political science; Sociology; Computer science","score_opus":0.2843044164649911,"score_gpt":0.5111892279361946,"score_spread":0.22688481147120348,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1569252761","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.16846041,0.00076728343,0.1952271,0.062169574,0.012724369,0.004735348,0.00008276101,0.0005188336,0.5553143],"genre_scores_gemma":[0.9078361,0.00006659588,0.0045792065,0.0028118482,0.0007122841,0.00031883872,0.000071883005,0.000027942864,0.083575346],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9956877,0.0003977679,0.00053022377,0.0005117814,0.0026241962,0.0002483266],"domain_scores_gemma":[0.9965988,0.0009419183,0.00030526437,0.0006957818,0.0013222275,0.00013604078],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.010212685,0.00016349897,0.00021920323,0.00038972197,0.00055864465,0.0003484067,0.00036644356,0.000104453335,0.014008028],"category_scores_gemma":[0.0026833683,0.00012683889,0.00019767841,0.00072455686,0.000038188682,0.0007469842,0.000038958144,0.00010765733,0.0023594436],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000031366428,0.000054823882,0.0043027694,0.000002792,0.000021555767,1.5861616e-8,0.0013351412,0.00040627617,0.00039873936,0.0024497386,0.38133636,0.6096604],"study_design_scores_gemma":[0.0011819873,0.00014201445,0.023521986,0.000011064852,0.000079810125,0.0000025990744,0.00030165288,0.22105752,0.002567693,0.046035934,0.7049364,0.00016134958],"about_ca_topic_score_codex":0.00008594883,"about_ca_topic_score_gemma":0.00037661055,"teacher_disagreement_score":0.73937565,"about_ca_system_score_codex":0.00017458298,"about_ca_system_score_gemma":0.00041495118,"threshold_uncertainty_score":0.9984173},"labels":[],"label_agreement":null},{"id":"W1781948974","doi":"10.1002/ev.20111","title":"How Does Accreditation Fit Into the Picture?","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Accounting Education and Careers","field":"Business, Management and Accounting","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Accreditation; Blank; Computer science; Public relations; Political science; Law; Engineering","score_opus":0.06017099692544828,"score_gpt":0.31944720650743275,"score_spread":0.2592762095819845,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1781948974","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6506878,0.00079164945,0.037149582,0.21929233,0.03355635,0.005199547,0.0000074631453,0.0012035815,0.052111693],"genre_scores_gemma":[0.9837978,0.0000029936411,0.0006875604,0.001757642,0.006268106,0.00034026938,0.0001702523,0.000023335282,0.0069520595],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9990979,0.00001793932,0.00013022513,0.0001861324,0.0004416143,0.0001261766],"domain_scores_gemma":[0.9985556,0.00006755922,0.00016479744,0.00019682077,0.0009993584,0.00001583758],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010864857,0.000101706006,0.000072472794,0.00016126958,0.00036699008,0.00068710267,0.00014307478,0.00005497132,0.000093623195],"category_scores_gemma":[0.0017970457,0.00006644284,0.00006520136,0.00045245214,0.000024824856,0.0012122961,0.0000235261,0.00006614211,0.00007939389],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000048726153,0.00007155378,0.0033640605,0.000039770515,0.00004956599,4.899269e-8,0.0032066044,0.0009326217,0.00017794357,0.011426648,0.7180243,0.26265812],"study_design_scores_gemma":[0.0008415886,0.000008802595,0.0049795466,0.000009937213,0.00019129169,4.3771095e-7,0.008812946,0.017615454,0.00008623333,0.039791245,0.92751956,0.0001429671],"about_ca_topic_score_codex":0.0003587237,"about_ca_topic_score_gemma":0.00090913277,"teacher_disagreement_score":0.33310997,"about_ca_system_score_codex":0.00012539093,"about_ca_system_score_gemma":0.00017618766,"threshold_uncertainty_score":0.6625747},"labels":[],"label_agreement":null},{"id":"W1929107339","doi":"10.1002/ev.20078","title":"Cross‐Case Analysis and Implications for Research, Theory, and Practice","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Leverage (statistics); Theory of change; Organizational change; Affect (linguistics); Thematic analysis; Context (archaeology); Order (exchange); Evaluation methods; Organization development; Knowledge management; Management science; Public relations; Sociology; Computer science; Political science; Management; Business; Qualitative research; Social science; Engineering; Economics","score_opus":0.46899379184883316,"score_gpt":0.6700118587679306,"score_spread":0.2010180669190974,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1929107339","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.16821696,0.00037160795,0.8029016,0.014942619,0.000306054,0.0024834492,0.00006344786,0.000056910794,0.010657331],"genre_scores_gemma":[0.97459424,0.0000751635,0.018669145,0.0002709866,0.00022749831,0.0010364907,0.00005721204,0.000012077852,0.0050572157],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9963924,0.0015691604,0.00046021005,0.00051489886,0.0008598513,0.00020344359],"domain_scores_gemma":[0.9725251,0.023042355,0.00022071245,0.00046855595,0.0036005194,0.0001427526],"candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.066208236,0.00009258839,0.00017106887,0.0007911834,0.001371108,0.0006310081,0.00012488608,0.000071628674,0.00025703447],"category_scores_gemma":[0.05830085,0.00007662498,0.00009255832,0.0016372924,0.0001235872,0.00078315916,0.000043060332,0.000083338615,0.000014930996],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00018846082,0.000059619564,0.0076149683,0.0000052409646,0.00013129688,4.2490072e-8,0.00086670864,0.00086525496,0.00022649315,0.081115566,0.00452521,0.9044011],"study_design_scores_gemma":[0.0019016346,0.00028072068,0.15956196,0.0000046446476,0.0009821125,0.00006834728,0.0013353026,0.18739453,0.00014879818,0.40618458,0.24197708,0.00016025809],"about_ca_topic_score_codex":0.00010353562,"about_ca_topic_score_gemma":0.0006853878,"teacher_disagreement_score":0.9042409,"about_ca_system_score_codex":0.00008131051,"about_ca_system_score_gemma":0.0002454568,"threshold_uncertainty_score":0.99992895},"labels":[],"label_agreement":null},{"id":"W1978303279","doi":"10.1002/ev.382","title":"Integrating a new evaluation unit with an old institution: See no evil; hear no evil; speak no evil","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Guelph","funders":"","keywords":"Unit (ring theory); CLARITY; Institution; Evaluation methods; Association (psychology); Psychology; Sociology; Computer science; Mathematics education; Social science; Engineering; Psychotherapist","score_opus":0.39098050530506484,"score_gpt":0.5034779046174458,"score_spread":0.11249739931238095,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1978303279","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.35947865,0.0008401606,0.122312106,0.0025218362,0.012616978,0.013830661,0.00009228828,0.0009065377,0.48740077],"genre_scores_gemma":[0.83747685,0.00017748134,0.09508414,0.0013198295,0.004451561,0.0017989994,0.00082730677,0.00014432208,0.05871948],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99045914,0.0012412453,0.0014722667,0.0011885612,0.0050962465,0.0005425385],"domain_scores_gemma":[0.9869133,0.0006028721,0.0008953673,0.0012966604,0.009825579,0.00046619197],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.017360294,0.00048490812,0.0005230155,0.0008294718,0.0011495486,0.0005766048,0.0006473174,0.00024707674,0.022103969],"category_scores_gemma":[0.008323997,0.00038155567,0.00023837572,0.0021613336,0.00013011084,0.003536137,0.00007265158,0.00038371966,0.004990346],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0015569028,0.00081251696,0.012331729,0.0000509591,0.0002271055,0.0000010925879,0.0073950593,0.012334615,0.0012205209,0.00660834,0.051664777,0.9057964],"study_design_scores_gemma":[0.006232469,0.0028454147,0.038905386,0.000318862,0.00068989,0.000027631837,0.001579132,0.73067474,0.00071280624,0.009590889,0.20760496,0.00081778626],"about_ca_topic_score_codex":0.0017020858,"about_ca_topic_score_gemma":0.00627782,"teacher_disagreement_score":0.9049786,"about_ca_system_score_codex":0.00078441214,"about_ca_system_score_gemma":0.0047859726,"threshold_uncertainty_score":0.9998636},"labels":[],"label_agreement":null},{"id":"W1986224130","doi":"10.1002/ev.133","title":"Background and history of the Joint Committee's Program Evaluation Standards","year":2004,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":27,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Queen's University","funders":"","keywords":"Joint (building); Context (archaeology); Political science; Library science; Engineering ethics; Computer science; Engineering; History; Archaeology; Civil engineering","score_opus":0.3548191126107928,"score_gpt":0.5240969201650518,"score_spread":0.16927780755425903,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1986224130","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.86351424,0.00509544,0.031235427,0.013740216,0.009536527,0.01705874,0.000098494245,0.00019509274,0.059525814],"genre_scores_gemma":[0.9904878,0.000048853803,0.004656688,0.0001779834,0.0001726649,0.0010214078,0.000029634204,0.000015389442,0.0033895394],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99416995,0.00046323636,0.0006914855,0.0003459838,0.0041742073,0.000155136],"domain_scores_gemma":[0.99522215,0.0002799276,0.00047953267,0.0005274784,0.0034167278,0.00007418441],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01762353,0.00012710881,0.00020121313,0.000237916,0.0002751182,0.00009502291,0.00020820252,0.00007783092,0.0011565782],"category_scores_gemma":[0.003021478,0.00008801436,0.00014460852,0.00056353275,0.00014282974,0.00047172073,0.00004642344,0.00010403926,0.000012890381],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000042603806,0.00017343486,0.0010507968,0.000010696426,0.000031497657,1.31522855e-8,0.0017770802,0.007806424,0.00068231864,0.003472627,0.032583088,0.9523694],"study_design_scores_gemma":[0.006239746,0.0005079239,0.13986789,0.00007369381,0.0004004885,0.0000068044133,0.0013747902,0.13020828,0.0015595215,0.10205649,0.617464,0.00024040656],"about_ca_topic_score_codex":0.00018356238,"about_ca_topic_score_gemma":0.0010729159,"teacher_disagreement_score":0.952129,"about_ca_system_score_codex":0.0021885536,"about_ca_system_score_gemma":0.0046141264,"threshold_uncertainty_score":0.9997565},"labels":[],"label_agreement":null},{"id":"W1992053221","doi":"10.1002/ev.325","title":"Enhancing disaster and emergency preparedness, response, and recovery through evaluation","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Preparedness; Disaster response; Emergency management; Disaster preparedness; Emergency response; Process management; Evaluation methods; Disaster recovery; Disaster planning; Computer science; Business; Risk analysis (engineering); Computer security; Public relations; Political science; Medical emergency; Poison control; Human factors and ergonomics; Engineering; Medicine; Reliability engineering","score_opus":0.16373296269347223,"score_gpt":0.5114703559569701,"score_spread":0.34773739326349784,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1992053221","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9752097,0.00038051026,0.0126752285,0.0017113662,0.0033245094,0.001658257,0.00001630038,0.00005184856,0.004972261],"genre_scores_gemma":[0.98703206,0.00014311355,0.006553541,0.00009989898,0.000386143,0.00055231544,0.000049258797,0.000019794079,0.005163873],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.99584615,0.00063459453,0.0007423634,0.0006284172,0.0019391015,0.00020937104],"domain_scores_gemma":[0.9964079,0.0011067799,0.00033842106,0.00048125876,0.0015551397,0.00011046343],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.019232864,0.00016832522,0.00019940322,0.0002752845,0.00058824907,0.0002575804,0.0001401196,0.00012767207,0.0036773614],"category_scores_gemma":[0.010586427,0.00014369974,0.000077253026,0.0005825302,0.00005707318,0.0015291893,0.00005406828,0.00014328705,0.00005409091],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0006838833,0.00010075947,0.007807197,0.000014282665,0.000049749196,6.299228e-8,0.010736519,0.0005569189,0.038565952,0.0010597986,0.016333222,0.92409164],"study_design_scores_gemma":[0.0050339997,0.00063246937,0.30664167,0.000050262213,0.00055872294,0.00002951039,0.005520883,0.26303568,0.005985218,0.2529367,0.15889345,0.000681449],"about_ca_topic_score_codex":0.00007801387,"about_ca_topic_score_gemma":0.0030419219,"teacher_disagreement_score":0.9234102,"about_ca_system_score_codex":0.00011710099,"about_ca_system_score_gemma":0.00076176564,"threshold_uncertainty_score":0.99774784},"labels":[],"label_agreement":null},{"id":"W1994122017","doi":"10.1002/ev.333","title":"Evaluation of disaster and emergency management: Do no harm, but do better","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Disaster Response and Management","field":"Health Professions","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Emergency management; Harm; Medical emergency; Business; Medicine; Psychology; Political science; Social psychology","score_opus":0.08876767161420009,"score_gpt":0.4590029695750237,"score_spread":0.3702352979608236,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1994122017","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8304724,0.00022721959,0.0028875787,0.0006661951,0.0045144986,0.00516342,0.00003291498,0.00007008088,0.15596569],"genre_scores_gemma":[0.9587472,0.00015611968,0.0028970684,0.00021589294,0.00085912383,0.0032826501,0.0001249099,0.000053415995,0.033663604],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9973167,0.0005691018,0.0005484735,0.00036326013,0.00093321863,0.00026922344],"domain_scores_gemma":[0.9979997,0.000117366406,0.00026693975,0.00043273164,0.0010903082,0.000092943934],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.00542736,0.00015503405,0.00017003733,0.00028666208,0.00044959938,0.000021125752,0.00011003332,0.00011847085,0.0061388942],"category_scores_gemma":[0.0004785154,0.00014673763,0.00008186319,0.00023907392,0.00003004353,0.00029481415,0.00008746587,0.00018504202,0.00024641855],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00024055438,0.00022139026,0.004223055,0.00034128389,0.0003180024,1.5293597e-7,0.0068274615,0.000040143674,0.004197447,0.0064877793,0.07542427,0.90167844],"study_design_scores_gemma":[0.013907472,0.00032769406,0.14512327,0.00033907127,0.0055082515,0.000001798709,0.009567867,0.023696281,0.00037043414,0.07108074,0.72936904,0.000708043],"about_ca_topic_score_codex":0.00007568279,"about_ca_topic_score_gemma":0.00023360881,"teacher_disagreement_score":0.9009704,"about_ca_system_score_codex":0.000103333114,"about_ca_system_score_gemma":0.00014203617,"threshold_uncertainty_score":0.99476963},"labels":[],"label_agreement":null},{"id":"W1999716079","doi":"10.1002/ev.243","title":"Infusing evaluative thinking as process use: The case of the International Development Research Centre (IDRC)","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":21,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Accountability; Process (computing); Political science; Sociology; Process management; Management; Business; Computer science","score_opus":0.45376368323160404,"score_gpt":0.6131986765418254,"score_spread":0.15943499331022132,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1999716079","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.97073287,0.000088618726,0.005401148,0.005605602,0.0018591895,0.0021244627,0.000007640155,0.000027014821,0.014153452],"genre_scores_gemma":[0.9925522,0.0000038376347,0.0021757549,0.00018869454,0.00022491481,0.000093993884,0.000016062926,0.00001182934,0.0047326833],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99184936,0.00079954765,0.0008792299,0.0003866346,0.0058032554,0.00028196335],"domain_scores_gemma":[0.9859741,0.003521078,0.00047735206,0.00048910244,0.00945406,0.00008435587],"candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.06376707,0.00012256455,0.00013640529,0.00044212598,0.0014433279,0.00038755563,0.0006892894,0.00007581469,0.0008620387],"category_scores_gemma":[0.020717338,0.00007020439,0.00009717375,0.0019873292,0.00012331706,0.00072254357,0.00015155594,0.00025335525,0.000059135837],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00021566892,0.00019928293,0.014877425,0.000009401295,0.0001178238,0.0000021175051,0.060796812,0.016938034,0.0005323885,0.006124012,0.016294807,0.88389224],"study_design_scores_gemma":[0.0045910636,0.00020652423,0.106915504,0.00024432348,0.00023473203,0.00024663637,0.1009031,0.28641787,0.034944553,0.25380877,0.21100812,0.0004788191],"about_ca_topic_score_codex":0.00032787875,"about_ca_topic_score_gemma":0.0063866493,"teacher_disagreement_score":0.88341343,"about_ca_system_score_codex":0.00071990327,"about_ca_system_score_gemma":0.003579946,"threshold_uncertainty_score":0.99985665},"labels":[],"label_agreement":null},{"id":"W2008070236","doi":"10.1002/ev.299","title":"Reflections on the dilemmas of conducting environmental evaluations","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Cognitive reframing; Multidisciplinary approach; Asset (computer security); Context (archaeology); Set (abstract data type); Face (sociological concept); Computer science; Management science; Theory of change; Work (physics); Sociology; Engineering ethics; Psychology; Social science; Social psychology; Economics; Engineering","score_opus":0.5676539202947379,"score_gpt":0.5954441216399692,"score_spread":0.027790201345231247,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2008070236","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.75038284,0.00029987976,0.037731193,0.043593157,0.0031011985,0.0056316517,0.00011168036,0.00016629962,0.15898208],"genre_scores_gemma":[0.9935433,0.00001929259,0.0015536534,0.00053680304,0.000249532,0.00019670125,0.00004855883,0.000008932285,0.0038432267],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9958905,0.0005305228,0.00074103504,0.00036842862,0.0022865906,0.0001828954],"domain_scores_gemma":[0.9964492,0.0019235753,0.00044422306,0.0006017363,0.000509823,0.000071463066],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.009315129,0.00013803007,0.00017837211,0.00036024328,0.00085021916,0.00011173378,0.0002908323,0.00006704957,0.004187814],"category_scores_gemma":[0.0045145196,0.000094105395,0.00018958209,0.0008748849,0.000062322935,0.00038536347,0.00001603735,0.00013781937,0.00018981971],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000097881464,0.00047238785,0.0005505753,0.0000017789201,0.00006305515,5.5793507e-8,0.0030911956,0.024284005,0.031147126,0.05045737,0.03608853,0.85374606],"study_design_scores_gemma":[0.0045049335,0.002190218,0.1032492,0.000067977024,0.0005718948,0.000017972887,0.009199553,0.23834185,0.048867468,0.38882503,0.20354925,0.0006146566],"about_ca_topic_score_codex":0.00001898199,"about_ca_topic_score_gemma":0.000079488134,"teacher_disagreement_score":0.85313135,"about_ca_system_score_codex":0.00023667119,"about_ca_system_score_gemma":0.00042744016,"threshold_uncertainty_score":0.9967225},"labels":[],"label_agreement":null},{"id":"W2013250096","doi":"10.1002/ev.20007","title":"When one must go: The Canadian experience with strategic review and judging program value","year":2012,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Valuation (finance); Function (biology); Government (linguistics); Value (mathematics); Citizen journalism; Public relations; Process (computing); Value for money; Public administration; Business; Political science; Economics; Public economics; Computer science; Accounting; Law","score_opus":0.38211926976493,"score_gpt":0.524848788433648,"score_spread":0.14272951866871803,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2013250096","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5310917,0.04930536,0.008846051,0.10464218,0.0038329044,0.030395804,0.000064510925,0.00051046285,0.271311],"genre_scores_gemma":[0.9895501,0.000372417,0.0050799525,0.001238898,0.00021974616,0.0014519378,0.000027448306,0.000012605479,0.0020469066],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9974876,0.00028447254,0.00036299377,0.00027355138,0.0012912492,0.00030010534],"domain_scores_gemma":[0.9982114,0.0002894883,0.00018674156,0.00039591195,0.00066985574,0.0002466147],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0075106025,0.000119651624,0.00016415729,0.00016297353,0.00089725514,0.00034020544,0.00019868347,0.00004593869,0.0012569275],"category_scores_gemma":[0.0005795008,0.000073465984,0.0000439995,0.00067489763,0.00007698065,0.00075787975,0.000016778446,0.00010517418,0.00007023394],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014444476,0.00011263824,0.014618085,0.000042452666,0.00004323125,5.1899615e-8,0.0050878134,0.00039331667,0.000027565133,0.008344883,0.010945407,0.9603701],"study_design_scores_gemma":[0.002063702,0.00063590537,0.19599095,0.00042303512,0.001104438,0.00004625676,0.00545183,0.08325389,0.00019627399,0.022606658,0.6875635,0.00066357065],"about_ca_topic_score_codex":0.008264406,"about_ca_topic_score_gemma":0.10604155,"teacher_disagreement_score":0.95970654,"about_ca_system_score_codex":0.00023403652,"about_ca_system_score_gemma":0.0010595373,"threshold_uncertainty_score":0.9996561},"labels":[],"label_agreement":null},{"id":"W2017680079","doi":"10.1002/ev.344","title":"Strategy evaluation: Experience at the International Development Research Centre","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"International Development Research Centre","funders":"","keywords":"Corporation; Program evaluation; Strategic planning; Modalities; International development; Sociology; Management; Business; Public relations; Political science; Economics; Public administration; Social science","score_opus":0.5427282247833871,"score_gpt":0.6215806160734704,"score_spread":0.07885239129008326,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2017680079","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.91039073,0.00014094659,0.0014902004,0.016930474,0.0060203075,0.0027575218,0.000012960522,0.00006168388,0.062195197],"genre_scores_gemma":[0.9515751,0.000011510159,0.0018905506,0.00021763319,0.0007345419,0.0011255036,0.00014056708,0.00001488827,0.04428971],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9894697,0.0006429789,0.00076087797,0.0006443333,0.008142579,0.00033953506],"domain_scores_gemma":[0.99081004,0.0016812054,0.00025631962,0.0007585415,0.0063509173,0.00014300107],"candidate_categories":["metaresearch","sts","insufficient_payload"],"consensus_categories":["metaresearch","insufficient_payload"],"category_scores_codex":[0.037252042,0.0001469221,0.00013060197,0.000405758,0.0017441866,0.00062192057,0.0008644863,0.000106243184,0.03820798],"category_scores_gemma":[0.008411651,0.00010012754,0.00009906555,0.0010996154,0.00013935176,0.00076802645,0.0001624214,0.00031341743,0.0012144872],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005336565,0.000089406385,0.0049950867,0.0000011643777,0.000027037691,6.855896e-8,0.0040438185,0.0028557577,0.003933436,0.0022790388,0.08839645,0.8933254],"study_design_scores_gemma":[0.0012715538,0.000038290716,0.05281446,0.0000072867156,0.000028728715,0.0000061117084,0.0034192987,0.15311489,0.008417612,0.01147851,0.7692514,0.00015183043],"about_ca_topic_score_codex":0.00007421249,"about_ca_topic_score_gemma":0.0074876235,"teacher_disagreement_score":0.8931735,"about_ca_system_score_codex":0.00066953537,"about_ca_system_score_gemma":0.002814644,"threshold_uncertainty_score":0.99994093},"labels":[],"label_agreement":null},{"id":"W2033749959","doi":"10.1002/ev.1190","title":"Evaluative inquiry in university‐school professional learning partnerships","year":2000,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Professional learning community; Professional development; Pedagogy; Psychology; Sociology","score_opus":0.37815566276918305,"score_gpt":0.5350127287443462,"score_spread":0.1568570659751632,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2033749959","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9267965,0.00014984483,0.002481498,0.0049139014,0.0015029907,0.0021973248,0.000007968982,0.00011750545,0.061832476],"genre_scores_gemma":[0.8720748,0.000051598465,0.001334831,0.0001586301,0.00032240388,0.0001090688,0.000079940124,0.0000134978645,0.12585524],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9950977,0.001463309,0.0005340239,0.0005073165,0.0021384237,0.00025923536],"domain_scores_gemma":[0.9973262,0.0011768168,0.00020252669,0.00028450377,0.00085402554,0.0001559027],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.011742452,0.00014234215,0.00020150297,0.000600313,0.000504633,0.00010643968,0.00026484876,0.000113055416,0.030804774],"category_scores_gemma":[0.0036946202,0.00012921577,0.000116816474,0.001591727,0.000051647403,0.0009103186,0.000024984824,0.00025963216,0.0011628204],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00036242625,0.00017203318,0.019669218,0.0000036477877,0.000023309416,6.2109024e-7,0.0044609667,0.06219227,0.00012365,0.001126557,0.031151507,0.8807138],"study_design_scores_gemma":[0.0042204238,0.00021430434,0.1445049,0.000046875113,0.0000694911,0.0000034846937,0.0071971817,0.31166738,0.00012061693,0.014794509,0.51690453,0.0002562752],"about_ca_topic_score_codex":0.000049506736,"about_ca_topic_score_gemma":0.00070522435,"teacher_disagreement_score":0.8804575,"about_ca_system_score_codex":0.000529168,"about_ca_system_score_gemma":0.0015628077,"threshold_uncertainty_score":0.9996149},"labels":[],"label_agreement":null},{"id":"W2038633574","doi":"10.1002/ev.1171","title":"Translating evaluation findings into “policy language”","year":2000,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Université du Québec à Montréal","funders":"","keywords":"Affect (linguistics); Welfare; Order (exchange); Policy learning; Policy analysis; Program evaluation; Language policy; Computer science; Management science; Political science; Public administration; Sociology; Pedagogy; Economics; Machine learning; Law; Finance","score_opus":0.19760218499848886,"score_gpt":0.5590826968278899,"score_spread":0.36148051182940105,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2038633574","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8476119,0.0007453475,0.01511153,0.01480731,0.0011529359,0.0038299651,0.000023426293,0.00024352386,0.116474055],"genre_scores_gemma":[0.9772639,0.00003327219,0.005804547,0.00039311696,0.00081355945,0.00061485526,0.00016282816,0.00002466972,0.014889245],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9942243,0.00056245556,0.0008258471,0.0005654122,0.0035335342,0.00028844117],"domain_scores_gemma":[0.99728805,0.0006400135,0.00018390776,0.0005170975,0.0012292553,0.00014170454],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.015486436,0.00018394327,0.00022291872,0.0006725255,0.00070887734,0.0003290183,0.00030582485,0.00011663148,0.02525264],"category_scores_gemma":[0.004425323,0.00016225078,0.00019058483,0.0018048945,0.00003870415,0.0009614254,0.000012515551,0.00012511032,0.0009265425],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000037213762,0.000042509666,0.00051013293,0.0000029625887,0.000016715243,3.2888373e-8,0.0065389206,0.012104154,0.0009759983,0.00044260567,0.0049649104,0.97436386],"study_design_scores_gemma":[0.0036857175,0.00017313921,0.017084036,0.000028352011,0.00021319283,0.000005516265,0.00175896,0.783249,0.0018582558,0.057034962,0.13459492,0.00031394477],"about_ca_topic_score_codex":0.0005207291,"about_ca_topic_score_gemma":0.0013140815,"teacher_disagreement_score":0.9740499,"about_ca_system_score_codex":0.0006330734,"about_ca_system_score_gemma":0.001744748,"threshold_uncertainty_score":0.99985135},"labels":[],"label_agreement":null},{"id":"W2055999462","doi":"10.1002/ev.392","title":"Internal evaluation a quarter‐century later: A conversation with Arnold J. Love","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Conversation; Evaluation methods; Perspective (graphical); Association (psychology); Quarter (Canadian coin); Sociology; Psychology; Computer science; Artificial intelligence; History; Engineering","score_opus":0.2229234698309984,"score_gpt":0.4641575431803136,"score_spread":0.2412340733493152,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2055999462","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.61333275,0.0005666818,0.20704406,0.0053913626,0.0062610894,0.009201546,0.000067196524,0.00030261884,0.15783268],"genre_scores_gemma":[0.99253136,0.000027642174,0.004075015,0.00021342552,0.00027601336,0.00032219975,0.00012036023,0.000022049002,0.0024119085],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99443346,0.0005540317,0.00078275666,0.000625641,0.0033279655,0.0002761534],"domain_scores_gemma":[0.9953578,0.00037370253,0.0005801295,0.00056597247,0.002970226,0.00015216986],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.009713752,0.00021947257,0.00024041109,0.00060544466,0.00021571667,0.00023610325,0.00030207416,0.000115095885,0.011453785],"category_scores_gemma":[0.0012237728,0.00016908732,0.0001462317,0.00087113434,0.000056685167,0.0012670308,0.000026319685,0.0001389492,0.0009926495],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0012857944,0.00048879173,0.025123613,0.00001552443,0.00024173464,5.3248465e-7,0.036016207,0.0030458206,0.0008227548,0.0018053349,0.038029734,0.89312416],"study_design_scores_gemma":[0.009654631,0.0013650034,0.17335416,0.00008060967,0.0007345598,0.000025682375,0.0073559466,0.7264556,0.0037076194,0.03146711,0.04524055,0.00055850414],"about_ca_topic_score_codex":0.00023813077,"about_ca_topic_score_gemma":0.0007713031,"teacher_disagreement_score":0.89256567,"about_ca_system_score_codex":0.00046724858,"about_ca_system_score_gemma":0.001016102,"threshold_uncertainty_score":0.9997852},"labels":[],"label_agreement":null},{"id":"W2066945648","doi":"10.1002/ev.315","title":"Knowledge translation: Implications for evaluation","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":49,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Institutes of Health Research","funders":"","keywords":"Knowledge translation; Computer science; Knowledge management; Field (mathematics); Translation (biology); Sustainability","score_opus":0.5370852562015647,"score_gpt":0.6053385358954619,"score_spread":0.06825327969389727,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2066945648","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.024740197,0.0029711446,0.79366606,0.05993576,0.0038833464,0.016123056,0.00019025935,0.00039587516,0.09809429],"genre_scores_gemma":[0.9679669,0.00005481833,0.020185275,0.00053759094,0.0011858479,0.0031942655,0.00073627225,0.000027752021,0.0061112624],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99596775,0.00040230624,0.0010093215,0.00069385814,0.0016165207,0.00031026494],"domain_scores_gemma":[0.9918218,0.0019598578,0.00039619816,0.000698105,0.0049603456,0.00016366558],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.015362313,0.00021100276,0.0002687965,0.0006156423,0.0009147095,0.00032058152,0.0003619714,0.00014666481,0.001384666],"category_scores_gemma":[0.004748076,0.00019146148,0.00030834554,0.001474273,0.000031131935,0.0009602904,0.000009574584,0.000090978116,0.00016938997],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000056097317,0.00014006207,0.00022805786,0.0000027104884,0.000017781096,2.254718e-9,0.00070717616,0.0040647923,0.00074062834,0.0083224485,0.04093262,0.9447876],"study_design_scores_gemma":[0.004305592,0.00031401013,0.05412227,0.0000121403355,0.00029988188,0.000002486509,0.00021093886,0.38600978,0.0005475296,0.2717831,0.28216392,0.00022834691],"about_ca_topic_score_codex":0.000008872319,"about_ca_topic_score_gemma":0.0003081708,"teacher_disagreement_score":0.9445593,"about_ca_system_score_codex":0.0004359178,"about_ca_system_score_gemma":0.001680196,"threshold_uncertainty_score":0.9995282},"labels":[],"label_agreement":null},{"id":"W2067449005","doi":"10.1002/ev.326","title":"Real‐time evaluation in humanitarian emergencies","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Formative assessment; Credibility; Evaluation methods; Program evaluation; Field (mathematics); Emergency management; Process management; Computer science; Monitoring and evaluation; Risk analysis (engineering); Political science; Business; Psychology; Engineering; Public administration","score_opus":0.23693256838441462,"score_gpt":0.526100458986193,"score_spread":0.28916789060177833,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2067449005","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9112027,0.000050018385,0.0015300631,0.0025357888,0.003859328,0.0028195567,0.00001626215,0.00010384174,0.077882424],"genre_scores_gemma":[0.9831802,0.000030389698,0.0045473333,0.00008297061,0.0006410053,0.0009268737,0.00018749232,0.000023607397,0.010380094],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99451846,0.00049047306,0.0009112529,0.0005576141,0.003247,0.00027518996],"domain_scores_gemma":[0.9958813,0.0007165552,0.00032337313,0.0006398227,0.0023267057,0.00011225195],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.024730623,0.00016679899,0.00022405921,0.0008356414,0.0004360841,0.000225463,0.0003461498,0.00014403694,0.02111136],"category_scores_gemma":[0.007601687,0.00014939815,0.00013542143,0.0013498528,0.000043472257,0.000930701,0.000033547472,0.0002025159,0.0009041436],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006143743,0.00024194793,0.00970449,0.0000038043308,0.000025481993,1.0096473e-7,0.0025765721,0.0067117508,0.03248901,0.006500219,0.037780326,0.90390486],"study_design_scores_gemma":[0.0022908766,0.00014543645,0.20895372,0.000010120862,0.00011585243,0.0000026306896,0.0006367078,0.61596113,0.0016407167,0.09887509,0.071104,0.00026370623],"about_ca_topic_score_codex":0.0005338709,"about_ca_topic_score_gemma":0.0154734785,"teacher_disagreement_score":0.90364116,"about_ca_system_score_codex":0.0002848825,"about_ca_system_score_gemma":0.0014098817,"threshold_uncertainty_score":0.99987376},"labels":[],"label_agreement":null},{"id":"W2070292114","doi":"10.1002/ev.393","title":"Internal evaluation, historically speaking","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":20,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Centre for Advancing Health Outcomes; University of British Columbia","funders":"","keywords":"Context (archaeology); Conservatism; Speculation; Government (linguistics); Function (biology); Evaluation methods; Internal validity; Political science; Public administration; Sociology; Public relations; Law; Business; History","score_opus":0.45966699897342617,"score_gpt":0.5336734648069541,"score_spread":0.07400646583352793,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2070292114","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08032717,0.0007554456,0.3456487,0.0020103045,0.011211468,0.004128161,0.000017113476,0.00026957344,0.55563205],"genre_scores_gemma":[0.96903145,0.000018073748,0.018405825,0.0002803786,0.00068460876,0.00054735306,0.00004555079,0.000023926166,0.0109628355],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9941237,0.0005208685,0.0008935625,0.0005509414,0.0036650556,0.00024587102],"domain_scores_gemma":[0.9943635,0.0005012636,0.00046783258,0.00057144905,0.003943295,0.00015264572],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.01812725,0.00016361428,0.00020834751,0.0005522726,0.00042340692,0.00016272158,0.00042750023,0.00008961283,0.020662947],"category_scores_gemma":[0.007109472,0.00013786968,0.00019061229,0.0009179246,0.000037804002,0.0008693263,0.000042727825,0.00011707883,0.00067341403],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006994035,0.000106494765,0.0047185146,0.0000017700013,0.000031867366,8.036219e-8,0.0028740086,0.00064233865,0.00019893453,0.0023458034,0.038618345,0.9503919],"study_design_scores_gemma":[0.0030791464,0.0003135103,0.116277464,0.000024925645,0.0003393954,0.000010531442,0.0007121035,0.36990052,0.0012607474,0.16594063,0.3417998,0.0003412171],"about_ca_topic_score_codex":0.00036491107,"about_ca_topic_score_gemma":0.00091064576,"teacher_disagreement_score":0.9500507,"about_ca_system_score_codex":0.00080362195,"about_ca_system_score_gemma":0.0012487881,"threshold_uncertainty_score":0.9802323},"labels":[],"label_agreement":null},{"id":"W2097486283","doi":"10.1002/ev.20045","title":"Rethinking the Relationship Among Monitoring, Evaluation, and Results‐Based Management: Observations From Canada","year":2013,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Public Policy and Administration Research","field":"Social Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Accountability; Government (linguistics); Monitoring and evaluation; Dual (grammatical number); Performance management; Process management; Political science; Public administration; Business; Marketing","score_opus":0.24376069185164306,"score_gpt":0.42792132003167843,"score_spread":0.18416062818003537,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2097486283","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.86054796,0.00031202947,0.0016132348,0.07823433,0.0012781528,0.0057560317,0.00009611248,0.00015190567,0.052010242],"genre_scores_gemma":[0.9912386,0.000025826253,0.0018381833,0.00016947713,0.0005342852,0.0013200359,0.00021693067,0.000009943825,0.004646721],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9971409,0.00068356504,0.00035661663,0.0002756471,0.0012934805,0.00024979538],"domain_scores_gemma":[0.99653506,0.0019205459,0.00018453006,0.00027258365,0.00093569484,0.00015161208],"candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.00461092,0.00009442323,0.00007597301,0.00011324773,0.0026547536,0.00041218888,0.00019330344,0.00008620907,0.00024613063],"category_scores_gemma":[0.0079408055,0.00008622287,0.00003714042,0.0006512862,0.00012630441,0.0005952404,0.000018750296,0.00013287873,0.0000056214894],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000067116125,0.00015025494,0.37690115,0.000029588273,0.00026608596,3.8599066e-7,0.02375833,0.00439118,0.000081811384,0.105173856,0.28259557,0.20658468],"study_design_scores_gemma":[0.00094387087,0.000011428653,0.8559817,0.00003602444,0.00010698946,6.6398975e-8,0.0022097528,0.022831788,0.00003714984,0.07689896,0.04083137,0.00011086718],"about_ca_topic_score_codex":0.7622146,"about_ca_topic_score_gemma":0.7844575,"teacher_disagreement_score":0.4790806,"about_ca_system_score_codex":0.0005385614,"about_ca_system_score_gemma":0.0026493918,"threshold_uncertainty_score":0.99864364},"labels":[],"label_agreement":null},{"id":"W2101260272","doi":"10.1002/ev.313","title":"Policy implementation: Implications for evaluation","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University; Douglas Mental Health University Institute","funders":"","keywords":"Accountability; Toolbox; Corporate governance; Context (archaeology); Process (computing); Government (linguistics); Sustainability; Political science; Public relations; Process management; Public administration; Sociology; Computer science; Business; Management; Economics","score_opus":0.3635462965884845,"score_gpt":0.6388161775626578,"score_spread":0.2752698809741734,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2101260272","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.07374496,0.00069677946,0.60759455,0.23746827,0.0037387987,0.025750782,0.00051988964,0.0005055534,0.049980417],"genre_scores_gemma":[0.97384626,0.000045642548,0.013234507,0.0018462659,0.0017148161,0.0038943223,0.0011391685,0.000026240645,0.0042527984],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9954602,0.00034872463,0.0010618655,0.00067296816,0.0020919237,0.00036428488],"domain_scores_gemma":[0.992761,0.0009590209,0.0005403029,0.0007275004,0.0048364247,0.00017572414],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0135816615,0.00020770733,0.00024875384,0.00087736116,0.00097587414,0.0003909091,0.00037745922,0.00010767638,0.0018362255],"category_scores_gemma":[0.005147405,0.00019040937,0.00026938223,0.0017724562,0.000027515933,0.0010022811,0.000017921118,0.000075096264,0.00011029302],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000040160245,0.00007711609,0.0007934293,0.0000017445924,0.000020099655,2.913662e-9,0.00038423736,0.002747438,0.0007377552,0.031873405,0.0545567,0.90876794],"study_design_scores_gemma":[0.005524888,0.00043213734,0.18522179,0.0000072370644,0.00026444555,0.0000036403565,0.00059999275,0.09917181,0.0010330003,0.56849855,0.13898616,0.0002563681],"about_ca_topic_score_codex":0.000066864566,"about_ca_topic_score_gemma":0.0005281173,"teacher_disagreement_score":0.9085115,"about_ca_system_score_codex":0.0007423776,"about_ca_system_score_gemma":0.003105188,"threshold_uncertainty_score":0.99907625},"labels":[],"label_agreement":null},{"id":"W2101892546","doi":"10.1002/ev.316","title":"Knowledge theories can inform evaluation practice: What can a complexity lens add?","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":60,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Competence (human resources); Fidelity; Knowledge management; Computer science; Process (computing); Context (archaeology); Complex adaptive system; Paradigm shift; Psychology; Epistemology; Artificial intelligence; Social psychology","score_opus":0.37554862024845537,"score_gpt":0.565178999495196,"score_spread":0.18963037924674064,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2101892546","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.20226106,0.0069217067,0.045617845,0.18890312,0.019849624,0.02303287,0.00030402813,0.0011128265,0.5119969],"genre_scores_gemma":[0.9877691,0.00027370782,0.0038435138,0.0023133783,0.0007249766,0.00058375765,0.0004694851,0.00002148346,0.004000544],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.99270236,0.0010355408,0.0010938247,0.00065177,0.0041124267,0.00040408046],"domain_scores_gemma":[0.98911935,0.001797782,0.0007161945,0.0008224178,0.0073528606,0.00019139524],"candidate_categories":["metaresearch","scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.023628587,0.00028440726,0.00035859365,0.00066575164,0.001089137,0.001187649,0.00041385926,0.00015745181,0.0032407104],"category_scores_gemma":[0.014891838,0.00024237475,0.00021619398,0.0017383805,0.00010671814,0.0042701135,0.000040687206,0.000217217,0.00023391297],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000118689684,0.00020985205,0.00020764252,0.0000036382814,0.00004033529,6.504539e-8,0.0074647125,0.0023225325,0.00009852554,0.023697067,0.02615753,0.9396794],"study_design_scores_gemma":[0.0042050118,0.00073179265,0.03972557,0.00007056997,0.0005773504,0.0000177249,0.013632263,0.29093575,0.0007459727,0.33321208,0.31564128,0.0005046349],"about_ca_topic_score_codex":0.00014934556,"about_ca_topic_score_gemma":0.0075420365,"teacher_disagreement_score":0.9391748,"about_ca_system_score_codex":0.0010515157,"about_ca_system_score_gemma":0.0035641717,"threshold_uncertainty_score":0.9998492},"labels":[],"label_agreement":null},{"id":"W2125410695","doi":"10.1002/ev.221","title":"The partnership of <i>new directions for evaluation</i> and the American evaluation association","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Association (psychology); General partnership; Political science; Computer science; Sociology; Psychology; Law","score_opus":0.21838504639813228,"score_gpt":0.5369309968417352,"score_spread":0.3185459504436029,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2125410695","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5878284,0.009630384,0.20287034,0.08705153,0.011363352,0.044238314,0.00014805535,0.00037526834,0.05649438],"genre_scores_gemma":[0.98035777,0.0004153797,0.0025899415,0.00049642933,0.0011904417,0.002527056,0.00014581822,0.000038955397,0.012238192],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.98942703,0.0018327346,0.0015968974,0.0006445365,0.006022083,0.00047668867],"domain_scores_gemma":[0.9735965,0.015857397,0.0020884972,0.0007377647,0.0075501776,0.000169655],"candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.121397234,0.00024084326,0.0004173595,0.00041389957,0.0018357547,0.00042475754,0.00040066402,0.000116550465,0.00028313257],"category_scores_gemma":[0.042096213,0.00015333366,0.00033392344,0.001967295,0.00023943264,0.00059771095,0.000043523178,0.00017574777,0.000026560183],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00062348205,0.000063570405,0.0040616393,0.0000042613074,0.00013628756,5.4737774e-9,0.0016692926,0.0026879767,0.00014231981,0.004862284,0.050348476,0.9354004],"study_design_scores_gemma":[0.012209879,0.00033359247,0.10805281,0.000024069535,0.0015555527,0.0000030442486,0.003718077,0.45060465,0.0014719767,0.137317,0.2844199,0.0002894347],"about_ca_topic_score_codex":0.0004474797,"about_ca_topic_score_gemma":0.0069186585,"teacher_disagreement_score":0.935111,"about_ca_system_score_codex":0.00096684886,"about_ca_system_score_gemma":0.002148149,"threshold_uncertainty_score":0.99946374},"labels":[],"label_agreement":null},{"id":"W2139159602","doi":"10.1002/ev.92","title":"PIE à la Mode: Mainstreaming Evaluation and Accountability in Each Program in Every County of a Statewide School Readiness Initiative","year":2003,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Accountability; Mandate; Mainstreaming; Legislature; Program evaluation; Plan (archaeology); Mainstream; Benchmarking; Quality (philosophy); Public administration; Political science; Public relations; Medical education; Psychology; Business; Pedagogy; Special education; Medicine","score_opus":0.2119641125322008,"score_gpt":0.5393907107995318,"score_spread":0.327426598267331,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2139159602","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.98210937,0.00018831757,0.0027732728,0.00019257599,0.00027478216,0.0034440882,0.000032928037,0.000027695702,0.0109569775],"genre_scores_gemma":[0.99452746,0.000055186498,0.0037799543,0.000038879563,0.000026755646,0.0013294251,0.00007134672,0.000011652381,0.00015934033],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.9943261,0.001667312,0.0010709392,0.0005440626,0.002153022,0.00023855153],"domain_scores_gemma":[0.9955288,0.0017365307,0.00041634138,0.0003542086,0.0018787419,0.00008536489],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.027279759,0.00016566992,0.00030932573,0.00076898735,0.00013090347,0.00014455822,0.00013214721,0.00012610806,0.0004491635],"category_scores_gemma":[0.013508805,0.00015063235,0.000061816616,0.0017149854,0.00007781114,0.0012546488,0.000022206565,0.0001723772,0.000006877861],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00021016084,0.0005914194,0.2649134,0.000027827922,0.00002262746,1.5727095e-7,0.0031826752,0.024247803,0.00031867274,0.0008954399,0.00047523575,0.7051146],"study_design_scores_gemma":[0.004210941,0.00014913382,0.5022712,0.00007911863,0.00007120272,0.0000026770376,0.0038223832,0.43560478,0.0006198055,0.050855793,0.0021303445,0.00018266654],"about_ca_topic_score_codex":0.00056548486,"about_ca_topic_score_gemma":0.006039717,"teacher_disagreement_score":0.7049319,"about_ca_system_score_codex":0.00076483964,"about_ca_system_score_gemma":0.0021745216,"threshold_uncertainty_score":0.9948008},"labels":[],"label_agreement":null},{"id":"W2140465263","doi":"10.1002/ev.240","title":"Going through the process: An examination of the operationalization of process use in empirical research on evaluation","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":74,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Operationalization; Process (computing); Empirical research; Empirical examination; Process management; Research methodology; Psychology; Management science; Computer science; Sociology; Epistemology; Business; Actuarial science; Engineering","score_opus":0.6748434947289444,"score_gpt":0.6733453144103931,"score_spread":0.001498180318551312,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2140465263","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9813769,0.000051532646,0.009700155,0.0011799309,0.0004178843,0.0033300424,0.000008975244,0.000014058818,0.0039205323],"genre_scores_gemma":[0.9985067,0.000007810736,0.00039470024,0.000110008106,0.00013540682,0.00037813673,0.000067022425,0.000013013911,0.00038717742],"study_design_codex":"simulation_or_modeling","study_design_gemma":"observational","domain_scores_codex":[0.9887614,0.002412768,0.0012260202,0.00047911072,0.0068864324,0.00023426254],"domain_scores_gemma":[0.98722726,0.0033135866,0.00056874315,0.0006070317,0.0082423845,0.00004099082],"candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04806614,0.00012549986,0.00018253624,0.00061130815,0.000539225,0.00014168727,0.00046469906,0.00012063411,0.00032320528],"category_scores_gemma":[0.017565576,0.000076621276,0.00008133943,0.003995198,0.00013870366,0.0015985633,0.000028361046,0.00020772747,0.0000091055845],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0003842263,0.0009781404,0.085231856,0.000043762768,0.00002950153,4.0312486e-8,0.030426534,0.55634403,0.0014380395,0.01084527,0.0014849453,0.31279367],"study_design_scores_gemma":[0.0012861461,0.00026660494,0.50764436,0.00007751615,0.000046251967,9.898677e-7,0.007140429,0.441682,0.00993726,0.030862842,0.0009655764,0.00009003848],"about_ca_topic_score_codex":0.00009816256,"about_ca_topic_score_gemma":0.003767385,"teacher_disagreement_score":0.42241248,"about_ca_system_score_codex":0.0004021439,"about_ca_system_score_gemma":0.0016397331,"threshold_uncertainty_score":0.9907099},"labels":[],"label_agreement":null},{"id":"W2278381810","doi":"10.1002/ev.20158","title":"Assessing the Practice Impact of Research on Evaluation","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Evaluation methods; Program evaluation; Psychology; Applied psychology; Computer science; Political science; Public administration; Reliability engineering; Engineering","score_opus":0.8085447410014557,"score_gpt":0.7537715169629834,"score_spread":0.05477322403847229,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2278381810","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.70269233,0.00078611786,0.021272914,0.018694196,0.003574813,0.00707513,0.000020962658,0.00009542397,0.2457881],"genre_scores_gemma":[0.99587554,0.000012628257,0.0016867296,0.00009992368,0.00042095894,0.0003986408,0.000037553007,0.000013541783,0.0014544608],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.98442227,0.004374872,0.00078656536,0.00044096666,0.009705986,0.0002693584],"domain_scores_gemma":[0.9732386,0.008652723,0.0005587357,0.0008078654,0.016597202,0.00014491197],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.13632615,0.00012462372,0.00018436185,0.00072531746,0.0005902221,0.0005644747,0.00040227128,0.00008864773,0.00096877385],"category_scores_gemma":[0.0782761,0.000075042764,0.00018163363,0.0025628554,0.000097575365,0.0017776948,0.00004383408,0.00024184523,0.0002397557],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":true,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00020844191,0.00026527254,0.0020550825,0.0000024553053,0.000068227884,5.3648638e-8,0.002552198,0.18361993,0.00024556243,0.0024258113,0.10867351,0.69988346],"study_design_scores_gemma":[0.0028332812,0.0010435717,0.062934116,0.000037927424,0.00020433175,0.0000069576154,0.010940687,0.8076981,0.00063449756,0.07064318,0.042884644,0.00013872793],"about_ca_topic_score_codex":0.00045540484,"about_ca_topic_score_gemma":0.00013619999,"teacher_disagreement_score":0.6997447,"about_ca_system_score_codex":0.0011479032,"about_ca_system_score_gemma":0.005921238,"threshold_uncertainty_score":0.99994445},"labels":[],"label_agreement":null},{"id":"W2512463988","doi":"10.1002/ev.20199","title":"Building Evaluation Capacity Through CLIPs: Communities of Learning, Inquiry, and Practice","year":2016,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"CLIPS; Context (archaeology); Work (physics); Community of practice; Medical education; Professional development; Higher education; Pedagogy; Knowledge management; Computer science; Sociology; Public relations; Psychology; Political science; Medicine; Engineering","score_opus":0.5953867694134086,"score_gpt":0.5760182223031797,"score_spread":0.01936854711022895,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2512463988","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7530027,0.0004714587,0.22322667,0.0063555953,0.0014649073,0.0018853132,0.000015053681,0.00007808533,0.013500206],"genre_scores_gemma":[0.97594464,0.00026138744,0.022255298,0.00011085566,0.00021743223,0.00025958437,0.000015032424,0.000013472286,0.0009223027],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9939329,0.00223033,0.00076347834,0.00031110697,0.0025807538,0.00018142596],"domain_scores_gemma":[0.98803496,0.0068095415,0.00074501685,0.00043368884,0.003912688,0.00006411213],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.027903566,0.00013928166,0.00023189605,0.00030110104,0.00051679177,0.00013040236,0.00020788885,0.0000971628,0.0010881282],"category_scores_gemma":[0.026964903,0.0000980987,0.00008998557,0.00054928235,0.00016458565,0.0020252913,0.000056985857,0.0001096805,0.000031152675],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00017888854,0.00011236208,0.008914937,0.000011895924,0.00006578926,1.6732297e-8,0.010674404,0.003148324,0.004252719,0.011055281,0.0052791378,0.9563062],"study_design_scores_gemma":[0.009569176,0.0011183295,0.06909404,0.000244745,0.0009758353,0.000032002987,0.027924566,0.19454497,0.0061150747,0.2463345,0.44347987,0.00056688866],"about_ca_topic_score_codex":0.00039491645,"about_ca_topic_score_gemma":0.0006101837,"teacher_disagreement_score":0.9557394,"about_ca_system_score_codex":0.00026677427,"about_ca_system_score_gemma":0.00057234656,"threshold_uncertainty_score":0.999825},"labels":[],"label_agreement":null},{"id":"W2625529206","doi":"10.1002/ev.20241","title":"Evaluation at the Time of Health Systems Reform: Chinese Policymakers’ Need for a Robust System of Evaluations to Assess Progress in the Implementation of Reform Efforts","year":2017,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Healthcare Systems and Reforms","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"China; Capacity building; Program evaluation; Political science; Evaluation methods; Economic growth; Public administration; Public relations; Economics; Engineering","score_opus":0.14607408309884093,"score_gpt":0.43423215831068135,"score_spread":0.28815807521184045,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2625529206","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.962664,0.0019512794,0.0045606378,0.0065225405,0.0014382325,0.019284472,0.00092048495,0.000024494091,0.0026338205],"genre_scores_gemma":[0.9955105,0.00002128464,0.00037954308,0.00002420306,0.00017230917,0.0034282254,0.00024797424,0.000025330417,0.00019063993],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.9966524,0.00029614265,0.001947334,0.00032667237,0.00048460934,0.00029284382],"domain_scores_gemma":[0.99499923,0.00008797005,0.0029108862,0.0008400692,0.0011017236,0.000060143637],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.024769206,0.00016702292,0.00059486006,0.00042844552,0.0007006835,0.00006283446,0.0003314779,0.00011161476,0.000022883101],"category_scores_gemma":[0.0005070747,0.00010676228,0.00019657398,0.00046025877,0.00005591646,0.00031384162,0.00004042106,0.000067184184,0.0000057996313],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00057368947,0.0009082225,0.15290752,0.007503154,0.001185186,1.2851307e-7,0.052474305,0.03321675,0.00028934403,0.10406182,0.0038292923,0.6430506],"study_design_scores_gemma":[0.004793185,0.0008884722,0.75811946,0.0004616552,0.00013366417,0.000008541157,0.011624772,0.22027045,0.00015638456,0.0020262098,0.0012679757,0.00024920775],"about_ca_topic_score_codex":0.047915492,"about_ca_topic_score_gemma":0.009729856,"teacher_disagreement_score":0.6428014,"about_ca_system_score_codex":0.0028173001,"about_ca_system_score_gemma":0.00070255925,"threshold_uncertainty_score":0.9584245},"labels":[],"label_agreement":null},{"id":"W2626410734","doi":"10.1002/ev.20240","title":"Conceptual Indicators Framework for Strengthening the Chinese Health System","year":2017,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Healthcare Systems and Reforms","field":"Economics, Econometrics and Finance","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"China; Government (linguistics); Plan (archaeology); Conceptual framework; Process (computing); Political science; Equity (law); Monitoring and evaluation; Process management; Public relations; Public administration; Economic growth; Business; Sociology; Computer science; Economics; Geography","score_opus":0.08227829248233864,"score_gpt":0.3703221274091979,"score_spread":0.28804383492685925,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2626410734","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.32011053,0.009922449,0.5962879,0.02621294,0.019021798,0.011516431,0.0010659646,0.00034316126,0.015518859],"genre_scores_gemma":[0.9940448,0.000044726876,0.0030394616,0.00010169698,0.0011693859,0.00073322013,0.00004297708,0.000025798767,0.00079797796],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.9987464,0.000034468445,0.00060551526,0.0002865982,0.000074086995,0.00025294643],"domain_scores_gemma":[0.9981787,0.00011952721,0.0009975175,0.0005143558,0.00008494388,0.00010495832],"candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.0029629408,0.00011491317,0.0002874711,0.00014326579,0.0021219992,0.00015234454,0.00022621849,0.00012767341,0.000027667791],"category_scores_gemma":[0.0010788966,0.00008184707,0.0001544978,0.00013479532,0.000051176634,0.00019001678,0.000019159763,0.00012038169,0.000031213338],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018114206,0.000025685467,0.014147032,0.00009780756,0.00007761869,2.4133955e-8,0.0032515603,0.00012230447,5.912313e-7,0.7355856,0.0013749453,0.24529877],"study_design_scores_gemma":[0.0041817334,0.0005494954,0.30936894,0.00036230878,0.000034254692,0.000008634839,0.0045315903,0.03009602,0.000027309987,0.13224919,0.5179996,0.0005909136],"about_ca_topic_score_codex":0.003054057,"about_ca_topic_score_gemma":0.00038605928,"teacher_disagreement_score":0.6739342,"about_ca_system_score_codex":0.00039035958,"about_ca_system_score_gemma":0.00020551591,"threshold_uncertainty_score":0.9991771},"labels":[],"label_agreement":null},{"id":"W2795416648","doi":"10.1002/ev.20297","title":"My Entry to the Field","year":2018,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Academic and Historical Perspectives in Psychology","field":"Psychology","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Field (mathematics); Permission; Advice (programming); Key (lock); Citizen journalism; Sociology; Work (physics); Public relations; Computer science; Psychology; Engineering ethics; Epistemology; World Wide Web; Political science; Computer security; Philosophy","score_opus":0.05542443802674325,"score_gpt":0.4559950890704666,"score_spread":0.40057065104372336,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2795416648","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.005242857,0.0008470888,0.16269067,0.034385603,0.015887912,0.0013341698,0.0000072393323,0.00013840132,0.77946603],"genre_scores_gemma":[0.8059088,0.000026536363,0.0028327813,0.015478813,0.012494417,0.0011585358,0.000016494281,0.000039572198,0.16204403],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99914116,0.00010625452,0.00014976917,0.0002758672,0.00015020695,0.00017676035],"domain_scores_gemma":[0.99906415,0.00034404,0.000049000424,0.0002963923,0.00018016002,0.00006625013],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0005441877,0.00007302469,0.000075302196,0.000088196895,0.0003379116,0.00000989862,0.00016058543,0.0001329547,0.005481534],"category_scores_gemma":[0.0007465925,0.00005794948,0.000061354374,0.00029656533,0.000049663457,0.000037686583,0.000016153874,0.00013785729,0.0014282598],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008229946,0.00003051938,0.00007727175,3.3559772e-7,0.000024862327,2.9977112e-8,0.0035402514,0.0000030614424,0.000038108778,0.015744818,0.7790779,0.20138057],"study_design_scores_gemma":[0.00055166456,0.00035862185,0.0042956346,0.000002735298,0.00006418767,0.0000046544105,0.00048364006,0.00014210117,0.00007192102,0.01437361,0.97957695,0.00007428972],"about_ca_topic_score_codex":0.00017831258,"about_ca_topic_score_gemma":0.00011852962,"teacher_disagreement_score":0.800666,"about_ca_system_score_codex":0.0001618345,"about_ca_system_score_gemma":0.0000460359,"threshold_uncertainty_score":0.99934924},"labels":[],"label_agreement":null},{"id":"W2891656379","doi":"10.1002/ev.20346","title":"The Evolving Market for Systematic Evaluation in Canada","year":2018,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"University of Ottawa","funders":"","keywords":"Accreditation; Professionalization; Context (archaeology); Supply and demand; Government (linguistics); Supply side; Business; Exploratory research; Affect (linguistics); Public relations; Marketing; Economics; Political science; Economic growth; Sociology; Social science","score_opus":0.1834456181021273,"score_gpt":0.4959395639226441,"score_spread":0.3124939458205168,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2891656379","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.41836742,0.0061579496,0.22318864,0.038059477,0.041391246,0.10270749,0.00021633905,0.00029810375,0.16961336],"genre_scores_gemma":[0.9877036,0.000012046026,0.0012804323,0.00018338757,0.00044263963,0.0039826515,0.000027533262,0.000015735643,0.0063519394],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9941405,0.0009443959,0.0010449042,0.00040011416,0.0031749427,0.00029515845],"domain_scores_gemma":[0.9890132,0.0055111474,0.00047506476,0.0005685717,0.004353782,0.00007822949],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.038302407,0.00013575707,0.00023300492,0.00025401052,0.0009370116,0.00031913872,0.00036372023,0.000050148443,0.0014594729],"category_scores_gemma":[0.025401084,0.00009128386,0.00009584875,0.0009488863,0.000034456723,0.0004608813,0.000023618535,0.00006540828,0.000040592287],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":true,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0002174891,0.00006305654,0.0070759673,0.00026505307,0.00008390297,4.7255096e-8,0.0014814213,0.0056862626,0.000134621,0.0017108815,0.42640573,0.5568756],"study_design_scores_gemma":[0.001457052,0.000084952786,0.027425444,0.0002014257,0.00010786698,0.0000013431214,0.0017519824,0.9301785,0.00014258845,0.020584507,0.017945861,0.00011843412],"about_ca_topic_score_codex":0.07362279,"about_ca_topic_score_gemma":0.9436849,"teacher_disagreement_score":0.9244923,"about_ca_system_score_codex":0.0021972624,"about_ca_system_score_gemma":0.0076341,"threshold_uncertainty_score":0.9994533},"labels":[],"label_agreement":null},{"id":"W2948770661","doi":"10.1002/ev.20365","title":"Sustainability‐Ready Evaluation: A Call to Action","year":2019,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Evaluation Society","funders":"","keywords":"Sustainability; Transformative learning; Sustainability science; Sustainability organizations; Relevance (law); Process management; Sustainable development; Social sustainability; Environmental resource management; Checklist; Natural resource; Action (physics); Business; Engineering ethics; Management science; Environmental planning; Computer science; Risk analysis (engineering); Political science; Sociology; Engineering; Psychology; Economics; Ecology","score_opus":0.30589920481363936,"score_gpt":0.5825784932667877,"score_spread":0.2766792884531483,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2948770661","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.83559984,0.00019866828,0.04872486,0.027987193,0.007995368,0.016761953,0.000041432497,0.00028262852,0.06240806],"genre_scores_gemma":[0.9581212,0.0000068356494,0.0020481322,0.00057507347,0.00049277797,0.0015924507,0.00014739559,0.000024503172,0.036991633],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.992732,0.00071975694,0.0008573946,0.0008293165,0.004516198,0.000345329],"domain_scores_gemma":[0.98970264,0.0010164471,0.00031406948,0.00095054065,0.0077840574,0.00023221648],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.021227662,0.00020381423,0.00028512196,0.0007400702,0.00038946548,0.00035308406,0.00035382668,0.0001286374,0.009956214],"category_scores_gemma":[0.009593405,0.00017861587,0.00017561366,0.0017560208,0.000022362945,0.0010064319,0.000053098385,0.00013104638,0.002598305],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00013637483,0.00011490639,0.0045897816,0.000010249631,0.000028245351,3.1901962e-8,0.001018908,0.048995893,0.00068390474,0.0018771377,0.0689676,0.87357694],"study_design_scores_gemma":[0.002839396,0.00045823984,0.063536875,0.000014235476,0.00017161571,0.0000039128554,0.0019603206,0.36249912,0.00089678785,0.043063417,0.52425504,0.00030105785],"about_ca_topic_score_codex":0.00020871444,"about_ca_topic_score_gemma":0.001072809,"teacher_disagreement_score":0.87327594,"about_ca_system_score_codex":0.001652014,"about_ca_system_score_gemma":0.002891407,"threshold_uncertainty_score":0.9987492},"labels":[],"label_agreement":null},{"id":"W3011218257","doi":"10.1002/ev.20394","title":"Advancing the Measurement of Collective Community Capacity and the Evaluation of Community Capacity‐Building Models","year":2020,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Community Health and Development","field":"Health Professions","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Capacity building; Work (physics); Collective efficacy; Community resilience; Adaptation (eye); Public relations; Psychological resilience; Political science; Sociology; Psychology; Computer science; Engineering; Social psychology; Social science","score_opus":0.4752172519940911,"score_gpt":0.47498330087062063,"score_spread":0.0002339511234704661,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3011218257","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9599781,0.00048658284,0.016542455,0.007754753,0.00039179798,0.008376143,0.000041092993,0.000051463423,0.006377591],"genre_scores_gemma":[0.99633545,0.000047428282,0.0015893213,0.0007917849,0.00006926483,0.001123621,0.000014962579,0.00001575344,0.000012399352],"study_design_codex":"qualitative","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9809496,0.016500684,0.0008573019,0.0001184586,0.0012948981,0.0002790875],"domain_scores_gemma":[0.98921716,0.005267253,0.00069101463,0.0005896988,0.0041026454,0.00013220498],"candidate_categories":["metaresearch","sts"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.045642577,0.00015818574,0.0003795667,0.00010619418,0.005662041,0.00001051449,0.00029539494,0.00011517223,0.0000438011],"category_scores_gemma":[0.012262949,0.000111998095,0.00009594806,0.0006313116,0.00022686261,0.00019818422,0.00017395901,0.0013452106,0.0000010219691],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0022137526,0.0005155972,0.0015266509,0.0014605618,0.00045868446,7.567798e-9,0.8345047,0.033413954,0.0034590345,0.01837504,0.009104199,0.09496778],"study_design_scores_gemma":[0.014389632,0.00038354174,0.030269763,0.00048412016,0.0011124352,0.000001043093,0.082623474,0.5918018,0.0019087227,0.27570206,0.0010844725,0.00023893475],"about_ca_topic_score_codex":0.016333412,"about_ca_topic_score_gemma":0.017728427,"teacher_disagreement_score":0.75188124,"about_ca_system_score_codex":0.0012284381,"about_ca_system_score_gemma":0.0027246187,"threshold_uncertainty_score":0.99605715},"labels":[],"label_agreement":null},{"id":"W3011548789","doi":"10.1002/ev.20398","title":"Finding the Impact: Methods for Assessing the Contribution of Collective Impact to Systems and Population Change in a Multi‐Site Study","year":2020,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Scale (ratio); Context (archaeology); Interim; Population; Work (physics); SPARK (programming language); Management science; Computer science; Political science; Sociology; Engineering","score_opus":0.5624400587745525,"score_gpt":0.6718887697007279,"score_spread":0.10944871092617536,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3011548789","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.77961814,0.00023250777,0.20941041,0.00176398,0.0003291203,0.008569719,0.000030907977,0.000015616508,0.000029599043],"genre_scores_gemma":[0.995693,0.0000040954865,0.002425569,0.00007293177,0.00013399834,0.001577101,0.00003713378,0.000009356584,0.000046765384],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9967269,0.0014154185,0.0006408828,0.00032235586,0.0007215775,0.00017288922],"domain_scores_gemma":[0.9949687,0.0031578713,0.00043921004,0.00021399057,0.0011433341,0.00007689281],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01809795,0.00012658749,0.00027751413,0.00029140335,0.0005374275,0.00034344377,0.00014670867,0.000052246614,0.000035973422],"category_scores_gemma":[0.009708745,0.000069077396,0.000115324605,0.0017297111,0.000016356826,0.0005876988,0.000034897246,0.000085463136,0.0000022493002],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00034007124,0.00019812894,0.5428273,0.00000989335,0.00010754883,3.0553412e-8,0.047929373,0.07795132,0.002071462,0.00016078605,0.00075978594,0.3276443],"study_design_scores_gemma":[0.001155353,0.00028007413,0.487921,0.000012244456,0.000057616973,5.09522e-7,0.0026199582,0.5073546,0.000052683212,0.00036689668,0.0001332614,0.000045796776],"about_ca_topic_score_codex":0.0023466907,"about_ca_topic_score_gemma":0.0012720851,"teacher_disagreement_score":0.4294033,"about_ca_system_score_codex":0.0005032026,"about_ca_system_score_gemma":0.00036571897,"threshold_uncertainty_score":0.9986329},"labels":[],"label_agreement":null},{"id":"W3196514161","doi":"10.1002/ev.20464","title":"Supporting systems transformation through design‐driven evaluation","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Complex Systems and Decision Making","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Economics Association","funders":"","keywords":"Cognitive reframing; Computer science; Process (computing); Process management; Service (business); New product development; Knowledge management; Product (mathematics); Value (mathematics); Management science; Engineering management; Systems engineering; Engineering; Business","score_opus":0.42191478588644576,"score_gpt":0.5201704823327796,"score_spread":0.09825569644633386,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3196514161","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.010233462,0.0022186814,0.96625334,0.00062078226,0.0037396408,0.002948854,0.000022269887,0.0001240497,0.013838938],"genre_scores_gemma":[0.9768347,0.000012797217,0.016855575,0.00007917305,0.00066682656,0.00067811174,0.0001014694,0.000030092822,0.00474123],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99135107,0.001594084,0.0016909359,0.0007548037,0.0042545306,0.00035458917],"domain_scores_gemma":[0.9906854,0.002388693,0.0007348344,0.00071908167,0.0053554503,0.00011651428],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.017348154,0.00020372534,0.00039035972,0.00036466954,0.0007646474,0.00091204233,0.00028273463,0.00014182508,0.0014474749],"category_scores_gemma":[0.013195943,0.00018159547,0.00028186853,0.0017125183,0.000022096547,0.0013873364,0.000036494286,0.000111970774,0.00024517966],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007972401,0.00010909398,0.00023838249,0.000020070831,0.00007523369,0.0000010753404,0.004907487,0.38189173,0.0038302476,0.016354566,0.08946788,0.5030245],"study_design_scores_gemma":[0.0016279753,0.00005986518,0.00089153857,0.00005080715,0.00016576493,0.000041339597,0.0026420555,0.80095154,0.0010913706,0.10128295,0.09100044,0.00019436495],"about_ca_topic_score_codex":0.00014090078,"about_ca_topic_score_gemma":0.00023791209,"teacher_disagreement_score":0.96660125,"about_ca_system_score_codex":0.00043873792,"about_ca_system_score_gemma":0.0012393594,"threshold_uncertainty_score":0.99946535},"labels":[],"label_agreement":null},{"id":"W3203262347","doi":"10.1002/ev.20477","title":"A learning agenda for the advocacy evaluation field's future","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Field (mathematics); Context (archaeology); Value (mathematics); Set (abstract data type); Political science; Public relations; Engineering ethics; Sociology; Computer science; Engineering","score_opus":0.25942037969970505,"score_gpt":0.5452853487151225,"score_spread":0.2858649690154174,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3203262347","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.10239966,0.0077450993,0.6805682,0.136775,0.02429066,0.013016364,0.000048213664,0.00033812915,0.034818664],"genre_scores_gemma":[0.93547463,0.00042165848,0.010638191,0.002486283,0.004643876,0.004459319,0.00051284523,0.000049191935,0.041314006],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99535906,0.0006964528,0.0006113049,0.0005176824,0.0025755751,0.0002399354],"domain_scores_gemma":[0.9898591,0.0045217625,0.0003291367,0.00054726814,0.0046718987,0.000070816175],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.016003728,0.00014982987,0.00018549108,0.00018687166,0.0012677602,0.00041476937,0.00026758373,0.000113987626,0.005554328],"category_scores_gemma":[0.01832935,0.00010710282,0.0002743444,0.0010305893,0.0000197179,0.00047915222,0.000037762144,0.00017051397,0.0001073433],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000048559996,0.00003679707,0.00043244715,0.0000037481054,0.000039840972,3.842633e-8,0.0012352517,0.018360065,0.00032065855,0.0006668158,0.0830245,0.8958313],"study_design_scores_gemma":[0.0016407135,0.00014156605,0.0068341214,0.000009000799,0.00025089007,0.00000393466,0.00379956,0.4461832,0.001436178,0.013257341,0.52633494,0.000108549044],"about_ca_topic_score_codex":0.000034869052,"about_ca_topic_score_gemma":0.0006329285,"teacher_disagreement_score":0.89572275,"about_ca_system_score_codex":0.00022823278,"about_ca_system_score_gemma":0.0025307469,"threshold_uncertainty_score":0.9953547},"labels":[],"label_agreement":null},{"id":"W3203797720","doi":"10.1002/ev.20478","title":"An introduction to policy advocacy evaluation: The concepts, history, and literature of the field","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Field (mathematics); Adaptation (eye); Policy development; Engineering ethics; Management science; Evaluation methods; Computer science; Political science; Public administration; Psychology","score_opus":0.12003077569204998,"score_gpt":0.5220412336486495,"score_spread":0.4020104579565995,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3203797720","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.38657135,0.0057557183,0.01649167,0.56118995,0.011273864,0.0056260647,0.000038413844,0.00008037855,0.012972602],"genre_scores_gemma":[0.9823692,0.000067471214,0.00159819,0.0032523458,0.0025276393,0.0003723172,0.00006203856,0.000012702308,0.00973811],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99612737,0.0009714849,0.0004700996,0.0004101978,0.0018944457,0.00012638325],"domain_scores_gemma":[0.9946122,0.00080834236,0.00025117016,0.0008498271,0.0033993244,0.00007914507],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0076503833,0.00010723218,0.00014317097,0.00025737032,0.00038443165,0.00016312761,0.0002743523,0.00007397093,0.0019677945],"category_scores_gemma":[0.013627946,0.00006515134,0.000100028024,0.0017048424,0.000052963635,0.00046496626,0.000046976867,0.00011954533,0.0000113371125],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005500391,0.00007916832,0.0010412021,0.0000062784065,0.000025519776,3.3810874e-8,0.009098953,0.006305797,0.0032933508,0.011303944,0.27106616,0.6977246],"study_design_scores_gemma":[0.0011572224,0.00022863958,0.039314326,0.000025219788,0.00016668053,0.000014332527,0.00226073,0.06779349,0.0061487043,0.022240108,0.86051416,0.00013640425],"about_ca_topic_score_codex":0.00007740122,"about_ca_topic_score_gemma":0.0004808959,"teacher_disagreement_score":0.6975882,"about_ca_system_score_codex":0.0003630283,"about_ca_system_score_gemma":0.0027962695,"threshold_uncertainty_score":0.9989445},"labels":[],"label_agreement":null},{"id":"W3204738611","doi":"10.1002/ev.20471","title":"Contribution analysis: A promising method for assessing advocacy's impact","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Context (archaeology); Policy analysis; Outcome (game theory); Political science; Public relations; Management science; Public administration; Economics","score_opus":0.24016093895790824,"score_gpt":0.6095421407491777,"score_spread":0.36938120179126943,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3204738611","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.059366457,0.00034170324,0.9338968,0.0025913422,0.00077582605,0.0015571958,0.0000421319,0.00007009049,0.0013584786],"genre_scores_gemma":[0.8494359,0.000016447486,0.14572912,0.0002562421,0.00048451117,0.0007995724,0.00085433986,0.000021228187,0.0024025908],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9956559,0.00080566993,0.0008594065,0.00064597046,0.0017292602,0.00030377644],"domain_scores_gemma":[0.98983455,0.0025923094,0.0005245784,0.00051565835,0.0063903397,0.0001425436],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.017453302,0.00017277052,0.0004162315,0.0006952653,0.0007474457,0.0009706353,0.00017180202,0.00011597277,0.0015171642],"category_scores_gemma":[0.015724326,0.00014396434,0.0006780688,0.003289677,0.000016662572,0.0011123528,0.000026934364,0.00009355635,0.000027413405],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011985172,0.0001849386,0.008311507,0.000008261439,0.0007409541,1.8543366e-7,0.00070979516,0.082318075,0.009030892,0.0011636168,0.0124599105,0.884952],"study_design_scores_gemma":[0.0024917761,0.00009078541,0.06795547,0.000016867394,0.0018378923,0.000005528679,0.0006834483,0.86821884,0.006704398,0.025034849,0.026769836,0.00019032702],"about_ca_topic_score_codex":0.000070714785,"about_ca_topic_score_gemma":0.0004935118,"teacher_disagreement_score":0.8847617,"about_ca_system_score_codex":0.0005745921,"about_ca_system_score_gemma":0.0030782877,"threshold_uncertainty_score":0.9993956},"labels":[],"label_agreement":null},{"id":"W4213266427","doi":"10.1002/ev.20337","title":"Issue Information","year":2019,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Computer science; Citation; Action (physics); World Wide Web; Information retrieval","score_opus":0.20791329578082282,"score_gpt":0.5385400636795997,"score_spread":0.3306267678987769,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4213266427","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00024144747,0.0009448001,0.07355205,0.0045913914,0.06122267,0.0054220525,0.00044038362,0.00009016799,0.85349506],"genre_scores_gemma":[0.0024065133,0.00029006903,0.0018545921,0.000993277,0.0023534442,0.0008362252,0.0044923048,0.000036184305,0.9867374],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.993071,0.00040294646,0.0014343439,0.000562716,0.0042070136,0.00032198595],"domain_scores_gemma":[0.9927869,0.0010831314,0.0011773505,0.0010132203,0.0037969283,0.00014246858],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.008838729,0.0003340359,0.00050516607,0.0012814364,0.00041582587,0.00085002056,0.0006003324,0.0004449764,0.11123613],"category_scores_gemma":[0.003589746,0.0002840584,0.0003441547,0.001224117,0.000029955749,0.0021803929,0.000066058856,0.00029133208,0.26169512],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002342681,0.000016409407,0.000013124994,0.000014773145,0.000025489455,3.1654679e-9,0.0002860884,0.009440855,0.0000067757483,0.00011850827,0.67054886,0.31950566],"study_design_scores_gemma":[0.0014609336,0.00014032953,0.00039785934,0.000043049637,0.00012222133,0.0000014910187,0.00031785396,0.07297323,0.00013815651,0.0024111066,0.92169315,0.0003006422],"about_ca_topic_score_codex":0.00013313135,"about_ca_topic_score_gemma":0.00012014309,"teacher_disagreement_score":0.31920502,"about_ca_system_score_codex":0.0005915871,"about_ca_system_score_gemma":0.0034036806,"threshold_uncertainty_score":0.99996114},"labels":[],"label_agreement":null},{"id":"W4214553715","doi":"10.1002/ev.20486","title":"How does teaching with cases support the development of evaluation competencies?","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Adaptability; Craft; Evaluation methods; Professional development; Medical education; Psychology; Computer science; Knowledge management; Pedagogy; Engineering; Medicine; Management","score_opus":0.2469212988547501,"score_gpt":0.49329140127841276,"score_spread":0.24637010242366267,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4214553715","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8607902,0.0002918822,0.1043453,0.013260816,0.0018444513,0.002941698,0.000031070762,0.00007475683,0.016419848],"genre_scores_gemma":[0.9648371,0.0000063739,0.02707852,0.00011468968,0.00015439397,0.0004217245,0.00014619323,0.000012248909,0.0072287996],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99401885,0.0007884622,0.0006669287,0.00041772588,0.0039183707,0.00018963215],"domain_scores_gemma":[0.99317056,0.0015887308,0.0005124198,0.0005450607,0.0041137454,0.00006948125],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.017232427,0.00014608008,0.00021787168,0.00021609459,0.0008875165,0.00035044368,0.00020652989,0.000052883664,0.0015876264],"category_scores_gemma":[0.006729027,0.00007824304,0.00010254022,0.00071639643,0.000057343226,0.0006435508,0.000043081756,0.00012007604,0.000020273525],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000057560286,0.00014328444,0.0049280836,0.000011571336,0.000092286326,3.2130592e-7,0.00588848,0.003547771,0.0024132817,0.0015831402,0.005159792,0.9761744],"study_design_scores_gemma":[0.005898815,0.0005534716,0.13618669,0.00012202798,0.00095734297,0.00009666706,0.042952344,0.22881223,0.0636058,0.019312836,0.500869,0.00063278864],"about_ca_topic_score_codex":0.000026136799,"about_ca_topic_score_gemma":0.0042983885,"teacher_disagreement_score":0.97554165,"about_ca_system_score_codex":0.00026127408,"about_ca_system_score_gemma":0.004420108,"threshold_uncertainty_score":0.99932504},"labels":[],"label_agreement":null},{"id":"W4224927296","doi":"10.1002/ev.20494","title":"Evaluation policy and organizational evaluation capacity building: A study of international aid agency evaluation policies","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":11,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Agency (philosophy); Pace; Evaluation methods; Program evaluation; Policy analysis; Capacity building; Public relations; Management science; Knowledge management; Process management; Business; Political science; Computer science; Public administration; Economics; Sociology; Economic growth","score_opus":0.3393749749893569,"score_gpt":0.529793832087028,"score_spread":0.1904188570976711,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4224927296","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9749468,0.00039913933,0.003908445,0.0026099854,0.0025698238,0.008554827,0.00012835817,0.00007449843,0.006808142],"genre_scores_gemma":[0.9930107,0.000029605768,0.0014496872,0.00015240994,0.000669078,0.0036071287,0.0004806008,0.00003867874,0.0005620984],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9721263,0.0048717037,0.0018619674,0.0011220168,0.019641984,0.00037601858],"domain_scores_gemma":[0.9817331,0.0011339502,0.0014060073,0.000785499,0.014772064,0.00016936866],"candidate_categories":["metaresearch","metaepi_narrow","sts","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.0702406,0.00033194732,0.00042626722,0.0023284494,0.0015243079,0.000310774,0.00065656146,0.00010993042,0.011292953],"category_scores_gemma":[0.027306203,0.00033553416,0.0001647515,0.0035454251,0.0000897113,0.0012023353,0.0002698326,0.00025253746,0.000020236392],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":true,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00021945807,0.0016786032,0.028784622,0.000010364674,0.00036884355,7.767655e-8,0.017744599,0.2687704,0.0035334576,0.0058064116,0.0047651427,0.66831803],"study_design_scores_gemma":[0.00819758,0.00071289035,0.14001735,0.000010782363,0.0010982841,0.000012913982,0.005615712,0.75866777,0.00041943265,0.07785241,0.0070916153,0.00030326084],"about_ca_topic_score_codex":0.001098213,"about_ca_topic_score_gemma":0.001091184,"teacher_disagreement_score":0.66801476,"about_ca_system_score_codex":0.0031241442,"about_ca_system_score_gemma":0.005966497,"threshold_uncertainty_score":0.99990964},"labels":[],"label_agreement":null},{"id":"W4233585958","doi":"10.1002/ev.20355","title":"Issue Information","year":2020,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of California, Los Angeles; Sierra Health Foundation; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; University of Hawai'i; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Library science; Information retrieval","score_opus":0.07860107879638363,"score_gpt":0.28719641824578224,"score_spread":0.20859533944939862,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4233585958","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.000009646113,0.0028237663,0.00652585,0.0012569176,0.03621064,0.0011314148,0.002868951,0.000050219813,0.9491226],"genre_scores_gemma":[0.00018999136,0.00076780125,0.00079961796,0.00043220044,0.0015378075,0.00034258296,0.0023548254,0.000023437397,0.99355173],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99864435,0.000011857477,0.00067229837,0.00039962115,0.00006688435,0.00020499474],"domain_scores_gemma":[0.9988837,0.000043554148,0.0005963282,0.00024197648,0.00014733501,0.00008706711],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.00066747615,0.00019208831,0.0004149348,0.00036807283,0.00026816482,0.00021280574,0.00016617011,0.00019057337,0.08283056],"category_scores_gemma":[0.00039825562,0.00024622702,0.00021065566,0.00027255915,0.000029171433,0.0006325591,0.000044695043,0.000120976314,0.46441674],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000100393745,0.000010535739,0.0000043853706,0.000037077407,0.00009833122,7.632642e-9,0.0005881328,0.00046343938,2.6469763e-7,0.0030297223,0.9836786,0.012079472],"study_design_scores_gemma":[0.0008011677,0.000035765268,0.000031805943,0.000014074234,0.000053250904,3.0936926e-7,0.00017494825,0.007056733,0.0000120409195,0.0025477374,0.98902494,0.00024721114],"about_ca_topic_score_codex":0.00028039125,"about_ca_topic_score_gemma":0.000014805989,"teacher_disagreement_score":0.3815862,"about_ca_system_score_codex":0.000362023,"about_ca_system_score_gemma":0.00014514753,"threshold_uncertainty_score":0.999999},"labels":[],"label_agreement":null},{"id":"W4234990433","doi":"10.1002/ev.20161","title":"Issue Information","year":2016,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Human auditory perception and evaluation","field":"Engineering","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of Colorado Denver; University of Illinois at Urbana-Champaign; Centers for Disease Control and Prevention; Sierra Health Foundation; Claremont Graduate University; College of Engineering, Michigan State University; Northern Arizona University; Syracuse University; Vanderbilt University; University of Toronto; RMIT University; Weizmann Institute of Science; Michigan State University; Syddansk Universitet; International Development Research Centre; University of Hawai'i at Mānoa; University of Hawai'i; University of Minnesota; University of Ottawa; George Mason University","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.033987135490722076,"score_gpt":0.3297103506522229,"score_spread":0.29572321516150085,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4234990433","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00021793673,0.0010391477,0.1987812,0.0005239742,0.06783925,0.0034374201,0.00041764133,0.0008707561,0.7268727],"genre_scores_gemma":[0.0009712428,0.0018363975,0.001656987,0.00022172894,0.014958772,0.0022752525,0.009377478,0.00016935098,0.9685328],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9983966,0.0000739414,0.00050431676,0.00020268586,0.00057375804,0.00024870806],"domain_scores_gemma":[0.99884266,0.00007446202,0.00014422213,0.00031421694,0.0005163029,0.00010811861],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.00065456977,0.00028824076,0.00024300213,0.0005189231,0.00021247493,0.00012955489,0.00012454655,0.00041307538,0.21477991],"category_scores_gemma":[0.00016653635,0.00028157828,0.00016356933,0.00020498091,0.000020118925,0.0008296751,0.000010025805,0.0001663238,0.29517102],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000005945664,0.0000045067704,1.7505643e-7,0.000071408016,0.000036742636,3.329478e-9,0.00019618907,0.00677589,0.00016945964,0.0000042474912,0.725366,0.26736942],"study_design_scores_gemma":[0.0010787003,0.000036117966,0.00007329287,0.00009603649,0.00015501033,0.0000011107754,0.000029384091,0.039931588,0.0002957267,0.0002761197,0.9577045,0.00032244922],"about_ca_topic_score_codex":0.0000041533895,"about_ca_topic_score_gemma":0.00002717474,"teacher_disagreement_score":0.26704696,"about_ca_system_score_codex":0.00087884825,"about_ca_system_score_gemma":0.00027532427,"threshold_uncertainty_score":0.99996364},"labels":[],"label_agreement":null},{"id":"W4236527672","doi":"10.1002/ev.20273","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4236527672","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00003435367,0.002113493,0.004931094,0.0003169737,0.05299304,0.0010317265,0.0023091952,0.00004083091,0.9362293],"genre_scores_gemma":[0.00012845024,0.00052843284,0.0008068433,0.0001820641,0.0020895111,0.00031783918,0.0017247224,0.000020879159,0.99420124],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986079,0.000012206888,0.0006784026,0.00039626574,0.00006746652,0.0002378011],"domain_scores_gemma":[0.9986594,0.00004275919,0.0006441558,0.0003298316,0.00025697926,0.00006690674],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0011527749,0.00019401344,0.00037817724,0.0005338805,0.00035519595,0.00022449912,0.00017483844,0.00022727632,0.14612591],"category_scores_gemma":[0.0003479774,0.00024239076,0.00019350799,0.0002466833,0.000054079497,0.0007033957,0.000043824253,0.00009012481,0.5442282],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000091665115,0.000013623907,0.000005492795,0.000025715992,0.00009306354,3.4559478e-9,0.00047206949,0.00013927101,1.7210422e-7,0.0019822125,0.9877389,0.0095203025],"study_design_scores_gemma":[0.0007716732,0.000044982666,0.000045223944,0.000019227788,0.000049987513,4.177001e-7,0.00013748209,0.0037127587,0.000015945328,0.0037563234,0.9911927,0.0002532647],"about_ca_topic_score_codex":0.00038198577,"about_ca_topic_score_gemma":0.00003390296,"teacher_disagreement_score":0.39810228,"about_ca_system_score_codex":0.0004358186,"about_ca_system_score_gemma":0.00014183887,"threshold_uncertainty_score":0.9884408},"labels":[],"label_agreement":null},{"id":"W4236772578","doi":"10.1002/ev.239","title":"Editor's notes","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Citation; Computer science; Library science; Sociology","score_opus":0.2516694502734112,"score_gpt":0.5602664481446863,"score_spread":0.3085969978712751,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4236772578","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09408283,0.0003643102,0.78037757,0.008008094,0.03957959,0.0023888242,0.000019443743,0.00026195502,0.0749174],"genre_scores_gemma":[0.9634696,0.000016384964,0.012544029,0.00044022384,0.0112263,0.00013915423,0.000061683124,0.000018947874,0.012083666],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99644333,0.00011313596,0.0005797027,0.00033679596,0.002305403,0.00022162122],"domain_scores_gemma":[0.99552995,0.0022052699,0.00021769461,0.00037464246,0.0015481065,0.00012431819],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.017940685,0.00010065,0.00013062023,0.0004196514,0.00038106207,0.00017164854,0.00021068913,0.00008040529,0.002860931],"category_scores_gemma":[0.0075505897,0.00008242649,0.000120734585,0.000900167,0.000026114512,0.0005144787,0.000019891539,0.00007535439,0.0005750006],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000037853293,0.000046949368,0.0036393807,0.0000011029562,0.000010891697,3.572027e-8,0.00033746337,0.0011076169,0.000579137,0.0014145261,0.15451081,0.83831424],"study_design_scores_gemma":[0.00127118,0.00011762092,0.06442342,0.000006218946,0.0000543384,0.0000020340801,0.0003828021,0.042939857,0.0039025382,0.04095733,0.8457961,0.00014657092],"about_ca_topic_score_codex":0.000059401827,"about_ca_topic_score_gemma":0.000854121,"teacher_disagreement_score":0.8693868,"about_ca_system_score_codex":0.00018842024,"about_ca_system_score_gemma":0.00048747126,"threshold_uncertainty_score":0.9980506},"labels":[],"label_agreement":null},{"id":"W4238213994","doi":"10.1002/ev.20356","title":"Issue Information","year":2020,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of California, Los Angeles; Sierra Health Foundation; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; University of Hawai'i; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Library science; Internet privacy","score_opus":0.2442056670308427,"score_gpt":0.5390197065552901,"score_spread":0.2948140395244474,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4238213994","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00009861552,0.0010203358,0.11267402,0.01985888,0.03588432,0.004990023,0.00058292935,0.00015181213,0.82473904],"genre_scores_gemma":[0.0034495215,0.000627753,0.0060588527,0.00404276,0.0067661125,0.0018334531,0.0090071885,0.00006886006,0.9681455],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9931979,0.0004119988,0.0014589432,0.0005714062,0.004073548,0.00028621446],"domain_scores_gemma":[0.9940382,0.00083759805,0.0010794112,0.0006929029,0.0031222596,0.00022961055],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0058329254,0.00032999142,0.0004939335,0.000866345,0.0004993931,0.0008652741,0.00059518043,0.00036866695,0.08940218],"category_scores_gemma":[0.005661698,0.00028665064,0.00033624246,0.0015057054,0.00003406556,0.0018798627,0.00007165631,0.00031630576,0.14825784],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027093245,0.000012651171,0.000005128698,0.000012311127,0.000025584626,8.414348e-9,0.0004528151,0.0030915835,0.0000079922365,0.00011288971,0.65496,0.34129193],"study_design_scores_gemma":[0.0012395603,0.00015144008,0.000208353,0.0000291444,0.00015472097,0.0000014067589,0.0003539149,0.081128776,0.00018378737,0.0032324344,0.9130333,0.00028315024],"about_ca_topic_score_codex":0.000096071664,"about_ca_topic_score_gemma":0.0001028896,"teacher_disagreement_score":0.34100878,"about_ca_system_score_codex":0.00045925903,"about_ca_system_score_gemma":0.0030333088,"threshold_uncertainty_score":0.9999586},"labels":[],"label_agreement":null},{"id":"W4238813095","doi":"10.1002/ev.20357","title":"Issue Information","year":2020,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of California, Los Angeles; Sierra Health Foundation; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; University of Hawai'i; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.2442056670308427,"score_gpt":0.5390197065552901,"score_spread":0.2948140395244474,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4238813095","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00009861552,0.0010203358,0.11267402,0.01985888,0.03588432,0.004990023,0.00058292935,0.00015181213,0.82473904],"genre_scores_gemma":[0.0034495215,0.000627753,0.0060588527,0.00404276,0.0067661125,0.0018334531,0.0090071885,0.00006886006,0.9681455],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9931979,0.0004119988,0.0014589432,0.0005714062,0.004073548,0.00028621446],"domain_scores_gemma":[0.9940382,0.00083759805,0.0010794112,0.0006929029,0.0031222596,0.00022961055],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0058329254,0.00032999142,0.0004939335,0.000866345,0.0004993931,0.0008652741,0.00059518043,0.00036866695,0.08940218],"category_scores_gemma":[0.005661698,0.00028665064,0.00033624246,0.0015057054,0.00003406556,0.0018798627,0.00007165631,0.00031630576,0.14825784],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000027093245,0.000012651171,0.000005128698,0.000012311127,0.000025584626,8.414348e-9,0.0004528151,0.0030915835,0.0000079922365,0.00011288971,0.65496,0.34129193],"study_design_scores_gemma":[0.0012395603,0.00015144008,0.000208353,0.0000291444,0.00015472097,0.0000014067589,0.0003539149,0.081128776,0.00018378737,0.0032324344,0.9130333,0.00028315024],"about_ca_topic_score_codex":0.000096071664,"about_ca_topic_score_gemma":0.0001028896,"teacher_disagreement_score":0.34100878,"about_ca_system_score_codex":0.00045925903,"about_ca_system_score_gemma":0.0030333088,"threshold_uncertainty_score":0.9999586},"labels":[],"label_agreement":null},{"id":"W4242601229","doi":"10.1002/ev.20107","title":"Editors' Notes","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Accreditation; Charter; Certification; Context (archaeology); Credentialing; Conversation; Public relations; Professional association; Scholarship; Task (project management); Computer science; Political science; Engineering ethics; Management; Psychology; Law; History","score_opus":0.41995704850429344,"score_gpt":0.5640301890940794,"score_spread":0.144073140589786,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4242601229","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.17682941,0.0010381868,0.47109112,0.05104867,0.08142015,0.005859904,0.00006818633,0.000688639,0.21195573],"genre_scores_gemma":[0.95684934,0.000011709371,0.012923772,0.0004691032,0.009742195,0.00046844405,0.00010362746,0.00002372671,0.019408057],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9962952,0.00022496503,0.00045517617,0.00031712363,0.0025472993,0.00016022494],"domain_scores_gemma":[0.99589014,0.00096759136,0.00018980424,0.00038311307,0.0023793513,0.00019001601],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.01075822,0.00009687734,0.00013681214,0.00030286048,0.00022453218,0.0002173573,0.00021595767,0.000068402136,0.0015037485],"category_scores_gemma":[0.013407272,0.00007723775,0.00009613941,0.0007739752,0.000023729006,0.0006379562,0.000024785337,0.00006319691,0.0009479864],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003273582,0.000049701703,0.0029438608,8.349187e-7,0.0000110058745,2.6604063e-8,0.0007136293,0.005496763,0.000106022315,0.0010438847,0.53061336,0.45898816],"study_design_scores_gemma":[0.0016101375,0.00014512871,0.0070351595,0.000004504304,0.00004858347,0.0000019060271,0.0004888114,0.1392865,0.0009171578,0.07170197,0.7786421,0.00011800618],"about_ca_topic_score_codex":0.00008553909,"about_ca_topic_score_gemma":0.00024312778,"teacher_disagreement_score":0.78001994,"about_ca_system_score_codex":0.00021809063,"about_ca_system_score_gemma":0.001232757,"threshold_uncertainty_score":0.9998299},"labels":[],"label_agreement":null},{"id":"W4245992123","doi":"10.1002/ev.20367","title":"Issue Information","year":2021,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.2232214289072087,"score_gpt":0.5437897844404399,"score_spread":0.3205683555332312,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4245992123","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00028106972,0.002638979,0.0847837,0.0071842866,0.04907193,0.0036923827,0.00045597472,0.00009903762,0.85179263],"genre_scores_gemma":[0.0012094196,0.000647451,0.004061347,0.0012194306,0.0033115642,0.0011455831,0.008266319,0.00003742157,0.98010147],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99292034,0.0005301491,0.0014697207,0.0005822469,0.0041873017,0.00031024034],"domain_scores_gemma":[0.99143666,0.0009826337,0.0010077797,0.00093942584,0.005464302,0.00016918452],"candidate_categories":["metaepi_narrow","scholarly_communication","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.008122634,0.0003200991,0.0005006663,0.001027674,0.00055630616,0.0011104688,0.00048909563,0.00042427346,0.15065265],"category_scores_gemma":[0.005698301,0.00028436226,0.00037249477,0.0015982938,0.00003311803,0.0018537543,0.000074996904,0.00029302324,0.0650481],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012799608,0.00002062087,0.000006590805,0.000011599469,0.000029005947,1.4051583e-8,0.00030537436,0.0036494315,0.000009650434,0.00010443917,0.64091533,0.35493514],"study_design_scores_gemma":[0.0012239398,0.0000845906,0.00031476395,0.000054610904,0.00016399393,0.0000033413785,0.0005944876,0.037765533,0.000340976,0.002380333,0.95677525,0.00029817264],"about_ca_topic_score_codex":0.00013113157,"about_ca_topic_score_gemma":0.00026737803,"teacher_disagreement_score":0.35463697,"about_ca_system_score_codex":0.0005672496,"about_ca_system_score_gemma":0.004462644,"threshold_uncertainty_score":0.99996084},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":["insufficient_payload"],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"opus","categories":["insufficient_payload"],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"},{"id":"W4247121542","doi":"10.1002/ev.20279","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4247121542","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00003435367,0.002113493,0.004931094,0.0003169737,0.05299304,0.0010317265,0.0023091952,0.00004083091,0.9362293],"genre_scores_gemma":[0.00012845024,0.00052843284,0.0008068433,0.0001820641,0.0020895111,0.00031783918,0.0017247224,0.000020879159,0.99420124],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986079,0.000012206888,0.0006784026,0.00039626574,0.00006746652,0.0002378011],"domain_scores_gemma":[0.9986594,0.00004275919,0.0006441558,0.0003298316,0.00025697926,0.00006690674],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0011527749,0.00019401344,0.00037817724,0.0005338805,0.00035519595,0.00022449912,0.00017483844,0.00022727632,0.14612591],"category_scores_gemma":[0.0003479774,0.00024239076,0.00019350799,0.0002466833,0.000054079497,0.0007033957,0.000043824253,0.00009012481,0.5442282],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000091665115,0.000013623907,0.000005492795,0.000025715992,0.00009306354,3.4559478e-9,0.00047206949,0.00013927101,1.7210422e-7,0.0019822125,0.9877389,0.0095203025],"study_design_scores_gemma":[0.0007716732,0.000044982666,0.000045223944,0.000019227788,0.000049987513,4.177001e-7,0.00013748209,0.0037127587,0.000015945328,0.0037563234,0.9911927,0.0002532647],"about_ca_topic_score_codex":0.00038198577,"about_ca_topic_score_gemma":0.00003390296,"teacher_disagreement_score":0.39810228,"about_ca_system_score_codex":0.0004358186,"about_ca_system_score_gemma":0.00014183887,"threshold_uncertainty_score":0.9884408},"labels":[],"label_agreement":null},{"id":"W4249000756","doi":"10.1002/ev.220","title":"Editor's notes","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Citation; Library science; Computer science; World Wide Web","score_opus":0.2516694502734112,"score_gpt":0.5602664481446863,"score_spread":0.3085969978712751,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4249000756","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.09408283,0.0003643102,0.78037757,0.008008094,0.03957959,0.0023888242,0.000019443743,0.00026195502,0.0749174],"genre_scores_gemma":[0.9634696,0.000016384964,0.012544029,0.00044022384,0.0112263,0.00013915423,0.000061683124,0.000018947874,0.012083666],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99644333,0.00011313596,0.0005797027,0.00033679596,0.002305403,0.00022162122],"domain_scores_gemma":[0.99552995,0.0022052699,0.00021769461,0.00037464246,0.0015481065,0.00012431819],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.017940685,0.00010065,0.00013062023,0.0004196514,0.00038106207,0.00017164854,0.00021068913,0.00008040529,0.002860931],"category_scores_gemma":[0.0075505897,0.00008242649,0.000120734585,0.000900167,0.000026114512,0.0005144787,0.000019891539,0.00007535439,0.0005750006],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000037853293,0.000046949368,0.0036393807,0.0000011029562,0.000010891697,3.572027e-8,0.00033746337,0.0011076169,0.000579137,0.0014145261,0.15451081,0.83831424],"study_design_scores_gemma":[0.00127118,0.00011762092,0.06442342,0.000006218946,0.0000543384,0.0000020340801,0.0003828021,0.042939857,0.0039025382,0.04095733,0.8457961,0.00014657092],"about_ca_topic_score_codex":0.000059401827,"about_ca_topic_score_gemma":0.000854121,"teacher_disagreement_score":0.8693868,"about_ca_system_score_codex":0.00018842024,"about_ca_system_score_gemma":0.00048747126,"threshold_uncertainty_score":0.9980506},"labels":[],"label_agreement":null},{"id":"W4249389693","doi":"10.1002/ev.20282","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4249389693","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00003435367,0.002113493,0.004931094,0.0003169737,0.05299304,0.0010317265,0.0023091952,0.00004083091,0.9362293],"genre_scores_gemma":[0.00012845024,0.00052843284,0.0008068433,0.0001820641,0.0020895111,0.00031783918,0.0017247224,0.000020879159,0.99420124],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986079,0.000012206888,0.0006784026,0.00039626574,0.00006746652,0.0002378011],"domain_scores_gemma":[0.9986594,0.00004275919,0.0006441558,0.0003298316,0.00025697926,0.00006690674],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0011527749,0.00019401344,0.00037817724,0.0005338805,0.00035519595,0.00022449912,0.00017483844,0.00022727632,0.14612591],"category_scores_gemma":[0.0003479774,0.00024239076,0.00019350799,0.0002466833,0.000054079497,0.0007033957,0.000043824253,0.00009012481,0.5442282],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000091665115,0.000013623907,0.000005492795,0.000025715992,0.00009306354,3.4559478e-9,0.00047206949,0.00013927101,1.7210422e-7,0.0019822125,0.9877389,0.0095203025],"study_design_scores_gemma":[0.0007716732,0.000044982666,0.000045223944,0.000019227788,0.000049987513,4.177001e-7,0.00013748209,0.0037127587,0.000015945328,0.0037563234,0.9911927,0.0002532647],"about_ca_topic_score_codex":0.00038198577,"about_ca_topic_score_gemma":0.00003390296,"teacher_disagreement_score":0.39810228,"about_ca_system_score_codex":0.0004358186,"about_ca_system_score_gemma":0.00014183887,"threshold_uncertainty_score":0.9884408},"labels":[],"label_agreement":null},{"id":"W4251068696","doi":"10.1002/ev.324","title":"Editors' notes","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Disaster Management and Resilience","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Citation; Library science; Preparedness; Sociology; Computer science; Political science; Law","score_opus":0.03532045501342495,"score_gpt":0.3837244448720017,"score_spread":0.34840398985857673,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4251068696","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.22581685,0.0000814575,0.011633996,0.01574612,0.056781802,0.0026426609,0.00000574062,0.0004773576,0.686814],"genre_scores_gemma":[0.95353276,0.000015521055,0.0024216617,0.00007149248,0.012990292,0.00016040317,0.000015619438,0.0000088924735,0.030783381],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99929565,0.000042692056,0.00007631381,0.00012222498,0.0003392931,0.00012381413],"domain_scores_gemma":[0.9995259,0.0001323269,0.00003633068,0.0000947351,0.00015446352,0.00005624109],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009108152,0.000039657963,0.000037938702,0.00006775745,0.00055544206,0.00008487476,0.0000950826,0.000049638238,0.0006814982],"category_scores_gemma":[0.0011139563,0.00003876496,0.00004028109,0.000212705,0.000062707775,0.0002453535,0.000008587048,0.00005038461,0.00006978362],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009764285,0.00007583132,0.0017480429,0.000004621652,0.000015060917,3.7825462e-8,0.0058210692,0.00006210954,0.0023335589,0.04020354,0.20437673,0.74534965],"study_design_scores_gemma":[0.00026510598,0.000015023458,0.004295449,0.0000028292525,0.00004111091,5.524315e-8,0.0006127753,0.0009018683,0.00026740952,0.012374918,0.9811576,0.000065834174],"about_ca_topic_score_codex":0.00080985425,"about_ca_topic_score_gemma":0.0089204665,"teacher_disagreement_score":0.7767809,"about_ca_system_score_codex":0.000029577437,"about_ca_system_score_gemma":0.0001616552,"threshold_uncertainty_score":0.74619275},"labels":[],"label_agreement":null},{"id":"W4252169279","doi":"10.1002/ev.309","title":"Editors' notes","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Community Health and Development","field":"Health Professions","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Citation; Library science; Population; Intervention (counseling); State (computer science); Sociology; Medicine; Computer science; Nursing","score_opus":0.1565093687785537,"score_gpt":0.5236589003624142,"score_spread":0.36714953158386054,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4252169279","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.25649247,0.0024704835,0.08416079,0.23742639,0.08452646,0.029019045,0.000077164324,0.0026495287,0.30317768],"genre_scores_gemma":[0.9000586,0.00023304949,0.031850792,0.013606274,0.02901332,0.0026101843,0.0007098879,0.00005752857,0.021860361],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986604,0.0002832273,0.00033788284,0.00014307506,0.0002848285,0.00029062183],"domain_scores_gemma":[0.99817383,0.0008667855,0.000111237816,0.00023583506,0.0004587686,0.0001535156],"candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.0016356953,0.00008423744,0.00011582713,0.0001405188,0.001497819,0.000007938289,0.00007453472,0.00012107909,0.00073572993],"category_scores_gemma":[0.0020538538,0.000082428734,0.00004702525,0.00025963137,0.0000070024876,0.00011396605,0.000013119525,0.00024342512,0.0002621702],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000065150816,0.00007328443,0.0012268753,0.000025680598,0.0000077781,2.8969225e-8,0.0025463006,0.000034202825,0.00012138347,0.0014097523,0.45048913,0.54400045],"study_design_scores_gemma":[0.0018883322,0.00014666402,0.08288344,0.00006369031,0.000042138956,4.2313317e-7,0.000428246,0.0018605356,0.00008687768,0.026007207,0.8864725,0.000119944496],"about_ca_topic_score_codex":0.00038422167,"about_ca_topic_score_gemma":0.0006363915,"teacher_disagreement_score":0.64356613,"about_ca_system_score_codex":0.0003993582,"about_ca_system_score_gemma":0.0013558605,"threshold_uncertainty_score":0.9998021},"labels":[],"label_agreement":null},{"id":"W4256226640","doi":"10.1002/ev.20276","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4256226640","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00003435367,0.002113493,0.004931094,0.0003169737,0.05299304,0.0010317265,0.0023091952,0.00004083091,0.9362293],"genre_scores_gemma":[0.00012845024,0.00052843284,0.0008068433,0.0001820641,0.0020895111,0.00031783918,0.0017247224,0.000020879159,0.99420124],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986079,0.000012206888,0.0006784026,0.00039626574,0.00006746652,0.0002378011],"domain_scores_gemma":[0.9986594,0.00004275919,0.0006441558,0.0003298316,0.00025697926,0.00006690674],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0011527749,0.00019401344,0.00037817724,0.0005338805,0.00035519595,0.00022449912,0.00017483844,0.00022727632,0.14612591],"category_scores_gemma":[0.0003479774,0.00024239076,0.00019350799,0.0002466833,0.000054079497,0.0007033957,0.000043824253,0.00009012481,0.5442282],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000091665115,0.000013623907,0.000005492795,0.000025715992,0.00009306354,3.4559478e-9,0.00047206949,0.00013927101,1.7210422e-7,0.0019822125,0.9877389,0.0095203025],"study_design_scores_gemma":[0.0007716732,0.000044982666,0.000045223944,0.000019227788,0.000049987513,4.177001e-7,0.00013748209,0.0037127587,0.000015945328,0.0037563234,0.9911927,0.0002532647],"about_ca_topic_score_codex":0.00038198577,"about_ca_topic_score_gemma":0.00003390296,"teacher_disagreement_score":0.39810228,"about_ca_system_score_codex":0.0004358186,"about_ca_system_score_gemma":0.00014183887,"threshold_uncertainty_score":0.9884408},"labels":[],"label_agreement":null},{"id":"W4293180673","doi":"10.1002/ev.20490","title":"The importance of implementation: Putting evaluation policy to work","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Work (physics); Government (linguistics); White paper; Public administration; Public relations; Early adopter; Program evaluation; Evaluation methods; Policy analysis; Business; Political science; Marketing","score_opus":0.2980319978182459,"score_gpt":0.5961758186242162,"score_spread":0.2981438208059703,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4293180673","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7828459,0.001607929,0.03320837,0.13422324,0.0071068397,0.016124846,0.00016824828,0.00017501465,0.024539622],"genre_scores_gemma":[0.9893297,0.000011052466,0.0031975934,0.0007444617,0.00044667287,0.0032922735,0.00011314333,0.000017273487,0.002847841],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9924928,0.00092516723,0.0010636223,0.0004202659,0.0048391996,0.0002589448],"domain_scores_gemma":[0.9951912,0.0013433011,0.0006796181,0.00059563154,0.0020966614,0.00009358202],"candidate_categories":["metaresearch","sts","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.032118186,0.000117078824,0.00016020292,0.00048968697,0.0018534898,0.0001744437,0.00045791568,0.00002514043,0.004750548],"category_scores_gemma":[0.0053201625,0.000093036164,0.00014039913,0.0033966824,0.000026385773,0.00032347848,0.00012327038,0.00010004611,0.000041214716],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000054827553,0.00003213319,0.013882514,0.0000011670263,0.00002573282,1.2460131e-8,0.0013693314,0.03815974,0.0001446965,0.0032667504,0.05568473,0.88737833],"study_design_scores_gemma":[0.003172344,0.000496369,0.18513095,0.000009426855,0.00023120327,0.000004388635,0.010390384,0.10720741,0.0011449698,0.11851981,0.57342863,0.0002641207],"about_ca_topic_score_codex":0.000102137805,"about_ca_topic_score_gemma":0.00088735716,"teacher_disagreement_score":0.8871142,"about_ca_system_score_codex":0.00085165305,"about_ca_system_score_gemma":0.0024305116,"threshold_uncertainty_score":0.999446},"labels":[],"label_agreement":null},{"id":"W4307053074","doi":"10.1002/ev.20511","title":"Empowering the search for pleasure, health and well‐being outside heteronormative definitions: The role of evaluation in shaping structurally sensitive programming for 2SGBTQ+ men who <i>Party and Play</i> in Ontario","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"HIV, Drug Use, Sexual Risk","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Casey House","funders":"","keywords":"Phenomenon; Agency (philosophy); Transformative learning; Pleasure; Sociology; Public health; Public relations; Health care; Qualitative research; Gender studies; Social psychology; Psychology; Medicine; Nursing; Political science; Epistemology; Social science","score_opus":0.10428937933277302,"score_gpt":0.39257313186623666,"score_spread":0.2882837525334636,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4307053074","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.98029214,0.0008700623,0.002691385,0.0022531208,0.00012708869,0.012899025,0.00006527697,0.00002697255,0.0007749417],"genre_scores_gemma":[0.99301064,0.000036755802,0.0032351166,0.00014721495,0.000048850387,0.0031144246,0.00030962305,0.00002507338,0.00007229992],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9979066,0.0004808707,0.00049114286,0.00030623714,0.0005372961,0.00027781914],"domain_scores_gemma":[0.9981733,0.0009901825,0.00021331268,0.00016330609,0.00039561314,0.000064263295],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0054066833,0.00014216719,0.00027299975,0.00024833635,0.00072845095,0.0000436931,0.000050488543,0.00004278232,0.000009913434],"category_scores_gemma":[0.0006598015,0.000114700895,0.000058266247,0.00030800438,0.000055771125,0.00020951538,0.000050252966,0.00027946537,1.6560158e-7],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0014287214,0.00013981314,0.015559607,0.00022627159,0.00017721237,2.228403e-7,0.40591723,0.016934192,0.001384932,0.0011581527,0.0002833802,0.55679023],"study_design_scores_gemma":[0.011872378,0.0019947118,0.10345275,0.00050662726,0.0004863581,0.00005665022,0.29513907,0.5668893,0.0027646369,0.011888496,0.0046256683,0.00032335237],"about_ca_topic_score_codex":0.011008203,"about_ca_topic_score_gemma":0.083807655,"teacher_disagreement_score":0.55646694,"about_ca_system_score_codex":0.0010479086,"about_ca_system_score_gemma":0.0011515145,"threshold_uncertainty_score":0.9955776},"labels":[],"label_agreement":null},{"id":"W4307053258","doi":"10.1002/ev.20514","title":"Identity as a compass when navigating uncharted equitable spaces: Our queer evaluation practices","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan; Saskatchewan Health","funders":"","keywords":"Queer; Dialogic; Situated; Sociology; Normative; Identity (music); Epistemology; Norm (philosophy); Social psychology; Psychology; Computer science; Aesthetics; Gender studies; Pedagogy","score_opus":0.37946722076513645,"score_gpt":0.5847148136120114,"score_spread":0.20524759284687494,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4307053258","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.80582666,0.0019333476,0.06489756,0.05557442,0.012585679,0.012019774,0.00017755994,0.0005451508,0.046439826],"genre_scores_gemma":[0.97967243,0.000016730251,0.004318316,0.0006521001,0.00052723126,0.002972892,0.00040223126,0.00003389493,0.0114041995],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9861952,0.0024654856,0.0011017299,0.0008573209,0.008921117,0.00045914954],"domain_scores_gemma":[0.99234587,0.0011687668,0.0019942794,0.00074758456,0.0035723848,0.0001711227],"candidate_categories":["metaresearch","sts","scholarly_communication","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.051013894,0.000233402,0.00032406437,0.00045279213,0.0026006922,0.0010415043,0.00063263736,0.000080544814,0.019863738],"category_scores_gemma":[0.015418283,0.00022619047,0.00021377247,0.0018706643,0.000024309324,0.0035016218,0.00023747071,0.0004018565,0.0005076299],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0003123172,0.00056424306,0.00774694,0.000022054855,0.00021558389,6.091701e-7,0.0125857955,0.17334187,0.0023594282,0.0066846665,0.18165368,0.6145128],"study_design_scores_gemma":[0.0030252028,0.00035162366,0.004598951,0.00002438069,0.00041230506,0.000016318372,0.02225355,0.6849001,0.0002623454,0.10770027,0.17615089,0.00030407615],"about_ca_topic_score_codex":0.0016499187,"about_ca_topic_score_gemma":0.002260426,"teacher_disagreement_score":0.61420876,"about_ca_system_score_codex":0.0012901152,"about_ca_system_score_gemma":0.0027114328,"threshold_uncertainty_score":0.9999955},"labels":[],"label_agreement":null},{"id":"W4307053273","doi":"10.1002/ev.20510","title":"At the intersection of co‐creation: Exploring LGBTQ2S evaluation with youth","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"University of Prince Edward Island; University of Winnipeg; Youth Services Bureau of Ottawa; Parks Canada","funders":"","keywords":"Intersection (aeronautics); Sociology; Context (archaeology); Public relations; Program evaluation; Positive Youth Development; Youth studies; Psychology; Political science; Gender studies; Public administration; Engineering","score_opus":0.46878928555177146,"score_gpt":0.5291504336284258,"score_spread":0.06036114807665438,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4307053273","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9439751,0.00021247836,0.016392145,0.0019280729,0.0020208328,0.003215263,0.00008095982,0.00006748029,0.03210769],"genre_scores_gemma":[0.99199605,0.000010455595,0.00040506473,0.00008422627,0.00019761921,0.0024963461,0.00033350487,0.000017754852,0.004459007],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9927616,0.0010505274,0.0007180612,0.000455304,0.0048330794,0.00018140055],"domain_scores_gemma":[0.9961001,0.00076261687,0.0006539192,0.0005969613,0.001825643,0.00006070901],"candidate_categories":["sts","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.017694706,0.00014685771,0.0001949984,0.00036826186,0.0015129464,0.00010793516,0.0003047447,0.000032938675,0.0095843235],"category_scores_gemma":[0.0012866192,0.000102496815,0.0001411229,0.0014746861,0.00007002493,0.0006777277,0.00007830914,0.00014141627,0.00004754341],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0006549892,0.00016005208,0.0070647467,0.0000062924487,0.00009764642,5.1510607e-8,0.015421011,0.2569839,0.00096890767,0.0013920321,0.01191483,0.70533556],"study_design_scores_gemma":[0.008349707,0.0019271638,0.06614615,0.000041503386,0.0011645718,0.00003235981,0.06650653,0.7054949,0.01409585,0.014211895,0.12151393,0.0005154756],"about_ca_topic_score_codex":0.00018074414,"about_ca_topic_score_gemma":0.00089106447,"teacher_disagreement_score":0.70482004,"about_ca_system_score_codex":0.0013052708,"about_ca_system_score_gemma":0.001031544,"threshold_uncertainty_score":0.999787},"labels":[],"label_agreement":null},{"id":"W4378223319","doi":"10.1002/ev.20539","title":"Meeting the challenges of educating internal evaluators","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Institute for Clinical Evaluative Sciences; Queen's University","funders":"","keywords":"Front line; Public relations; Geopolitics; Coronavirus disease 2019 (COVID-19); Pandemic; Program evaluation; Political science; Business; Medicine; Public administration","score_opus":0.3888663697506948,"score_gpt":0.5659171151344419,"score_spread":0.1770507453837471,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223319","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.81679654,0.002155344,0.0053557097,0.032410588,0.0065827146,0.002995843,0.000019475445,0.00027154636,0.13341226],"genre_scores_gemma":[0.9955015,0.00014315336,0.00076143583,0.00005847383,0.00041959816,0.0002911571,0.0000131421575,0.000012777656,0.0027987494],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9960158,0.00050998747,0.0006999962,0.00031110246,0.0022760285,0.00018709348],"domain_scores_gemma":[0.9951709,0.0025355215,0.0004472402,0.00043268382,0.0013577846,0.000055878445],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.024371296,0.000100550555,0.0001610651,0.0004154885,0.00047797454,0.00009592403,0.00037412968,0.000047153564,0.0007444051],"category_scores_gemma":[0.010487166,0.000066315064,0.00015030542,0.0012658661,0.0000421683,0.00028712075,0.000056722296,0.00008789517,0.00021985687],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000016667489,0.000040345905,0.0019444905,0.000007415414,0.000033568267,1.683456e-8,0.004373027,0.010803508,0.0004864877,0.006708819,0.022143131,0.9534425],"study_design_scores_gemma":[0.0013548174,0.00019682081,0.10862375,0.00008520552,0.00015900457,0.000004377574,0.018435454,0.7021917,0.0034182116,0.09723347,0.06809587,0.00020128334],"about_ca_topic_score_codex":0.00009024751,"about_ca_topic_score_gemma":0.00043902232,"teacher_disagreement_score":0.9532412,"about_ca_system_score_codex":0.000102066435,"about_ca_system_score_gemma":0.0005498398,"threshold_uncertainty_score":0.9978479},"labels":[],"label_agreement":null},{"id":"W4378223534","doi":"10.1002/ev.20536","title":"Learning by linking the Canadian Evaluation Society's student case competition within a graduate evaluation course","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Queen's University","funders":"","keywords":"Experiential learning; Dialogic; Pedagogy; Psychology; Nature versus nurture; Situated; Competition (biology); Professional development; Assessment for learning; Sociology; Formative assessment; Computer science","score_opus":0.34293943692583495,"score_gpt":0.5475533572533501,"score_spread":0.20461392032751513,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223534","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.95149624,0.0005276616,0.0061716596,0.019021247,0.0053449147,0.009802668,0.00008894522,0.0003878107,0.0071588443],"genre_scores_gemma":[0.9921185,0.00004963144,0.00041968687,0.00039644452,0.000512786,0.002359136,0.0015035017,0.000040056966,0.0026002913],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.98632854,0.0025590588,0.0011682152,0.0008428421,0.008563664,0.0005376657],"domain_scores_gemma":[0.99048924,0.00152772,0.00075859914,0.0006627764,0.0062870984,0.00027458568],"candidate_categories":["metaresearch","sts","scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.08201619,0.0002943852,0.0002813505,0.00055217004,0.0041687996,0.0011071658,0.00041369032,0.00019529849,0.001570499],"category_scores_gemma":[0.0046274257,0.00023439845,0.00029382302,0.0029893944,0.00010113196,0.0008964866,0.00005741287,0.00045271206,0.0005705869],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024592073,0.000098087155,0.0033222707,0.000008447749,0.00015582205,0.0000014189,0.016023908,0.35544565,0.00028585494,0.0008382401,0.08079413,0.5430016],"study_design_scores_gemma":[0.0023268582,0.00016665782,0.014739879,0.00003401322,0.00070203963,0.000043097258,0.013097922,0.93604726,0.00014607846,0.011793256,0.020627752,0.00027520285],"about_ca_topic_score_codex":0.012895173,"about_ca_topic_score_gemma":0.25234655,"teacher_disagreement_score":0.58060163,"about_ca_system_score_codex":0.0020827928,"about_ca_system_score_gemma":0.004821155,"threshold_uncertainty_score":0.9999298},"labels":[],"label_agreement":null},{"id":"W4378223570","doi":"10.1002/ev.20540","title":"What we can learn from the international program for development evaluation training (IPDET)","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Program evaluation; Training (meteorology); Monitoring and evaluation; Diversity (politics); Medical education; Corporate governance; Political science; Psychology; Business; Medicine; Geography; Public administration; Finance","score_opus":0.5304740430548962,"score_gpt":0.569693978952799,"score_spread":0.03921993589790285,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223570","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.60155666,0.0015055815,0.051582206,0.26209196,0.043193158,0.02989428,0.0002483742,0.0013844905,0.008543247],"genre_scores_gemma":[0.8953398,0.00060191785,0.032613263,0.0017155617,0.0039056386,0.026778976,0.0059145438,0.00010506078,0.033025254],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9940389,0.0003564642,0.0008220404,0.0006102448,0.0038630227,0.00030932165],"domain_scores_gemma":[0.99392647,0.0026959223,0.00039128863,0.00042918773,0.002459302,0.00009782404],"candidate_categories":["scholarly_communication","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.020275827,0.00018073295,0.0001945055,0.00038146926,0.00095626886,0.0011836049,0.0005673315,0.00009727187,0.0017426191],"category_scores_gemma":[0.0061981454,0.0001325587,0.00017733841,0.00117908,0.000038535414,0.000978679,0.000061704166,0.00011496855,0.0002619385],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000037922997,0.00003987788,0.00033755956,0.0000015549485,0.000062554784,2.0138792e-8,0.008020009,0.0072679254,0.00012977568,0.00021686904,0.05327784,0.9306081],"study_design_scores_gemma":[0.0014806249,0.00006568262,0.013364019,0.00003419938,0.0000887798,5.442189e-7,0.0072771306,0.4473603,0.0003893526,0.022468109,0.5073485,0.00012280054],"about_ca_topic_score_codex":0.00006608532,"about_ca_topic_score_gemma":0.0036262511,"teacher_disagreement_score":0.9304853,"about_ca_system_score_codex":0.0005514317,"about_ca_system_score_gemma":0.0024848708,"threshold_uncertainty_score":0.99985325},"labels":[],"label_agreement":null},{"id":"W4378223607","doi":"10.1002/ev.20543","title":"Guest Editors’ notes","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Timeline; Adventure; Ethos; Dream; Psychology; Sociology; Operations research; Library science; Computer science; Law; History; Political science; Artificial intelligence; Engineering","score_opus":0.3729067314618274,"score_gpt":0.5685543924994344,"score_spread":0.195647661037607,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223607","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.39055502,0.000508165,0.10093265,0.09566969,0.26543757,0.009821494,0.00022382937,0.0031329677,0.13371861],"genre_scores_gemma":[0.9092156,0.00008943823,0.0035820547,0.0004772673,0.0332639,0.0011923923,0.00050736556,0.000048512164,0.051623456],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9964512,0.00019500857,0.0004903719,0.00037627292,0.002273031,0.00021410575],"domain_scores_gemma":[0.99618614,0.0019022638,0.00017878752,0.00041474932,0.0012154243,0.00010264577],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0095142545,0.000105270956,0.00014256449,0.00058234687,0.00046021756,0.00022915732,0.00023636596,0.000074619886,0.0022602496],"category_scores_gemma":[0.0091219675,0.00008691567,0.00012689825,0.0020370882,0.000025389765,0.00053289393,0.0000340383,0.00006901746,0.0037450655],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013402688,0.000025848683,0.0024919445,0.0000017472369,0.000011801628,5.273941e-8,0.0004013477,0.0034714262,0.0004200109,0.00056781503,0.6949603,0.2976343],"study_design_scores_gemma":[0.0008332748,0.000077747696,0.037245747,0.0000068107324,0.000040975192,9.617311e-7,0.0002740351,0.16106911,0.001229236,0.026005384,0.7730998,0.000116912925],"about_ca_topic_score_codex":0.000058404905,"about_ca_topic_score_gemma":0.00025851128,"teacher_disagreement_score":0.5186606,"about_ca_system_score_codex":0.00012568003,"about_ca_system_score_gemma":0.0005420085,"threshold_uncertainty_score":0.9992246},"labels":[],"label_agreement":null},{"id":"W4388306673","doi":"10.1002/ev.20564","title":"Artificial intelligence and the future of evaluation education: Possibilities and prototypes","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Fraser Health","funders":"","keywords":"Chatbot; Evaluation methods; Engineering ethics; Literacy; Computer science; Program evaluation; Management science; Psychology; Pedagogy; Artificial intelligence; Political science; Engineering","score_opus":0.2331504610129478,"score_gpt":0.5285796286386468,"score_spread":0.29542916762569904,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4388306673","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.92493236,0.005009631,0.008771664,0.037902303,0.0041904515,0.012171764,0.000029939401,0.00012421433,0.006867656],"genre_scores_gemma":[0.9944193,0.00028498506,0.0010264373,0.00009483676,0.0005414897,0.0015889743,0.000042919604,0.000008171827,0.0019928867],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.99694914,0.000518244,0.0005869541,0.00030462322,0.0015329632,0.00010808988],"domain_scores_gemma":[0.99639004,0.0012500217,0.00026043758,0.00029895312,0.0017531373,0.000047394013],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.017285567,0.00009295601,0.00015872052,0.00032584355,0.0003833795,0.0001679127,0.00010992475,0.000056948036,0.00047319444],"category_scores_gemma":[0.0032217961,0.00006000196,0.00005839018,0.0010346783,0.00016321105,0.00037779508,0.000033141918,0.00006323629,0.000026523765],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009152033,0.000023590937,0.0005221862,0.000009302708,0.000010928699,3.4237106e-9,0.0027270888,0.0007941752,0.000039307386,0.036056466,0.0019884575,0.95773697],"study_design_scores_gemma":[0.00041533372,0.000088342415,0.020256178,0.000016972896,0.00010542194,0.000002574249,0.00967973,0.32230806,0.0005779981,0.6381781,0.008294486,0.00007680805],"about_ca_topic_score_codex":0.00006165556,"about_ca_topic_score_gemma":0.00039799913,"teacher_disagreement_score":0.95766014,"about_ca_system_score_codex":0.000054958946,"about_ca_system_score_gemma":0.0010919448,"threshold_uncertainty_score":0.5990867},"labels":[],"label_agreement":null},{"id":"W4392918018","doi":"10.1002/ev.20454","title":"Issue Information","year":2023,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Computer science; Psychology","score_opus":0.090513352209208,"score_gpt":0.3024423793431357,"score_spread":0.21192902713392772,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4392918018","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.000021101938,0.0016604309,0.0027912324,0.00051398226,0.06399715,0.0011123505,0.0032493458,0.00009140453,0.926563],"genre_scores_gemma":[0.00006492665,0.0009283246,0.00031592546,0.00011892298,0.0011745569,0.0004719561,0.0024256625,0.000029128074,0.9944706],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99859774,0.000011398229,0.0006856667,0.00038701037,0.00007181203,0.00024639833],"domain_scores_gemma":[0.99882966,0.00007244905,0.00056660996,0.0002998587,0.00016803005,0.00006339605],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0012315182,0.00018660031,0.0003856768,0.0006931963,0.00031972388,0.00021394895,0.00016191762,0.00021552603,0.06158159],"category_scores_gemma":[0.00045103751,0.00023868243,0.00020944327,0.0003830084,0.000030558887,0.0005967315,0.000047100355,0.000106895575,0.68909067],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000061293067,0.000009866851,0.0000037534073,0.00003432326,0.000100473386,7.894801e-9,0.00042337063,0.00087821146,1.3856071e-7,0.0025846295,0.98409206,0.011867015],"study_design_scores_gemma":[0.00070833426,0.000024413781,0.0000784635,0.00002070298,0.000045907414,2.626979e-7,0.00021056451,0.0056796893,0.0000064375367,0.004328458,0.9886546,0.00024216733],"about_ca_topic_score_codex":0.0005931383,"about_ca_topic_score_gemma":0.000048644888,"teacher_disagreement_score":0.6275091,"about_ca_system_score_codex":0.00042441266,"about_ca_system_score_gemma":0.00013162535,"threshold_uncertainty_score":0.97331876},"labels":[],"label_agreement":null},{"id":"W4403250772","doi":"10.1002/ev.20615","title":"Using contribution analysis to assess evaluation capacity outcomes in community organizations","year":2024,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université de Montréal; University of Ottawa","funders":"","keywords":"Evaluation methods; Capacity building; Program evaluation; Process management; Business; Political science; Public administration; Economic growth; Economics; Engineering","score_opus":0.538010031454944,"score_gpt":0.5850516251312451,"score_spread":0.04704159367630112,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403250772","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6357742,0.00007385741,0.35830238,0.0015973938,0.0011988044,0.0019515802,0.00007559378,0.00010114634,0.0009250299],"genre_scores_gemma":[0.9932098,0.0000061050723,0.0051238067,0.00015875477,0.000118300406,0.0004079335,0.00038061323,0.00001832878,0.00057639665],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99297214,0.002189526,0.0010415817,0.00052166707,0.0029958952,0.00027917096],"domain_scores_gemma":[0.99182653,0.002260649,0.00020936821,0.0006749226,0.004878069,0.0001504596],"candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.031351037,0.00020067222,0.0004017927,0.002616893,0.0007694029,0.00070138124,0.00032238226,0.00013322102,0.0021422023],"category_scores_gemma":[0.018385787,0.00017777407,0.00022039514,0.0117064705,0.000030590563,0.0011165452,0.00005686581,0.0002654117,0.00015739731],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000039143637,0.00040045963,0.11956669,0.0000137535435,0.00062750577,1.6486634e-7,0.0051685954,0.735638,0.0037271937,0.0059494907,0.0027402118,0.12612881],"study_design_scores_gemma":[0.00069763244,0.000053140902,0.2391568,0.000019520548,0.001082934,0.000001079018,0.0006029268,0.73930687,0.0005010961,0.01564433,0.00277178,0.00016191676],"about_ca_topic_score_codex":0.0013326773,"about_ca_topic_score_gemma":0.014989953,"teacher_disagreement_score":0.35743555,"about_ca_system_score_codex":0.0017428473,"about_ca_system_score_gemma":0.0015896104,"threshold_uncertainty_score":0.99877},"labels":[],"label_agreement":null},{"id":"W4408984956","doi":"10.1002/ev.20499","title":"Issue Information","year":2024,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Human auditory perception and evaluation","field":"Engineering","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"Consortium of International Agricultural Research Centers; Universiteit Stellenbosch; York University; W.K. Kellogg Foundation; University of Illinois at Urbana-Champaign; Western Michigan University; University of Bern; University of Connecticut; Inter-American Development Bank","keywords":"Computer science; Process management; Knowledge management; Business","score_opus":0.03557066694896771,"score_gpt":0.33923249572305814,"score_spread":0.3036618287740904,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408984956","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00032473725,0.004371213,0.06389365,0.00054201344,0.12704751,0.0038965386,0.00043761317,0.0015777942,0.7979089],"genre_scores_gemma":[0.0011115097,0.0015387002,0.0013922316,0.00019138733,0.013715766,0.002220359,0.016171826,0.00019397565,0.96346426],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99842834,0.000054370434,0.00049903494,0.00022281586,0.00057493633,0.00022049395],"domain_scores_gemma":[0.9991249,0.00004952657,0.00008352076,0.00027261238,0.00037452334,0.00009495992],"candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0006517385,0.0002961646,0.00023352576,0.00066279946,0.00016667272,0.00027322484,0.00010899568,0.00041342073,0.14607544],"category_scores_gemma":[0.00010644055,0.00033092688,0.00018866522,0.00035436478,0.000015876441,0.00063856295,0.000012066748,0.00029842,0.43263224],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000038574185,0.0000045850757,5.0596995e-8,0.00021880408,0.00005909704,9.7189155e-9,0.0004664145,0.036259945,0.00008429062,0.0000070225997,0.7934582,0.1694377],"study_design_scores_gemma":[0.0004155668,0.000028951377,0.00001934508,0.000100585836,0.00028032577,0.000001534043,0.000054366283,0.19006743,0.00014536372,0.00037141168,0.8082493,0.0002658295],"about_ca_topic_score_codex":0.000007118152,"about_ca_topic_score_gemma":0.000042145704,"teacher_disagreement_score":0.28655678,"about_ca_system_score_codex":0.0008679677,"about_ca_system_score_gemma":0.00030451803,"threshold_uncertainty_score":0.9999143},"labels":[],"label_agreement":null},{"id":"W4411428767","doi":"10.1002/ev.70000","title":"Issue Information","year":2025,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Consortium of International Agricultural Research Centers; Universiteit Stellenbosch; York University; W.K. Kellogg Foundation; University of Illinois at Urbana-Champaign; Western Michigan University; University of Bern; University of Connecticut; Inter-American Development Bank","keywords":"Computer science","score_opus":0.05774558077803174,"score_gpt":0.29368375071581176,"score_spread":0.23593816993778002,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4411428767","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.000008258448,0.0036791533,0.0070452876,0.0005347903,0.046918225,0.0010980442,0.002759277,0.000038355647,0.9379186],"genre_scores_gemma":[0.000069033566,0.0008503583,0.00058272626,0.00023238242,0.0007110856,0.00039637473,0.0018703409,0.0000113591705,0.99527633],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986656,0.000012159217,0.0006791655,0.0003750995,0.00005682946,0.00021115737],"domain_scores_gemma":[0.9988626,0.00006367939,0.0005154321,0.00030974214,0.00019914247,0.00004935187],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.00095118745,0.00018571777,0.0003972845,0.0006962733,0.00031498668,0.00020585788,0.0001679222,0.000220776,0.084016524],"category_scores_gemma":[0.0004032883,0.0002382117,0.00020782568,0.00031613192,0.000028569428,0.0005639774,0.000046809484,0.000108073575,0.28162867],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000008005067,0.000014107213,0.0000053292247,0.000049491708,0.00010950558,3.5971068e-9,0.00027002153,0.0005680877,1.01549105e-7,0.004976966,0.9740316,0.019966781],"study_design_scores_gemma":[0.0008031294,0.000019541032,0.000039561222,0.00003162481,0.00006212598,1.7589056e-7,0.00013575079,0.0040687374,0.00001056758,0.003558435,0.9910526,0.00021770966],"about_ca_topic_score_codex":0.0004811857,"about_ca_topic_score_gemma":0.000023349761,"teacher_disagreement_score":0.19761214,"about_ca_system_score_codex":0.00048131414,"about_ca_system_score_gemma":0.00022131394,"threshold_uncertainty_score":0.97139907},"labels":[],"label_agreement":null},{"id":"W4413128915","doi":"10.1002/ev.70009","title":"Community‐Based Research on Evaluation: Beyond Academic and Evaluator Perspectives","year":2025,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa; University of Waterloo","funders":"Canadian Mental Health Association","keywords":"Evaluation methods; Program evaluation; Computer science; Sociology; Engineering ethics; Management science; Psychology; Political science; Public administration; Engineering","score_opus":0.6424948000151569,"score_gpt":0.6631852882198059,"score_spread":0.020690488204649027,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4413128915","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.67417395,0.004150063,0.016882418,0.060728762,0.0032735248,0.010576932,0.0000677313,0.00025296627,0.22989365],"genre_scores_gemma":[0.987997,0.00007788343,0.0012152693,0.00083466514,0.00026241018,0.001514025,0.00008553801,0.000017914548,0.007995293],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.98656046,0.0056574047,0.00082757405,0.00072891155,0.0058525335,0.0003730962],"domain_scores_gemma":[0.9860942,0.0069090463,0.00024212805,0.0009942732,0.005604618,0.00015575415],"candidate_categories":["metaresearch","sts","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.076589204,0.00021573737,0.0002927718,0.0020287598,0.0023818885,0.00038024122,0.0005549098,0.00022109007,0.0018778932],"category_scores_gemma":[0.019417228,0.0001873534,0.00013415588,0.0029899478,0.0002292463,0.000567909,0.00009967574,0.00088153867,0.000114452756],"study_design_candidate":"design_other","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0003698355,0.00035710147,0.0024832613,0.000020644815,0.000079673315,5.2871634e-8,0.004655154,0.011670796,0.0011284911,0.026913749,0.11467541,0.8376458],"study_design_scores_gemma":[0.005863603,0.00064648106,0.050362073,0.00012224975,0.00028527484,0.0000013978984,0.015753767,0.58153534,0.0025586,0.2541974,0.0883907,0.0002831476],"about_ca_topic_score_codex":0.00010566431,"about_ca_topic_score_gemma":0.00029324915,"teacher_disagreement_score":0.8373627,"about_ca_system_score_codex":0.00080321427,"about_ca_system_score_gemma":0.0036505095,"threshold_uncertainty_score":0.9990345},"labels":[],"label_agreement":null},{"id":"W7125971164","doi":"10.1002/ev.70012","title":"Issue Information","year":2025,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Human auditory perception and evaluation","field":"Engineering","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Consortium of International Agricultural Research Centers; York University; W.K. Kellogg Foundation; University of Connecticut; University of Illinois at Urbana-Champaign; University of Bern; Western Michigan University; Inter-American Development Bank; Universiteit Stellenbosch; National Science Foundation","keywords":"Information system; Data collection; Key (lock); Research methodology; Information technology; Utilization; Program evaluation","score_opus":0.02251690894749914,"score_gpt":0.32453244346504384,"score_spread":0.3020155345175447,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W7125971164","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.030197514,0.00036302194,0.6516431,0.0014120691,0.018116076,0.0026807257,0.000018329692,0.0018818161,0.29368734],"genre_scores_gemma":[0.86659473,0.00038482994,0.017074864,0.0018105976,0.0042590178,0.0035287135,0.0024124268,0.00009773384,0.10383707],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9993955,0.000026647607,0.00021345171,0.0000754006,0.00018861463,0.00010038659],"domain_scores_gemma":[0.9995588,0.00003479131,0.000024419172,0.00012977284,0.00022014073,0.000032043055],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.00045081886,0.00007929214,0.000069649686,0.00028837536,0.00015232158,0.00006171033,0.0000471225,0.00006952429,0.006337925],"category_scores_gemma":[0.00016924443,0.00009098171,0.000049692444,0.00028932476,0.000008379885,0.0005236201,0.0000041698563,0.000053172673,0.0028220494],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000005026815,0.0000058501037,0.000018187608,0.000022576118,0.000017325388,1.8271662e-9,0.00028807676,0.028552651,0.00051738723,0.00016672032,0.5470483,0.42335793],"study_design_scores_gemma":[0.0006442486,0.000010185584,0.00380455,0.000012554803,0.000055366036,1.883037e-7,0.00007745757,0.33087873,0.0006274096,0.0018805149,0.66194016,0.00006860444],"about_ca_topic_score_codex":0.0000032011349,"about_ca_topic_score_gemma":0.000033987344,"teacher_disagreement_score":0.83639723,"about_ca_system_score_codex":0.00027575638,"about_ca_system_score_gemma":0.0000884889,"threshold_uncertainty_score":0.99795437},"labels":[],"label_agreement":null}]}