{"meta":{"query_hash":"3e0a82fc70b4","filters":{"venue":"New Directions for Evaluation"},"cohort_total":73,"direct_labels_cover":1,"predictions_cover":73,"exported":73,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/3e0a82fc70b4","api":"https://metacan.xera.ac/api/v1/cohort?venue=New+Directions+for+Evaluation"},"results":[{"id":"W1505207325","doi":"10.1002/ev.20084","title":"Credible Judgment: Combining Truth, Beauty, and Justice","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Université du Québec à Montréal","funders":"","keywords":"Beauty; Argumentation theory; Economic Justice; Process (computing); Variety (cybernetics); Psychology; Social psychology; Public relations; Computer science; Law; Epistemology; Political science","score_opus":0.26060324573329097,"score_gpt":0.5132600924299058,"score_spread":0.2526568466966148,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1505207325","genre_codex":"other","genre_gemma":"methods","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19383098,0.0061282353,0.24728167,0.055748686,0.000607064,0.00089452835,0.00011477446,0.0002170994,0.49517697],"genre_scores_gemma":[0.9811183,0.00042080163,0.016552597,0.0007280466,0.000084071915,0.000098320495,0.000016240258,0.000028016317,0.00095354044],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.77591133,0.17904244,0.0059933304,0.005239694,0.029340195,0.0044729197],"domain_scores_gemma":[0.6437319,0.29281074,0.020743605,0.014064765,0.024324015,0.0043251053],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.11897355,0.00072653947,0.0013043905,0.008952597,0.007388193,0.023022458,0.0025381213,0.0041001067,0.004495417],"category_scores_gemma":[0.22100055,0.0007243973,0.00085849233,0.0043820473,0.0783371,0.025082124,0.013310086,0.0057293638,0.00033693627],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009596281,0.000091655114,0.0049045277,0.0004253691,0.000061307,0.00025527074,0.04351084,0.0014064871,0.00024904392,0.9062387,0.002382814,0.040378105],"study_design_scores_gemma":[0.00003025347,0.00009619659,0.0030130867,0.0010817666,0.000051699444,0.00020084284,0.024141664,0.003951899,0.00076759886,0.94861424,0.017980402,0.000070349335],"about_ca_topic_score_codex":0.0042620194,"about_ca_topic_score_gemma":0.003921961,"teacher_disagreement_score":0.88102645,"about_ca_system_score_codex":0.010921066,"about_ca_system_score_gemma":0.012150626,"threshold_uncertainty_score":0.6291998},"labels":[],"label_agreement":null},{"id":"W1534140708","doi":"10.1002/ev.20088","title":"House With a View: Validity and Evaluative Argument","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"123 Certification (Canada)","funders":"","keywords":"CLARITY; Argument (complex analysis); Style (visual arts); Epistemology; Narrative; Evaluation methods; Psychology; Computer science; Sociology; Linguistics; Philosophy; History","score_opus":0.24014051371560705,"score_gpt":0.5046303264375429,"score_spread":0.26448981272193584,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1534140708","genre_codex":"commentary","genre_gemma":"methods","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.01973745,0.016641084,0.06048966,0.5924214,0.007104952,0.00035221622,0.00009211645,0.00011789212,0.3030433],"genre_scores_gemma":[0.84831494,0.008094476,0.035702344,0.07275525,0.0048668045,0.00088264514,0.000094717434,0.00031121133,0.02897766],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.84216607,0.13075548,0.002637188,0.0041049803,0.018198112,0.0021381758],"domain_scores_gemma":[0.7050767,0.2628883,0.00541187,0.010216669,0.014830764,0.0015757513],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.10040923,0.0006014818,0.0012105464,0.0025695614,0.006011734,0.019025022,0.0025261946,0.0070749745,0.006318588],"category_scores_gemma":[0.2555495,0.0007013882,0.0007399375,0.0021426568,0.045042045,0.026663445,0.006939893,0.010517413,0.00081168837],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000025121482,0.00002312422,0.00017705209,0.000109669316,0.000016969814,0.00007809785,0.006656816,0.00013030734,0.00005472607,0.96701205,0.016630141,0.0090859085],"study_design_scores_gemma":[0.00004446672,0.000056531982,0.00065875205,0.0016172824,0.000033615663,0.000086093605,0.012565251,0.0013686536,0.000664743,0.84102315,0.14183752,0.000043842338],"about_ca_topic_score_codex":0.003517228,"about_ca_topic_score_gemma":0.0023540827,"teacher_disagreement_score":0.8995908,"about_ca_system_score_codex":0.008304209,"about_ca_system_score_gemma":0.007916982,"threshold_uncertainty_score":0.5310211},"labels":[],"label_agreement":null},{"id":"W1541185587","doi":"10.1002/ev.20112","title":"Credentialed Evaluator Designation Program, the Canadian Experience","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":5,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Value (mathematics); Professional development; Service (business); Administration (probate law); Political science; Public relations; Management; Engineering ethics; Sociology; Pedagogy; Computer science; Engineering; Law; Business; Marketing; Economics","score_opus":0.4863368872817085,"score_gpt":0.5792195495990002,"score_spread":0.09288266231729175,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1541185587","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.19419515,0.014601635,0.016630424,0.39332134,0.0043954384,0.0011212152,0.00076248584,0.00039388245,0.37457845],"genre_scores_gemma":[0.8433982,0.0090414835,0.016985638,0.017542731,0.0002040386,0.00033188832,0.00029980124,0.00019836903,0.11199792],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.9746241,0.009173535,0.0010267196,0.0012939435,0.009312254,0.0045693954],"domain_scores_gemma":[0.9292789,0.011960083,0.0015015523,0.002013017,0.032235567,0.023010863],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.037257075,0.00030940995,0.00032125396,0.0019555031,0.020188803,0.0073765474,0.0026195939,0.0021523067,0.008462339],"category_scores_gemma":[0.045955647,0.00047690186,0.00031949038,0.004317825,0.010349644,0.003870797,0.005922341,0.0040944987,0.0005946788],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":true,"about_ca_system_consensus":true,"study_design_scores_codex":[0.00022493752,0.00076954416,0.021852223,0.000551784,0.000017866723,0.0011560625,0.13606988,0.0012286109,0.00081385433,0.24458592,0.29262948,0.30009985],"study_design_scores_gemma":[0.00006953974,0.00014807934,0.016045291,0.00044188896,0.000012693001,0.0003639553,0.04568359,0.0007466643,0.00068073184,0.0057458994,0.9299415,0.000120226905],"about_ca_topic_score_codex":0.964298,"about_ca_topic_score_gemma":0.9854418,"teacher_disagreement_score":0.9627429,"about_ca_system_score_codex":0.14815925,"about_ca_system_score_gemma":0.37224182,"threshold_uncertainty_score":0.9880145},"labels":[],"label_agreement":null},{"id":"W1543074450","doi":"10.1002/ev.20076","title":"Framing the Capacity to Do and Use Evaluation","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Framing (construction); Conceptualization; Situated; Construct (python library); Capacity building; Organizational change; Sociology; Management science; Knowledge management; Public relations; Computer science; Political science; Economics","score_opus":0.33725614035289037,"score_gpt":0.5217974321371192,"score_spread":0.1845412917842288,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1543074450","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.06639417,0.00757985,0.2284453,0.14452644,0.0012524794,0.0005096663,0.00017468777,0.00021132304,0.55090606],"genre_scores_gemma":[0.9661865,0.0017058798,0.022144908,0.0024948828,0.00027711326,0.0004177698,0.00005099316,0.00006223343,0.0066597653],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9557816,0.034913454,0.0012541219,0.0016499592,0.0041953255,0.0022055905],"domain_scores_gemma":[0.90098345,0.07559327,0.0038179061,0.0052747712,0.011064532,0.0032661124],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.041458633,0.00088368513,0.00058087654,0.004957188,0.0050615207,0.018028855,0.0020369377,0.0052275,0.007687629],"category_scores_gemma":[0.06174754,0.00068566477,0.00052075746,0.0026631788,0.059706047,0.017366238,0.0115216365,0.0065382738,0.0005711409],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007568898,0.000011677303,0.00029918578,0.000045081084,0.0000028512998,0.000032285665,0.0042784703,0.00044005155,0.000080456295,0.98824126,0.0010942316,0.0054668216],"study_design_scores_gemma":[0.000016593714,0.00002561242,0.0007192267,0.0007522631,0.000011663308,0.00006353661,0.012614057,0.0026464085,0.00064559636,0.91337556,0.06908432,0.000045200828],"about_ca_topic_score_codex":0.009178755,"about_ca_topic_score_gemma":0.004943858,"teacher_disagreement_score":0.9585414,"about_ca_system_score_codex":0.016627043,"about_ca_system_score_gemma":0.015253942,"threshold_uncertainty_score":0.21925682},"labels":[],"label_agreement":null},{"id":"W1557717928","doi":"10.1002/ev.20082","title":"The Value in Validity","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"123 Certification (Canada)","funders":"","keywords":"Beauty; Prioritization; Economic Justice; Value (mathematics); Sociology; Balance (ability); Management science; Psychology; Social psychology; Epistemology; Computer science; Political science; Law; Economics","score_opus":0.36226017726620263,"score_gpt":0.5450118219863205,"score_spread":0.18275164472011785,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1557717928","genre_codex":"other","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.021086447,0.029108236,0.1352781,0.31812772,0.0035776624,0.00033950034,0.00020608277,0.00013725087,0.4921389],"genre_scores_gemma":[0.9234975,0.0067071845,0.037817623,0.019090662,0.002161998,0.0004995837,0.00006927065,0.00017265117,0.009983559],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.7128092,0.22632627,0.0070621185,0.008287271,0.041334853,0.004180312],"domain_scores_gemma":[0.6576813,0.28168708,0.0076138186,0.02238187,0.026715718,0.0039202655],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.13564959,0.0009395056,0.0017846455,0.0056382073,0.007040419,0.021834057,0.0029869685,0.0059609446,0.0050979895],"category_scores_gemma":[0.21293801,0.00070360926,0.0010614459,0.0038926846,0.116228126,0.021833658,0.012953847,0.010295587,0.0007903806],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018375133,0.000009838239,0.00046051154,0.00010902818,0.000015874593,0.00002733467,0.0014328787,0.00026916215,0.000039651015,0.98364073,0.003337322,0.010639313],"study_design_scores_gemma":[0.000012933749,0.000019505702,0.0003078637,0.00067846733,0.000011523235,0.000048903297,0.0011743358,0.0005626293,0.00020116428,0.9700435,0.026919777,0.000019394316],"about_ca_topic_score_codex":0.006053765,"about_ca_topic_score_gemma":0.0032058337,"teacher_disagreement_score":0.13564959,"about_ca_system_score_codex":0.017437622,"about_ca_system_score_gemma":0.016243039,"threshold_uncertainty_score":0.7173922},"labels":[],"label_agreement":null},{"id":"W1569252761","doi":"10.1002/ev.20079","title":"Postscript: That Was Then, This Is Now","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec; Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Sustainability; Public relations; Organizational communication; Knowledge management; Political science; Sociology; Computer science","score_opus":0.2843044164649911,"score_gpt":0.5111892279361946,"score_spread":0.22688481147120348,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1569252761","genre_codex":"commentary","genre_gemma":"commentary","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"commentary","genre_consensus":"commentary","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0028055157,0.0020378567,0.0031972248,0.45348808,0.43469158,0.000277929,0.0013374083,0.0007636557,0.10140084],"genre_scores_gemma":[0.03731368,0.0010899311,0.0015403127,0.09238013,0.026667465,0.00021168296,0.00047226547,0.00073821435,0.8395864],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99741924,0.00049818214,0.00020646633,0.00031605814,0.0011741237,0.0003860035],"domain_scores_gemma":[0.9832895,0.002848485,0.00056046736,0.0008050229,0.01121486,0.0012816398],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0029161852,0.0005094245,0.00049434684,0.00070787675,0.004438649,0.0054086824,0.0012252097,0.0028062477,0.12214718],"category_scores_gemma":[0.02709484,0.00021159384,0.00043955081,0.00044338367,0.0019795839,0.003515801,0.0021291624,0.009142771,0.05810899],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006135527,0.000017401808,0.00017079417,0.00009259309,0.0000035841103,0.0001278021,0.00072798925,0.000033757748,0.0005379266,0.010460794,0.9775352,0.010230859],"study_design_scores_gemma":[0.0000050758463,0.000026142687,0.00045034182,0.00011006629,0.0000052519454,0.00008433275,0.0011149757,0.00006547635,0.0005769581,0.001961258,0.99558085,0.000019260347],"about_ca_topic_score_codex":0.013647565,"about_ca_topic_score_gemma":0.01235449,"teacher_disagreement_score":0.12214718,"about_ca_system_score_codex":0.0041183988,"about_ca_system_score_gemma":0.003951143,"threshold_uncertainty_score":0.40862298},"labels":[],"label_agreement":null},{"id":"W1781948974","doi":"10.1002/ev.20111","title":"How Does Accreditation Fit Into the Picture?","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Accounting Education and Careers","field":"Business, Management and Accounting","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Victoria","funders":"","keywords":"Accreditation; Blank; Computer science; Public relations; Political science; Law; Engineering","score_opus":0.06017099692544828,"score_gpt":0.31944720650743275,"score_spread":0.2592762095819845,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1781948974","genre_codex":"commentary","genre_gemma":"commentary","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"commentary","genre_consensus":"commentary","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0032267368,0.042937897,0.009411978,0.8938583,0.0047231168,0.00004612456,0.00011524814,0.00014922213,0.045531444],"genre_scores_gemma":[0.60571146,0.079877995,0.023233842,0.26162326,0.014511084,0.00036169315,0.0004073542,0.0007313827,0.013541981],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.9288412,0.044792354,0.0038615346,0.003111777,0.015760344,0.0036326929],"domain_scores_gemma":[0.82560015,0.10857657,0.006768341,0.0063014245,0.045712326,0.0070410194],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0816688,0.0008534237,0.0020541726,0.0097895,0.006907308,0.028633565,0.0028347664,0.011060888,0.010056603],"category_scores_gemma":[0.18416613,0.00074934575,0.001305825,0.007753699,0.02697164,0.060936667,0.007664841,0.01864472,0.002372281],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006507406,0.0001226057,0.0053206426,0.0012526815,0.00008242324,0.00012657246,0.0033236202,0.0006271497,0.000112791124,0.7279004,0.12019978,0.14086618],"study_design_scores_gemma":[0.00002542652,0.00011069284,0.008014156,0.00915292,0.00007206246,0.00027615248,0.021266121,0.0015764721,0.00025138215,0.6532779,0.30583546,0.00014136768],"about_ca_topic_score_codex":0.012840184,"about_ca_topic_score_gemma":0.010309741,"teacher_disagreement_score":0.0816688,"about_ca_system_score_codex":0.012820755,"about_ca_system_score_gemma":0.021501364,"threshold_uncertainty_score":0.43191105},"labels":[],"label_agreement":null},{"id":"W1929107339","doi":"10.1002/ev.20078","title":"Cross‐Case Analysis and Implications for Research, Theory, and Practice","year":2014,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université du Québec en Outaouais; University of Ottawa","funders":"","keywords":"Leverage (statistics); Theory of change; Organizational change; Affect (linguistics); Thematic analysis; Context (archaeology); Order (exchange); Evaluation methods; Organization development; Knowledge management; Management science; Public relations; Sociology; Computer science; Political science; Management; Business; Qualitative research; Social science; Engineering; Economics","score_opus":0.46899379184883316,"score_gpt":0.6700118587679306,"score_spread":0.2010180669190974,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1929107339","genre_codex":"empirical","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.55610955,0.01125931,0.18500903,0.025810083,0.00078406726,0.006658667,0.00064637914,0.00021524499,0.21350768],"genre_scores_gemma":[0.9089054,0.0035521826,0.07459771,0.0011465027,0.00011165042,0.0036703101,0.0003797244,0.000052273786,0.0075842612],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"qualitative","domain_scores_codex":[0.91155493,0.07308399,0.0029063805,0.0019883723,0.008336085,0.0021301974],"domain_scores_gemma":[0.7665394,0.20876619,0.004003285,0.006596324,0.013135414,0.00095937506],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.060237095,0.00063436,0.0010134112,0.0072483523,0.006627776,0.008287329,0.0030934482,0.0021325469,0.011352998],"category_scores_gemma":[0.11509279,0.0005686231,0.00081774767,0.009334709,0.005971221,0.010471991,0.0051727025,0.002305213,0.00073056074],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00029148333,0.0026929525,0.044552665,0.0019031743,0.00022516097,0.0025544649,0.21243253,0.0040575326,0.00094150053,0.49736512,0.012260449,0.22072288],"study_design_scores_gemma":[0.00010183647,0.00057479227,0.04062124,0.0065802764,0.0001810851,0.0014416483,0.57342005,0.017073749,0.0027250685,0.24866356,0.108481936,0.00013482291],"about_ca_topic_score_codex":0.0057848054,"about_ca_topic_score_gemma":0.007603145,"teacher_disagreement_score":0.060237095,"about_ca_system_score_codex":0.01041882,"about_ca_system_score_gemma":0.004112858,"threshold_uncertainty_score":0.318568},"labels":[],"label_agreement":null},{"id":"W1978303279","doi":"10.1002/ev.382","title":"Integrating a new evaluation unit with an old institution: See no evil; hear no evil; speak no evil","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Guelph","funders":"","keywords":"Unit (ring theory); CLARITY; Institution; Evaluation methods; Association (psychology); Psychology; Sociology; Computer science; Mathematics education; Social science; Engineering; Psychotherapist","score_opus":0.39098050530506484,"score_gpt":0.5034779046174458,"score_spread":0.11249739931238095,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1978303279","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2279489,0.0022450937,0.14315668,0.19376627,0.009341985,0.003202911,0.00011233307,0.0033261976,0.41689965],"genre_scores_gemma":[0.8273485,0.00077776826,0.09592788,0.022413472,0.0012651152,0.001782231,0.000062853636,0.0005186198,0.049903605],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.928192,0.04689859,0.004059466,0.001315328,0.01694284,0.0025917527],"domain_scores_gemma":[0.9073618,0.0389775,0.008430242,0.005974495,0.018413913,0.02084206],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.05355232,0.00044006165,0.00076320797,0.001161552,0.0051825168,0.0129218595,0.002172179,0.0037944934,0.01159493],"category_scores_gemma":[0.10006285,0.00038755502,0.0004521878,0.00059238635,0.006101722,0.007667326,0.010478282,0.0064197266,0.0035741616],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0010261603,0.003586912,0.029378077,0.0012490938,0.00009222425,0.0013450368,0.023825996,0.00206715,0.004903742,0.14589266,0.16215868,0.6244743],"study_design_scores_gemma":[0.00069422374,0.0037342927,0.04828169,0.00412796,0.00025303586,0.0020448745,0.065180115,0.020239847,0.025109893,0.08010741,0.7491906,0.0010360824],"about_ca_topic_score_codex":0.0014666603,"about_ca_topic_score_gemma":0.00450404,"teacher_disagreement_score":0.05355232,"about_ca_system_score_codex":0.0044070263,"about_ca_system_score_gemma":0.012483813,"threshold_uncertainty_score":0.2832151},"labels":[],"label_agreement":null},{"id":"W1986224130","doi":"10.1002/ev.133","title":"Background and history of the Joint Committee's Program Evaluation Standards","year":2004,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":27,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Queen's University","funders":"","keywords":"Joint (building); Context (archaeology); Political science; Library science; Engineering ethics; Computer science; Engineering; History; Archaeology; Civil engineering","score_opus":0.3548191126107928,"score_gpt":0.5240969201650518,"score_spread":0.16927780755425903,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1986224130","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00448101,0.08445513,0.11144498,0.3109683,0.0105205,0.0017404042,0.0021285985,0.001080728,0.47318032],"genre_scores_gemma":[0.25997716,0.123506986,0.21622322,0.07889466,0.012299552,0.004690323,0.0043654894,0.0023223744,0.29772016],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.8385463,0.043368492,0.013763431,0.0074065547,0.08941179,0.0075035184],"domain_scores_gemma":[0.66994864,0.07358265,0.00935027,0.01953854,0.21991779,0.0076621296],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1708321,0.0011377047,0.001510802,0.017924864,0.011207257,0.023901068,0.005849861,0.007421306,0.0058999723],"category_scores_gemma":[0.20294617,0.0019025701,0.0014039736,0.020786026,0.01951945,0.0068729115,0.0059042,0.015989305,0.001859508],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003120284,0.0000706906,0.0010461815,0.0005757432,0.00002667048,0.000058906753,0.0029043518,0.001602531,0.00030869202,0.7112929,0.14922467,0.13285744],"study_design_scores_gemma":[0.000009573116,0.000030375755,0.0028241032,0.0025403467,0.000021395626,0.000046071844,0.00088499504,0.0006134872,0.0004379679,0.042163223,0.95034266,0.000085852764],"about_ca_topic_score_codex":0.6719794,"about_ca_topic_score_gemma":0.62610275,"teacher_disagreement_score":0.6719794,"about_ca_system_score_codex":0.11552066,"about_ca_system_score_gemma":0.28782988,"threshold_uncertainty_score":0.9034573},"labels":[],"label_agreement":null},{"id":"W1992053221","doi":"10.1002/ev.325","title":"Enhancing disaster and emergency preparedness, response, and recovery through evaluation","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Preparedness; Disaster response; Emergency management; Disaster preparedness; Emergency response; Process management; Evaluation methods; Disaster recovery; Disaster planning; Computer science; Business; Risk analysis (engineering); Computer security; Public relations; Political science; Medical emergency; Poison control; Human factors and ergonomics; Engineering; Medicine; Reliability engineering","score_opus":0.16373296269347223,"score_gpt":0.5114703559569701,"score_spread":0.34773739326349784,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1992053221","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14580928,0.01593145,0.34065855,0.09418296,0.0005817396,0.0028068086,0.00020974485,0.00052020757,0.3992993],"genre_scores_gemma":[0.91314673,0.004007331,0.07818524,0.0014523022,0.000089480855,0.0005127036,0.00004352311,0.000019448262,0.002543337],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.92588145,0.065462984,0.0013048729,0.0005805253,0.005559944,0.0012102315],"domain_scores_gemma":[0.9468051,0.041522823,0.0032345478,0.0014234353,0.005011362,0.0020027466],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.056342144,0.00056525815,0.0007059893,0.002700732,0.0016909561,0.009153529,0.0009931433,0.0013911725,0.0031808065],"category_scores_gemma":[0.048768885,0.0002053618,0.00051543146,0.0018593718,0.005216806,0.0051380703,0.0046705795,0.0016441176,0.00026928243],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00034294612,0.00087385235,0.0096678175,0.0021646365,0.00011033446,0.00014871953,0.0045488887,0.012355261,0.00085826806,0.6403917,0.010821733,0.31771585],"study_design_scores_gemma":[0.00069518614,0.0024785923,0.019040646,0.012594235,0.00045054313,0.0005422341,0.021268284,0.06841223,0.007942213,0.7217634,0.14452615,0.00028635643],"about_ca_topic_score_codex":0.0021072428,"about_ca_topic_score_gemma":0.0024843663,"teacher_disagreement_score":0.056342144,"about_ca_system_score_codex":0.005687425,"about_ca_system_score_gemma":0.014307735,"threshold_uncertainty_score":0.29796928},"labels":[],"label_agreement":null},{"id":"W1994122017","doi":"10.1002/ev.333","title":"Evaluation of disaster and emergency management: Do no harm, but do better","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Disaster Response and Management","field":"Health Professions","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Emergency management; Harm; Medical emergency; Business; Medicine; Psychology; Political science; Social psychology","score_opus":0.08876767161420009,"score_gpt":0.4590029695750237,"score_spread":0.3702352979608236,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1994122017","genre_codex":"commentary","genre_gemma":"commentary","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"commentary","genre_consensus":"commentary","domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0064781294,0.08460758,0.011654791,0.84375286,0.011115115,0.0003037924,0.00009247159,0.00018614226,0.04180898],"genre_scores_gemma":[0.507943,0.10162489,0.055580214,0.30535114,0.013902758,0.0009337173,0.00015063648,0.0002929766,0.014220628],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.85342807,0.114472024,0.007581585,0.0014530546,0.021851724,0.001213547],"domain_scores_gemma":[0.74559015,0.17355058,0.011789113,0.011916995,0.04802983,0.009123298],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.111032486,0.0012571424,0.0024374756,0.0032169167,0.0031350488,0.015021957,0.0013198359,0.005115042,0.0043353],"category_scores_gemma":[0.22723761,0.00054953055,0.00088996766,0.0017172946,0.012034925,0.01462227,0.0068270224,0.009057412,0.0012366122],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0007054266,0.0004888189,0.0074642403,0.008559809,0.00061800366,0.0002591996,0.0054959725,0.0010796399,0.0005714139,0.12063377,0.24954984,0.60457385],"study_design_scores_gemma":[0.00071015314,0.0012029223,0.012789087,0.071437426,0.00088276534,0.0016855474,0.026096363,0.0033516893,0.0014918355,0.41613507,0.46382987,0.0003873442],"about_ca_topic_score_codex":0.0032949287,"about_ca_topic_score_gemma":0.004787893,"teacher_disagreement_score":0.8889675,"about_ca_system_score_codex":0.0072031626,"about_ca_system_score_gemma":0.016288254,"threshold_uncertainty_score":0.5872029},"labels":[],"label_agreement":null},{"id":"W1999716079","doi":"10.1002/ev.243","title":"Infusing evaluative thinking as process use: The case of the International Development Research Centre (IDRC)","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":21,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Accountability; Process (computing); Political science; Sociology; Process management; Management; Business; Computer science","score_opus":0.45376368323160404,"score_gpt":0.6131986765418254,"score_spread":0.15943499331022132,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1999716079","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.310509,0.0028675487,0.040236685,0.108794816,0.00034735,0.0006891305,0.00007226004,0.00030779815,0.53617543],"genre_scores_gemma":[0.97520876,0.0005779939,0.007978215,0.002624129,0.00004526568,0.00016915849,0.0000121253,0.00011475324,0.013269668],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"qualitative","domain_scores_codex":[0.89419293,0.080885656,0.00110162,0.0020748524,0.013359313,0.008385685],"domain_scores_gemma":[0.9013883,0.07718405,0.0034073505,0.0046545966,0.009119047,0.004246599],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.075108916,0.00079563854,0.00086469925,0.0031419382,0.032689158,0.03302682,0.0037551317,0.008534928,0.004342149],"category_scores_gemma":[0.0624779,0.00072112697,0.0008868957,0.0033132776,0.0562201,0.011207045,0.010546647,0.013305315,0.0005061783],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_system_candidate":true,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009712376,0.00016650489,0.0027392418,0.00015049997,0.00001732002,0.004585906,0.3968565,0.0014422522,0.00047107082,0.5642857,0.007429224,0.021758681],"study_design_scores_gemma":[0.00015783998,0.00022816237,0.004181713,0.0013343276,0.00009082045,0.002409658,0.5062369,0.008259964,0.003150814,0.13028671,0.34346282,0.00020030848],"about_ca_topic_score_codex":0.26077476,"about_ca_topic_score_gemma":0.24701832,"teacher_disagreement_score":0.94064486,"about_ca_system_score_codex":0.059355155,"about_ca_system_score_gemma":0.046000455,"threshold_uncertainty_score":0.5185138},"labels":[],"label_agreement":null},{"id":"W2008070236","doi":"10.1002/ev.299","title":"Reflections on the dilemmas of conducting environmental evaluations","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Cognitive reframing; Multidisciplinary approach; Asset (computer security); Context (archaeology); Set (abstract data type); Face (sociological concept); Computer science; Management science; Theory of change; Work (physics); Sociology; Engineering ethics; Psychology; Social science; Social psychology; Economics; Engineering","score_opus":0.5676539202947379,"score_gpt":0.5954441216399692,"score_spread":0.027790201345231247,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2008070236","genre_codex":"commentary","genre_gemma":"commentary","domain_codex":"methods","domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"commentary","genre_consensus":"commentary","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0054298956,0.003965845,0.010828068,0.96035,0.0023731275,0.0002984233,0.000043045246,0.00009310204,0.0166186],"genre_scores_gemma":[0.5010082,0.009382629,0.088825084,0.37662435,0.0061386614,0.0050139865,0.00008852689,0.00080240227,0.012116214],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.27856824,0.6316417,0.021594314,0.009080901,0.051690046,0.0074248295],"domain_scores_gemma":[0.14910601,0.77607554,0.008158681,0.0110236835,0.047335047,0.008301123],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.59502614,0.001021566,0.0025110287,0.0038654397,0.014278121,0.03957076,0.0080017755,0.020924514,0.0052475613],"category_scores_gemma":[0.6618935,0.0018299705,0.0020327216,0.0040071653,0.06077994,0.037140373,0.017437518,0.03868965,0.0014527239],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0002574074,0.00025980562,0.0022143687,0.001814513,0.00012249885,0.00071887276,0.117635585,0.0021556735,0.00032354737,0.51858604,0.2539023,0.102009445],"study_design_scores_gemma":[0.00023693212,0.0002549067,0.0016980586,0.008625879,0.00005147261,0.00063194666,0.12680684,0.0037823624,0.0009956168,0.39789212,0.45874923,0.0002745647],"about_ca_topic_score_codex":0.007999527,"about_ca_topic_score_gemma":0.007478223,"teacher_disagreement_score":0.59502614,"about_ca_system_score_codex":0.02578024,"about_ca_system_score_gemma":0.04382318,"threshold_uncertainty_score":0.4994049},"labels":[],"label_agreement":null},{"id":"W2013250096","doi":"10.1002/ev.20007","title":"When one must go: The Canadian experience with strategic review and judging program value","year":2012,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Valuation (finance); Function (biology); Government (linguistics); Value (mathematics); Citizen journalism; Public relations; Process (computing); Value for money; Public administration; Business; Political science; Economics; Public economics; Computer science; Accounting; Law","score_opus":0.38211926976493,"score_gpt":0.524848788433648,"score_spread":0.14272951866871803,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2013250096","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.12728845,0.024796387,0.017407846,0.23719688,0.0018542319,0.0006175029,0.00027122686,0.00026469285,0.5903028],"genre_scores_gemma":[0.93894017,0.012318186,0.015654627,0.008676007,0.00026730567,0.00017569265,0.000120022865,0.00015088708,0.023697134],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"qualitative","domain_scores_codex":[0.87040424,0.06674352,0.003536206,0.003643278,0.045927413,0.009745293],"domain_scores_gemma":[0.816703,0.09481774,0.00465567,0.0045852656,0.06610148,0.013136836],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.09253305,0.00055005064,0.0010549673,0.005499267,0.02775499,0.024354352,0.0034082686,0.0034873427,0.004857157],"category_scores_gemma":[0.1252026,0.00057279767,0.0005773462,0.009955995,0.02187442,0.00622281,0.0067273085,0.006237126,0.00046216525],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":true,"about_ca_system_consensus":true,"study_design_scores_codex":[0.00027955693,0.00020886693,0.011648881,0.0009037015,0.000088932575,0.0006647203,0.12085008,0.0022680403,0.0008110615,0.40968627,0.09912063,0.35346922],"study_design_scores_gemma":[0.00008693124,0.0001821595,0.018235644,0.0015630876,0.00008567153,0.00025535823,0.091567956,0.0023890256,0.0014438996,0.06471133,0.8190959,0.00038297713],"about_ca_topic_score_codex":0.9346248,"about_ca_topic_score_gemma":0.96714836,"teacher_disagreement_score":0.90746695,"about_ca_system_score_codex":0.18534002,"about_ca_system_score_gemma":0.31775114,"threshold_uncertainty_score":0.9448901},"labels":[],"label_agreement":null},{"id":"W2017680079","doi":"10.1002/ev.344","title":"Strategy evaluation: Experience at the International Development Research Centre","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"International Development Research Centre","funders":"","keywords":"Corporation; Program evaluation; Strategic planning; Modalities; International development; Sociology; Management; Business; Public relations; Political science; Economics; Public administration; Social science","score_opus":0.5427282247833871,"score_gpt":0.6215806160734704,"score_spread":0.07885239129008326,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2017680079","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.25236556,0.02227874,0.022863008,0.06835711,0.001559808,0.0017967669,0.00042407075,0.00079045264,0.62956446],"genre_scores_gemma":[0.88697284,0.011567136,0.023901427,0.0071000904,0.00038922668,0.00088569755,0.00048010188,0.00067447184,0.06802896],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.86783475,0.09384147,0.0033221005,0.0033797172,0.021038283,0.010583653],"domain_scores_gemma":[0.9057304,0.03804176,0.0023528382,0.005983589,0.028720353,0.019171143],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.12441589,0.00079619297,0.0010101353,0.003101812,0.011493101,0.017532868,0.0034660941,0.0035693487,0.00911707],"category_scores_gemma":[0.06483799,0.00064098556,0.0006686727,0.006110033,0.0076875966,0.005926758,0.009161646,0.009480849,0.0020931163],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00072253996,0.005232947,0.010586985,0.0011633667,0.00008096587,0.0019626506,0.23832211,0.0028459288,0.0016888712,0.12716727,0.1356118,0.47461453],"study_design_scores_gemma":[0.00014848368,0.001335245,0.0055947234,0.0011178163,0.0000442267,0.0005222415,0.08343735,0.0017766408,0.0033213806,0.0048645944,0.89766324,0.0001741485],"about_ca_topic_score_codex":0.036930174,"about_ca_topic_score_gemma":0.031728256,"teacher_disagreement_score":0.8755841,"about_ca_system_score_codex":0.028633473,"about_ca_system_score_gemma":0.047596753,"threshold_uncertainty_score":0.657982},"labels":[],"label_agreement":null},{"id":"W2033749959","doi":"10.1002/ev.1190","title":"Evaluative inquiry in university‐school professional learning partnerships","year":2000,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Queen's University","funders":"","keywords":"Professional learning community; Professional development; Pedagogy; Psychology; Sociology","score_opus":0.37815566276918305,"score_gpt":0.5350127287443462,"score_spread":0.1568570659751632,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2033749959","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5466221,0.008619864,0.08265789,0.07620462,0.00071587815,0.0015181707,0.000054464985,0.00024339218,0.2833637],"genre_scores_gemma":[0.9855182,0.0008479455,0.009908015,0.00063304463,0.00005026175,0.00059232727,0.000010130921,0.000013565309,0.0024264455],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"qualitative","domain_scores_codex":[0.7794903,0.20900355,0.0015138648,0.0013652658,0.0052191247,0.0034078762],"domain_scores_gemma":[0.6830457,0.27888173,0.0090376595,0.0060445354,0.009991335,0.01299908],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.12564859,0.00038373398,0.0006862238,0.003030009,0.011993465,0.018357165,0.0021194639,0.0033827687,0.0051747616],"category_scores_gemma":[0.17484953,0.00046746415,0.0003127631,0.003423545,0.022495713,0.012075928,0.016968971,0.0038135792,0.00035696093],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00033264238,0.0016131384,0.013739259,0.00077858526,0.000048869,0.0006572384,0.14708287,0.0037124038,0.00026610884,0.6782577,0.010680665,0.14283048],"study_design_scores_gemma":[0.00031725303,0.0007261592,0.005964408,0.0016841413,0.00003420753,0.00034509125,0.33641905,0.0074335025,0.0011116515,0.5543759,0.09150466,0.00008394744],"about_ca_topic_score_codex":0.0019983153,"about_ca_topic_score_gemma":0.003189683,"teacher_disagreement_score":0.12564859,"about_ca_system_score_codex":0.010398584,"about_ca_system_score_gemma":0.025252864,"threshold_uncertainty_score":0.6645012},"labels":[],"label_agreement":null},{"id":"W2038633574","doi":"10.1002/ev.1171","title":"Translating evaluation findings into “policy language”","year":2000,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Université du Québec à Montréal","funders":"","keywords":"Affect (linguistics); Welfare; Order (exchange); Policy learning; Policy analysis; Program evaluation; Language policy; Computer science; Management science; Political science; Public administration; Sociology; Pedagogy; Economics; Machine learning; Law; Finance","score_opus":0.19760218499848886,"score_gpt":0.5590826968278899,"score_spread":0.36148051182940105,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2038633574","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.029073428,0.011637059,0.22131133,0.3360745,0.008373706,0.0019256594,0.0021656335,0.001470447,0.38796824],"genre_scores_gemma":[0.69268924,0.020179389,0.2197573,0.036620546,0.0016765747,0.0029926735,0.0011624508,0.00090875337,0.024013089],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.819006,0.14297545,0.0069969827,0.0021399714,0.026656238,0.0022252772],"domain_scores_gemma":[0.6446816,0.24549496,0.0064675612,0.01373955,0.08783863,0.0017777907],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.15634297,0.0007346299,0.0013015994,0.006424267,0.0025092904,0.017653706,0.002182998,0.0017882234,0.008775703],"category_scores_gemma":[0.30007192,0.0005081587,0.00067274575,0.0047393823,0.00950209,0.007992976,0.005105081,0.0055929846,0.0016132126],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00014135781,0.00032591348,0.0024377631,0.0036428005,0.00014792981,0.00026975767,0.01692639,0.0055340314,0.0018908876,0.65800226,0.07408161,0.23659936],"study_design_scores_gemma":[0.00023092443,0.00032517553,0.006708131,0.026743736,0.00029881005,0.00012701044,0.042148974,0.008441988,0.011484318,0.33700117,0.5662746,0.00021527347],"about_ca_topic_score_codex":0.038294595,"about_ca_topic_score_gemma":0.027900442,"teacher_disagreement_score":0.15634297,"about_ca_system_score_codex":0.025862707,"about_ca_system_score_gemma":0.053100053,"threshold_uncertainty_score":0.8268305},"labels":[],"label_agreement":null},{"id":"W2055999462","doi":"10.1002/ev.392","title":"Internal evaluation a quarter‐century later: A conversation with Arnold J. Love","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Conversation; Evaluation methods; Perspective (graphical); Association (psychology); Quarter (Canadian coin); Sociology; Psychology; Computer science; Artificial intelligence; History; Engineering","score_opus":0.2229234698309984,"score_gpt":0.4641575431803136,"score_spread":0.2412340733493152,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2055999462","genre_codex":"commentary","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0008269026,0.019303538,0.00075546384,0.9642933,0.0076681203,0.000010052199,0.0000068203453,0.000011122145,0.0071247094],"genre_scores_gemma":[0.081466466,0.0453897,0.0025495852,0.8189036,0.015436091,0.000121130935,0.000025291267,0.00021093056,0.035897106],"study_design_codex":"not_applicable","study_design_gemma":"qualitative","domain_scores_codex":[0.89687884,0.07444954,0.0038131562,0.0037095349,0.016653957,0.0044950396],"domain_scores_gemma":[0.87017816,0.083542906,0.003248955,0.0024325869,0.0236155,0.016981928],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.09419937,0.0006579602,0.0018722428,0.002343828,0.012439204,0.02158207,0.0016465223,0.015171719,0.004861577],"category_scores_gemma":[0.11931596,0.0008502112,0.0013561661,0.0024716589,0.026157,0.018428177,0.009645966,0.043706346,0.0017115746],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000056113986,0.0001596456,0.00072846096,0.0004044189,0.000027789196,0.0007254373,0.04633964,0.00015015434,0.00022208437,0.1414363,0.7515459,0.058204077],"study_design_scores_gemma":[0.000015734797,0.00004968336,0.0005336078,0.0018587488,0.000011945097,0.00057810964,0.019811198,0.00016831838,0.00017587845,0.019596625,0.95712334,0.000076829514],"about_ca_topic_score_codex":0.009212085,"about_ca_topic_score_gemma":0.011180598,"teacher_disagreement_score":0.09419937,"about_ca_system_score_codex":0.025852272,"about_ca_system_score_gemma":0.021902556,"threshold_uncertainty_score":0.4981798},"labels":[],"label_agreement":null},{"id":"W2066945648","doi":"10.1002/ev.315","title":"Knowledge translation: Implications for evaluation","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":49,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Institutes of Health Research","funders":"","keywords":"Knowledge translation; Computer science; Knowledge management; Field (mathematics); Translation (biology); Sustainability","score_opus":0.5370852562015647,"score_gpt":0.6053385358954619,"score_spread":0.06825327969389727,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2066945648","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.007774075,0.09006819,0.087141916,0.6963973,0.0091395015,0.0030935258,0.00073069194,0.00043754512,0.10521726],"genre_scores_gemma":[0.7018377,0.05530583,0.14529559,0.06968587,0.008317306,0.011780006,0.00062886794,0.00030546647,0.0068433317],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.5179054,0.4376667,0.013447683,0.005042946,0.022533447,0.0034038783],"domain_scores_gemma":[0.18376282,0.7369837,0.0134116355,0.015345411,0.04593169,0.0045647766],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.34841308,0.0029713644,0.006076108,0.010812934,0.006256331,0.026509887,0.0068970546,0.015117353,0.023838468],"category_scores_gemma":[0.66347134,0.0010901947,0.0027381333,0.01682822,0.03192297,0.037642684,0.010877289,0.009222144,0.002351412],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00088935136,0.00038803098,0.00222179,0.009355695,0.00034814168,0.00044869172,0.004393993,0.0042272354,0.000095909345,0.64677054,0.06249544,0.26836514],"study_design_scores_gemma":[0.0003823108,0.00019286256,0.0011953247,0.016534762,0.00021359242,0.00017233618,0.008403186,0.0076554646,0.00031107108,0.9227388,0.042081974,0.000118368574],"about_ca_topic_score_codex":0.011011935,"about_ca_topic_score_gemma":0.0051717334,"teacher_disagreement_score":0.34841308,"about_ca_system_score_codex":0.029121144,"about_ca_system_score_gemma":0.044795383,"threshold_uncertainty_score":0.8035227},"labels":[],"label_agreement":null},{"id":"W2067449005","doi":"10.1002/ev.326","title":"Real‐time evaluation in humanitarian emergencies","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Formative assessment; Credibility; Evaluation methods; Program evaluation; Field (mathematics); Emergency management; Process management; Computer science; Monitoring and evaluation; Risk analysis (engineering); Political science; Business; Psychology; Engineering; Public administration","score_opus":0.23693256838441462,"score_gpt":0.526100458986193,"score_spread":0.28916789060177833,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2067449005","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.4876526,0.037896924,0.30224043,0.014553656,0.0021847496,0.0029660852,0.0003976052,0.0009528152,0.15115519],"genre_scores_gemma":[0.95339274,0.00314715,0.03760653,0.0003982829,0.00015417035,0.00047880696,0.000083761595,0.000038255996,0.00470033],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9339301,0.060141098,0.0014014582,0.0006246209,0.0034510014,0.00045165455],"domain_scores_gemma":[0.9371894,0.048982043,0.0041814484,0.001630723,0.0068647168,0.0011517332],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.03589217,0.00043578362,0.00044835082,0.0013330145,0.00062354904,0.0030958415,0.00059300713,0.00078940025,0.0035599235],"category_scores_gemma":[0.06829165,0.00019031894,0.00038213286,0.0008453292,0.0014279921,0.0018997823,0.0013962813,0.00094350096,0.00052343006],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00435597,0.0017272193,0.011505323,0.0033264162,0.00025047315,0.0006754783,0.00626591,0.05308495,0.004820307,0.0716366,0.0184058,0.8239456],"study_design_scores_gemma":[0.003274831,0.021960149,0.05349191,0.01255002,0.0007146026,0.0034571595,0.02663531,0.23572254,0.055831738,0.22481158,0.36068213,0.00086806825],"about_ca_topic_score_codex":0.0010392042,"about_ca_topic_score_gemma":0.0009825756,"teacher_disagreement_score":0.9641078,"about_ca_system_score_codex":0.0016178663,"about_ca_system_score_gemma":0.0018206727,"threshold_uncertainty_score":0.1898182},"labels":[],"label_agreement":null},{"id":"W2070292114","doi":"10.1002/ev.393","title":"Internal evaluation, historically speaking","year":2011,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":20,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Centre for Advancing Health Outcomes; University of British Columbia","funders":"","keywords":"Context (archaeology); Conservatism; Speculation; Government (linguistics); Function (biology); Evaluation methods; Internal validity; Political science; Public administration; Sociology; Public relations; Law; Business; History","score_opus":0.45966699897342617,"score_gpt":0.5336734648069541,"score_spread":0.07400646583352793,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2070292114","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.04296228,0.04430625,0.06992962,0.051199775,0.0026115905,0.00033437568,0.0002332645,0.0003409967,0.7880818],"genre_scores_gemma":[0.9067696,0.01796104,0.019412603,0.0059241424,0.001268406,0.000359186,0.00018279866,0.0002093843,0.047912862],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9694826,0.016542764,0.0017152313,0.0020242173,0.009029339,0.0012059393],"domain_scores_gemma":[0.9543013,0.018466301,0.0041268663,0.003221247,0.018517861,0.0013664368],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02384737,0.0006609487,0.00070171955,0.0044288067,0.0030397824,0.0135988165,0.0011184312,0.0012526559,0.0048345965],"category_scores_gemma":[0.045980755,0.00026633157,0.00046465264,0.003956595,0.015089988,0.0068849283,0.0050362903,0.0026366324,0.00096440746],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00007345891,0.00010281936,0.005645331,0.0007370454,0.000028707092,0.00012042326,0.007165432,0.0008467087,0.000432552,0.68178743,0.020840032,0.28222007],"study_design_scores_gemma":[0.000055982826,0.00032472252,0.01880724,0.009207484,0.00010984964,0.00085726375,0.017231038,0.0035238012,0.0046791355,0.3831814,0.561885,0.00013711852],"about_ca_topic_score_codex":0.0035143844,"about_ca_topic_score_gemma":0.0028835395,"teacher_disagreement_score":0.97615266,"about_ca_system_score_codex":0.012942725,"about_ca_system_score_gemma":0.009247742,"threshold_uncertainty_score":0.12611848},"labels":[],"label_agreement":null},{"id":"W2097486283","doi":"10.1002/ev.20045","title":"Rethinking the Relationship Among Monitoring, Evaluation, and Results‐Based Management: Observations From Canada","year":2013,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Public Policy and Administration Research","field":"Social Sciences","cited_by":24,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Accountability; Government (linguistics); Monitoring and evaluation; Dual (grammatical number); Performance management; Process management; Political science; Public administration; Business; Marketing","score_opus":0.24376069185164306,"score_gpt":0.42792132003167843,"score_spread":0.18416062818003537,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2097486283","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.94072443,0.00244807,0.0012506964,0.022294588,0.00007430742,0.00019343519,0.00088192144,0.00004920736,0.032083385],"genre_scores_gemma":[0.9920089,0.0011279715,0.0010757,0.0012017996,0.000012510512,0.000025668189,0.00015447955,0.00003710049,0.0043557156],"study_design_codex":"qualitative","study_design_gemma":"qualitative","domain_scores_codex":[0.9887639,0.0027441876,0.000459452,0.00066738663,0.0049969386,0.0023680865],"domain_scores_gemma":[0.95467424,0.015612444,0.0022435742,0.0010098746,0.02328623,0.003173626],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.011465997,0.00040118644,0.00064406206,0.00302773,0.015962739,0.0067425906,0.0023678232,0.0015562508,0.0018873295],"category_scores_gemma":[0.033864442,0.00062616373,0.00041974572,0.008812846,0.0055416045,0.0019195399,0.0035340555,0.0037061682,0.00017389149],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":true,"about_ca_system_consensus":true,"study_design_scores_codex":[0.00054398633,0.00039826968,0.35679886,0.00046872595,0.00010977571,0.0022821703,0.47278926,0.0019182278,0.0024202142,0.027627762,0.031990573,0.10265216],"study_design_scores_gemma":[0.00005242232,0.00012776552,0.44001502,0.0007114069,0.00009533844,0.00021654989,0.43247074,0.003592517,0.0018400545,0.0020914825,0.118539445,0.0002473549],"about_ca_topic_score_codex":0.997729,"about_ca_topic_score_gemma":0.9988122,"teacher_disagreement_score":0.8352543,"about_ca_system_score_codex":0.16474566,"about_ca_system_score_gemma":0.20250255,"threshold_uncertainty_score":0.96877664},"labels":[],"label_agreement":null},{"id":"W2101260272","doi":"10.1002/ev.313","title":"Policy implementation: Implications for evaluation","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":96,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University; Douglas Mental Health University Institute","funders":"","keywords":"Accountability; Toolbox; Corporate governance; Context (archaeology); Process (computing); Government (linguistics); Sustainability; Political science; Public relations; Process management; Public administration; Sociology; Computer science; Business; Management; Economics","score_opus":0.3635462965884845,"score_gpt":0.6388161775626578,"score_spread":0.2752698809741734,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2101260272","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":"methods","domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.011660262,0.07371935,0.059885606,0.71189445,0.008755189,0.010599201,0.0020147269,0.0005922159,0.120879054],"genre_scores_gemma":[0.6265844,0.062304873,0.15924577,0.09050732,0.004668756,0.04422284,0.0012462583,0.00033873616,0.010881011],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.43743,0.5136371,0.01552967,0.0040821573,0.024475304,0.004845687],"domain_scores_gemma":[0.19606355,0.71348125,0.016679335,0.01441785,0.05197144,0.0073866034],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.45408645,0.0033717009,0.00587556,0.0073505947,0.005036974,0.028149975,0.007697774,0.01594864,0.02746675],"category_scores_gemma":[0.69589037,0.0010737083,0.0023467722,0.012725505,0.014433255,0.02382317,0.0084104,0.00875962,0.0020570916],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0022957942,0.0017014294,0.007585606,0.01691569,0.0006862576,0.00044697284,0.0038653219,0.008790869,0.00016014896,0.4755763,0.08701752,0.3949581],"study_design_scores_gemma":[0.0019204363,0.0017028002,0.0065513514,0.076454625,0.0010552261,0.00022695505,0.022606034,0.022477537,0.0010689613,0.7217121,0.1439165,0.00030742004],"about_ca_topic_score_codex":0.011902106,"about_ca_topic_score_gemma":0.009014539,"teacher_disagreement_score":0.45408645,"about_ca_system_score_codex":0.032236077,"about_ca_system_score_gemma":0.079622224,"threshold_uncertainty_score":0.6732086},"labels":[],"label_agreement":null},{"id":"W2101892546","doi":"10.1002/ev.316","title":"Knowledge theories can inform evaluation practice: What can a complexity lens add?","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":60,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Competence (human resources); Fidelity; Knowledge management; Computer science; Process (computing); Context (archaeology); Complex adaptive system; Paradigm shift; Psychology; Epistemology; Artificial intelligence; Social psychology","score_opus":0.37554862024845537,"score_gpt":0.565178999495196,"score_spread":0.18963037924674064,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2101892546","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.008178153,0.10776946,0.2630899,0.55451035,0.006585613,0.0013745246,0.0003544737,0.0006124529,0.057525065],"genre_scores_gemma":[0.4621877,0.07241422,0.39490202,0.051250607,0.008398079,0.0063587907,0.00023572748,0.00045273337,0.003800199],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.6143786,0.33592108,0.013819647,0.0035155928,0.029088836,0.0032763693],"domain_scores_gemma":[0.18585683,0.74346787,0.010437975,0.02107557,0.03487414,0.0042876415],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.35224256,0.0033061176,0.0067472425,0.030623948,0.0079228,0.046662863,0.0070927944,0.011223835,0.015076941],"category_scores_gemma":[0.49937993,0.0022468807,0.0038687976,0.013057161,0.067514814,0.084324636,0.023451144,0.017216278,0.0018302229],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00018049551,0.00018902801,0.0033267187,0.0069969287,0.00031041342,0.00018415642,0.008726928,0.0023081866,0.00011530436,0.76633537,0.020153299,0.19117317],"study_design_scores_gemma":[0.000087857,0.00012505159,0.0010921657,0.015064176,0.00014769132,0.00012179672,0.009821593,0.0030242219,0.00025171033,0.9247552,0.045376413,0.0001322275],"about_ca_topic_score_codex":0.0075864187,"about_ca_topic_score_gemma":0.010205195,"teacher_disagreement_score":0.35224256,"about_ca_system_score_codex":0.026557706,"about_ca_system_score_gemma":0.027439937,"threshold_uncertainty_score":0.79880023},"labels":[],"label_agreement":null},{"id":"W2125410695","doi":"10.1002/ev.221","title":"The partnership of <i>new directions for evaluation</i> and the American evaluation association","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Association (psychology); General partnership; Political science; Computer science; Sociology; Psychology; Law","score_opus":0.21838504639813228,"score_gpt":0.5369309968417352,"score_spread":0.3185459504436029,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2125410695","genre_codex":"commentary","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00073106296,0.028242817,0.005920232,0.9071005,0.016135398,0.00005427205,0.000052984717,0.00008391891,0.041678794],"genre_scores_gemma":[0.13513982,0.15542313,0.07003165,0.38557193,0.06253102,0.0009056284,0.00060474535,0.0004566179,0.18933542],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.94402486,0.03095425,0.0026124127,0.0017709759,0.017788427,0.002849032],"domain_scores_gemma":[0.8367263,0.065120935,0.0059061763,0.0054267035,0.063339144,0.023480782],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.08235093,0.00045984806,0.00077961787,0.0027566655,0.0036921129,0.017418401,0.0014114253,0.007814776,0.009785045],"category_scores_gemma":[0.083763644,0.0004867261,0.00075231545,0.0024682917,0.00592852,0.009478055,0.0057261786,0.015417405,0.0027541642],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005967045,0.0001367935,0.0014560191,0.00028822743,0.000017420558,0.00006274784,0.0005736451,0.00026053638,0.00020686488,0.2130001,0.6665818,0.11735629],"study_design_scores_gemma":[0.000017629036,0.000039429277,0.0009895654,0.00063043,0.000009932887,0.00006570239,0.0006068162,0.00039269918,0.00012245114,0.03485003,0.9622445,0.0000307763],"about_ca_topic_score_codex":0.007620287,"about_ca_topic_score_gemma":0.013074901,"teacher_disagreement_score":0.08235093,"about_ca_system_score_codex":0.010287704,"about_ca_system_score_gemma":0.05143477,"threshold_uncertainty_score":0.43551856},"labels":[],"label_agreement":null},{"id":"W2139159602","doi":"10.1002/ev.92","title":"PIE à la Mode: Mainstreaming Evaluation and Accountability in Each Program in Every County of a Statewide School Readiness Initiative","year":2003,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Accountability; Mandate; Mainstreaming; Legislature; Program evaluation; Plan (archaeology); Mainstream; Benchmarking; Quality (philosophy); Public administration; Political science; Public relations; Medical education; Psychology; Business; Pedagogy; Special education; Medicine","score_opus":0.2119641125322008,"score_gpt":0.5393907107995318,"score_spread":0.327426598267331,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2139159602","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.41713628,0.0021861242,0.2319594,0.06332153,0.0006521919,0.004629376,0.00053503405,0.0020991066,0.277481],"genre_scores_gemma":[0.7874777,0.001078818,0.1812191,0.004048491,0.00011696155,0.0018635202,0.00039748367,0.0001190358,0.023678953],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.9866915,0.008635495,0.0004469853,0.0004961949,0.0027446058,0.0009853212],"domain_scores_gemma":[0.98520875,0.00784455,0.00054452464,0.0008814234,0.0037237327,0.0017970459],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.031137958,0.00022395146,0.00022950298,0.0017981234,0.0033197193,0.0069229994,0.0010595315,0.0008154989,0.0041900957],"category_scores_gemma":[0.029440865,0.000358882,0.000271639,0.0014735634,0.0018130961,0.004899811,0.0059984215,0.003086574,0.00035781134],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00018576483,0.0011798504,0.021287702,0.00025871742,0.000024972405,0.0001232495,0.011715477,0.0032389543,0.002310762,0.12240869,0.04477871,0.7924871],"study_design_scores_gemma":[0.0005224952,0.0033969139,0.16626881,0.0022263317,0.00016465322,0.0004969281,0.07111351,0.043543685,0.02020607,0.08959783,0.6021252,0.00033757303],"about_ca_topic_score_codex":0.010490428,"about_ca_topic_score_gemma":0.035728253,"teacher_disagreement_score":0.031137958,"about_ca_system_score_codex":0.007069761,"about_ca_system_score_gemma":0.021011151,"threshold_uncertainty_score":0.16467524},"labels":[],"label_agreement":null},{"id":"W2140465263","doi":"10.1002/ev.240","title":"Going through the process: An examination of the operationalization of process use in empirical research on evaluation","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":74,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Operationalization; Process (computing); Empirical research; Empirical examination; Process management; Research methodology; Psychology; Management science; Computer science; Sociology; Epistemology; Business; Actuarial science; Engineering","score_opus":0.6748434947289444,"score_gpt":0.6733453144103931,"score_spread":0.001498180318551312,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2140465263","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"methods","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":"methods","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6563143,0.0039531523,0.27133673,0.008134892,0.000135219,0.0014878057,0.00011804229,0.0002794402,0.05824037],"genre_scores_gemma":[0.97088665,0.00051341136,0.027373089,0.00024353743,0.000016205948,0.0005777811,0.00003887075,0.000053258766,0.00029732453],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.8412625,0.1203737,0.011150094,0.0031430493,0.021697989,0.002372661],"domain_scores_gemma":[0.34694257,0.5728718,0.031685498,0.019008322,0.027670953,0.0018208142],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.11627305,0.0007493747,0.00089177437,0.011799896,0.0040814783,0.01359049,0.002098031,0.0026498858,0.0018915056],"category_scores_gemma":[0.37962437,0.0006970566,0.0014878666,0.011766743,0.028971242,0.020557113,0.0071137496,0.0043823035,0.0001493087],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0002561738,0.0004901417,0.08066736,0.001598976,0.00020292273,0.0002054672,0.28812572,0.002244,0.001877612,0.4763933,0.0007293436,0.14720897],"study_design_scores_gemma":[0.00012628849,0.0012252283,0.15668416,0.007287987,0.0004199436,0.0007805972,0.29789168,0.030079061,0.0070733554,0.46595663,0.032112584,0.0003624251],"about_ca_topic_score_codex":0.0060048024,"about_ca_topic_score_gemma":0.0036908851,"teacher_disagreement_score":0.88372695,"about_ca_system_score_codex":0.00840466,"about_ca_system_score_gemma":0.00929725,"threshold_uncertainty_score":0.614918},"labels":[],"label_agreement":null},{"id":"W2278381810","doi":"10.1002/ev.20158","title":"Assessing the Practice Impact of Research on Evaluation","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Evaluation methods; Program evaluation; Psychology; Applied psychology; Computer science; Political science; Public administration; Reliability engineering; Engineering","score_opus":0.8085447410014557,"score_gpt":0.7537715169629834,"score_spread":0.05477322403847229,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2278381810","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":"methods","domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.11905708,0.12778299,0.1389456,0.3180472,0.0054542995,0.005183393,0.00093083776,0.0003815254,0.284217],"genre_scores_gemma":[0.8945893,0.023960501,0.062569864,0.013037999,0.0012376664,0.0024496014,0.0002093139,0.00011484768,0.0018309036],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.20695063,0.63660675,0.028469441,0.0100454455,0.113349795,0.0045780637],"domain_scores_gemma":[0.047679532,0.84913665,0.032972895,0.023458142,0.04469252,0.0020602087],"candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.5486119,0.0019866063,0.0032790068,0.015160202,0.004930666,0.023244556,0.004394612,0.0068866275,0.008151031],"category_scores_gemma":[0.8136303,0.0012866546,0.0024427162,0.014517742,0.022180505,0.025614142,0.018286804,0.009095642,0.0009189603],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00044803304,0.00041060025,0.043142743,0.014543055,0.0017551577,0.00022560013,0.01012744,0.005406774,0.0006142105,0.44987887,0.010908556,0.462539],"study_design_scores_gemma":[0.00040476857,0.0030918142,0.081441596,0.0845484,0.0029354284,0.0005632733,0.0378311,0.015356779,0.0048855245,0.643161,0.12544242,0.00033785572],"about_ca_topic_score_codex":0.00488868,"about_ca_topic_score_gemma":0.0051830187,"teacher_disagreement_score":0.45138812,"about_ca_system_score_codex":0.031195842,"about_ca_system_score_gemma":0.036840495,"threshold_uncertainty_score":0.55664194},"labels":[],"label_agreement":null},{"id":"W2512463988","doi":"10.1002/ev.20199","title":"Building Evaluation Capacity Through CLIPs: Communities of Learning, Inquiry, and Practice","year":2016,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"CLIPS; Context (archaeology); Work (physics); Community of practice; Medical education; Professional development; Higher education; Pedagogy; Knowledge management; Computer science; Sociology; Public relations; Psychology; Political science; Medicine; Engineering","score_opus":0.5953867694134086,"score_gpt":0.5760182223031797,"score_spread":0.01936854711022895,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2512463988","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.20115109,0.0014864795,0.46654522,0.051125403,0.00043878696,0.004961348,0.00013405543,0.0012711951,0.2728865],"genre_scores_gemma":[0.8337615,0.00047604044,0.15752904,0.0011703321,0.000073762276,0.0016939908,0.000074406904,0.000106927284,0.00511397],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.88506323,0.098992236,0.0018023064,0.002573375,0.008637002,0.0029319385],"domain_scores_gemma":[0.8364749,0.12361313,0.004166293,0.014695226,0.013082763,0.007967575],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.08931211,0.000736178,0.00073853164,0.005123136,0.006891178,0.014517279,0.0034402334,0.0026404308,0.0074406667],"category_scores_gemma":[0.10429899,0.0006946943,0.0008111704,0.0022161228,0.02695804,0.016351221,0.019349283,0.003219095,0.0008213714],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0002058944,0.0011712381,0.009489201,0.0008384563,0.00008535632,0.00041202133,0.08412867,0.006040373,0.00124859,0.58212626,0.014827872,0.29942614],"study_design_scores_gemma":[0.000324463,0.00093908067,0.0047480995,0.0024849735,0.00008651006,0.0004525713,0.106740445,0.03103945,0.0036366337,0.69227666,0.1570551,0.00021601551],"about_ca_topic_score_codex":0.0040634014,"about_ca_topic_score_gemma":0.0045992546,"teacher_disagreement_score":0.08931211,"about_ca_system_score_codex":0.009246946,"about_ca_system_score_gemma":0.023426125,"threshold_uncertainty_score":0.4723332},"labels":[],"label_agreement":null},{"id":"W2625529206","doi":"10.1002/ev.20241","title":"Evaluation at the Time of Health Systems Reform: Chinese Policymakers’ Need for a Robust System of Evaluations to Assess Progress in the Implementation of Reform Efforts","year":2017,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Healthcare Systems and Reforms","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"China; Capacity building; Program evaluation; Political science; Evaluation methods; Economic growth; Public administration; Public relations; Economics; Engineering","score_opus":0.14607408309884093,"score_gpt":0.43423215831068135,"score_spread":0.28815807521184045,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2625529206","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.3045489,0.01894003,0.07578576,0.49053043,0.0014041719,0.004971537,0.0008615531,0.00053665356,0.102420986],"genre_scores_gemma":[0.9742045,0.0009981904,0.015453082,0.006406554,0.00014079124,0.0014852497,0.00012207296,0.00003671631,0.0011529701],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.71868783,0.22549956,0.013901008,0.007656405,0.023566121,0.010689075],"domain_scores_gemma":[0.5910687,0.2555616,0.020073865,0.024986247,0.09533241,0.012977171],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.3998632,0.00087949773,0.0023096576,0.007065792,0.008777276,0.019130701,0.004198353,0.0040744985,0.0052601337],"category_scores_gemma":[0.31232506,0.0009906702,0.0010807825,0.0054733194,0.019557092,0.018986423,0.01558735,0.006509815,0.00023340031],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0009305603,0.00057787495,0.11217229,0.004837655,0.00065654237,0.00046289223,0.09111272,0.014327755,0.002811313,0.36437008,0.039966665,0.36777365],"study_design_scores_gemma":[0.0013160019,0.0011379992,0.2603628,0.017549252,0.0010723525,0.00020338412,0.14794734,0.053119753,0.010084981,0.32386386,0.18212977,0.0012124445],"about_ca_topic_score_codex":0.14713852,"about_ca_topic_score_gemma":0.09829311,"teacher_disagreement_score":0.3998632,"about_ca_system_score_codex":0.06793053,"about_ca_system_score_gemma":0.21578023,"threshold_uncertainty_score":0.7400755},"labels":[],"label_agreement":null},{"id":"W2626410734","doi":"10.1002/ev.20240","title":"Conceptual Indicators Framework for Strengthening the Chinese Health System","year":2017,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Healthcare Systems and Reforms","field":"Economics, Econometrics and Finance","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"China; Government (linguistics); Plan (archaeology); Conceptual framework; Process (computing); Political science; Equity (law); Monitoring and evaluation; Process management; Public relations; Public administration; Economic growth; Business; Sociology; Computer science; Economics; Geography","score_opus":0.08227829248233864,"score_gpt":0.3703221274091979,"score_spread":0.28804383492685925,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2626410734","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.06576832,0.005181246,0.54302883,0.07253626,0.00093428313,0.0036067762,0.002742717,0.00064911164,0.3055525],"genre_scores_gemma":[0.85001194,0.0020633896,0.13592444,0.0012101693,0.00028347562,0.002959594,0.0009908257,0.00006581612,0.006490342],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.98575264,0.0086320145,0.000945089,0.0010172653,0.0026288538,0.0010240623],"domain_scores_gemma":[0.99081826,0.0028547347,0.0012103961,0.00056261296,0.0036153055,0.00093868864],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.026178857,0.0014232473,0.00077629555,0.0156336,0.003362134,0.009469719,0.0033313644,0.0020297302,0.008259454],"category_scores_gemma":[0.017800888,0.0004507431,0.0010774609,0.014844851,0.014302981,0.012444039,0.0073866444,0.002918381,0.0004625114],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000004363117,0.000010699323,0.0009051201,0.0000679259,0.000007932611,0.000023986118,0.00074829557,0.0021331336,0.000027059232,0.9908275,0.0013345188,0.003909415],"study_design_scores_gemma":[0.000057822905,0.0000861784,0.0062638805,0.00068508665,0.00009423145,0.00007048852,0.0066462075,0.042235497,0.00035901196,0.86221653,0.08120938,0.00007569976],"about_ca_topic_score_codex":0.048100244,"about_ca_topic_score_gemma":0.02278261,"teacher_disagreement_score":0.048100244,"about_ca_system_score_codex":0.03317743,"about_ca_system_score_gemma":0.03300628,"threshold_uncertainty_score":0.24072027},"labels":[],"label_agreement":null},{"id":"W2795416648","doi":"10.1002/ev.20297","title":"My Entry to the Field","year":2018,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Academic and Historical Perspectives in Psychology","field":"Psychology","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Field (mathematics); Permission; Advice (programming); Key (lock); Citizen journalism; Sociology; Work (physics); Public relations; Computer science; Psychology; Engineering ethics; Epistemology; World Wide Web; Political science; Computer security; Philosophy","score_opus":0.05542443802674325,"score_gpt":0.4559950890704666,"score_spread":0.40057065104372336,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2795416648","genre_codex":"other","genre_gemma":"commentary","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"commentary","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.002018703,0.010572406,0.002099909,0.391423,0.106746405,0.0002227867,0.0020933214,0.0003422883,0.48448116],"genre_scores_gemma":[0.016556485,0.004786909,0.00079669367,0.0368253,0.012805938,0.00024697348,0.0007338611,0.0002699804,0.9269778],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9984919,0.00047340008,0.000068254136,0.00023756086,0.00040964517,0.00031927],"domain_scores_gemma":[0.99284136,0.0009781438,0.00027578132,0.00039936093,0.0024367124,0.003068609],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0027615814,0.00067745824,0.0006906073,0.00165555,0.007337094,0.0069561037,0.0013684181,0.002852042,0.35869882],"category_scores_gemma":[0.01319366,0.0002871548,0.0004343184,0.001118907,0.0021883661,0.005396453,0.005587304,0.006222401,0.1392265],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000011359453,0.000011119776,0.00007724561,0.000036233858,6.127343e-7,0.000047812002,0.0008362021,0.00000551492,0.000045741377,0.012522505,0.9716411,0.014764597],"study_design_scores_gemma":[0.000002591777,0.000006857816,0.000103336606,0.00014858492,3.6373459e-7,0.00003945919,0.0015187296,0.000006038989,0.000016208047,0.0018827217,0.99627244,0.000002639598],"about_ca_topic_score_codex":0.0030851334,"about_ca_topic_score_gemma":0.004671479,"teacher_disagreement_score":0.35869882,"about_ca_system_score_codex":0.0032493086,"about_ca_system_score_gemma":0.0052892454,"threshold_uncertainty_score":0.9147383},"labels":[],"label_agreement":null},{"id":"W2891656379","doi":"10.1002/ev.20346","title":"The Evolving Market for Systematic Evaluation in Canada","year":2018,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":10,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"University of Ottawa","funders":"","keywords":"Accreditation; Professionalization; Context (archaeology); Supply and demand; Government (linguistics); Supply side; Business; Exploratory research; Affect (linguistics); Public relations; Marketing; Economics; Political science; Economic growth; Sociology; Social science","score_opus":0.1834456181021273,"score_gpt":0.4959395639226441,"score_spread":0.3124939458205168,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2891656379","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.14530544,0.022566224,0.0063383183,0.54654425,0.001308289,0.00042844692,0.0020565158,0.00037178418,0.27508077],"genre_scores_gemma":[0.90829015,0.009976946,0.007400031,0.02703498,0.0003220742,0.00011143173,0.00056345644,0.00015765429,0.04614318],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.97216743,0.0034037873,0.00076889765,0.001756662,0.015854312,0.0060489182],"domain_scores_gemma":[0.86910224,0.026646238,0.00414524,0.0021149798,0.07055154,0.027439784],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02246012,0.0003545073,0.00060825446,0.0053264094,0.012648979,0.01817266,0.0032589824,0.0035332497,0.016377818],"category_scores_gemma":[0.044237826,0.0006729079,0.00064761087,0.0064986977,0.008398074,0.0035113366,0.0040617962,0.003842383,0.00077196385],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":true,"about_ca_topic_consensus":true,"about_ca_system_candidate":true,"about_ca_system_consensus":true,"study_design_scores_codex":[0.00028210174,0.0002435487,0.033778116,0.00079296634,0.000049024617,0.0010522689,0.010211929,0.002144793,0.0016607583,0.49133778,0.2053912,0.25305548],"study_design_scores_gemma":[0.00008565362,0.00012522656,0.08197357,0.0016977098,0.000042292413,0.0002412947,0.022257095,0.0059798234,0.0010238646,0.026189474,0.8601332,0.00025094312],"about_ca_topic_score_codex":0.98623276,"about_ca_topic_score_gemma":0.9915679,"teacher_disagreement_score":0.9775399,"about_ca_system_score_codex":0.31247154,"about_ca_system_score_gemma":0.496807,"threshold_uncertainty_score":0.7974356},"labels":[],"label_agreement":null},{"id":"W2948770661","doi":"10.1002/ev.20365","title":"Sustainability‐Ready Evaluation: A Call to Action","year":2019,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":30,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Evaluation Society","funders":"","keywords":"Sustainability; Transformative learning; Sustainability science; Sustainability organizations; Relevance (law); Process management; Sustainable development; Social sustainability; Environmental resource management; Checklist; Natural resource; Action (physics); Business; Engineering ethics; Management science; Environmental planning; Computer science; Risk analysis (engineering); Political science; Sociology; Engineering; Psychology; Economics; Ecology","score_opus":0.30589920481363936,"score_gpt":0.5825784932667877,"score_spread":0.2766792884531483,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2948770661","genre_codex":"commentary","genre_gemma":"commentary","domain_codex":"methods","domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"commentary","genre_consensus":"commentary","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00074307417,0.0061361436,0.025009945,0.952478,0.0042465706,0.00071558193,0.00005346606,0.0003053509,0.01031191],"genre_scores_gemma":[0.10202846,0.01735385,0.3152071,0.5364714,0.0093511725,0.0067846053,0.00037464328,0.0007095642,0.011719214],"study_design_codex":"not_applicable","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.34747028,0.5337562,0.030758398,0.0074777557,0.07155301,0.008984444],"domain_scores_gemma":[0.1656388,0.65873706,0.015116846,0.031805005,0.10008847,0.0286138],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.6195292,0.0027388756,0.005974706,0.007788901,0.013698829,0.042186633,0.010294567,0.040152308,0.014299209],"category_scores_gemma":[0.5939354,0.0022020356,0.0047506997,0.005039988,0.047435973,0.049383674,0.027848464,0.051552653,0.0032484483],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00025059725,0.00079140055,0.0017168622,0.0048817866,0.00016175247,0.00040109214,0.008684715,0.0018675674,0.00055653823,0.23350492,0.4462766,0.30090612],"study_design_scores_gemma":[0.00028565014,0.00039423478,0.0021266236,0.024456872,0.00012665636,0.00041485322,0.016720148,0.0024031536,0.0007515468,0.4055137,0.54631406,0.0004924672],"about_ca_topic_score_codex":0.009849535,"about_ca_topic_score_gemma":0.010955283,"teacher_disagreement_score":0.6195292,"about_ca_system_score_codex":0.029997118,"about_ca_system_score_gemma":0.17745037,"threshold_uncertainty_score":0.46918827},"labels":[],"label_agreement":null},{"id":"W3011218257","doi":"10.1002/ev.20394","title":"Advancing the Measurement of Collective Community Capacity and the Evaluation of Community Capacity‐Building Models","year":2020,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Community Health and Development","field":"Health Professions","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Capacity building; Work (physics); Collective efficacy; Community resilience; Adaptation (eye); Public relations; Psychological resilience; Political science; Sociology; Psychology; Computer science; Engineering; Social psychology; Social science","score_opus":0.4752172519940911,"score_gpt":0.47498330087062063,"score_spread":0.0002339511234704661,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3011218257","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.31936568,0.009765209,0.42565665,0.03960966,0.001477014,0.004532781,0.0014626868,0.00073764334,0.19739263],"genre_scores_gemma":[0.8738543,0.0021454701,0.11853381,0.00083955703,0.0001021408,0.0025996817,0.00027567788,0.0000714631,0.0015777871],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.83768463,0.13721678,0.003044364,0.003036474,0.017283672,0.0017341533],"domain_scores_gemma":[0.6190163,0.28417635,0.019095078,0.02769468,0.04434738,0.005670193],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.13760681,0.0009717617,0.0011367038,0.0052437866,0.0029237806,0.009706336,0.0027328844,0.0014922674,0.0058684954],"category_scores_gemma":[0.2800628,0.000583032,0.0012590254,0.005236973,0.008505627,0.011926749,0.010006808,0.0039469623,0.0005219663],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00021008386,0.0008424,0.11816031,0.0026409065,0.00064690394,0.000063247644,0.022161845,0.014344084,0.00045348483,0.36390755,0.015670788,0.46089846],"study_design_scores_gemma":[0.00023358558,0.0014271049,0.12792574,0.016182609,0.00047113505,0.000146841,0.0748893,0.08866965,0.0038983207,0.5966482,0.08895989,0.0005476354],"about_ca_topic_score_codex":0.019514708,"about_ca_topic_score_gemma":0.024791887,"teacher_disagreement_score":0.13760681,"about_ca_system_score_codex":0.010421486,"about_ca_system_score_gemma":0.01961646,"threshold_uncertainty_score":0.7277431},"labels":[],"label_agreement":null},{"id":"W3011548789","doi":"10.1002/ev.20398","title":"Finding the Impact: Methods for Assessing the Contribution of Collective Impact to Systems and Population Change in a Multi‐Site Study","year":2020,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":17,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Scale (ratio); Context (archaeology); Interim; Population; Work (physics); SPARK (programming language); Management science; Computer science; Political science; Sociology; Engineering","score_opus":0.5624400587745525,"score_gpt":0.6718887697007279,"score_spread":0.10944871092617536,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3011548789","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.06755281,0.0014130834,0.8995656,0.0015981685,0.00041949833,0.013593711,0.0015567803,0.00055575825,0.013744581],"genre_scores_gemma":[0.17820542,0.0004577779,0.7900923,0.00032707755,0.00006835969,0.028923525,0.0003779718,0.0001497588,0.0013979004],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.7363665,0.23667562,0.007560759,0.006175041,0.012420479,0.0008015945],"domain_scores_gemma":[0.4712714,0.46238315,0.02331467,0.022143101,0.019334363,0.0015533854],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.18346988,0.0016096868,0.0019540472,0.009344366,0.0026028203,0.006816916,0.0033771116,0.0018280483,0.00645298],"category_scores_gemma":[0.30336508,0.0010939462,0.003634382,0.009802483,0.005649056,0.004720349,0.0069339117,0.0033781244,0.0006333433],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.001277818,0.0014834362,0.16395454,0.008790137,0.005448068,0.00050723494,0.055557802,0.022406284,0.002963448,0.12923716,0.017100865,0.5912732],"study_design_scores_gemma":[0.0015775877,0.0063849115,0.2350095,0.011535383,0.0049439403,0.0006945196,0.07999187,0.19228005,0.008601093,0.3595571,0.09800996,0.0014141316],"about_ca_topic_score_codex":0.0058153593,"about_ca_topic_score_gemma":0.007199498,"teacher_disagreement_score":0.18346988,"about_ca_system_score_codex":0.004073071,"about_ca_system_score_gemma":0.0049094507,"threshold_uncertainty_score":0.97029305},"labels":[],"label_agreement":null},{"id":"W3196514161","doi":"10.1002/ev.20464","title":"Supporting systems transformation through design‐driven evaluation","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Complex Systems and Decision Making","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Economics Association","funders":"","keywords":"Cognitive reframing; Computer science; Process (computing); Process management; Service (business); New product development; Knowledge management; Product (mathematics); Value (mathematics); Management science; Engineering management; Systems engineering; Engineering; Business","score_opus":0.42191478588644576,"score_gpt":0.5201704823327796,"score_spread":0.09825569644633386,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3196514161","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.017351782,0.00033038823,0.9492179,0.0026228284,0.00015520495,0.0022404646,0.00014639624,0.0014294599,0.026505534],"genre_scores_gemma":[0.3721265,0.00031546675,0.6220856,0.00052338233,0.00004025236,0.0024362623,0.00025614753,0.00023930742,0.0019771408],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9308261,0.054892924,0.0019957102,0.001699875,0.009439312,0.0011460787],"domain_scores_gemma":[0.8685998,0.101595946,0.0042433254,0.010741425,0.0133679705,0.0014515813],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.056388937,0.0012908342,0.0008962627,0.002214781,0.0014040078,0.00742562,0.0023332038,0.0019250574,0.0067682858],"category_scores_gemma":[0.10597143,0.0006905892,0.0008494824,0.0011257184,0.0052956645,0.0052043465,0.005736333,0.0025239582,0.000821056],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00041354436,0.0009170544,0.004804641,0.0018964569,0.00018285715,0.0002709607,0.00313423,0.13483137,0.006732847,0.5249842,0.006987066,0.31484482],"study_design_scores_gemma":[0.00069343526,0.0008458593,0.0011306615,0.0013428774,0.00011157084,0.00013263663,0.0015849211,0.52201575,0.018774152,0.38813844,0.06509055,0.00013910863],"about_ca_topic_score_codex":0.0029323602,"about_ca_topic_score_gemma":0.0023086271,"teacher_disagreement_score":0.056388937,"about_ca_system_score_codex":0.0053064837,"about_ca_system_score_gemma":0.011495955,"threshold_uncertainty_score":0.2982167},"labels":[],"label_agreement":null},{"id":"W3203262347","doi":"10.1002/ev.20477","title":"A learning agenda for the advocacy evaluation field's future","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Field (mathematics); Context (archaeology); Value (mathematics); Set (abstract data type); Political science; Public relations; Engineering ethics; Sociology; Computer science; Engineering","score_opus":0.25942037969970505,"score_gpt":0.5452853487151225,"score_spread":0.2858649690154174,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3203262347","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00073263975,0.0052856975,0.008427773,0.9717011,0.0048470045,0.00009253343,0.000028650597,0.00006425227,0.008820285],"genre_scores_gemma":[0.19664593,0.0421768,0.23360392,0.43632755,0.027047014,0.0035091543,0.0005851896,0.00045640726,0.05964811],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.8874842,0.08530278,0.0040766275,0.0032163118,0.014569354,0.005350698],"domain_scores_gemma":[0.7169568,0.20731424,0.0044920375,0.009005079,0.041140605,0.021091338],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.23091614,0.0014987355,0.002192324,0.0045289584,0.01670988,0.04696918,0.0060420213,0.03519886,0.01551521],"category_scores_gemma":[0.12227282,0.001138386,0.0023055715,0.0027998602,0.033470687,0.042745527,0.020827034,0.047389716,0.0035844285],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010148551,0.0004040356,0.00039642875,0.0005499346,0.00001859656,0.00018626281,0.0069373534,0.0006915218,0.00023245889,0.7886399,0.13979891,0.06204302],"study_design_scores_gemma":[0.00009011104,0.00014273533,0.0002542187,0.0031624169,0.000021587886,0.00011205209,0.021766627,0.0015121228,0.00037429572,0.4849784,0.48748916,0.00009629417],"about_ca_topic_score_codex":0.0053630266,"about_ca_topic_score_gemma":0.0050735055,"teacher_disagreement_score":0.23091614,"about_ca_system_score_codex":0.023225812,"about_ca_system_score_gemma":0.08922476,"threshold_uncertainty_score":0.9484173},"labels":[],"label_agreement":null},{"id":"W3203797720","doi":"10.1002/ev.20478","title":"An introduction to policy advocacy evaluation: The concepts, history, and literature of the field","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":25,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Field (mathematics); Adaptation (eye); Policy development; Engineering ethics; Management science; Evaluation methods; Computer science; Political science; Public administration; Psychology","score_opus":0.12003077569204998,"score_gpt":0.5220412336486495,"score_spread":0.4020104579565995,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3203797720","genre_codex":"review","genre_gemma":"review","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"review","genre_consensus":"review","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0021492771,0.49311006,0.11209678,0.26511914,0.024421789,0.0009063494,0.0007069637,0.00040992393,0.101079695],"genre_scores_gemma":[0.06920272,0.6011276,0.1878337,0.06786297,0.0373251,0.0033852432,0.0005965391,0.00057379936,0.032092325],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"not_applicable","domain_scores_codex":[0.9788159,0.014933281,0.0016924834,0.0008532145,0.0032732284,0.00043192357],"domain_scores_gemma":[0.9158549,0.073945284,0.002183668,0.0016299362,0.005074736,0.001311441],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.024138834,0.0010699837,0.0012309572,0.008949646,0.004066679,0.013001649,0.0015747607,0.0058463486,0.008969664],"category_scores_gemma":[0.038593676,0.0007639079,0.0012394218,0.0081378035,0.014006103,0.011548903,0.004717198,0.010706973,0.00257341],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000059849226,0.00016739858,0.0009841742,0.0033988284,0.000029328792,0.00020115933,0.0043575405,0.0008645074,0.00040412138,0.607442,0.16270956,0.21938147],"study_design_scores_gemma":[0.000010686496,0.00006617831,0.00082143233,0.009038984,0.00001010298,0.00034321187,0.0014944194,0.0005413572,0.0001567873,0.20399098,0.78347296,0.000052950923],"about_ca_topic_score_codex":0.003155035,"about_ca_topic_score_gemma":0.0040046386,"teacher_disagreement_score":0.024138834,"about_ca_system_score_codex":0.00736585,"about_ca_system_score_gemma":0.010225242,"threshold_uncertainty_score":0.12765992},"labels":[],"label_agreement":null},{"id":"W3204738611","doi":"10.1002/ev.20471","title":"Contribution analysis: A promising method for assessing advocacy's impact","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Impact","funders":"","keywords":"Context (archaeology); Policy analysis; Outcome (game theory); Political science; Public relations; Management science; Public administration; Economics","score_opus":0.24016093895790824,"score_gpt":0.6095421407491777,"score_spread":0.36938120179126943,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W3204738611","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.033632882,0.0010315088,0.8915009,0.0027412835,0.00057808566,0.0047950335,0.0018917887,0.0021547691,0.06167375],"genre_scores_gemma":[0.35530335,0.00051971537,0.6301973,0.00053574215,0.00017421306,0.008461528,0.0005099965,0.00045214625,0.0038460207],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.80054855,0.15563601,0.007905859,0.006075598,0.028572597,0.001261403],"domain_scores_gemma":[0.42756796,0.50194436,0.01801471,0.023292689,0.027511492,0.0016687614],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.15864317,0.0025990745,0.0020629615,0.02199066,0.0034022683,0.008886466,0.0033808867,0.0022489117,0.017076278],"category_scores_gemma":[0.38583562,0.0010121651,0.0033552914,0.014081496,0.0067678406,0.011198437,0.0070231427,0.0039681382,0.0018370601],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0020603633,0.00072732155,0.037675854,0.0046216124,0.001673442,0.0002639436,0.017323636,0.007871129,0.0038361275,0.30951184,0.016036939,0.5983978],"study_design_scores_gemma":[0.0009098262,0.0030157922,0.039566368,0.0045473957,0.0015895261,0.0009388039,0.01849456,0.1118024,0.01836095,0.7010918,0.09889362,0.00078890007],"about_ca_topic_score_codex":0.003145752,"about_ca_topic_score_gemma":0.002746035,"teacher_disagreement_score":0.15864317,"about_ca_system_score_codex":0.004120034,"about_ca_system_score_gemma":0.0059584496,"threshold_uncertainty_score":0.83899534},"labels":[],"label_agreement":null},{"id":"W4213266427","doi":"10.1002/ev.20337","title":"Issue Information","year":2019,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Computer science; Citation; Action (physics); World Wide Web; Information retrieval","score_opus":0.20791329578082282,"score_gpt":0.5385400636795997,"score_spread":0.3306267678987769,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4213266427","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00015154992,0.0009841216,0.0008045414,0.0036339147,0.012101359,0.0004584335,0.025235545,0.0017672679,0.9548633],"genre_scores_gemma":[0.00048331422,0.00070949626,0.0004309557,0.0011576405,0.0015216803,0.00013943456,0.0112668965,0.0003308537,0.98395985],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9987716,0.00012926666,0.00007329327,0.00014224621,0.000755447,0.00012811765],"domain_scores_gemma":[0.9948487,0.0006195813,0.00028628652,0.00043657684,0.002466348,0.0013425807],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.001300779,0.0017622925,0.0012070892,0.0047775027,0.0016616336,0.008005891,0.0022531247,0.0030848696,0.9175557],"category_scores_gemma":[0.007713025,0.0006803456,0.0011057183,0.004325318,0.00066209806,0.003885877,0.002442514,0.0021708193,0.86585104],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000010208411,0.000016802662,0.000022601333,0.00010093151,8.787976e-7,0.000007747744,0.0000028962802,0.000024048533,0.00004234793,0.0006592114,0.9777256,0.021386804],"study_design_scores_gemma":[0.000014392048,0.000011266216,0.00017104456,0.00009628765,0.0000014043374,0.000012947711,0.000012786668,0.00003793503,0.000044776723,0.0008421743,0.99875104,0.0000038884273],"about_ca_topic_score_codex":0.00412773,"about_ca_topic_score_gemma":0.008643029,"teacher_disagreement_score":0.9175557,"about_ca_system_score_codex":0.0016158905,"about_ca_system_score_gemma":0.003911298,"threshold_uncertainty_score":0.117596745},"labels":[],"label_agreement":null},{"id":"W4214553715","doi":"10.1002/ev.20486","title":"How does teaching with cases support the development of evaluation competencies?","year":2021,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":8,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Adaptability; Craft; Evaluation methods; Professional development; Medical education; Psychology; Computer science; Knowledge management; Pedagogy; Engineering; Medicine; Management","score_opus":0.2469212988547501,"score_gpt":0.49329140127841276,"score_spread":0.24637010242366267,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4214553715","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.89038867,0.00080379355,0.034290392,0.013134503,0.00018385134,0.00045745217,0.00007500233,0.00033518064,0.060331296],"genre_scores_gemma":[0.9887673,0.00033031782,0.00963357,0.00027971403,0.000030755797,0.000059573078,0.000025320656,0.000017984203,0.0008555561],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.95344657,0.036869984,0.0012958545,0.0012334706,0.0056467564,0.0015073733],"domain_scores_gemma":[0.7016456,0.24825528,0.017871581,0.009741073,0.01459912,0.0078872815],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.033301853,0.0002597346,0.00035963173,0.0020557775,0.0011668386,0.0067286594,0.0015891697,0.0013891509,0.0056252433],"category_scores_gemma":[0.22724766,0.00030730857,0.00026157408,0.001161307,0.0022307308,0.0076669804,0.0032066933,0.0009818648,0.00085819256],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00042227158,0.0033642692,0.17681934,0.0007187786,0.000051337323,0.0014121596,0.05586551,0.0016092177,0.0032669199,0.029590081,0.012235007,0.7146452],"study_design_scores_gemma":[0.0004984447,0.0040587164,0.31535175,0.006357336,0.0003319883,0.005376136,0.2348783,0.04325372,0.021954913,0.12736739,0.24019957,0.00037168764],"about_ca_topic_score_codex":0.0018748973,"about_ca_topic_score_gemma":0.0029458152,"teacher_disagreement_score":0.033301853,"about_ca_system_score_codex":0.0030227148,"about_ca_system_score_gemma":0.0041987575,"threshold_uncertainty_score":0.17611909},"labels":[],"label_agreement":null},{"id":"W4224927296","doi":"10.1002/ev.20494","title":"Evaluation policy and organizational evaluation capacity building: A study of international aid agency evaluation policies","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":11,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Agency (philosophy); Pace; Evaluation methods; Program evaluation; Policy analysis; Capacity building; Public relations; Management science; Knowledge management; Process management; Business; Political science; Computer science; Public administration; Economics; Sociology; Economic growth","score_opus":0.3393749749893569,"score_gpt":0.529793832087028,"score_spread":0.1904188570976711,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4224927296","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9334589,0.00038771218,0.0053393263,0.0113770515,0.00003200043,0.00029619064,0.0000453042,0.000029029883,0.049034443],"genre_scores_gemma":[0.99797446,0.00011353646,0.00074579235,0.00030351206,0.000006481695,0.00014694293,0.00000950154,0.0000064734163,0.0006933911],"study_design_codex":"qualitative","study_design_gemma":"qualitative","domain_scores_codex":[0.9159221,0.07088581,0.00183486,0.0016866106,0.0042706444,0.005400045],"domain_scores_gemma":[0.77367574,0.16988231,0.021975983,0.008099886,0.018091947,0.008274129],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.082806796,0.00030888504,0.00046972165,0.0032946935,0.011133468,0.013180184,0.002181374,0.0023573213,0.003479148],"category_scores_gemma":[0.15446293,0.0006378003,0.00031236498,0.0049201497,0.015833646,0.010315118,0.007108046,0.005545232,0.00028522327],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0003491015,0.0024800822,0.12890804,0.00039771106,0.00006919956,0.0005781027,0.45781988,0.004647769,0.00067741045,0.3280821,0.006954959,0.06903557],"study_design_scores_gemma":[0.00008995754,0.00038377216,0.072721966,0.0007132109,0.000033392997,0.00016735046,0.83111656,0.007592757,0.0008959815,0.035981577,0.050204996,0.0000985444],"about_ca_topic_score_codex":0.02502399,"about_ca_topic_score_gemma":0.014470901,"teacher_disagreement_score":0.082806796,"about_ca_system_score_codex":0.029809989,"about_ca_system_score_gemma":0.031401772,"threshold_uncertainty_score":0.43792945},"labels":[],"label_agreement":null},{"id":"W4233585958","doi":"10.1002/ev.20355","title":"Issue Information","year":2020,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of California, Los Angeles; Sierra Health Foundation; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; University of Hawai'i; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Library science; Information retrieval","score_opus":0.07860107879638363,"score_gpt":0.28719641824578224,"score_spread":0.20859533944939862,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4233585958","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00022594274,0.0011439944,0.00058485544,0.0038691072,0.018012214,0.00045959777,0.029214567,0.002229562,0.9442602],"genre_scores_gemma":[0.00042264798,0.00042311763,0.00017871233,0.0009447042,0.001419923,0.000093681396,0.008107338,0.00038138335,0.9880284],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9985306,0.00016633456,0.00009003017,0.00017705865,0.00084396603,0.000192079],"domain_scores_gemma":[0.9957917,0.00059954706,0.00022805113,0.00034225176,0.001708644,0.0013297711],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0015760814,0.0023047754,0.0016966381,0.004017818,0.0018893767,0.010062111,0.002427786,0.0038307507,0.92151546],"category_scores_gemma":[0.0067781536,0.0008954027,0.0012383767,0.0034878831,0.0006767204,0.004506144,0.00232007,0.0028410936,0.9030208],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000020410864,0.00001678281,0.000019277973,0.000073670206,9.540689e-7,0.000006238845,0.0000021500975,0.000023789324,0.00003175419,0.0003479053,0.98706204,0.012395037],"study_design_scores_gemma":[0.00004330209,0.000024898689,0.00028770173,0.00011926185,0.0000022975128,0.000015829528,0.000016064154,0.000095547155,0.00006129088,0.00081856665,0.9985079,0.000007327311],"about_ca_topic_score_codex":0.0058753337,"about_ca_topic_score_gemma":0.012026707,"teacher_disagreement_score":0.078484535,"about_ca_system_score_codex":0.002114901,"about_ca_system_score_gemma":0.0031954963,"threshold_uncertainty_score":0.11194861},"labels":[],"label_agreement":null},{"id":"W4234990433","doi":"10.1002/ev.20161","title":"Issue Information","year":2016,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Human auditory perception and evaluation","field":"Engineering","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of Colorado Denver; University of Illinois at Urbana-Champaign; Centers for Disease Control and Prevention; Sierra Health Foundation; Claremont Graduate University; College of Engineering, Michigan State University; Northern Arizona University; Syracuse University; Vanderbilt University; University of Toronto; RMIT University; Weizmann Institute of Science; Michigan State University; Syddansk Universitet; International Development Research Centre; University of Hawai'i at Mānoa; University of Hawai'i; University of Minnesota; University of Ottawa; George Mason University","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.033987135490722076,"score_gpt":0.3297103506522229,"score_spread":0.29572321516150085,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4234990433","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00028966926,0.00084070355,0.0019275984,0.0030638343,0.011209076,0.0004582593,0.012775488,0.002573175,0.96686226],"genre_scores_gemma":[0.0005481855,0.0003512865,0.00046534836,0.0006292243,0.0008322737,0.00007856498,0.003841499,0.000434119,0.9928195],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9987424,0.0001438367,0.000071141,0.00015935018,0.00077189,0.00011138066],"domain_scores_gemma":[0.99517435,0.00061521225,0.00018484719,0.00045304437,0.002352681,0.0012199567],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0015982585,0.0011010986,0.0012176904,0.0038692767,0.001350646,0.0072826482,0.0016207842,0.002087869,0.89962584],"category_scores_gemma":[0.008987399,0.00047421333,0.0007565907,0.0029645548,0.0005159614,0.0032401648,0.0022177799,0.0017419021,0.8502993],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018894618,0.000031927648,0.000049464943,0.00012869984,0.00000138825,0.000015037777,0.000009426389,0.000039845785,0.00013339089,0.0014206924,0.9448915,0.053259693],"study_design_scores_gemma":[0.000013609016,0.000015116118,0.00019390952,0.00007385195,0.0000017433598,0.000021928374,0.000023926128,0.00008641731,0.00011710774,0.0015201138,0.99792767,0.0000046981027],"about_ca_topic_score_codex":0.0017391664,"about_ca_topic_score_gemma":0.004148077,"teacher_disagreement_score":0.10037416,"about_ca_system_score_codex":0.0010858772,"about_ca_system_score_gemma":0.0029065139,"threshold_uncertainty_score":0.14317155},"labels":[],"label_agreement":null},{"id":"W4236527672","doi":"10.1002/ev.20273","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4236527672","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00019243093,0.0011349258,0.00080483087,0.004303252,0.016345628,0.0005194983,0.04407755,0.002483122,0.93013877],"genre_scores_gemma":[0.00088032323,0.0009915077,0.00039620232,0.0018458,0.0027998649,0.00016745979,0.018994838,0.0009745884,0.9729495],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986148,0.00015435783,0.00011464665,0.00021595,0.00073634554,0.00016395132],"domain_scores_gemma":[0.9932482,0.00084729516,0.00028800537,0.000680989,0.0033003574,0.0016350915],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0014182586,0.0015930095,0.0016822658,0.0037598407,0.0018894164,0.009196392,0.0020464289,0.002387124,0.9296991],"category_scores_gemma":[0.009938553,0.00058184343,0.000942528,0.0045082388,0.0005631555,0.0054673557,0.0022714722,0.0022743498,0.8993487],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000112129055,0.000009218323,0.00002443147,0.00009423848,8.649879e-7,0.0000063952243,0.000004886648,0.000013835452,0.000034003755,0.0005426423,0.98670536,0.012552741],"study_design_scores_gemma":[0.000008586674,0.000007635449,0.00013533846,0.00009273872,9.849172e-7,0.000013777232,0.000016902124,0.000020849395,0.000036633934,0.00046678502,0.9991967,0.0000031167247],"about_ca_topic_score_codex":0.003166578,"about_ca_topic_score_gemma":0.0051580593,"teacher_disagreement_score":0.07030088,"about_ca_system_score_codex":0.001717687,"about_ca_system_score_gemma":0.0028308749,"threshold_uncertainty_score":0.100275695},"labels":[],"label_agreement":null},{"id":"W4236772578","doi":"10.1002/ev.239","title":"Editor's notes","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa","funders":"","keywords":"Citation; Computer science; Library science; Sociology","score_opus":0.2516694502734112,"score_gpt":0.5602664481446863,"score_spread":0.3085969978712751,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4236772578","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00009934859,0.00301481,0.00021689467,0.15160197,0.8315672,0.000059617654,0.00030029129,0.00008976536,0.013050088],"genre_scores_gemma":[0.00284713,0.004210832,0.0011525358,0.31178123,0.46873,0.00016032755,0.00033711398,0.00016039163,0.2106205],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99343747,0.0008682173,0.0006638069,0.00079972256,0.0036053762,0.00062536984],"domain_scores_gemma":[0.9732171,0.006282153,0.0021705034,0.0018235788,0.013740078,0.0027665775],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0072997636,0.0013344378,0.0017398696,0.00257294,0.0025031131,0.0047568227,0.0044717817,0.012160909,0.09518548],"category_scores_gemma":[0.052755028,0.0007844376,0.0018172044,0.0016092467,0.0013053102,0.0024912946,0.002005491,0.011092013,0.051634096],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000124726175,0.0000056582744,0.0000203563,0.000049510058,0.0000032764513,0.000063406274,0.0000075468733,0.000008066747,0.00003533172,0.00026637208,0.9958527,0.0036753402],"study_design_scores_gemma":[0.000024669707,0.000009483757,0.00020331888,0.00014083272,0.000013986241,0.00013548104,0.000034114917,0.0000461028,0.0001328876,0.00048538524,0.99876237,0.00001123045],"about_ca_topic_score_codex":0.0047309618,"about_ca_topic_score_gemma":0.011823394,"teacher_disagreement_score":0.90481454,"about_ca_system_score_codex":0.0028651566,"about_ca_system_score_gemma":0.0050600753,"threshold_uncertainty_score":0.3184272},"labels":[],"label_agreement":null},{"id":"W4238213994","doi":"10.1002/ev.20356","title":"Issue Information","year":2020,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of California, Los Angeles; Sierra Health Foundation; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; University of Hawai'i; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Library science; Internet privacy","score_opus":0.2442056670308427,"score_gpt":0.5390197065552901,"score_spread":0.2948140395244474,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4238213994","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0007259439,0.00058181334,0.00522488,0.013615707,0.010178403,0.00072815607,0.011109039,0.0011071429,0.95672894],"genre_scores_gemma":[0.0068917684,0.0011830393,0.0032618148,0.0058102314,0.003088925,0.0006392276,0.00927147,0.00078070804,0.9690729],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99313617,0.0013385107,0.00065819215,0.0004520652,0.00402553,0.00038956114],"domain_scores_gemma":[0.97620344,0.008743526,0.0011366934,0.0027226978,0.009739134,0.0014544852],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.007217741,0.0010102036,0.000977424,0.006389304,0.0025575322,0.010938256,0.0020133436,0.0060756044,0.5578128],"category_scores_gemma":[0.03584341,0.00061348506,0.00076960796,0.005647087,0.0009213533,0.01032594,0.0045787604,0.0034838908,0.40969166],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000032568518,0.000032831595,0.00011410932,0.0002826208,0.0000023070838,0.00006744377,0.00018249267,0.00006894762,0.0002943363,0.014264592,0.9308539,0.05380379],"study_design_scores_gemma":[0.0000042983766,0.0000086453965,0.0001386379,0.00007434628,0.0000017094629,0.000028879589,0.00007255905,0.00006076943,0.00011195878,0.0015725883,0.9979213,0.00000433724],"about_ca_topic_score_codex":0.0015805846,"about_ca_topic_score_gemma":0.0025572171,"teacher_disagreement_score":0.4421872,"about_ca_system_score_codex":0.0016707098,"about_ca_system_score_gemma":0.0047284276,"threshold_uncertainty_score":0.63072634},"labels":[],"label_agreement":null},{"id":"W4238813095","doi":"10.1002/ev.20357","title":"Issue Information","year":2020,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of California, Los Angeles; Sierra Health Foundation; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; University of Hawai'i; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.2442056670308427,"score_gpt":0.5390197065552901,"score_spread":0.2948140395244474,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4238813095","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0007259439,0.00058181334,0.00522488,0.013615707,0.010178403,0.00072815607,0.011109039,0.0011071429,0.95672894],"genre_scores_gemma":[0.0068917684,0.0011830393,0.0032618148,0.0058102314,0.003088925,0.0006392276,0.00927147,0.00078070804,0.9690729],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99313617,0.0013385107,0.00065819215,0.0004520652,0.00402553,0.00038956114],"domain_scores_gemma":[0.97620344,0.008743526,0.0011366934,0.0027226978,0.009739134,0.0014544852],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.007217741,0.0010102036,0.000977424,0.006389304,0.0025575322,0.010938256,0.0020133436,0.0060756044,0.5578128],"category_scores_gemma":[0.03584341,0.00061348506,0.00076960796,0.005647087,0.0009213533,0.01032594,0.0045787604,0.0034838908,0.40969166],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000032568518,0.000032831595,0.00011410932,0.0002826208,0.0000023070838,0.00006744377,0.00018249267,0.00006894762,0.0002943363,0.014264592,0.9308539,0.05380379],"study_design_scores_gemma":[0.0000042983766,0.0000086453965,0.0001386379,0.00007434628,0.0000017094629,0.000028879589,0.00007255905,0.00006076943,0.00011195878,0.0015725883,0.9979213,0.00000433724],"about_ca_topic_score_codex":0.0015805846,"about_ca_topic_score_gemma":0.0025572171,"teacher_disagreement_score":0.4421872,"about_ca_system_score_codex":0.0016707098,"about_ca_system_score_gemma":0.0047284276,"threshold_uncertainty_score":0.63072634},"labels":[],"label_agreement":null},{"id":"W4242601229","doi":"10.1002/ev.20107","title":"Editors' Notes","year":2015,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Accreditation; Charter; Certification; Context (archaeology); Credentialing; Conversation; Public relations; Professional association; Scholarship; Task (project management); Computer science; Political science; Engineering ethics; Management; Psychology; Law; History","score_opus":0.41995704850429344,"score_gpt":0.5640301890940794,"score_spread":0.144073140589786,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4242601229","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0002297917,0.008953522,0.0015551324,0.110180244,0.7011521,0.00041154475,0.0039409176,0.0010378099,0.17253889],"genre_scores_gemma":[0.0028137693,0.009594838,0.0032888767,0.07299873,0.15046845,0.0006257473,0.0033802958,0.0011341843,0.7556951],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9904056,0.0008678977,0.0010904985,0.0013816761,0.005446929,0.0008073403],"domain_scores_gemma":[0.9544216,0.0078430325,0.0024573372,0.0038820868,0.027128045,0.00426794],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.007956757,0.0014004203,0.0015255429,0.0046876697,0.0035888113,0.008597897,0.0057062022,0.0059312605,0.4067391],"category_scores_gemma":[0.06994286,0.0007954877,0.0015150027,0.004095411,0.0014007204,0.005555568,0.0035158594,0.0068893307,0.3229369],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000007503414,0.0000063848793,0.000014726995,0.00008431599,0.0000010024454,0.000021763966,0.000025650188,0.000009565534,0.000036761685,0.00090639107,0.98743814,0.011447731],"study_design_scores_gemma":[0.000004531754,0.0000031471952,0.00003969228,0.00010963897,0.0000014415398,0.000024105464,0.000025622474,0.000008948015,0.000042582873,0.00033728417,0.9993993,0.0000036427557],"about_ca_topic_score_codex":0.003783897,"about_ca_topic_score_gemma":0.0066051446,"teacher_disagreement_score":0.4067391,"about_ca_system_score_codex":0.0036890095,"about_ca_system_score_gemma":0.009225461,"threshold_uncertainty_score":0.84621465},"labels":[],"label_agreement":null},{"id":"W4245992123","doi":"10.1002/ev.20367","title":"Issue Information","year":2021,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.2232214289072087,"score_gpt":0.5437897844404399,"score_spread":0.3205683555332312,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4245992123","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0001146831,0.00054918235,0.00041829247,0.0020575956,0.006045516,0.0002926902,0.026246617,0.0013800788,0.9628953],"genre_scores_gemma":[0.0004070469,0.000453581,0.00032422418,0.00066535483,0.00071783125,0.00009453295,0.01034456,0.00034821665,0.98664457],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99870205,0.00009379484,0.00007119765,0.00014738053,0.0008325616,0.00015302778],"domain_scores_gemma":[0.99520147,0.0005448972,0.0002294907,0.0004384623,0.002468265,0.0011174654],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0011592988,0.0017377472,0.0013236816,0.00542718,0.0021652335,0.0098492615,0.0020910555,0.002915466,0.9245241],"category_scores_gemma":[0.0068394393,0.00079703185,0.0010855715,0.0058473623,0.000641134,0.0038291428,0.002246916,0.002257409,0.8840906],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009014626,0.00001399109,0.000023773015,0.00007724569,7.2453315e-7,0.0000072328594,0.0000038028732,0.0000229487,0.00003376752,0.00054817914,0.9818897,0.017369635],"study_design_scores_gemma":[0.000010638585,0.000006777242,0.00019978009,0.000070149,0.0000011420454,0.000008665761,0.000013176159,0.000032279513,0.000030569354,0.00046483366,0.9991584,0.0000035100836],"about_ca_topic_score_codex":0.015913634,"about_ca_topic_score_gemma":0.03715463,"teacher_disagreement_score":0.07547587,"about_ca_system_score_codex":0.002403858,"about_ca_system_score_gemma":0.00615756,"threshold_uncertainty_score":0},"labels":[{"model":"gpt","categories":[],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"grok","categories":["insufficient_payload"],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"high"},{"model":"opus","categories":["insufficient_payload"],"domain":null,"study_design":"not_applicable","genre":"other","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"},{"id":"W4247121542","doi":"10.1002/ev.20279","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4247121542","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00019243093,0.0011349258,0.00080483087,0.004303252,0.016345628,0.0005194983,0.04407755,0.002483122,0.93013877],"genre_scores_gemma":[0.00088032323,0.0009915077,0.00039620232,0.0018458,0.0027998649,0.00016745979,0.018994838,0.0009745884,0.9729495],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986148,0.00015435783,0.00011464665,0.00021595,0.00073634554,0.00016395132],"domain_scores_gemma":[0.9932482,0.00084729516,0.00028800537,0.000680989,0.0033003574,0.0016350915],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0014182586,0.0015930095,0.0016822658,0.0037598407,0.0018894164,0.009196392,0.0020464289,0.002387124,0.9296991],"category_scores_gemma":[0.009938553,0.00058184343,0.000942528,0.0045082388,0.0005631555,0.0054673557,0.0022714722,0.0022743498,0.8993487],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000112129055,0.000009218323,0.00002443147,0.00009423848,8.649879e-7,0.0000063952243,0.000004886648,0.000013835452,0.000034003755,0.0005426423,0.98670536,0.012552741],"study_design_scores_gemma":[0.000008586674,0.000007635449,0.00013533846,0.00009273872,9.849172e-7,0.000013777232,0.000016902124,0.000020849395,0.000036633934,0.00046678502,0.9991967,0.0000031167247],"about_ca_topic_score_codex":0.003166578,"about_ca_topic_score_gemma":0.0051580593,"teacher_disagreement_score":0.07030088,"about_ca_system_score_codex":0.001717687,"about_ca_system_score_gemma":0.0028308749,"threshold_uncertainty_score":0.100275695},"labels":[],"label_agreement":null},{"id":"W4249000756","doi":"10.1002/ev.220","title":"Editor's notes","year":2007,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Citation; Library science; Computer science; World Wide Web","score_opus":0.2516694502734112,"score_gpt":0.5602664481446863,"score_spread":0.3085969978712751,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4249000756","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00009934859,0.00301481,0.00021689467,0.15160197,0.8315672,0.000059617654,0.00030029129,0.00008976536,0.013050088],"genre_scores_gemma":[0.00284713,0.004210832,0.0011525358,0.31178123,0.46873,0.00016032755,0.00033711398,0.00016039163,0.2106205],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99343747,0.0008682173,0.0006638069,0.00079972256,0.0036053762,0.00062536984],"domain_scores_gemma":[0.9732171,0.006282153,0.0021705034,0.0018235788,0.013740078,0.0027665775],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0072997636,0.0013344378,0.0017398696,0.00257294,0.0025031131,0.0047568227,0.0044717817,0.012160909,0.09518548],"category_scores_gemma":[0.052755028,0.0007844376,0.0018172044,0.0016092467,0.0013053102,0.0024912946,0.002005491,0.011092013,0.051634096],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000124726175,0.0000056582744,0.0000203563,0.000049510058,0.0000032764513,0.000063406274,0.0000075468733,0.000008066747,0.00003533172,0.00026637208,0.9958527,0.0036753402],"study_design_scores_gemma":[0.000024669707,0.000009483757,0.00020331888,0.00014083272,0.000013986241,0.00013548104,0.000034114917,0.0000461028,0.0001328876,0.00048538524,0.99876237,0.00001123045],"about_ca_topic_score_codex":0.0047309618,"about_ca_topic_score_gemma":0.011823394,"teacher_disagreement_score":0.09518548,"about_ca_system_score_codex":0.0028651566,"about_ca_system_score_gemma":0.0050600753,"threshold_uncertainty_score":0.3184272},"labels":[],"label_agreement":null},{"id":"W4249389693","doi":"10.1002/ev.20282","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Computer science; Citation; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4249389693","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00019243093,0.0011349258,0.00080483087,0.004303252,0.016345628,0.0005194983,0.04407755,0.002483122,0.93013877],"genre_scores_gemma":[0.00088032323,0.0009915077,0.00039620232,0.0018458,0.0027998649,0.00016745979,0.018994838,0.0009745884,0.9729495],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986148,0.00015435783,0.00011464665,0.00021595,0.00073634554,0.00016395132],"domain_scores_gemma":[0.9932482,0.00084729516,0.00028800537,0.000680989,0.0033003574,0.0016350915],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0014182586,0.0015930095,0.0016822658,0.0037598407,0.0018894164,0.009196392,0.0020464289,0.002387124,0.9296991],"category_scores_gemma":[0.009938553,0.00058184343,0.000942528,0.0045082388,0.0005631555,0.0054673557,0.0022714722,0.0022743498,0.8993487],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0000112129055,0.000009218323,0.00002443147,0.00009423848,8.649879e-7,0.0000063952243,0.000004886648,0.000013835452,0.000034003755,0.0005426423,0.98670536,0.012552741],"study_design_scores_gemma":[0.000008586674,0.000007635449,0.00013533846,0.00009273872,9.849172e-7,0.000013777232,0.000016902124,0.000020849395,0.000036633934,0.00046678502,0.9991967,0.0000031167247],"about_ca_topic_score_codex":0.003166578,"about_ca_topic_score_gemma":0.0051580593,"teacher_disagreement_score":0.9296991,"about_ca_system_score_codex":0.001717687,"about_ca_system_score_gemma":0.0028308749,"threshold_uncertainty_score":0.100275695},"labels":[],"label_agreement":null},{"id":"W4251068696","doi":"10.1002/ev.324","title":"Editors' notes","year":2010,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Disaster Management and Resilience","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Social Sciences and Humanities Research Council","funders":"","keywords":"Citation; Library science; Preparedness; Sociology; Computer science; Political science; Law","score_opus":0.03532045501342495,"score_gpt":0.3837244448720017,"score_spread":0.34840398985857673,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4251068696","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00023756502,0.005843495,0.00046521492,0.24625209,0.7112253,0.00008889297,0.0005121402,0.00015452431,0.035220947],"genre_scores_gemma":[0.0038433166,0.004759437,0.0012987447,0.3037597,0.2996362,0.00022955894,0.00040866167,0.00021872736,0.3858457],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9945851,0.0007161168,0.0005169465,0.00083287206,0.002751624,0.0005973585],"domain_scores_gemma":[0.97794366,0.0055439207,0.0016184684,0.0015514022,0.010737341,0.0026051982],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006130853,0.0009285428,0.0011405672,0.0017970727,0.0023564862,0.0046786615,0.004025815,0.008993666,0.10913146],"category_scores_gemma":[0.04436534,0.00052148267,0.0012316334,0.0014459926,0.0011787736,0.002676965,0.002179849,0.009436477,0.06170758],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013753541,0.0000063013235,0.00003478509,0.000048870592,0.0000022138506,0.000054895976,0.000020294958,0.000011046011,0.00005411318,0.0006126981,0.99169356,0.0074473713],"study_design_scores_gemma":[0.000012628587,0.0000067207775,0.00014690221,0.00008363078,0.000006449449,0.000081020946,0.000041905314,0.000025405758,0.000099220386,0.00044573954,0.99904424,0.0000062528206],"about_ca_topic_score_codex":0.0049401857,"about_ca_topic_score_gemma":0.011199222,"teacher_disagreement_score":0.10913146,"about_ca_system_score_codex":0.0024633433,"about_ca_system_score_gemma":0.004607864,"threshold_uncertainty_score":0.36508107},"labels":[],"label_agreement":null},{"id":"W4252169279","doi":"10.1002/ev.309","title":"Editors' notes","year":2009,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Community Health and Development","field":"Health Professions","cited_by":1,"is_retracted":false,"has_abstract":false,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Citation; Library science; Population; Intervention (counseling); State (computer science); Sociology; Medicine; Computer science; Nursing","score_opus":0.1565093687785537,"score_gpt":0.5236589003624142,"score_spread":0.36714953158386054,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4252169279","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00012592573,0.0038505078,0.0003254184,0.3130656,0.6668798,0.00008102233,0.00026154114,0.00012440715,0.015285809],"genre_scores_gemma":[0.0017896581,0.0028528592,0.001007245,0.5299438,0.32032514,0.00024462695,0.00020636816,0.00015009096,0.14348017],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.98738956,0.0017372353,0.0013243405,0.0015326231,0.00668472,0.0013315108],"domain_scores_gemma":[0.9436357,0.015919052,0.004394293,0.0032491067,0.02689587,0.0059060045],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.013162,0.0011798807,0.0018173497,0.0023144346,0.003217611,0.0064702816,0.0063220584,0.020657657,0.080148436],"category_scores_gemma":[0.10291096,0.00086907344,0.002214249,0.0017423119,0.0018700609,0.00380384,0.0030825327,0.018850954,0.05411454],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000009081058,0.00000536287,0.000029460367,0.000043933884,0.0000024117733,0.000059808,0.00001829761,0.000006814546,0.00002429624,0.0003704508,0.99518126,0.004248771],"study_design_scores_gemma":[0.00002036546,0.000008434516,0.00015691954,0.00018324771,0.000010581766,0.00012265671,0.000061653154,0.00002796389,0.000079312704,0.0004944667,0.9988237,0.0000106856605],"about_ca_topic_score_codex":0.0053996122,"about_ca_topic_score_gemma":0.014120821,"teacher_disagreement_score":0.080148436,"about_ca_system_score_codex":0.0035107692,"about_ca_system_score_gemma":0.007849287,"threshold_uncertainty_score":0.2681232},"labels":[],"label_agreement":null},{"id":"W4256226640","doi":"10.1002/ev.20276","title":"Issue Information","year":2018,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"University of California, Los Angeles; National Science Foundation; Claremont Graduate University; Duquesne University; Weizmann Institute of Science; Michigan State University; Sierra Health Foundation; Northwestern University; Gordon and Betty Moore Foundation; University of Connecticut; College of Engineering, Michigan State University; École nationale d'administration publique; Syracuse University; University of Southern California; John D. and Catherine T. MacArthur Foundation","keywords":"Citation; Computer science; World Wide Web; Information retrieval; Library science","score_opus":0.0711548467400742,"score_gpt":0.2947114174565125,"score_spread":0.2235565707164383,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4256226640","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00020841674,0.0012503098,0.0008358347,0.00401598,0.015351852,0.00049663684,0.040785775,0.0026195107,0.9344356],"genre_scores_gemma":[0.00091488287,0.0010414724,0.00040478917,0.0016554655,0.002553956,0.000154356,0.01817797,0.0009729802,0.9741243],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99873835,0.0001436175,0.000105493,0.00020164582,0.0006578239,0.00015317454],"domain_scores_gemma":[0.9938408,0.00081609597,0.00027033014,0.0006526128,0.0028328022,0.0015872376],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0013766788,0.0015687817,0.0015995526,0.0034951367,0.0017351841,0.009092397,0.0019609325,0.002255672,0.9353011],"category_scores_gemma":[0.009411073,0.0005610083,0.00094576547,0.0041674315,0.00055599323,0.0053459224,0.0021662526,0.0021937557,0.8994907],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012861909,0.000010013838,0.000024537334,0.00010034166,9.244048e-7,0.0000067771166,0.000004928544,0.000015636195,0.0000372804,0.00056945364,0.9847178,0.014499445],"study_design_scores_gemma":[0.000009150229,0.000008131694,0.00013300047,0.00009435131,0.0000010656986,0.000014861837,0.000015858865,0.000022487366,0.000038659782,0.00047458656,0.99918467,0.0000031261088],"about_ca_topic_score_codex":0.0030718893,"about_ca_topic_score_gemma":0.0047967373,"teacher_disagreement_score":0.064698875,"about_ca_system_score_codex":0.0016541342,"about_ca_system_score_gemma":0.0026126823,"threshold_uncertainty_score":0.0922851},"labels":[],"label_agreement":null},{"id":"W4293180673","doi":"10.1002/ev.20490","title":"The importance of implementation: Putting evaluation policy to work","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Work (physics); Government (linguistics); White paper; Public administration; Public relations; Early adopter; Program evaluation; Evaluation methods; Policy analysis; Business; Political science; Marketing","score_opus":0.2980319978182459,"score_gpt":0.5961758186242162,"score_spread":0.2981438208059703,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4293180673","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":"methods","domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.004683529,0.0062412014,0.027834171,0.9123735,0.0024211572,0.0006969701,0.00003898644,0.00018764369,0.045523],"genre_scores_gemma":[0.608847,0.01369932,0.11471592,0.24271898,0.0033014724,0.004469274,0.00016513046,0.0006058203,0.011477045],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.2811696,0.61683714,0.02221506,0.011054202,0.049435716,0.019288305],"domain_scores_gemma":[0.14881612,0.7270937,0.014474628,0.025516802,0.062980525,0.021118132],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.6139299,0.0016591247,0.0034656462,0.007433902,0.023528885,0.06750646,0.008175779,0.028363211,0.010009714],"category_scores_gemma":[0.63208205,0.0023608676,0.0023581223,0.006064767,0.069425605,0.078748,0.024977138,0.040613487,0.0020099883],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00022666328,0.000620101,0.0044051283,0.0032921243,0.00013596113,0.0002627535,0.03435894,0.0015706543,0.00048529715,0.637095,0.06867458,0.24887274],"study_design_scores_gemma":[0.00038182526,0.0006357049,0.00533867,0.029126802,0.00023740358,0.0002660223,0.090626396,0.002505256,0.0021639003,0.5402797,0.32814124,0.00029711242],"about_ca_topic_score_codex":0.01588233,"about_ca_topic_score_gemma":0.01142473,"teacher_disagreement_score":0.6139299,"about_ca_system_score_codex":0.05244821,"about_ca_system_score_gemma":0.24724607,"threshold_uncertainty_score":0.47609317},"labels":[],"label_agreement":null},{"id":"W4307053074","doi":"10.1002/ev.20511","title":"Empowering the search for pleasure, health and well‐being outside heteronormative definitions: The role of evaluation in shaping structurally sensitive programming for 2SGBTQ+ men who <i>Party and Play</i> in Ontario","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"HIV, Drug Use, Sexual Risk","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Casey House","funders":"","keywords":"Phenomenon; Agency (philosophy); Transformative learning; Pleasure; Sociology; Public health; Public relations; Health care; Qualitative research; Gender studies; Social psychology; Psychology; Medicine; Nursing; Political science; Epistemology; Social science","score_opus":0.10428937933277302,"score_gpt":0.39257313186623666,"score_spread":0.2882837525334636,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4307053074","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.586773,0.0017070553,0.026969638,0.06612931,0.00026972304,0.0017360033,0.00009795317,0.00013880683,0.31617853],"genre_scores_gemma":[0.98430324,0.00046585218,0.0050804513,0.0012873231,0.0000131927445,0.00030500037,0.000015203318,0.000037255257,0.0084925415],"study_design_codex":"qualitative","study_design_gemma":"qualitative","domain_scores_codex":[0.9784475,0.015798252,0.00026397777,0.0005879525,0.0020751327,0.0028271894],"domain_scores_gemma":[0.9891299,0.00543409,0.0005294266,0.00050748815,0.0016931418,0.0027059903],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.019410746,0.0003985815,0.00040836397,0.00069405645,0.016960423,0.009319324,0.0019386818,0.0012043737,0.004571783],"category_scores_gemma":[0.014916119,0.00033503442,0.00042655235,0.0006286351,0.038634196,0.0025456182,0.01182676,0.0027852883,0.00022043732],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","about_ca_topic_candidate":true,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011767087,0.00033945462,0.010308591,0.00040108204,0.000021642274,0.00050111744,0.7444818,0.0008961026,0.0019927095,0.14994033,0.008107147,0.082892425],"study_design_scores_gemma":[0.00009053445,0.00030943696,0.021711314,0.0013624721,0.000056413093,0.00016366692,0.62671095,0.0016913795,0.0024359839,0.062151954,0.28321263,0.00010325759],"about_ca_topic_score_codex":0.4781729,"about_ca_topic_score_gemma":0.7452886,"teacher_disagreement_score":0.5218271,"about_ca_system_score_codex":0.06756195,"about_ca_system_score_gemma":0.13047297,"threshold_uncertainty_score":0.95077926},"labels":[],"label_agreement":null},{"id":"W4307053258","doi":"10.1002/ev.20514","title":"Identity as a compass when navigating uncharted equitable spaces: Our queer evaluation practices","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Saskatchewan; Saskatchewan Health","funders":"","keywords":"Queer; Dialogic; Situated; Sociology; Normative; Identity (music); Epistemology; Norm (philosophy); Social psychology; Psychology; Computer science; Aesthetics; Gender studies; Pedagogy","score_opus":0.37946722076513645,"score_gpt":0.5847148136120114,"score_spread":0.20524759284687494,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4307053258","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.29407215,0.0068185288,0.14481401,0.19171377,0.00091888406,0.00058240484,0.000056199322,0.00037364094,0.3606505],"genre_scores_gemma":[0.9713679,0.0006625893,0.0077812984,0.0034782332,0.000056484896,0.00013794232,0.000008960072,0.000109369415,0.016397186],"study_design_codex":"qualitative","study_design_gemma":"qualitative","domain_scores_codex":[0.7739963,0.20351614,0.002787008,0.0044667823,0.008900447,0.006333375],"domain_scores_gemma":[0.9092591,0.06736936,0.004076526,0.005910384,0.008814386,0.0045702714],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.12272829,0.0005840787,0.000990733,0.0032032249,0.034517594,0.030400183,0.0028067878,0.005050606,0.0060369032],"category_scores_gemma":[0.09185512,0.0006062889,0.00070257,0.0020383117,0.09518934,0.021604078,0.020985292,0.009117198,0.0008051072],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003401524,0.000049663184,0.0010690906,0.00007399299,0.000010272874,0.00037991337,0.569746,0.00031083927,0.00038338196,0.4056766,0.00392617,0.018339962],"study_design_scores_gemma":[0.0000201081,0.00008138898,0.0010287839,0.00069550757,0.000027839485,0.0003548729,0.6105668,0.0013007007,0.0018192456,0.17244008,0.2115526,0.000112170885],"about_ca_topic_score_codex":0.030774968,"about_ca_topic_score_gemma":0.036658224,"teacher_disagreement_score":0.12272829,"about_ca_system_score_codex":0.02690808,"about_ca_system_score_gemma":0.020760996,"threshold_uncertainty_score":0.649057},"labels":[],"label_agreement":null},{"id":"W4307053273","doi":"10.1002/ev.20510","title":"At the intersection of co‐creation: Exploring LGBTQ2S evaluation with youth","year":2022,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"University of Prince Edward Island; University of Winnipeg; Youth Services Bureau of Ottawa; Parks Canada","funders":"","keywords":"Intersection (aeronautics); Sociology; Context (archaeology); Public relations; Program evaluation; Positive Youth Development; Youth studies; Psychology; Political science; Gender studies; Public administration; Engineering","score_opus":0.46878928555177146,"score_gpt":0.5291504336284258,"score_spread":0.06036114807665438,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4307053273","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.91610503,0.001149373,0.010227865,0.019547835,0.00015903832,0.00052420225,0.000052966207,0.0000877645,0.052145917],"genre_scores_gemma":[0.99319094,0.00030006867,0.002787734,0.0007850827,0.000021293166,0.00021450853,0.000017345465,0.000035755653,0.0026472865],"study_design_codex":"qualitative","study_design_gemma":"qualitative","domain_scores_codex":[0.8439943,0.13799137,0.001546973,0.0017750778,0.0057793865,0.008912901],"domain_scores_gemma":[0.9398753,0.03917284,0.0031794098,0.002081067,0.006805184,0.008886165],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.08169101,0.00075850234,0.00093043974,0.002959014,0.028528858,0.02238078,0.0030161378,0.0025117132,0.004734807],"category_scores_gemma":[0.04712327,0.00078900123,0.00060441723,0.00231095,0.029749839,0.009665422,0.03120348,0.0051259603,0.00052471436],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000058575148,0.00027450462,0.0095499195,0.00012709967,0.0000128208285,0.0005546886,0.956522,0.00011601852,0.00032873853,0.008712699,0.0018396745,0.021903211],"study_design_scores_gemma":[0.00000962957,0.00007105233,0.0023012483,0.0001782731,0.000007670768,0.000102135,0.9834442,0.00018389962,0.0003896439,0.0018801264,0.011418088,0.000013943653],"about_ca_topic_score_codex":0.027504515,"about_ca_topic_score_gemma":0.0675269,"teacher_disagreement_score":0.08169101,"about_ca_system_score_codex":0.026170045,"about_ca_system_score_gemma":0.027590847,"threshold_uncertainty_score":0.43202853},"labels":[],"label_agreement":null},{"id":"W4378223319","doi":"10.1002/ev.20539","title":"Meeting the challenges of educating internal evaluators","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Institute for Clinical Evaluative Sciences; Queen's University","funders":"","keywords":"Front line; Public relations; Geopolitics; Coronavirus disease 2019 (COVID-19); Pandemic; Program evaluation; Political science; Business; Medicine; Public administration","score_opus":0.3888663697506948,"score_gpt":0.5659171151344419,"score_spread":0.1770507453837471,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223319","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.083802156,0.014684737,0.071721956,0.76352036,0.0043730275,0.000584449,0.00006500858,0.00079135655,0.06045701],"genre_scores_gemma":[0.7817616,0.01076962,0.08398671,0.09212858,0.0030770334,0.0010501026,0.00012258446,0.00045818463,0.02664566],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.85422975,0.11510977,0.0038892638,0.0036153267,0.016071368,0.007084548],"domain_scores_gemma":[0.59851396,0.24490304,0.017936341,0.015171752,0.08644867,0.037026223],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.17755692,0.00087834115,0.0010646811,0.002188819,0.008922302,0.023677541,0.0028298858,0.005399364,0.007904977],"category_scores_gemma":[0.18269081,0.0007156722,0.0007141907,0.0009225427,0.009942832,0.012611985,0.015002983,0.01348913,0.0028179716],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00021000602,0.0014747367,0.018179959,0.0017242528,0.00014807988,0.0004887077,0.071878694,0.0021657804,0.0017242596,0.070080236,0.26161352,0.5703118],"study_design_scores_gemma":[0.00028945852,0.0008883237,0.013419819,0.0083546555,0.00010396119,0.0009286759,0.15983182,0.0061244457,0.0045426637,0.14174926,0.6634736,0.00029330075],"about_ca_topic_score_codex":0.0027129902,"about_ca_topic_score_gemma":0.0053114416,"teacher_disagreement_score":0.17755692,"about_ca_system_score_codex":0.008452885,"about_ca_system_score_gemma":0.029420016,"threshold_uncertainty_score":0.93902194},"labels":[],"label_agreement":null},{"id":"W4378223534","doi":"10.1002/ev.20536","title":"Learning by linking the Canadian Evaluation Society's student case competition within a graduate evaluation course","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"Queen's University","funders":"","keywords":"Experiential learning; Dialogic; Pedagogy; Psychology; Nature versus nurture; Situated; Competition (biology); Professional development; Assessment for learning; Sociology; Formative assessment; Computer science","score_opus":0.34293943692583495,"score_gpt":0.5475533572533501,"score_spread":0.20461392032751513,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223534","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.7353532,0.00031834698,0.011242261,0.030110948,0.0012575717,0.0009359054,0.00027377365,0.00026757072,0.22024046],"genre_scores_gemma":[0.9365143,0.00013979794,0.006224395,0.0018201668,0.0000820241,0.00022409657,0.00014631136,0.00012554551,0.054723293],"study_design_codex":"qualitative","study_design_gemma":"not_applicable","domain_scores_codex":[0.9739604,0.0111135915,0.0004891384,0.0016999504,0.007248088,0.0054887924],"domain_scores_gemma":[0.9720063,0.0074897944,0.0009049813,0.0014111369,0.008212745,0.009974995],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.021708632,0.0006018087,0.0004985437,0.0021129025,0.019451944,0.01359042,0.002550939,0.0027640644,0.009603619],"category_scores_gemma":[0.031300712,0.00034115303,0.00053891743,0.0015007943,0.007964598,0.0026522535,0.009988921,0.0047217607,0.0013427851],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000268575,0.0012769593,0.025123125,0.00016077222,0.000026494487,0.0040555396,0.5868398,0.0020178214,0.005327994,0.066322535,0.13654208,0.1720383],"study_design_scores_gemma":[0.000028397357,0.000395936,0.016412875,0.0002391196,0.0000141685205,0.0008184853,0.5330135,0.0027540075,0.0026009423,0.006024853,0.43750027,0.00019744129],"about_ca_topic_score_codex":0.31727856,"about_ca_topic_score_gemma":0.66644615,"teacher_disagreement_score":0.31727856,"about_ca_system_score_codex":0.041495744,"about_ca_system_score_gemma":0.045987874,"threshold_uncertainty_score":0.6308636},"labels":[],"label_agreement":null},{"id":"W4378223570","doi":"10.1002/ev.20540","title":"What we can learn from the international program for development evaluation training (IPDET)","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Program evaluation; Training (meteorology); Monitoring and evaluation; Diversity (politics); Medical education; Corporate governance; Political science; Psychology; Business; Medicine; Geography; Public administration; Finance","score_opus":0.5304740430548962,"score_gpt":0.569693978952799,"score_spread":0.03921993589790285,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223570","genre_codex":"commentary","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0022577872,0.042620372,0.016427994,0.8713218,0.013973818,0.00034581323,0.0003891783,0.0005690255,0.05209426],"genre_scores_gemma":[0.14093596,0.1689,0.16862918,0.44618803,0.0221651,0.0038769362,0.0023214105,0.0014065517,0.04557685],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.8960354,0.077986605,0.0043902523,0.0031714248,0.0131342765,0.0052820924],"domain_scores_gemma":[0.75609106,0.13073711,0.0070233447,0.019516157,0.05611706,0.03051524],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.14139847,0.0010375802,0.0013677113,0.0036997786,0.004764757,0.022890829,0.0040027373,0.0074709854,0.017082939],"category_scores_gemma":[0.22259797,0.00068175694,0.0015871957,0.0042994143,0.008882074,0.030569652,0.012524304,0.016243609,0.0067313784],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005264946,0.00030610582,0.0021825382,0.002567788,0.000048319664,0.00020155373,0.0065155886,0.00031673117,0.00015373263,0.068893336,0.54850155,0.37026012],"study_design_scores_gemma":[0.00004515281,0.00016384477,0.002310506,0.012722819,0.000031241263,0.00022296813,0.007388561,0.0003002708,0.0003074848,0.05119148,0.9252431,0.00007255779],"about_ca_topic_score_codex":0.0045608785,"about_ca_topic_score_gemma":0.007751058,"teacher_disagreement_score":0.14139847,"about_ca_system_score_codex":0.009910089,"about_ca_system_score_gemma":0.033676118,"threshold_uncertainty_score":0.7477956},"labels":[],"label_agreement":null},{"id":"W4378223607","doi":"10.1002/ev.20543","title":"Guest Editors’ notes","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Timeline; Adventure; Ethos; Dream; Psychology; Sociology; Operations research; Library science; Computer science; Law; History; Political science; Artificial intelligence; Engineering","score_opus":0.3729067314618274,"score_gpt":0.5685543924994344,"score_spread":0.195647661037607,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4378223607","genre_codex":"editorial","genre_gemma":"editorial","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"editorial","genre_consensus":"editorial","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0002514425,0.004476465,0.0013773901,0.082326844,0.8476825,0.0002833709,0.0012798432,0.0010180675,0.061304037],"genre_scores_gemma":[0.0054283957,0.006673466,0.0039891843,0.07715555,0.35934982,0.0008984807,0.0020257637,0.002347938,0.54213136],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9871104,0.001619672,0.0011633869,0.0018134563,0.007019895,0.0012731921],"domain_scores_gemma":[0.9314541,0.011097775,0.003980978,0.0060141194,0.037981365,0.0094717],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.013784205,0.0016558502,0.001433176,0.0036227903,0.0037963525,0.012497247,0.004984005,0.006270865,0.27928305],"category_scores_gemma":[0.09551546,0.00084612507,0.0014799181,0.0026911267,0.0017879715,0.006175213,0.0065838685,0.009190381,0.19321989],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013087679,0.000008644119,0.000022917011,0.00009177791,0.000001350594,0.00003925272,0.000055450473,0.000016902071,0.00010202687,0.0009928804,0.98828083,0.0103747435],"study_design_scores_gemma":[0.000004531181,0.000005727545,0.00004042852,0.000097613854,0.0000011859054,0.00003582455,0.00004323368,0.000014391318,0.000067631074,0.0003793928,0.99930537,0.000004650009],"about_ca_topic_score_codex":0.001671881,"about_ca_topic_score_gemma":0.0028902832,"teacher_disagreement_score":0.27928305,"about_ca_system_score_codex":0.0036934463,"about_ca_system_score_gemma":0.009378722,"threshold_uncertainty_score":0.9342949},"labels":[],"label_agreement":null},{"id":"W4388306673","doi":"10.1002/ev.20564","title":"Artificial intelligence and the future of evaluation education: Possibilities and prototypes","year":2023,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Fraser Health","funders":"","keywords":"Chatbot; Evaluation methods; Engineering ethics; Literacy; Computer science; Program evaluation; Management science; Psychology; Pedagogy; Artificial intelligence; Political science; Engineering","score_opus":0.2331504610129478,"score_gpt":0.5285796286386468,"score_spread":0.29542916762569904,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4388306673","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.13171364,0.050047778,0.18312535,0.27105623,0.0018868245,0.0007982326,0.00013360755,0.0011509155,0.36008742],"genre_scores_gemma":[0.8873028,0.0088486895,0.091248035,0.0032004307,0.00029226375,0.0005289186,0.00006772172,0.00008733855,0.008423749],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9748905,0.021192916,0.0006471915,0.0005376567,0.001770107,0.0009616725],"domain_scores_gemma":[0.9442628,0.046243258,0.000890327,0.0028959084,0.0035443772,0.0021633639],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.046526518,0.00052868796,0.00070007803,0.00203292,0.0029246134,0.014776283,0.002962243,0.004164731,0.006922479],"category_scores_gemma":[0.032923292,0.00041457842,0.0006387548,0.0011818003,0.019585883,0.016958714,0.004686157,0.0043381043,0.0006021291],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00022071722,0.00056694524,0.001591782,0.00052910665,0.000015033081,0.00020573774,0.007126331,0.0024066614,0.0005081817,0.8734421,0.004766324,0.10862114],"study_design_scores_gemma":[0.000217884,0.0008176088,0.0012727844,0.0028829754,0.000027876442,0.00041039713,0.014221656,0.018020123,0.0020406554,0.7973178,0.16264632,0.0001237952],"about_ca_topic_score_codex":0.0031530873,"about_ca_topic_score_gemma":0.001776822,"teacher_disagreement_score":0.046526518,"about_ca_system_score_codex":0.0053884867,"about_ca_system_score_gemma":0.005827419,"threshold_uncertainty_score":0.2460587},"labels":[],"label_agreement":null},{"id":"W4392918018","doi":"10.1002/ev.20454","title":"Issue Information","year":2023,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"","keywords":"Computer science; Psychology","score_opus":0.090513352209208,"score_gpt":0.3024423793431357,"score_spread":0.21192902713392772,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4392918018","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00021048648,0.0012545862,0.00070924347,0.004370431,0.021781674,0.00045374894,0.028365914,0.0021137751,0.9407401],"genre_scores_gemma":[0.00045964718,0.00046960442,0.00024700956,0.0012727173,0.0016648525,0.00011122619,0.008359079,0.0004449858,0.9869709],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9989526,0.00011467407,0.00005940028,0.00011679738,0.00060872076,0.00014763587],"domain_scores_gemma":[0.9953454,0.0005317816,0.00020274913,0.00031605078,0.0020743124,0.00152968],"candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.0013029998,0.0016713875,0.0015810679,0.0031280736,0.0016324859,0.007761436,0.0021127518,0.0029572407,0.9209316],"category_scores_gemma":[0.0071420725,0.0005965229,0.0009939304,0.0028492482,0.00053420226,0.0035811837,0.0022136034,0.0022374303,0.88129765],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000017169488,0.000012568055,0.00001669272,0.000066919674,7.4169384e-7,0.0000064385945,0.0000021331314,0.000022223501,0.000028049393,0.0003794578,0.9867534,0.012694128],"study_design_scores_gemma":[0.000024824818,0.000013390707,0.00017192312,0.00010047208,0.0000014602939,0.000013491961,0.000012937185,0.00005662965,0.00004226369,0.000772949,0.9987845,0.000005048193],"about_ca_topic_score_codex":0.005838225,"about_ca_topic_score_gemma":0.011855835,"teacher_disagreement_score":0.9209316,"about_ca_system_score_codex":0.001962628,"about_ca_system_score_gemma":0.003413986,"threshold_uncertainty_score":0.112781405},"labels":[],"label_agreement":null},{"id":"W4403250772","doi":"10.1002/ev.20615","title":"Using contribution analysis to assess evaluation capacity outcomes in community organizations","year":2024,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Université de Montréal; University of Ottawa","funders":"","keywords":"Evaluation methods; Capacity building; Program evaluation; Process management; Business; Political science; Public administration; Economic growth; Economics; Engineering","score_opus":0.538010031454944,"score_gpt":0.5850516251312451,"score_spread":0.04704159367630112,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403250772","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.8480289,0.0011874138,0.069081105,0.0018862208,0.0001464547,0.008467887,0.0015620174,0.00039064194,0.06924925],"genre_scores_gemma":[0.97142935,0.0002005192,0.022594295,0.00008135007,0.000016042322,0.0044452115,0.00035036998,0.000028736758,0.0008540196],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.8738123,0.10086641,0.005151742,0.0023979542,0.015225028,0.0025465616],"domain_scores_gemma":[0.602225,0.29099613,0.03155084,0.012605991,0.058516942,0.004105104],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.13317563,0.0008381415,0.0009471989,0.015749885,0.0026837473,0.00612035,0.001894073,0.00076184544,0.004338236],"category_scores_gemma":[0.2660719,0.00042069794,0.001299594,0.009617411,0.0027481357,0.0059542125,0.0074004564,0.0013524764,0.00044820513],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0022679449,0.0028857838,0.3183703,0.0055185803,0.0010845758,0.0002499357,0.04496251,0.012517179,0.0017672378,0.047735617,0.0068593863,0.555781],"study_design_scores_gemma":[0.00092973065,0.010256867,0.60570526,0.007188572,0.0020102207,0.0003805595,0.13280606,0.07195381,0.023073833,0.08735606,0.057627086,0.0007119321],"about_ca_topic_score_codex":0.005603034,"about_ca_topic_score_gemma":0.0060074213,"teacher_disagreement_score":0.13317563,"about_ca_system_score_codex":0.008764398,"about_ca_system_score_gemma":0.009895675,"threshold_uncertainty_score":0.70430845},"labels":[],"label_agreement":null},{"id":"W4408984956","doi":"10.1002/ev.20499","title":"Issue Information","year":2024,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Human auditory perception and evaluation","field":"Engineering","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":true,"ca_institutions":"","funders":"Consortium of International Agricultural Research Centers; Universiteit Stellenbosch; York University; W.K. Kellogg Foundation; University of Illinois at Urbana-Champaign; Western Michigan University; University of Bern; University of Connecticut; Inter-American Development Bank","keywords":"Computer science; Process management; Knowledge management; Business","score_opus":0.03557066694896771,"score_gpt":0.33923249572305814,"score_spread":0.3036618287740904,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4408984956","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0001798436,0.0011020656,0.0007284541,0.0031375154,0.01638343,0.00044069818,0.010128737,0.0017633997,0.96613586],"genre_scores_gemma":[0.00040353992,0.00063683814,0.000301455,0.0011334872,0.0017947264,0.000094487215,0.0040611383,0.00033245736,0.99124193],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.99914277,0.00007373289,0.000041463314,0.00010762974,0.00052766094,0.00010677047],"domain_scores_gemma":[0.99599683,0.00036620809,0.00017873806,0.00026447128,0.0018955471,0.0012982932],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.001129011,0.0018600774,0.0013613079,0.004313027,0.0017056625,0.0075632734,0.0019743238,0.0030398679,0.90728986],"category_scores_gemma":[0.0053913314,0.00073431776,0.0012582288,0.0034060292,0.00059436634,0.003429255,0.0019593923,0.0019498736,0.8640903],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000012042474,0.000023534121,0.000025607538,0.00007657732,0.0000010933215,0.000008483012,0.0000026744146,0.00002301721,0.000058707206,0.0004680475,0.9741702,0.025129892],"study_design_scores_gemma":[0.000019818071,0.0000166939,0.00022336065,0.000086500135,0.000002104644,0.000017131902,0.000016035227,0.00005121944,0.00005666405,0.0006439373,0.9988613,0.000005122283],"about_ca_topic_score_codex":0.0049125473,"about_ca_topic_score_gemma":0.010281332,"teacher_disagreement_score":0.09271014,"about_ca_system_score_codex":0.0013015468,"about_ca_system_score_gemma":0.0033148495,"threshold_uncertainty_score":0.1322397},"labels":[],"label_agreement":null},{"id":"W4411428767","doi":"10.1002/ev.70000","title":"Issue Information","year":2025,"lang":"en","type":"paratext","venue":"New Directions for Evaluation","topic":"Diverse Scientific and Economic Studies","field":"Economics, Econometrics and Finance","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Consortium of International Agricultural Research Centers; Universiteit Stellenbosch; York University; W.K. Kellogg Foundation; University of Illinois at Urbana-Champaign; Western Michigan University; University of Bern; University of Connecticut; Inter-American Development Bank","keywords":"Computer science","score_opus":0.05774558077803174,"score_gpt":0.29368375071581176,"score_spread":0.23593816993778002,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4411428767","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00020343674,0.0009610848,0.00054517906,0.0040679104,0.017794736,0.00032343113,0.032659687,0.0016471776,0.9417974],"genre_scores_gemma":[0.0005947088,0.0003897107,0.00020924286,0.001146324,0.0016327181,0.000106432526,0.008482479,0.00047552088,0.98696274],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9986064,0.0001432511,0.0000922368,0.00017945297,0.00079452415,0.0001841982],"domain_scores_gemma":[0.99524987,0.0007420722,0.00025279803,0.00045031065,0.0020948353,0.0012100704],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.001214704,0.001575737,0.0016270167,0.0045349,0.0017927326,0.009097095,0.0018906685,0.003245561,0.90932333],"category_scores_gemma":[0.008101003,0.00064207637,0.00093507837,0.003987469,0.0006035935,0.004145779,0.0023279141,0.0024033915,0.87158453],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000013955176,0.000010754095,0.000022322052,0.00008868509,7.6241986e-7,0.000008152474,0.0000054329357,0.000017786851,0.000036949208,0.0005486814,0.98722374,0.012022699],"study_design_scores_gemma":[0.000014303675,0.000009163646,0.00023391432,0.0001018077,0.0000015012075,0.000011730287,0.000016381926,0.000040205614,0.000043476986,0.00055758207,0.99896586,0.000003984963],"about_ca_topic_score_codex":0.0052499836,"about_ca_topic_score_gemma":0.009986455,"teacher_disagreement_score":0.090676665,"about_ca_system_score_codex":0.0019910224,"about_ca_system_score_gemma":0.0033288146,"threshold_uncertainty_score":0.12933916},"labels":[],"label_agreement":null},{"id":"W4413128915","doi":"10.1002/ev.70009","title":"Community‐Based Research on Evaluation: Beyond Academic and Evaluator Perspectives","year":2025,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Ottawa; University of Waterloo","funders":"Canadian Mental Health Association","keywords":"Evaluation methods; Program evaluation; Computer science; Sociology; Engineering ethics; Management science; Psychology; Political science; Public administration; Engineering","score_opus":0.6424948000151569,"score_gpt":0.6631852882198059,"score_spread":0.020690488204649027,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4413128915","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"evaluation","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":"evaluation","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.36386347,0.038502466,0.14109151,0.15708329,0.0017008943,0.0023796137,0.00020770743,0.00034573054,0.29482532],"genre_scores_gemma":[0.9780473,0.002966447,0.012985404,0.0030613975,0.00018603913,0.00075118564,0.00003142274,0.00007437327,0.0018964142],"study_design_codex":"qualitative","study_design_gemma":"qualitative","domain_scores_codex":[0.5651576,0.40292475,0.005797733,0.005340577,0.017119767,0.0036595636],"domain_scores_gemma":[0.3778877,0.54166496,0.018186942,0.021676464,0.031771764,0.008812181],"candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.2723688,0.00082861236,0.0017441497,0.010643755,0.010695735,0.02262862,0.0036448091,0.003877523,0.005984748],"category_scores_gemma":[0.29851195,0.0006422098,0.00066882634,0.0073435395,0.041092694,0.025007818,0.01881148,0.0061833565,0.00047275392],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00020561094,0.0010115901,0.019938918,0.003612632,0.00020855227,0.00038826687,0.3828173,0.0012133723,0.0006079891,0.37543756,0.00661499,0.20794319],"study_design_scores_gemma":[0.00017547896,0.00070602656,0.014243603,0.012487412,0.00013926241,0.00040243764,0.5558316,0.003391275,0.0021128033,0.28937748,0.12096217,0.00017045354],"about_ca_topic_score_codex":0.003629763,"about_ca_topic_score_gemma":0.0053415257,"teacher_disagreement_score":0.7276312,"about_ca_system_score_codex":0.014126192,"about_ca_system_score_gemma":0.023181317,"threshold_uncertainty_score":0.8972988},"labels":[],"label_agreement":null},{"id":"W7125971164","doi":"10.1002/ev.70012","title":"Issue Information","year":2025,"lang":"en","type":"article","venue":"New Directions for Evaluation","topic":"Human auditory perception and evaluation","field":"Engineering","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Consortium of International Agricultural Research Centers; York University; W.K. Kellogg Foundation; University of Connecticut; University of Illinois at Urbana-Champaign; University of Bern; Western Michigan University; Inter-American Development Bank; Universiteit Stellenbosch; National Science Foundation","keywords":"Information system; Data collection; Key (lock); Research methodology; Information technology; Utilization; Program evaluation","score_opus":0.02251690894749914,"score_gpt":0.32453244346504384,"score_spread":0.3020155345175447,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W7125971164","genre_codex":"other","genre_gemma":"other","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"other","genre_consensus":"other","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.00040648066,0.0029894146,0.0010376304,0.0058236443,0.026416613,0.00072931056,0.034497947,0.0022248179,0.92587405],"genre_scores_gemma":[0.0011273721,0.0017244853,0.00055933197,0.0020251607,0.0030685072,0.00016240329,0.012280368,0.0005317326,0.97852063],"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","domain_scores_codex":[0.9988674,0.0001278592,0.0000984066,0.00017673433,0.00058034854,0.00014927385],"domain_scores_gemma":[0.99495727,0.0007044555,0.00033994476,0.0004887543,0.0021816697,0.0013279183],"candidate_categories":["insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.0013657289,0.001558803,0.0016412594,0.005518619,0.0015176393,0.007906165,0.0018046716,0.0031393839,0.90430003],"category_scores_gemma":[0.0069710007,0.00071056164,0.001137508,0.0039642374,0.0005049553,0.003775641,0.002331576,0.001837444,0.8279703],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000030982646,0.00003048564,0.00006414552,0.00026954422,0.0000021294761,0.000022863022,0.000008324309,0.000026238744,0.00014509071,0.0006095268,0.95975345,0.039037105],"study_design_scores_gemma":[0.000014502807,0.000017726406,0.0002956256,0.00012833806,0.000002768469,0.00002515042,0.000017878663,0.000029204506,0.00007111441,0.00039845734,0.99899477,0.0000045098154],"about_ca_topic_score_codex":0.0025573606,"about_ca_topic_score_gemma":0.0047818823,"teacher_disagreement_score":0.095699966,"about_ca_system_score_codex":0.0010375694,"about_ca_system_score_gemma":0.0029229494,"threshold_uncertainty_score":0.13650441},"labels":[],"label_agreement":null}]}