{"id":"W3088466851","doi":"10.1093/reseval/rvaa024","title":"Understanding and evaluating the impact of integrated problem-oriented research programmes: Concepts and considerations","year":2020,"lang":"en","type":"article","venue":"Research Evaluation","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":44,"is_retracted":false,"has_abstract":true,"ca_institutions":"Royal Roads University","funders":"Social Sciences and Humanities Research Council of Canada; Canada Research Chairs","keywords":"Context (archaeology); Counterfactual thinking; Scale (ratio); Theory of change; Management science; Discipline; Computer science; Field (mathematics); Impact evaluation; Ex-ante; Impact assessment; Research program; Political science; Economics; Sociology; Psychology; Management; Social science","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1006083,0.0001282934,0.0002434517,0.0004838284,0.001147737,0.0006790437,0.0002516636,0.00008097177,0.001447812],"category_scores_gemma":[0.04543822,0.00007328518,0.00005653481,0.002831275,0.001040859,0.0005508438,0.0002508014,0.0006699691,0.00003201097],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004582906,"about_ca_system_score_gemma":0.002916306,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002530933,"about_ca_topic_score_gemma":0.0000953998,"domain_scores_codex":[0.9805496,0.008999654,0.0008301128,0.0006068106,0.008506783,0.0005069894],"domain_scores_gemma":[0.9819466,0.009232275,0.0002243316,0.0003951812,0.00794959,0.0002520253],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00139013,0.0005108977,0.1828406,0.00019041,0.0004596982,0.000005525078,0.1342084,0.02148522,0.06514219,0.1622057,0.04378114,0.3877801],"study_design_scores_gemma":[0.001164549,0.002040059,0.01740629,0.00006401355,0.0000166772,0.000005091206,0.03212816,0.8412883,0.0003314675,0.1052587,0.0002034112,0.00009326704],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9755163,0.0005313833,0.002984251,0.01386427,0.00003493584,0.003301704,0.00002027857,0.00002302378,0.003723898],"genre_scores_gemma":[0.9984961,0.00007651995,0.001024131,0.00004691432,0.00005419824,0.0002043512,0.00002189422,0.00001045785,0.00006544671],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8198031,"threshold_uncertainty_score":0.999465,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9197335360181665,"score_gpt":0.7290685971858931,"score_spread":0.1906649388322734,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}