{"id":"W3109000035","doi":"10.3386/w25460","title":"Addressing Cross-National Generalizability in Educational Impact Evaluation","year":2019,"lang":"en","type":"preprint","venue":"National Bureau of Economic Research","topic":"Evaluation and Performance Assessment","field":"Decision Sciences","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Generalizability theory; Variety (cybernetics); Cross country; External validity; Quarter (Canadian coin); Political science; Cross-cultural; Econometrics; Psychology; Regional science; Computer science; Economics; Geography; Demographic economics; Artificial intelligence; Social psychology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch","insufficient_payload"],"category_scores_codex":[0.08301709,0.0002667002,0.0005671273,0.002557484,0.0001793408,0.0006906352,0.001506788,0.0004315306,0.01772806],"category_scores_gemma":[0.01399659,0.0002374638,0.0003254306,0.0006596534,0.0003595934,0.000828013,0.0007419573,0.001025352,0.0011175],"about_ca_system_candidate":true,"about_ca_system_consensus":true,"about_ca_system_score_codex":0.006745976,"about_ca_system_score_gemma":0.02965126,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0007338489,"about_ca_topic_score_gemma":0.0003482115,"domain_scores_codex":[0.9825823,0.00211622,0.002189011,0.001246367,0.01139837,0.0004677536],"domain_scores_gemma":[0.9797602,0.006020819,0.0008906073,0.0007489418,0.01244455,0.0001348825],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00008247104,0.0001889934,0.2141033,0.00003330895,0.00004255374,7.993774e-8,0.0001665292,0.7007563,0.00007029223,0.06618258,0.01663265,0.001740888],"study_design_scores_gemma":[0.0004789091,0.00002401421,0.2664239,0.00003625747,0.000002377987,9.930576e-7,0.00002886823,0.2653649,0.00004387083,0.4672338,0.0002479319,0.0001140917],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.7562391,0.0004168865,0.00005615399,0.00572601,0.001402573,0.00183735,0.0003624477,0.000009316846,0.2339501],"genre_scores_gemma":[0.9950848,0.00004624608,0.0005289825,0.00007798984,0.0006495914,0.0003267479,0.001207918,0.00001951623,0.002058263],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4353914,"threshold_uncertainty_score":0.9996603,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.8923654787697515,"score_gpt":0.7649700817209754,"score_spread":0.1273953970487761,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}