{"id":"W3124201545","doi":"","title":"Evaluating Strategic Forecasters","year":2017,"lang":"en","type":"preprint","venue":"RePEc: Research Papers in Economics","topic":"Auction Theory and Applications","field":"Decision Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Social Sciences and Humanities Research Council of Canada; Rice University; Indian Statistical Institute; National Science Foundation","keywords":"Generality; Robustness (evolution); Principal (computer security); Computer science; Quality (philosophy); Mechanism design; Simple (philosophy); Perception; Prediction market; Event (particle physics); Principal–agent problem; Econometrics; Microeconomics; Operations research; Economics; Computer security; Mathematics; Psychology; Finance","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02464601,0.001048098,0.002032292,0.0009913542,0.0006923441,0.003972461,0.001792408,0.003353299,0.00472902],"category_scores_gemma":[0.1211027,0.0006367933,0.0005445259,0.0009761662,0.002051349,0.005979735,0.001570322,0.002122538,0.0006081351],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00226026,"about_ca_system_score_gemma":0.002765644,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001059312,"about_ca_topic_score_gemma":0.0008521407,"domain_scores_codex":[0.9880695,0.007779015,0.0005237929,0.001741873,0.001066332,0.0008195668],"domain_scores_gemma":[0.9455392,0.03761368,0.008361033,0.003812747,0.00298844,0.001684927],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.002247145,0.0007185447,0.0296114,0.0004727575,0.0005114294,0.0003217528,0.0007491886,0.2976421,0.003722723,0.448729,0.006638548,0.2086354],"study_design_scores_gemma":[0.0004989759,0.001030343,0.004759975,0.0001820737,0.0001544675,0.0001724577,0.0003910512,0.5044575,0.003819353,0.4801182,0.004319038,0.00009656238],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3782082,0.001601168,0.5810425,0.01554443,0.000412208,0.0005238191,0.0004029896,0.0004357573,0.021829],"genre_scores_gemma":[0.9472885,0.0002856657,0.04916609,0.0003144127,0.0000977144,0.0001195756,0.00006931606,0.00002615755,0.002632608],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.02464601,"threshold_uncertainty_score":0.1303421,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.476135220608249,"score_gpt":0.5345493742352441,"score_spread":0.05841415362699504,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}