{"id":"W3137084011","doi":"10.2139/ssrn.3606633","title":"Eliciting Human Judgment for Prediction Algorithms","year":2020,"lang":"en","type":"article","venue":"SSRN Electronic Journal","topic":"Forecasting Techniques and Applications","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":false,"ca_institutions":"Western University","funders":"","keywords":"Algorithm; Computer science; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003272395,0.00009573232,0.0001432626,0.00007669277,0.0005519073,0.0001667817,0.000515323,0.00006945194,0.00003625489],"category_scores_gemma":[0.0005044613,0.00007477095,0.000143655,0.0003278769,0.00002895283,0.0001453552,0.00005099753,0.0009279702,0.00002717304],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002359779,"about_ca_system_score_gemma":0.0003264701,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001123848,"about_ca_topic_score_gemma":0.00002644828,"domain_scores_codex":[0.9976932,0.00003828642,0.0005078652,0.0002683528,0.0005196614,0.0009726258],"domain_scores_gemma":[0.9991215,0.0001224403,0.0002718316,0.0001578204,0.0002146603,0.0001117571],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.00003059197,0.00006123258,0.000882933,0.000003324959,0.00005285803,6.910417e-7,0.0003469717,0.0002602319,0.006322527,0.4013104,0.01774382,0.5729844],"study_design_scores_gemma":[0.0003333466,0.0006209722,0.0001122394,0.000007395513,0.00001534821,0.00006519703,0.0009444115,0.01515167,0.0008027633,0.9431344,0.03871176,0.0001005097],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.06222855,0.0001946603,0.9250594,0.01071975,0.00008176885,0.0003281643,0.00004152965,0.0001329257,0.001213188],"genre_scores_gemma":[0.9911865,0.00008855169,0.006758409,0.000495235,0.0009168016,0.00004404859,0.00001175736,0.00001842274,0.0004802795],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9289579,"threshold_uncertainty_score":0.424488,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1493413844967695,"score_gpt":0.3968091790348126,"score_spread":0.2474677945380431,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}