{"id":"W3137084011","doi":"10.2139/ssrn.3606633","title":"Eliciting Human Judgment for Prediction Algorithms","year":2020,"lang":"en","type":"article","venue":"SSRN Electronic Journal","topic":"Forecasting Techniques and Applications","field":"Decision Sciences","cited_by":2,"is_retracted":false,"has_abstract":false,"ca_institutions":"Western University","funders":"","keywords":"Algorithm; Computer science; Artificial intelligence","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009405376,0.0005661896,0.0004976943,0.0006764447,0.0006671542,0.002590206,0.0006502991,0.001460198,0.006255833],"category_scores_gemma":[0.1220155,0.0002344115,0.00037809,0.0006734899,0.0005794736,0.002007552,0.001356726,0.001118402,0.001246293],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0008132426,"about_ca_system_score_gemma":0.001193732,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00130737,"about_ca_topic_score_gemma":0.002176837,"domain_scores_codex":[0.9876831,0.008795486,0.0005084983,0.0008904113,0.00178948,0.000333034],"domain_scores_gemma":[0.8708029,0.1114102,0.00377688,0.00709967,0.005824873,0.001085551],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.005876138,0.002461834,0.06366305,0.001702376,0.0003283233,0.0004760468,0.009818068,0.03324246,0.06755775,0.0337949,0.01739294,0.7636861],"study_design_scores_gemma":[0.000725936,0.00369054,0.07998902,0.0006238308,0.0003227034,0.0005425288,0.007944961,0.6158499,0.07466097,0.1850708,0.03026129,0.0003174996],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6053909,0.0006667252,0.3583143,0.00176165,0.0002405961,0.0005647676,0.0005359418,0.000632383,0.03189274],"genre_scores_gemma":[0.9427072,0.0001393146,0.05462934,0.0003212351,0.00006394876,0.0001796343,0.0003396937,0.00005128098,0.001568238],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.009405376,"threshold_uncertainty_score":0.04974097,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1493413844967695,"score_gpt":0.3968091790348126,"score_spread":0.2474677945380431,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}