{"id":"W4389518686","doi":"10.18653/v1/2023.emnlp-main.330","title":"Just Ask for Calibration: Strategies for Eliciting Calibrated Confidence Scores from Language Models Fine-Tuned with Human Feedback","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":83,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Office of Naval Research; Canadian Institute for Advanced Research; National Science Foundation","keywords":"Ask price; Calibration; Computer science; Tian; Artificial intelligence; Statistics; Art; Mathematics; Literature","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0001870884,0.000189716,0.0002255064,0.00009105275,0.0002631916,0.0006408629,0.0006390332,0.00007756353,0.00002298611],"category_scores_gemma":[0.00002543715,0.0001563176,0.00005774359,0.0003344561,0.00003973244,0.001457852,0.0001256881,0.00007634466,0.000005906682],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002036981,"about_ca_system_score_gemma":0.0001761082,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001266765,"about_ca_topic_score_gemma":0.001162992,"domain_scores_codex":[0.9984814,0.0000293223,0.0003199312,0.0005601289,0.0002293302,0.0003798702],"domain_scores_gemma":[0.9988749,0.0003189523,0.000106304,0.0004870734,0.0001312676,0.00008151863],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003272223,0.00001993262,0.00008238399,0.00009046043,0.00004653754,0.00001324615,0.004896921,0.110768,0.02048769,0.8595244,0.002200559,0.001837101],"study_design_scores_gemma":[0.0006508668,0.00008131113,0.00003987566,0.00005988576,0.000009775739,0.000001264086,0.001945917,0.9120332,0.009373458,0.0755607,0.00001980982,0.0002239207],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1782075,0.00002697587,0.8191372,0.0007794022,0.00008913848,0.0005210169,0.00003782972,0.0006804826,0.0005204598],"genre_scores_gemma":[0.8595603,0.000001068146,0.1388365,0.0003073493,0.0001799862,0.0001665231,0.0001088911,0.00002402213,0.0008154134],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8012652,"threshold_uncertainty_score":0.6374449,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.07316623696227058,"score_gpt":0.3050333361367429,"score_spread":0.2318670991744723,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}