{"id":"W4402667062","doi":"10.18653/v1/2024.acl-long.20","title":"Confidence Under the Hood: An Investigation into the Confidence-Probability Alignment in Large Language Models","year":2024,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Confidence interval; Artificial intelligence; Confidence distribution; Natural language processing; Statistics; Mathematics","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001788054,0.0001232174,0.00009828673,0.00004532025,0.0001239134,0.0004318705,0.001083293,0.0000532087,0.00003616274],"category_scores_gemma":[0.00003024446,0.00006690247,0.00003746698,0.0003287868,0.00009521302,0.0009158924,0.000293901,0.0002246969,0.00002774928],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001069521,"about_ca_system_score_gemma":0.0001654423,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003183168,"about_ca_topic_score_gemma":0.003459105,"domain_scores_codex":[0.998349,0.000258733,0.0002730853,0.0004913203,0.0003779473,0.0002499326],"domain_scores_gemma":[0.9986001,0.0002702256,0.00003118303,0.0009970131,0.00003732636,0.00006419884],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[6.024611e-7,0.00001079584,0.00006868885,0.00001058742,0.000004144219,0.000004063641,0.0292223,0.009392649,0.0003493127,0.9586403,0.00005317344,0.002243411],"study_design_scores_gemma":[0.00004113069,0.000009913623,0.0002046923,0.00002183907,0.000001948767,0.000002653204,0.002186412,0.5642404,0.0005699441,0.4326255,0.00003374378,0.00006180661],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2597123,0.0004361586,0.7278765,0.01074742,0.0001993495,0.0002775911,6.80088e-7,0.0001486007,0.0006013457],"genre_scores_gemma":[0.9887648,0.00001392809,0.008242697,0.00259512,0.00005013984,0.00005044867,0.000001482706,0.000005913987,0.0002754601],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.7290525,"threshold_uncertainty_score":0.4812019,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04172525238957015,"score_gpt":0.2864207796739621,"score_spread":0.244695527284392,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}