{"id":"W6891666200","doi":"10.48448/esq0-jx32","title":"TheoremQA: A Theorem-driven Question Answering Dataset","year":2023,"lang":"en","type":"other","venue":"Open MIND","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Benchmark (surveying); Question answering; Domain (mathematical analysis); Code (set theory); Diversity (politics)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002210906,0.002247475,0.0005770791,0.002559418,0.001103834,0.001553576,0.00385345,0.004000366,0.01649313],"category_scores_gemma":[0.01217362,0.0004443472,0.001877847,0.001979192,0.0008917443,0.003017156,0.00228001,0.002633432,0.01409087],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002399815,"about_ca_system_score_gemma":0.002153442,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02315729,"about_ca_topic_score_gemma":0.04849133,"domain_scores_codex":[0.9977394,0.0005699401,0.0002448212,0.0006367764,0.000668371,0.0001408018],"domain_scores_gemma":[0.9951806,0.002277162,0.0002267141,0.0008947343,0.001039787,0.0003809677],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0005064025,0.0006176144,0.009363526,0.002123042,0.0001465048,0.0004810901,0.0005040866,0.009574013,0.003271665,0.007701694,0.9181566,0.04755371],"study_design_scores_gemma":[0.0008641765,0.0004970017,0.02281381,0.0003917292,0.00009695765,0.001136953,0.0008760709,0.1057185,0.01066691,0.01692535,0.839844,0.000168614],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"dataset","genre_gemma":"dataset","genre_scores_codex":[0.04958151,0.002365731,0.01487963,0.002727225,0.0005913121,0.0008687238,0.8813118,0.02435637,0.02331772],"genre_scores_gemma":[0.03434238,0.0002429797,0.02101058,0.0007486556,0.0000562707,0.0005365827,0.938327,0.0004464731,0.004289104],"genre_candidate":"dataset","genre_consensus":"dataset","teacher_disagreement_score":0.02315729,"threshold_uncertainty_score":0.05517507,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04318410324575502,"score_gpt":0.3538773653443045,"score_spread":0.3106932620985495,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}