{"id":"W3101035550","doi":"","title":"Measuring Systematic Generalization in Neural Proof Generation with Transformers","year":2020,"lang":"en","type":"article","venue":"Neural Information Processing Systems","topic":"Topic Modeling","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"ca_institutions":"Polytechnique Montréal; McGill University","funders":"","keywords":"Mathematical proof; Computer science; Backward chaining; Generalization; Chaining; Artificial intelligence; Natural deduction; Automated theorem proving; Inference; Theoretical computer science; Programming language; Mathematics; Inference engine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0003131559,0.000181015,0.0002714008,0.0001707154,0.0001437998,0.0008570871,0.0003367032,0.00006555032,7.129203e-7],"category_scores_gemma":[0.00004370013,0.0001442096,0.00002867993,0.0006752679,0.00001266581,0.006753657,0.00002611815,0.0001395827,0.000007125581],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00009194788,"about_ca_system_score_gemma":0.00008707737,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003286376,"about_ca_topic_score_gemma":0.000006640852,"domain_scores_codex":[0.9980448,0.0001028121,0.0008358029,0.0002231104,0.0005475521,0.0002459147],"domain_scores_gemma":[0.9991764,0.00001143273,0.0003494249,0.0001634713,0.0002053046,0.00009389433],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001170882,0.00001057815,0.0004056916,0.01490876,0.000007915366,0.000004325508,0.01288859,0.9523719,0.001057917,0.001019333,0.00001568064,0.01729763],"study_design_scores_gemma":[0.000392025,0.00005228544,0.0000298228,0.0005939768,0.000006522026,0.00005001943,0.0003061461,0.9976006,0.000765188,0.000002381439,0.00002443986,0.0001765586],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04589287,0.0002816627,0.9512012,0.00085784,0.0002331622,0.001003814,9.115295e-7,0.0002351723,0.0002933465],"genre_scores_gemma":[0.9972578,0.000001576058,0.001898395,0.0005707069,0.0001019846,0.0001319473,0.00001757347,0.000009703942,0.00001033715],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9513649,"threshold_uncertainty_score":0.8264911,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06149766231254278,"score_gpt":0.229747275178059,"score_spread":0.1682496128655162,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}