{"id":"W2968398601","doi":"10.18653/v1/d19-1458","title":"CLUTRR: A Diagnostic Benchmark for Inductive Reasoning from Text","year":2019,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":17,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal; Mila - Quebec Artificial Intelligence Institute; McGill University","funders":"Samsung; Samsung Advanced Institute of Technology; Canadian Institute for Advanced Research","keywords":"Robustness (evolution); Computer science; Natural language understanding; Inductive logic programming; Artificial intelligence; Suite; Benchmark (surveying); Generalization; Machine learning; Theoretical computer science; Natural language; Mathematics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002810191,0.0003053126,0.0004187106,0.0001096657,0.00007732159,0.0003127564,0.001805703,0.0003518522,0.00006214502],"category_scores_gemma":[0.0007590551,0.0002866414,0.000184093,0.00008820684,0.00002100625,0.0002452397,0.002775029,0.0005482297,0.00008845457],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001369327,"about_ca_system_score_gemma":0.0002981249,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001292609,"about_ca_topic_score_gemma":0.00005320849,"domain_scores_codex":[0.9976655,0.00006171851,0.000357191,0.001253339,0.0002877872,0.0003745088],"domain_scores_gemma":[0.9962331,0.001641453,0.0002178462,0.001660233,0.0001475476,0.00009981958],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00003920827,0.0003237052,0.01385894,0.000505342,0.0007733096,0.00004410047,0.01355941,0.07657279,0.0002303479,0.46104,0.02127725,0.4117756],"study_design_scores_gemma":[0.0004280191,0.00004614089,0.002980261,0.0004853489,0.00003971179,0.000002023457,0.00007804084,0.8589169,0.0002966119,0.1326808,0.003376311,0.0006697937],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.02754144,0.000366587,0.9606209,0.0007483576,0.002880392,0.0009874739,0.00003374151,0.0002241289,0.006596952],"genre_scores_gemma":[0.4648218,0.00002503432,0.5333225,0.0003485557,0.0004996532,0.0002100544,0.00004109139,0.00002106398,0.0007102588],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.7823442,"threshold_uncertainty_score":0.9999586,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02735789800130105,"score_gpt":0.2664366112061615,"score_spread":0.2390787132048605,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}