{"id":"W2970946372","doi":"10.18653/v1/d19-1335","title":"How Reasonable are Common-Sense Reasoning Tasks: A Case-Study on the Winograd Schema Challenge and SWAG","year":2019,"lang":"en","type":"preprint","venue":"","topic":"Logic, Reasoning, and Knowledge","field":"Computer Science","cited_by":43,"is_retracted":false,"has_abstract":true,"ca_institutions":"McGill University","funders":"Natural Sciences and Engineering Research Council of Canada; Canadian Institute for Advanced Research; Microsoft Research","keywords":"Computer science; Schema (genetic algorithms); Common sense; Commonsense reasoning; Ask price; Artificial intelligence; Theoretical computer science; Machine learning; Epistemology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.001320191,0.000805982,0.0009323976,0.0001881215,0.0006221173,0.001764444,0.001579883,0.0004504722,0.00001559882],"category_scores_gemma":[0.0002944112,0.00051984,0.0002545304,0.000326758,0.0001505231,0.0003222959,0.004409403,0.001525861,0.00009160414],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001177561,"about_ca_system_score_gemma":0.0001563426,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008203851,"about_ca_topic_score_gemma":0.0007632423,"domain_scores_codex":[0.9958503,0.0005648033,0.0003843038,0.001785252,0.0006341573,0.0007811404],"domain_scores_gemma":[0.9948269,0.0007424747,0.000517206,0.003397475,0.0002523716,0.0002636074],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001917551,0.007229135,0.1056708,0.00174124,0.002486884,0.04412273,0.1065923,0.0003449978,0.00003840028,0.6130737,0.05579544,0.06271265],"study_design_scores_gemma":[0.01385202,0.008955911,0.05465229,0.009856227,0.001308835,0.03337547,0.2175,0.5077416,0.001154601,0.04177646,0.09274259,0.01708396],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8672587,0.005774414,0.01429317,0.009502723,0.001903168,0.003992984,0.00002344306,0.001132714,0.09611868],"genre_scores_gemma":[0.9927203,0.000209072,0.001944326,0.0003509956,0.0002890554,0.0001871012,0.000005393188,0.00005222039,0.004241516],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5712972,"threshold_uncertainty_score":0.9997253,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05092409937177934,"score_gpt":0.2665618099533431,"score_spread":0.2156377105815637,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}