{"id":"W6966533420","doi":"10.48448/hqbd-as52","title":"Studying word order through iterative shuffling","year":2021,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Mila - Quebec Artificial Intelligence Institute","funders":"","keywords":"Shuffling; Phrase; Word (group theory); Language model; Inference; Suite; Sentence; Benchmark (surveying)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.001189339,0.0007242573,0.0007566732,0.001006708,0.0006359702,0.0007470765,0.001551999,0.0002896128,0.008724479],"category_scores_gemma":[0.000767993,0.0006606234,0.0001032055,0.005930497,0.001939594,0.0005432859,0.0008612971,0.0008194461,0.002548897],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005611429,"about_ca_system_score_gemma":0.002015701,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006122541,"about_ca_topic_score_gemma":0.001885539,"domain_scores_codex":[0.9942379,0.0001655601,0.0005537931,0.001867112,0.001976578,0.001199064],"domain_scores_gemma":[0.9971905,0.0001436524,0.0005434784,0.001238917,0.000658776,0.0002247243],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00006349281,0.002294084,0.002242905,0.0005995488,0.001073009,0.001046104,0.02320473,0.002901181,0.03933641,0.04004618,0.8440122,0.04318016],"study_design_scores_gemma":[0.001161688,0.0001134651,0.00007908944,0.002090369,0.0001345108,0.00005691361,0.004856086,0.009427469,0.001179073,0.001268191,0.9775935,0.00203963],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.0004359319,0.003630395,0.01226274,0.0002751616,0.0020797,0.0009726188,0.0001298906,0.001002367,0.9792112],"genre_scores_gemma":[0.01664201,0.0001675834,0.2054204,0.001178968,0.002114097,0.00009080428,0.0003191947,0.002394221,0.7716727],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.2075385,"threshold_uncertainty_score":0.9995845,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06734167491659197,"score_gpt":0.3518259270461,"score_spread":0.284484252129508,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}