{"id":"W3213458975","doi":"10.1162/tacl_a_00419","title":"<scp>ParsiNLU</scp>: A Suite of Language Understanding Challenges for Persian","year":2021,"lang":"en","type":"article","venue":"Transactions of the Association for Computational Linguistics","topic":"Topic Modeling","field":"Computer Science","cited_by":18,"is_retracted":false,"has_abstract":true,"ca_institutions":"Microsoft (Canada); International Medias Data Services (Canada)","funders":"","keywords":"Computer science; Natural language understanding; Natural language processing; Suite; Benchmark (surveying); Artificial intelligence; Persian; Natural language; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002070852,0.00192997,0.0008625564,0.002915516,0.002256888,0.00218555,0.001647298,0.001675976,0.007875171],"category_scores_gemma":[0.009059928,0.0002945137,0.0006941662,0.002961193,0.001244059,0.003916285,0.002846814,0.001828892,0.005960732],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001636066,"about_ca_system_score_gemma":0.002414869,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02529729,"about_ca_topic_score_gemma":0.03280266,"domain_scores_codex":[0.9975258,0.001041478,0.0001929522,0.0006214657,0.0004382776,0.0001799329],"domain_scores_gemma":[0.9946214,0.002535433,0.0002010056,0.001035302,0.001267279,0.0003396248],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0007308775,0.0004417709,0.01457426,0.001732096,0.0001905504,0.003280389,0.003533613,0.01981629,0.00939104,0.00863805,0.6180795,0.3195915],"study_design_scores_gemma":[0.0004175819,0.000474011,0.05564914,0.0007659082,0.0001663215,0.005980846,0.01798126,0.1691344,0.05786688,0.05207434,0.6391106,0.0003787563],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6244645,0.01014314,0.04877105,0.01692512,0.002180739,0.0005722797,0.1891115,0.04811381,0.0597179],"genre_scores_gemma":[0.6013018,0.00115579,0.04344494,0.001726029,0.0003679951,0.0004199042,0.3403288,0.001735997,0.009518782],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02529729,"threshold_uncertainty_score":0.05030006,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05384418972214958,"score_gpt":0.281614772268062,"score_spread":0.2277705825459125,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}