{"id":"W4389519409","doi":"10.18653/v1/2023.emnlp-main.16","title":"GPTAraEval: A Comprehensive Evaluation of ChatGPT on Arabic NLP","year":2023,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":48,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"Alliance de recherche numérique du Canada; Social Sciences and Humanities Research Council of Canada; Natural Sciences and Engineering Research Council of Canada; Canada Research Chairs","keywords":"Arabic; Modern Standard Arabic; Computer science; Natural language processing; Focus (optics); Transformative learning; Artificial intelligence; Software deployment; Linguistics; Psychology; Software engineering","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005076138,0.00006689208,0.0001025498,0.0001394557,0.00003160425,0.00002066539,0.0003435893,0.00003017887,0.00004057298],"category_scores_gemma":[0.00005342972,0.00005777346,0.00004049303,0.0003943945,0.00001287192,0.0001287288,0.0001156816,0.0000521711,0.0002771347],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003073553,"about_ca_system_score_gemma":0.00005806239,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00003453211,"about_ca_topic_score_gemma":0.000005175184,"domain_scores_codex":[0.9987256,0.00008391929,0.0001642524,0.0002465059,0.000638547,0.0001411645],"domain_scores_gemma":[0.9990991,0.0000895254,0.00005130207,0.0004900421,0.0002378982,0.00003217861],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001086475,0.0001053176,0.0004832016,0.00005184878,0.00004992876,0.000007991669,0.003453217,0.1869006,0.007675171,0.2969373,0.004102572,0.500222],"study_design_scores_gemma":[0.000261788,0.00004470008,0.006820817,0.00001557485,0.000004901531,0.000001049447,0.00004411259,0.9804848,0.00231494,0.009557412,0.0003841049,0.00006575996],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8193572,0.00004523988,0.1680381,0.001572912,0.0005247375,0.0003018125,6.802401e-7,0.0003443025,0.009815033],"genre_scores_gemma":[0.9933717,0.000005356984,0.005989041,0.0002662542,0.00003368982,0.0000141622,0.000001634494,0.000004307729,0.0003138925],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7935842,"threshold_uncertainty_score":0.3562098,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.136394988307044,"score_gpt":0.3458940014193512,"score_spread":0.2094990131123072,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}