{"id":"W4416037765","doi":"10.18653/v1/2025.arabicnlp-sharedtasks.107","title":"PalmX 2025: The First Shared Task on Benchmarking LLMs on Arabic and Islamic Culture","year":2025,"lang":"","type":"article","venue":"","topic":"Mathematics, Computing, and Information Processing","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Alliance de recherche numérique du Canada; Social Sciences and Humanities Research Council of Canada; Natural Sciences and Engineering Research Council of Canada; Canada Research Chairs","keywords":"Task (project management); Benchmarking; Islam; Arabic; Islamic culture","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009583827,0.007199709,0.002914971,0.002190038,0.002168427,0.004674897,0.00557957,0.005717394,0.0194066],"category_scores_gemma":[0.02589429,0.00125023,0.003369899,0.001802342,0.001789301,0.00678069,0.01013148,0.006227622,0.02289034],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002925601,"about_ca_system_score_gemma":0.004480957,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.02211582,"about_ca_topic_score_gemma":0.02762479,"domain_scores_codex":[0.9908007,0.003806055,0.0008050239,0.00258979,0.001237945,0.0007605143],"domain_scores_gemma":[0.9863026,0.005656118,0.0003496622,0.003091913,0.003006924,0.001592873],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.002683684,0.003954495,0.01063958,0.00355772,0.001431503,0.001092137,0.00238186,0.02904049,0.0157578,0.001870722,0.5353313,0.3922588],"study_design_scores_gemma":[0.002610902,0.006477311,0.0317193,0.001043917,0.0008483303,0.001497837,0.004709181,0.4955259,0.04314262,0.01288031,0.3987152,0.0008290475],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"empirical","genre_gemma":"other","genre_scores_codex":[0.4555773,0.01249625,0.1232454,0.006606382,0.008867535,0.009472442,0.1080162,0.2172557,0.05846283],"genre_scores_gemma":[0.416745,0.001289682,0.1718594,0.004133729,0.0008534731,0.006033676,0.3475577,0.009661996,0.04186532],"genre_candidate":"other","genre_consensus":null,"teacher_disagreement_score":0.02211582,"threshold_uncertainty_score":0.06492162,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01208333510318101,"score_gpt":0.2448959339539454,"score_spread":0.2328125988507644,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}