{"id":"W7106826469","doi":"10.48448/mehj-w712","title":"PalmX 2025: The First Shared Task on Benchmarking LLMs on Arabic and Islamic Culture","year":2025,"lang":"","type":"other","venue":"Open MIND","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"","keywords":"Islam; Benchmarking; Islamic culture; Task (project management); Arabic; Competence (human resources)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009518187,0.006600956,0.00270389,0.002076143,0.002136303,0.004359852,0.005390641,0.005557472,0.01783179],"category_scores_gemma":[0.02655165,0.001167897,0.003105874,0.001733289,0.001773415,0.00666692,0.01017931,0.005928576,0.02061228],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002858868,"about_ca_system_score_gemma":0.004335817,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01973436,"about_ca_topic_score_gemma":0.02522954,"domain_scores_codex":[0.9904051,0.004048169,0.0008312853,0.002673434,0.001263999,0.0007780853],"domain_scores_gemma":[0.9858056,0.005940883,0.0003721646,0.003150243,0.003108347,0.001622748],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.002880139,0.004262409,0.01231947,0.003791691,0.001394362,0.001189611,0.002962106,0.02955095,0.01844898,0.002055598,0.5137696,0.4073751],"study_design_scores_gemma":[0.002624739,0.007046871,0.03661174,0.001076179,0.0008316121,0.001654528,0.005501031,0.4712417,0.04741292,0.01372019,0.4114119,0.0008665589],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"empirical","genre_gemma":"other","genre_scores_codex":[0.5001684,0.01095437,0.1209761,0.006194517,0.007668886,0.009396654,0.1023431,0.1848247,0.05747338],"genre_scores_gemma":[0.4476753,0.001135301,0.1671924,0.003872185,0.0007836857,0.006178944,0.3253808,0.008653775,0.0391276],"genre_candidate":"other","genre_consensus":null,"teacher_disagreement_score":0.01973436,"threshold_uncertainty_score":0.05965334,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0239591253103362,"score_gpt":0.2914931088194618,"score_spread":0.2675339835091256,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}