{"id":"W4415622293","doi":"10.1145/3773285","title":"TaskEval: Assessing Difficulty of Code Generation Tasks for Large Language Models","year":2025,"lang":"en","type":"article","venue":"ACM Transactions on Software Engineering and Methodology","topic":"Topic Modeling","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"Polytechnique Montréal; Huawei Technologies (Canada)","funders":"","keywords":"Benchmarking; Benchmark (surveying); Task (project management); Code (set theory); Program comprehension; Code generation; Source code","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005616354,0.000105711,0.0002048553,0.0001674645,0.00008990544,0.00002758394,0.0002313103,0.0001053676,0.000001283667],"category_scores_gemma":[0.000399037,0.0001052037,0.00005975364,0.000151655,0.000009635419,0.0001731525,0.00001504198,0.00012976,1.45917e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002204788,"about_ca_system_score_gemma":0.0000328237,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001267381,"about_ca_topic_score_gemma":0.000006754316,"domain_scores_codex":[0.9992033,0.00008683718,0.0001982438,0.0002711304,0.00006374789,0.0001767082],"domain_scores_gemma":[0.9984936,0.0009817585,0.0000368969,0.0004006638,0.00005437302,0.00003267315],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000009298812,0.00006796757,0.00001916633,0.0001820964,0.00008224546,0.000001480293,0.001571742,0.7312549,0.03386635,0.02664385,0.00002992857,0.2062709],"study_design_scores_gemma":[0.0004370121,0.00003490243,0.0001099811,0.00003874813,0.00003217266,0.000005580715,0.00006471713,0.9756721,0.02178732,0.00137333,0.0003288623,0.0001152734],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.02806752,0.0002606299,0.9706544,0.0001617376,0.0005139278,0.0001279086,0.00002129378,0.0001886929,0.000003853172],"genre_scores_gemma":[0.3106455,0.00001532094,0.6891541,0.00006650617,0.00001903777,0.00003147983,0.000003997955,0.000006312222,0.00005778395],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.282578,"threshold_uncertainty_score":0.4290081,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1129611490242461,"score_gpt":0.3607796288867569,"score_spread":0.2478184798625108,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}