{"id":"W4404481617","doi":"10.1145/3687123.3698286","title":"Evaluation of Code LLMs on Geospatial Code Generation","year":2024,"lang":"en","type":"article","venue":"","topic":"Semantic Web and Ontologies","field":"Computer Science","cited_by":23,"is_retracted":false,"has_abstract":true,"ca_institutions":"Artificial Intelligence in Medicine (Canada)","funders":"","keywords":"Geospatial analysis; Computer science; Code (set theory); Computer security; Programming language; Database; Geography; Remote sensing","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0008348427,0.00005250921,0.00006496235,0.00006154904,0.00002657779,0.00007846932,0.000193409,0.000033077,0.00006276529],"category_scores_gemma":[0.00008161673,0.00003914623,0.00002852819,0.0001259943,0.00001413888,0.0001696261,0.00003803175,0.000035353,0.00006780899],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003177544,"about_ca_system_score_gemma":0.0001084474,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00005700485,"about_ca_topic_score_gemma":0.000337124,"domain_scores_codex":[0.9990095,0.00007355878,0.0001267715,0.0001929501,0.0005181491,0.00007910462],"domain_scores_gemma":[0.9995483,0.00005784157,0.00002089586,0.0002272684,0.000130854,0.00001487624],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002966849,0.00005834249,0.0001290749,0.00001889981,0.00003732084,0.000004368079,0.0009040558,0.006732393,0.01646237,0.3748316,0.01265788,0.5881608],"study_design_scores_gemma":[0.00009817754,0.00007130111,0.001415421,0.00001188565,0.00001570463,0.000001851369,0.00001231616,0.9538499,0.03934385,0.004522002,0.0006078944,0.0000497275],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.216321,0.0003206196,0.7691113,0.00143396,0.001283366,0.0001495413,0.000002509809,0.0002094562,0.01116816],"genre_scores_gemma":[0.990813,0.000008446846,0.008735055,0.0001119241,0.00009939938,0.000008524655,0.000003162645,0.000002527001,0.0002180269],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9471174,"threshold_uncertainty_score":0.1596337,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1167048333718861,"score_gpt":0.3446862826511968,"score_spread":0.2279814492793107,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}