{"meta":{"query_hash":"cba7bb08c2d0","filters":{"venue":"25th International Conference on Software Engineering, 2003. Proceedings."},"cohort_total":6,"direct_labels_cover":1,"predictions_cover":6,"exported":6,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/cba7bb08c2d0","api":"https://metacan.xera.ac/api/v1/cohort?venue=25th+International+Conference+on+Software+Engineering%2C+2003.+Proceedings."},"results":[{"id":"W4229596932","doi":"10.1109/icse.2003.1201189","title":"Using benchmarking to advance research: a challenge to software engineering","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":150,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Benchmarking; Benchmark (surveying); Computer science; Data science; Software engineering; Software; Academic community; Engineering management; Management science; Engineering","score_opus":0.11012686057636706,"score_gpt":0.3553565504832152,"score_spread":0.24522968990684813,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4229596932","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":"methods","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":"methods","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.018883402,0.028022975,0.6365957,0.28054297,0.0042177844,0.00062923215,0.00033181533,0.0037208863,0.027055277],"genre_scores_gemma":[0.20736398,0.015213503,0.7508223,0.014939757,0.0035339652,0.0020416898,0.00077916763,0.0024742517,0.0028313866],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.6199469,0.25066832,0.02084692,0.015539281,0.088458814,0.004539864],"domain_scores_gemma":[0.2921697,0.4562198,0.027531328,0.11629769,0.09335408,0.014427461],"candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.30673128,0.0033124432,0.0054695522,0.012507649,0.006122894,0.030853327,0.01107002,0.009709315,0.0019050738],"category_scores_gemma":[0.52741086,0.0014535624,0.0017279002,0.018374143,0.021529878,0.05805392,0.02213132,0.018721705,0.0024956746],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00012503231,0.00035741655,0.0055687805,0.0019192201,0.00018523783,0.00017150791,0.0044922694,0.009868663,0.0013822116,0.45449317,0.03955973,0.4818767],"study_design_scores_gemma":[0.00008790466,0.0003992956,0.0015056969,0.002298149,0.000049074817,0.00016942837,0.0033330452,0.014983171,0.00163817,0.86365235,0.11167177,0.00021193683],"about_ca_topic_score_codex":0.0027564478,"about_ca_topic_score_gemma":0.0025146632,"teacher_disagreement_score":0.6932687,"about_ca_system_score_codex":0.007025863,"about_ca_system_score_gemma":0.024001725,"threshold_uncertainty_score":0.8549237},"labels":[],"label_agreement":null},{"id":"W4237741047","doi":"10.1109/icse.2003.1201251","title":"Empowering software engineers in human-centered design","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Usability and User Interface Design","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Usability; Computer science; Software engineering; Usability engineering; Social software engineering; Software development; Toolbox; Software construction; Human–computer interaction; Software; Programming language","score_opus":0.05692146588363678,"score_gpt":0.2961389041541138,"score_spread":0.23921743827047703,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4237741047","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.059808705,0.0015938575,0.88698983,0.012133659,0.00020400615,0.00055664126,0.000028510429,0.0011201277,0.037564687],"genre_scores_gemma":[0.4435472,0.00213241,0.54050326,0.0027641514,0.00015311399,0.00092888763,0.00009271966,0.0001890043,0.009689256],"study_design_codex":"design_other","study_design_gemma":"qualitative","domain_scores_codex":[0.98387456,0.01143019,0.00055825285,0.00090150355,0.0026891474,0.00054630585],"domain_scores_gemma":[0.9787339,0.013530866,0.00095409626,0.0028847558,0.0026471857,0.0012491678],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.014731789,0.0006573225,0.00043881542,0.0016382094,0.002179711,0.005686185,0.0010852143,0.0019067181,0.0020580825],"category_scores_gemma":[0.029315975,0.00066059356,0.000449861,0.0011179472,0.0066448227,0.0052491073,0.0053752293,0.0030177932,0.0009844899],"study_design_candidate":"qualitative","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00010149571,0.00092074607,0.009840881,0.0010530887,0.00006312605,0.0007005602,0.13688996,0.0083375,0.012343738,0.30326027,0.015177338,0.51131135],"study_design_scores_gemma":[0.00022307994,0.0006810344,0.0047706277,0.0015174673,0.000079511796,0.0014994914,0.021809747,0.029688837,0.009530321,0.5708585,0.35920292,0.00013833104],"about_ca_topic_score_codex":0.0011291368,"about_ca_topic_score_gemma":0.0013748627,"teacher_disagreement_score":0.014731789,"about_ca_system_score_codex":0.0011619134,"about_ca_system_score_gemma":0.0044021145,"threshold_uncertainty_score":0.077910125},"labels":[],"label_agreement":null},{"id":"W4238417811","doi":"10.1109/icse.2003.1201304","title":"FEAT a tool for locating, describing, and analyzing concerns in source code","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":33,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Computer science; Source code; Task (project management); Code (set theory); Programming language; Software engineering; Code review; KPI-driven code analysis; Open source; Static program analysis; Software development; Software; Systems engineering; Engineering","score_opus":0.05235386629265224,"score_gpt":0.29129267946729115,"score_spread":0.23893881317463891,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4238417811","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0048420103,0.00048672027,0.7262996,0.00024077056,0.00012984363,0.0004000089,0.0044078073,0.25755563,0.0056375773],"genre_scores_gemma":[0.045150016,0.0008788231,0.87848055,0.00037793114,0.000092378104,0.0009864848,0.02236527,0.03605144,0.015617109],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99812824,0.00039714287,0.00024705814,0.0003069327,0.00077713944,0.00014354552],"domain_scores_gemma":[0.99242485,0.004633057,0.0007637523,0.0012549461,0.00078235014,0.0001409545],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0029898083,0.0032065227,0.0010491137,0.007514638,0.0019055858,0.003243704,0.0029738704,0.0028549223,0.020302106],"category_scores_gemma":[0.014221382,0.0026804516,0.00229287,0.0025863785,0.0014932425,0.0070186383,0.0036722017,0.003253773,0.010013874],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0007338863,0.0005725955,0.015934482,0.0033665437,0.00051685405,0.0024131255,0.0044277245,0.0135106575,0.033591628,0.057239402,0.33583528,0.53185785],"study_design_scores_gemma":[0.00042232074,0.00046260466,0.008530466,0.0012371505,0.00036743196,0.0050790496,0.0009914377,0.15852979,0.05467959,0.057807725,0.71140265,0.0004897621],"about_ca_topic_score_codex":0.0058216075,"about_ca_topic_score_gemma":0.008650236,"teacher_disagreement_score":0.020302106,"about_ca_system_score_codex":0.0011694607,"about_ca_system_score_gemma":0.0023259725,"threshold_uncertainty_score":0.06791735},"labels":[],"label_agreement":null},{"id":"W4239921431","doi":"10.1109/icse.2003.1201214","title":"Design pattern rationale graphs: linking design to source","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Software design pattern; Computer science; Design pattern; Structural pattern; Flexibility (engineering); Pattern language (formal languages); Source code; TRACE (psycholinguistics); Code (set theory); Representation (politics); Software engineering; Human–computer interaction; Programming language; Software design; Software development; Software","score_opus":0.05281837649059293,"score_gpt":0.2709705174956941,"score_spread":0.2181521410051012,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4239921431","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0021429071,0.00018363154,0.98307216,0.00059670646,0.00006740093,0.0004029993,0.00081523857,0.008832308,0.0038867404],"genre_scores_gemma":[0.022885852,0.00068951846,0.9661985,0.00036891413,0.000047226273,0.00076161354,0.0033918207,0.0025211354,0.0031354977],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.9934676,0.0028303522,0.0006144592,0.00067917403,0.0021989788,0.00020943438],"domain_scores_gemma":[0.9730483,0.016089791,0.0023185192,0.005203233,0.0029279913,0.00041208093],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.007895775,0.0022589648,0.00073829835,0.00860691,0.001504263,0.004369649,0.002866488,0.0027972176,0.009306025],"category_scores_gemma":[0.044769086,0.002085609,0.001869139,0.0059175147,0.0021146857,0.00749288,0.004954023,0.003089373,0.003393876],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0001856561,0.00039850784,0.0043287515,0.0019330947,0.00019954344,0.0014165007,0.0049978197,0.02851656,0.008240688,0.23138304,0.04905101,0.6693489],"study_design_scores_gemma":[0.00017285226,0.00015120344,0.0017980759,0.0012424266,0.00017681284,0.0011004183,0.0010857906,0.15819988,0.017102439,0.41695517,0.40178314,0.00023173462],"about_ca_topic_score_codex":0.007418426,"about_ca_topic_score_gemma":0.009502147,"teacher_disagreement_score":0.009306025,"about_ca_system_score_codex":0.0015291978,"about_ca_system_score_gemma":0.0040186155,"threshold_uncertainty_score":0.041757405},"labels":[],"label_agreement":null},{"id":"W4241947741","doi":"10.1109/icse.2003.1201219","title":"Hipikat: recommending pertinent software development artifacts","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":173,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"U.S. Consumer Product Safety Commission","keywords":"Eclipse; Computer science; Task (project management); Context (archaeology); Open source; Open source software; Software; Software project management; Open-source software development; Software engineering; Software development; World Wide Web; Data science; Software construction; Engineering; Systems engineering; Programming language","score_opus":0.04131285254867295,"score_gpt":0.27387316265481426,"score_spread":0.2325603101061413,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4241947741","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.25225955,0.0016736849,0.67373717,0.0027769292,0.00031565843,0.0030598736,0.0032401008,0.047715317,0.015221653],"genre_scores_gemma":[0.15544999,0.0005962084,0.8318896,0.00019931265,0.00006514585,0.00054425246,0.003267194,0.00064515835,0.0073430724],"study_design_codex":"design_other","study_design_gemma":"bench_or_experimental","domain_scores_codex":[0.99799734,0.00059668993,0.00019822121,0.0004392025,0.0006793251,0.00008924515],"domain_scores_gemma":[0.9868666,0.0074642133,0.0013094225,0.001879244,0.0017407752,0.0007397554],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0038040045,0.0012592431,0.0006703684,0.0050392016,0.0011790089,0.0025690042,0.0022278067,0.0017400947,0.005219799],"category_scores_gemma":[0.02931864,0.00077954103,0.0005792489,0.002375835,0.0004518373,0.0035078363,0.0023316504,0.00097189227,0.002398761],"study_design_candidate":"bench_or_experimental","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00048436812,0.0007509786,0.026977042,0.0021070123,0.0001863315,0.0012494235,0.008646168,0.0058830646,0.017359829,0.0030093796,0.037430875,0.89591557],"study_design_scores_gemma":[0.00087104493,0.0029420739,0.086575925,0.0019904221,0.0013533939,0.0060035703,0.019100754,0.25923538,0.07066793,0.028923485,0.5213999,0.00093613076],"about_ca_topic_score_codex":0.0046853297,"about_ca_topic_score_gemma":0.017569104,"teacher_disagreement_score":0.005219799,"about_ca_system_score_codex":0.0006533288,"about_ca_system_score_gemma":0.002566259,"threshold_uncertainty_score":0.0201177},"labels":[],"label_agreement":null},{"id":"W4243475163","doi":"10.1109/icse.2003.1201295","title":"/spl chi/Chek: A model checker for multi-valued reasoning","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Formal Methods in Verification","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Model checking; Symbolic trajectory evaluation; Computer science; Computation tree logic; Generalization; Extension (predicate logic); Temporal logic; Theoretical computer science; Programming language; Linear temporal logic; Algorithm; Mathematics","score_opus":0.08485247251597294,"score_gpt":0.32961700117439113,"score_spread":0.2447645286584182,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4243475163","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.004624398,0.00013117737,0.9715551,0.00022836126,0.00011868939,0.00014030929,0.0004569101,0.020768315,0.001976696],"genre_scores_gemma":[0.1256808,0.0003460225,0.86319816,0.00074218307,0.000082744045,0.0004196919,0.0022658533,0.0031625386,0.004101939],"study_design_codex":"simulation_or_modeling","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9962728,0.0009432839,0.0003532177,0.0004425069,0.0016730195,0.0003152129],"domain_scores_gemma":[0.9936737,0.0031529919,0.0003952547,0.0016346145,0.00094014226,0.00020328978],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003099305,0.0012016068,0.0011926112,0.0015660342,0.00088127406,0.0024835498,0.003995146,0.0016463883,0.0071600894],"category_scores_gemma":[0.009413043,0.0012432229,0.0020383417,0.0010798522,0.0023170027,0.0055756913,0.0034761226,0.005129006,0.0018889426],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0010949068,0.0006505024,0.005013171,0.0016025107,0.00058171834,0.0011911392,0.000605824,0.31681973,0.05029256,0.28425512,0.053041384,0.28485146],"study_design_scores_gemma":[0.00018575563,0.0000886535,0.0002487995,0.000098091,0.00012430696,0.00025148538,0.000030630817,0.885756,0.04215201,0.046876047,0.024119975,0.00006819915],"about_ca_topic_score_codex":0.0053805574,"about_ca_topic_score_gemma":0.007941934,"teacher_disagreement_score":0.0071600894,"about_ca_system_score_codex":0.0013923728,"about_ca_system_score_gemma":0.0029378152,"threshold_uncertainty_score":0.023952901},"labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"simulation_or_modeling","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"medium"},{"model":"gpt","categories":[],"domain":null,"study_design":"theoretical_or_conceptual","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"}]}