{"meta":{"query_hash":"cba7bb08c2d0","filters":{"venue":"25th International Conference on Software Engineering, 2003. Proceedings."},"cohort_total":6,"direct_labels_cover":1,"predictions_cover":6,"exported":6,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/cba7bb08c2d0","api":"https://metacan.xera.ac/api/v1/cohort?venue=25th+International+Conference+on+Software+Engineering%2C+2003.+Proceedings."},"results":[{"id":"W4229596932","doi":"10.1109/icse.2003.1201189","title":"Using benchmarking to advance research: a challenge to software engineering","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":150,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Benchmarking; Benchmark (surveying); Computer science; Data science; Software engineering; Software; Academic community; Engineering management; Management science; Engineering","score_opus":0.11012686057636706,"score_gpt":0.3553565504832152,"score_spread":0.24522968990684813,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4229596932","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.035544276,0.00012570001,0.95824564,0.00007472648,0.0026695472,0.0009707446,0.00002500254,0.0016144984,0.0007298615],"genre_scores_gemma":[0.13091749,0.000039940907,0.8674935,0.00020104983,0.00044534157,0.00035236866,0.000007128916,0.00014397329,0.00039920417],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99357855,0.00003417471,0.00066851074,0.0016026578,0.00239492,0.0017211864],"domain_scores_gemma":[0.99480003,0.00009815988,0.000114142036,0.00074200914,0.0032436703,0.0010019767],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0016060624,0.0007129067,0.0005186179,0.0020093198,0.00025527773,0.00086891,0.0028211295,0.00026095193,0.00018125473],"category_scores_gemma":[0.02405716,0.0008155166,0.00011682752,0.0036841724,0.00004172446,0.0010628244,0.00080157164,0.0013213662,0.00032901403],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00002660777,0.00014080518,0.0033393928,0.00015275509,0.00009051163,0.00009192485,0.0011506547,0.04131754,0.0005259609,0.94958913,0.0028297305,0.00074496097],"study_design_scores_gemma":[0.00081936776,0.0008566037,0.0040678456,0.0017476295,0.000011179387,0.00013445385,0.0001256912,0.8810512,0.0062270425,0.00012301496,0.102801576,0.0020344416],"about_ca_topic_score_codex":0.000021989657,"about_ca_topic_score_gemma":0.0000030617095,"teacher_disagreement_score":0.94946617,"about_ca_system_score_codex":0.0011144234,"about_ca_system_score_gemma":0.00036323545,"threshold_uncertainty_score":0.9994296},"labels":[],"label_agreement":null},{"id":"W4237741047","doi":"10.1109/icse.2003.1201251","title":"Empowering software engineers in human-centered design","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Usability and User Interface Design","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Concordia University","funders":"","keywords":"Usability; Computer science; Software engineering; Usability engineering; Social software engineering; Software development; Toolbox; Software construction; Human–computer interaction; Software; Programming language","score_opus":0.05692146588363678,"score_gpt":0.2961389041541138,"score_spread":0.23921743827047703,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4237741047","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0261269,0.00005059004,0.9696379,0.000026715205,0.001469393,0.0004511718,0.000010170698,0.000829771,0.0013974303],"genre_scores_gemma":[0.4134693,0.000023320628,0.58548266,0.00018225599,0.00006872037,0.0001338864,0.000009124604,0.00005330751,0.000577415],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9971233,0.000030319141,0.0006058166,0.00087998103,0.00068622665,0.0006743293],"domain_scores_gemma":[0.998455,0.000014609465,0.00015842983,0.0003563054,0.0007957551,0.00021993133],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.00065085647,0.0004992296,0.00037416705,0.0005598503,0.0000957845,0.00048399068,0.0015995306,0.00021062163,0.0002717176],"category_scores_gemma":[0.0020678162,0.00054451026,0.00009289448,0.00086993224,0.000044881286,0.0010255805,0.00015198575,0.00063637487,0.000097371834],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000023486911,0.00021983434,0.0041965917,0.00006191764,0.000060230843,0.000038932016,0.0011317377,0.006549083,0.0004041943,0.984999,0.0021953525,0.0001196505],"study_design_scores_gemma":[0.0020837956,0.00058899244,0.0032192771,0.00081603474,0.000014530125,0.000081374674,0.00025820077,0.94410604,0.03419318,0.0007933844,0.0121360365,0.001709129],"about_ca_topic_score_codex":0.000011911048,"about_ca_topic_score_gemma":0.0000028372915,"teacher_disagreement_score":0.9842056,"about_ca_system_score_codex":0.00039852984,"about_ca_system_score_gemma":0.00016499618,"threshold_uncertainty_score":0.99970067},"labels":[],"label_agreement":null},{"id":"W4238417811","doi":"10.1109/icse.2003.1201304","title":"FEAT a tool for locating, describing, and analyzing concerns in source code","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":33,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Computer science; Source code; Task (project management); Code (set theory); Programming language; Software engineering; Code review; KPI-driven code analysis; Open source; Static program analysis; Software development; Software; Systems engineering; Engineering","score_opus":0.05235386629265224,"score_gpt":0.29129267946729115,"score_spread":0.23893881317463891,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4238417811","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.097715326,0.00014393122,0.90004283,0.000030274909,0.0005936649,0.0005330077,0.000016779115,0.00060274923,0.00032141086],"genre_scores_gemma":[0.49972692,0.00006298437,0.49866894,0.0001191,0.00013930016,0.00029439197,0.000010633519,0.000065748514,0.00091198407],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99749076,0.000012794672,0.0004429999,0.0008156144,0.00056948833,0.0006683251],"domain_scores_gemma":[0.99843794,0.000055543154,0.00012910918,0.00024809796,0.0009366769,0.00019260547],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006974671,0.00038493934,0.00032748806,0.00054714084,0.00009816846,0.0005527484,0.000945474,0.0001576494,0.00004420677],"category_scores_gemma":[0.0074061877,0.00041970474,0.00006075957,0.00081414776,0.000052966585,0.0006431326,0.00017421019,0.00047553072,0.000015853166],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000016664746,0.00006239725,0.0761839,0.00010561237,0.000052510477,0.000011206239,0.00065181765,0.0020170046,0.0001319003,0.9184999,0.0014860728,0.0007809982],"study_design_scores_gemma":[0.0012322203,0.0002043071,0.005111973,0.00048530422,0.000008257255,0.000042930387,0.00008644776,0.97470534,0.0027837434,0.00010286997,0.01452743,0.00070916634],"about_ca_topic_score_codex":0.000018981216,"about_ca_topic_score_gemma":0.0000056913054,"teacher_disagreement_score":0.9726883,"about_ca_system_score_codex":0.00031870417,"about_ca_system_score_gemma":0.00019558141,"threshold_uncertainty_score":0.9998255},"labels":[],"label_agreement":null},{"id":"W4239921431","doi":"10.1109/icse.2003.1201214","title":"Design pattern rationale graphs: linking design to source","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":19,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"","keywords":"Software design pattern; Computer science; Design pattern; Structural pattern; Flexibility (engineering); Pattern language (formal languages); Source code; TRACE (psycholinguistics); Code (set theory); Representation (politics); Software engineering; Human–computer interaction; Programming language; Software design; Software development; Software","score_opus":0.05281837649059293,"score_gpt":0.2709705174956941,"score_spread":0.2181521410051012,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4239921431","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0022184483,0.000047070123,0.99400413,0.000053267373,0.0012963783,0.00076822215,0.000005941551,0.0012651687,0.000341357],"genre_scores_gemma":[0.12763107,0.000027237005,0.8701012,0.0005073466,0.00015871954,0.00039738274,0.000010957376,0.00008900459,0.0010770428],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9963248,0.00004037441,0.00049703347,0.0009774066,0.0013676868,0.00079273566],"domain_scores_gemma":[0.99726456,0.00007659861,0.00013527955,0.0004023052,0.0017045687,0.00041670856],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001032416,0.00051309384,0.0003248473,0.000830862,0.00015932,0.00081284664,0.001937865,0.00018966982,0.00020656899],"category_scores_gemma":[0.005423454,0.0005502006,0.000084014835,0.001486561,0.000033175747,0.00076063845,0.00022713377,0.0006161601,0.00036721173],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000029145207,0.00016016404,0.0067263744,0.00006408311,0.00014662604,0.00004480292,0.0012765267,0.067887254,0.00041641216,0.9120719,0.009047709,0.0021290048],"study_design_scores_gemma":[0.0006502035,0.00036465956,0.0016388553,0.0003602357,0.00000872476,0.00007204171,0.00004165695,0.9687303,0.014158288,0.00026067378,0.012762868,0.00095152826],"about_ca_topic_score_codex":0.0000063905145,"about_ca_topic_score_gemma":4.271043e-7,"teacher_disagreement_score":0.91181123,"about_ca_system_score_codex":0.00032166817,"about_ca_system_score_gemma":0.00031539056,"threshold_uncertainty_score":0.99969494},"labels":[],"label_agreement":null},{"id":"W4241947741","doi":"10.1109/icse.2003.1201219","title":"Hipikat: recommending pertinent software development artifacts","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Software Engineering Research","field":"Computer Science","cited_by":173,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of British Columbia","funders":"U.S. Consumer Product Safety Commission","keywords":"Eclipse; Computer science; Task (project management); Context (archaeology); Open source; Open source software; Software; Software project management; Open-source software development; Software engineering; Software development; World Wide Web; Data science; Software construction; Engineering; Systems engineering; Programming language","score_opus":0.04131285254867295,"score_gpt":0.27387316265481426,"score_spread":0.2325603101061413,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4241947741","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.050784092,0.00010742775,0.93997425,0.00008664039,0.0035048125,0.00055952993,0.000011306668,0.002662751,0.0023092171],"genre_scores_gemma":[0.18436912,0.000043557862,0.8131911,0.00027512532,0.00017522636,0.00024881496,0.00002525957,0.00009524102,0.0015765306],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99582416,0.000019563644,0.00068193884,0.0010594814,0.001392533,0.0010223506],"domain_scores_gemma":[0.99740154,0.00003699644,0.0001986294,0.00043982168,0.0014475423,0.00047548534],"candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007418304,0.0006179772,0.0003940021,0.0007035332,0.00021481837,0.0007232466,0.0018684298,0.0002158052,0.0005021125],"category_scores_gemma":[0.0088881655,0.0006409551,0.000100401034,0.001115518,0.000040211147,0.00092782686,0.0003495677,0.0008012069,0.00048862974],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000018285342,0.00021864433,0.02652894,0.00012207139,0.00016218463,0.000058197227,0.0009845519,0.0009985984,0.00014850091,0.96093917,0.00788668,0.0019341747],"study_design_scores_gemma":[0.0025880053,0.00056105043,0.035498124,0.0015447629,0.000027643613,0.00032943187,0.00023270604,0.4884448,0.055859324,0.00026112277,0.41072708,0.0039259507],"about_ca_topic_score_codex":0.00000465133,"about_ca_topic_score_gemma":0.0000013741089,"teacher_disagreement_score":0.96067804,"about_ca_system_score_codex":0.00072360836,"about_ca_system_score_gemma":0.0004335513,"threshold_uncertainty_score":0.99960417},"labels":[],"label_agreement":null},{"id":"W4243475163","doi":"10.1109/icse.2003.1201295","title":"/spl chi/Chek: A model checker for multi-valued reasoning","year":2003,"lang":"en","type":"article","venue":"25th International Conference on Software Engineering, 2003. Proceedings.","topic":"Formal Methods in Verification","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Model checking; Symbolic trajectory evaluation; Computer science; Computation tree logic; Generalization; Extension (predicate logic); Temporal logic; Theoretical computer science; Programming language; Linear temporal logic; Algorithm; Mathematics","score_opus":0.08485247251597294,"score_gpt":0.32961700117439113,"score_spread":0.2447645286584182,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4243475163","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.0027181953,0.000031575284,0.9922743,0.0000016758589,0.0011158799,0.00047897352,0.00002116854,0.0006091866,0.0027490798],"genre_scores_gemma":[0.0032376358,0.000026528896,0.9946276,0.00021114343,0.000103179875,0.00033369928,0.000016490732,0.000049915507,0.0013938211],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977755,0.000012235791,0.00044273993,0.0007112952,0.00056140625,0.0004968052],"domain_scores_gemma":[0.9978247,5.444383e-7,0.00024195755,0.0002921201,0.0014620863,0.00017853749],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0007249785,0.00036922135,0.00026730244,0.0002740998,0.00013098198,0.0003534368,0.0011768191,0.00018353685,0.0000433712],"category_scores_gemma":[0.0038164505,0.00038466536,0.00009774892,0.0005100978,0.00003771175,0.0009235076,0.00010951242,0.00036182092,0.000041779225],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000014443711,0.00007265215,0.0002650342,0.000038809023,0.000036238896,0.0000011246451,0.00035747068,0.0022669786,0.000026705953,0.9962275,0.0006625518,0.000030483327],"study_design_scores_gemma":[0.00064995314,0.00008610203,0.00029706946,0.00012324998,0.000008888057,0.000017910146,0.00003151321,0.9879688,0.006716176,0.00007106805,0.0036038673,0.00042539224],"about_ca_topic_score_codex":0.0000031351758,"about_ca_topic_score_gemma":4.397157e-7,"teacher_disagreement_score":0.99615645,"about_ca_system_score_codex":0.00023695453,"about_ca_system_score_gemma":0.0002084768,"threshold_uncertainty_score":0.9998605},"labels":[{"model":"gemma","categories":[],"domain":null,"study_design":"simulation_or_modeling","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"medium"},{"model":"gpt","categories":[],"domain":null,"study_design":"theoretical_or_conceptual","genre":"methods","about_ca_system":false,"about_ca_topic":false,"confidence":"high"}],"label_agreement":"split"}]}