{"meta":{"page":1,"per_page":50,"max_per_page":100,"total":14,"total_is_capped":false,"direct_labels_cover":0,"predictions_cover":14,"direct_label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline (scores rank; they never assert a category)","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12","author_layer_release":"2026-06-26"},"query_hash":"0d7ccbc5d97c","filters":{"venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing"}},"results":[{"id":"W4306694236","doi":"10.1609/hcomp.v10i1.21986","title":"Eliciting and Learning with Soft Labels from Every Annotator","year":2022,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":26,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"","funders":"Engineering and Physical Sciences Research Council; Canadian Institute for Advanced Research","keywords":"Computer science; Categorical variable; Crowdsourcing; Artificial intelligence; Machine learning; Robustness (evolution); Generalization; Set (abstract data type); Soft skills; Natural language processing; World Wide Web","authors":[{"name":"Katherine M. Collins","is_ca":false},{"name":"Umang Bhatt","is_ca":false},{"name":"Adrian Weller","is_ca":false}],"retraction":null,"screen_n_in":null,"score":{"opus":0.02357610624059404,"gpt":0.240562109562208,"spread":0.216986003321614,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.04146983,0.002886866,0.002105475,0.002948971,0.004610458,0.004103038,0.003499218,0.003780185,0.008652726],"category_scores_gemma":[0.1106495,0.001626424,0.001822814,0.003641654,0.004149523,0.007918087,0.01155762,0.006739689,0.008014644],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003028555,"about_ca_system_score_gemma":0.008564414,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004838172,"about_ca_topic_score_gemma":0.01888795,"domain_scores_codex":[0.948999,0.03356538,0.002291614,0.008119404,0.005826782,0.001197835],"domain_scores_gemma":[0.8619913,0.06826486,0.007511863,0.04161943,0.01748443,0.003128052],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002411801,0.0009170792,0.03055581,0.003415169,0.0004273346,0.001057483,0.01717464,0.04485076,0.07421295,0.1057422,0.1450794,0.5741553],"study_design_scores_gemma":[0.0005005738,0.0005651163,0.008646972,0.001040115,0.0002817837,0.0009642007,0.007603456,0.2619488,0.04911944,0.4120473,0.2568296,0.0004526556],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.03114075,0.0005100684,0.9391757,0.004017902,0.0005023507,0.0009666719,0.00338992,0.003347327,0.01694942],"genre_scores_gemma":[0.2201826,0.0003900514,0.7511899,0.00258629,0.0004230202,0.004173973,0.007790813,0.001433832,0.01182956],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.04146983,"threshold_uncertainty_score":0.219316,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2794325666","doi":"10.1609/hcomp.v5i1.13307","title":"Deja Vu: Characterizing Worker Reliability Using Task Consistency","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada; National Science Foundation","keywords":"Consistency (knowledge bases); Task (project management); Reliability (semiconductor); Metric (unit); Computer science; Context (archaeology); Consistency model; Measure (data warehouse); Reliability engineering; Data mining; Artificial intelligence; Data consistency; Engineering; Geography; Operations management; Database","authors":[{"name":"Alex C. Williams","is_ca":true},{"name":"Joslin Goh","is_ca":true},{"name":"Charlie Willis","is_ca":false},{"name":"Aaron M. Ellison","is_ca":false},{"name":"James H. Brusuelas","is_ca":false},{"name":"Charles C. Davis","is_ca":false},{"name":"Edith Law","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.06568918211076574,"gpt":0.3058742671703849,"spread":0.2401850850596192,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.01816675,0.001331362,0.001265428,0.002913164,0.001400664,0.003067796,0.002640934,0.001570054,0.001309621],"category_scores_gemma":[0.1581714,0.0008792727,0.0009301109,0.002124333,0.001941698,0.004787448,0.004089996,0.001940113,0.0005855765],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001380721,"about_ca_system_score_gemma":0.002126821,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003528017,"about_ca_topic_score_gemma":0.002074333,"domain_scores_codex":[0.9813104,0.007092563,0.001878346,0.003536752,0.005277734,0.0009042301],"domain_scores_gemma":[0.859104,0.07534956,0.01985242,0.02804342,0.01520465,0.002446028],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002845871,0.001053393,0.2738426,0.001482548,0.00075294,0.0004937788,0.009629657,0.1947154,0.0479531,0.04782259,0.0112336,0.4081745],"study_design_scores_gemma":[0.0002600916,0.002020061,0.1031327,0.0002830276,0.0002817565,0.0008477825,0.002160174,0.7385901,0.04159852,0.09697043,0.0133722,0.0004832637],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2668017,0.0008505498,0.7207717,0.0005402839,0.0001862732,0.000716035,0.0009426808,0.003086894,0.006103843],"genre_scores_gemma":[0.8826841,0.0001139636,0.1143242,0.0001419201,0.0000716978,0.0006580505,0.0005861067,0.0003310711,0.001088945],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9818332,"threshold_uncertainty_score":0.09607613,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2793871201","doi":"10.1609/hcomp.v5i1.13309","title":"Lessons from an Online Massive Genomics Computer Game","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"McGill University","funders":"Canadian Institutes of Health Research; Genome Canada","keywords":"Computer science; Crowdsourcing; Casual; Task (project management); Data science; Matching (statistics); Citizen science; Artificial intelligence; Human–computer interaction; World Wide Web; Engineering; Biology","authors":[{"name":"Akash Singh","is_ca":true},{"name":"Faizy Ahsan","is_ca":true},{"name":"Mathieu Blanchette","is_ca":true},{"name":"Jérôme Waldispühl","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.09804089691815192,"gpt":0.3325677333469798,"spread":0.2345268364288279,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002451192,0.0005684962,0.0003319514,0.0009318375,0.0026519,0.004913573,0.001847546,0.002557483,0.01208772],"category_scores_gemma":[0.02004395,0.0002443642,0.0003631742,0.0006924024,0.002758628,0.00496656,0.002774019,0.002174542,0.002836496],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001599614,"about_ca_system_score_gemma":0.001593355,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01278135,"about_ca_topic_score_gemma":0.01600695,"domain_scores_codex":[0.9983087,0.0009742006,0.00004825368,0.0001864847,0.0002989864,0.0001835109],"domain_scores_gemma":[0.9924179,0.004776909,0.0002866594,0.0004127013,0.0008379792,0.001267909],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","study_design_scores_codex":[0.0008859328,0.002149946,0.04068881,0.0008210607,0.0001230181,0.003028673,0.02656085,0.01379778,0.002297646,0.3535061,0.2483878,0.3077525],"study_design_scores_gemma":[0.0002037735,0.0006823791,0.01653207,0.0005617151,0.00004672795,0.001523024,0.02394872,0.03226104,0.001922619,0.4822639,0.4398724,0.000181636],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"other","genre_gemma":"empirical","genre_scores_codex":[0.3504258,0.003742829,0.05539115,0.201293,0.002160606,0.0005696181,0.002031703,0.001035202,0.3833502],"genre_scores_gemma":[0.9399644,0.002031676,0.0167256,0.009025782,0.0004466167,0.0002634973,0.000676852,0.0003702278,0.03049535],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.01278135,"threshold_uncertainty_score":0.04043752,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4403434583","doi":"10.1609/hcomp.v12i1.31597","title":"Disclosures & Disclaimers: Investigating the Impact of Transparency Disclosures and Reliability Disclaimers on Learner-LLM Interactions","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Big Data and Business Intelligence","field":"Business, Management and Accounting","cited_by":9,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto","funders":"","keywords":"Transparency (behavior); Reliability (semiconductor); Psychology; Political science; Law; Physics; Thermodynamics","authors":[{"name":"Jessica Y. Bo","is_ca":true},{"name":"Harsh Kumar","is_ca":true},{"name":"Michael Liut","is_ca":true},{"name":"Ashton Anderson","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.1156158976501775,"gpt":0.3606875602226767,"spread":0.2450716625724992,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.04146833,0.0006335202,0.0005945633,0.0009505257,0.001871672,0.005190299,0.001790158,0.001907846,0.005974253],"category_scores_gemma":[0.3869017,0.0006310699,0.0004702776,0.0005740681,0.002022041,0.00579478,0.004048777,0.002768978,0.001081376],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001822291,"about_ca_system_score_gemma":0.0027783,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001020439,"about_ca_topic_score_gemma":0.00119491,"domain_scores_codex":[0.9302277,0.04836952,0.004829378,0.003648866,0.0113584,0.001566226],"domain_scores_gemma":[0.4130424,0.4647113,0.07535342,0.0237022,0.01733529,0.005855333],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.006734579,0.00664391,0.3739325,0.004095502,0.00045376,0.001559071,0.1531492,0.01242299,0.04642516,0.01713372,0.01180873,0.3656408],"study_design_scores_gemma":[0.002320608,0.02122117,0.5335383,0.00301695,0.00148653,0.002316588,0.09773187,0.1152278,0.09337744,0.04729911,0.08100658,0.001457093],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9775914,0.0002075791,0.01167886,0.001308116,0.00009773936,0.0004379415,0.0001626353,0.0004676525,0.00804802],"genre_scores_gemma":[0.9911179,0.00007228222,0.006429297,0.0002950579,0.00003960425,0.0003473402,0.00007964113,0.00007822103,0.001540658],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.04146833,"threshold_uncertainty_score":0.2193081,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2999004052","doi":"10.1609/hcomp.v7i1.5274","title":"A Hybrid Approach to Identifying Unknown Unknowns of Predictive Models","year":2019,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"","keywords":"Crowdsourcing; Computer science; Task (project management); Machine learning; Set (abstract data type); Artificial intelligence; Data mining; Engineering","authors":[{"name":"Colin Vandenhof","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.04897807387878888,"gpt":0.2707203747712563,"spread":0.2217423008924674,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006632691,0.002531257,0.003249929,0.004575905,0.001986999,0.005271351,0.005353463,0.004033286,0.002343828],"category_scores_gemma":[0.03103063,0.001545794,0.002354284,0.003405782,0.002975093,0.005578817,0.005159665,0.005545251,0.001186417],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001831484,"about_ca_system_score_gemma":0.00269895,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006949726,"about_ca_topic_score_gemma":0.008463151,"domain_scores_codex":[0.99291,0.00250021,0.0004458264,0.00202149,0.001773628,0.0003487235],"domain_scores_gemma":[0.973235,0.01871432,0.001958644,0.003701077,0.00193412,0.0004568896],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0006211922,0.0005451645,0.009186961,0.0005581885,0.0006004997,0.0008471843,0.001553748,0.5060061,0.005661767,0.05553769,0.007979921,0.4109017],"study_design_scores_gemma":[0.00002781004,0.00005159332,0.0004198454,0.00005146399,0.00005221513,0.0001436955,0.00009049723,0.9437753,0.0014552,0.05164784,0.002248238,0.0000362841],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.01093142,0.0003771367,0.9851692,0.0007948717,0.00004036806,0.0001420761,0.0001956201,0.0009317782,0.001417542],"genre_scores_gemma":[0.4088714,0.0003950439,0.5838196,0.0009624808,0.0003914422,0.0004684638,0.001109429,0.0002128067,0.003769303],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.006949726,"threshold_uncertainty_score":0.03507739,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2780153469","doi":"10.1609/hcomp.v5i1.13300","title":"Drafty: Enlisting Users To Be Editors Who Maintain Structured Data","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto","funders":"Brown University","keywords":"Computer science; Construct (python library); Matching (statistics); World Wide Web; User modeling; Information retrieval; Data science; User interface","authors":[{"name":"Shaun Wallace","is_ca":false},{"name":"Lucy Van Kleunen","is_ca":false},{"name":"Marianne Aubin-Le Quere","is_ca":false},{"name":"Abraham Peterkin","is_ca":false},{"name":"Yirui Huang","is_ca":true},{"name":"Jeff Huang","is_ca":false}],"retraction":null,"screen_n_in":null,"score":{"opus":0.0971098124215099,"gpt":0.3317737254023328,"spread":0.2346639129808229,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.03413397,0.001569132,0.001127629,0.003361155,0.00211932,0.004998047,0.002925936,0.002112037,0.0126287],"category_scores_gemma":[0.2298464,0.001181405,0.0009705288,0.002509744,0.001993685,0.01219649,0.0107204,0.001911032,0.009307933],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006698364,"about_ca_system_score_gemma":0.004002551,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001506686,"about_ca_topic_score_gemma":0.003822531,"domain_scores_codex":[0.9807039,0.01055716,0.001984173,0.002866028,0.003170455,0.0007182133],"domain_scores_gemma":[0.7024991,0.1564536,0.01866192,0.09681355,0.0157835,0.009788305],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002800877,0.0007994961,0.04564272,0.004098396,0.0003140226,0.001038292,0.02932435,0.002046442,0.03563594,0.006644381,0.4001388,0.4715164],"study_design_scores_gemma":[0.0009476078,0.001403766,0.04258131,0.0007269387,0.0002022145,0.001408529,0.007225887,0.02162585,0.02535069,0.01760845,0.8803475,0.0005712232],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2708117,0.002714325,0.4161137,0.01463603,0.004703529,0.007146016,0.02277496,0.2237942,0.0373056],"genre_scores_gemma":[0.3050482,0.0009659579,0.6248381,0.004336048,0.002391436,0.004928906,0.02247527,0.01311894,0.02189724],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.03413397,"threshold_uncertainty_score":0.1805198,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1605921839","doi":"10.1609/hcomp.v1i1.13112","title":"Reducing Error in Context-Sensitive Crowdsourced Tasks","year":2013,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Luxmux Technology (Canada)","funders":"","keywords":"Computer science; Redundancy (engineering); Crowdsourcing; Task (project management); Workflow; Context (archaeology); Quality (philosophy); Hierarchy; Human–computer interaction; World Wide Web; Database; Engineering","authors":[{"name":"Daniel Haas","is_ca":false},{"name":"Matthew Greenstein","is_ca":false},{"name":"Kainar Kamalov","is_ca":false},{"name":"Adam Marcus","is_ca":false},{"name":"Marek Olszewski","is_ca":false},{"name":"Marc Piette","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.0393451378942388,"gpt":0.2767068485752072,"spread":0.2373617106809684,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009183922,0.001748459,0.001967615,0.001145143,0.001705658,0.001915287,0.003683349,0.001955168,0.001080743],"category_scores_gemma":[0.04687179,0.001343292,0.0008953311,0.001111486,0.002688774,0.003533411,0.006640292,0.002376317,0.0005183287],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001939114,"about_ca_system_score_gemma":0.004124768,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007468374,"about_ca_topic_score_gemma":0.004448405,"domain_scores_codex":[0.9910113,0.002780086,0.0004327221,0.002202208,0.002879038,0.0006946984],"domain_scores_gemma":[0.9709107,0.01462072,0.003690945,0.006201037,0.003369277,0.001207339],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001050265,0.0004668383,0.0080951,0.0003559599,0.0001833162,0.0003232905,0.003046314,0.7798774,0.01839077,0.01320135,0.002743416,0.172266],"study_design_scores_gemma":[0.00007103749,0.0002413901,0.00193468,0.00003856849,0.00004496257,0.00009962737,0.0002597152,0.9653032,0.007344685,0.02292512,0.001689401,0.00004757561],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08351608,0.0005988186,0.91215,0.0005133609,0.00009471822,0.0001809643,0.00006458705,0.001211311,0.00167011],"genre_scores_gemma":[0.8824542,0.0002118575,0.1154056,0.0002164272,0.0001062129,0.000141087,0.00007315745,0.0001931367,0.00119825],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.009183922,"threshold_uncertainty_score":0.04856986,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4403434584","doi":"10.1609/hcomp.v12i1.31596","title":"“Hi. I’m Molly, Your Virtual Interviewer!” Exploring the Impact of Race and Gender in AI-Powered Virtual Interview Experiences","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"AI in Service Interactions","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Lehigh Hanson (Canada)","funders":"","keywords":"Interview; Race (biology); Psychology; Applied psychology; Gender studies; Sociology; Anthropology","authors":[{"name":"Shreyan Biswas","is_ca":false},{"name":"Ji‐Youn Jung","is_ca":false},{"name":"Abhishek Unnam","is_ca":true},{"name":"Kuldeep Yadav","is_ca":true},{"name":"Shreyansh Gupta","is_ca":true},{"name":"Ujwal Gadiraju","is_ca":false}],"retraction":null,"screen_n_in":null,"score":{"opus":0.1423396126374964,"gpt":0.371991523183155,"spread":0.2296519105456585,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009901873,0.000269333,0.0002262796,0.0004116734,0.004171847,0.002677804,0.0005398606,0.0005912759,0.007910112],"category_scores_gemma":[0.02146327,0.00021345,0.0002704175,0.0003050961,0.00304527,0.0019245,0.002425898,0.001042435,0.001648882],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006521493,"about_ca_system_score_gemma":0.0007220973,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001261597,"about_ca_topic_score_gemma":0.00201802,"domain_scores_codex":[0.9911591,0.007555421,0.0001305872,0.0003701967,0.0004011934,0.0003835378],"domain_scores_gemma":[0.9902813,0.006380999,0.001173225,0.0007878643,0.0007173436,0.0006592056],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"observational","study_design_scores_codex":[0.001080623,0.0003324352,0.04962162,0.0006171468,0.00005815668,0.001084892,0.7724537,0.0003951369,0.01416456,0.01716656,0.02146458,0.1215606],"study_design_scores_gemma":[0.0001246121,0.001131649,0.03970154,0.0005121321,0.00007483542,0.001633108,0.7221439,0.002138949,0.006641889,0.01165312,0.21407,0.0001742435],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9290881,0.0005861123,0.02284964,0.005300431,0.0004609563,0.00045009,0.000212141,0.0001220355,0.04093049],"genre_scores_gemma":[0.9799594,0.0002390773,0.008496124,0.002156808,0.00007509618,0.0006299559,0.00006421815,0.0000439728,0.008335418],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.009901873,"threshold_uncertainty_score":0.05236667,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2811506820","doi":"10.1609/hcomp.v6i1.13323","title":"Towards Quantifying Behaviour in Social Crowdsourcing Communities","year":2018,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Calgary","funders":"","keywords":"Crowdsourcing; Quality (philosophy); Proxy (statistics); Process (computing); Psychology; Computer science; Social psychology; Knowledge management; Applied psychology; Machine learning; World Wide Web","authors":[{"name":"Khobaib Zaamout","is_ca":true},{"name":"Ken Barker","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.1231108802447632,"gpt":0.3553618345602627,"spread":0.2322509543154995,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005713595,0.001026266,0.0009127089,0.006964792,0.001099344,0.002923753,0.001213642,0.001290599,0.000956139],"category_scores_gemma":[0.0408471,0.0007016199,0.001248341,0.004232289,0.002548115,0.003436763,0.003082638,0.001391846,0.0003316073],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002401791,"about_ca_system_score_gemma":0.001851204,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01664956,"about_ca_topic_score_gemma":0.01194888,"domain_scores_codex":[0.9942049,0.002740169,0.0003147468,0.001131134,0.00131257,0.0002963853],"domain_scores_gemma":[0.9687996,0.01601679,0.007544097,0.003138454,0.003333232,0.001167709],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003447515,0.0005992206,0.4307844,0.0008611706,0.0007956657,0.0003605956,0.0166121,0.3019021,0.01641217,0.06016914,0.001960027,0.1691987],"study_design_scores_gemma":[0.00002923015,0.000167827,0.1050894,0.0001236353,0.00007326452,0.0001457132,0.003198583,0.7712845,0.003634527,0.1121151,0.003985293,0.0001529118],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.4456621,0.0002710666,0.5482338,0.0004159773,0.00002321808,0.0004839751,0.0008478667,0.0005492707,0.0035127],"genre_scores_gemma":[0.8478249,0.00008927868,0.1505553,0.00004577245,0.00002228955,0.0002523054,0.0005332535,0.0000610784,0.0006158106],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.01664956,"threshold_uncertainty_score":0.03310525,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W2180780611","doi":"10.1609/hcomp.v3i1.13261","title":"Acquiring Reliable Ratings from the Crowd","year":2015,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"Canadian Nautical Research Society","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Data science; Machine learning; World Wide Web","authors":[{"name":"Beatrice Valeri","is_ca":false},{"name":"Shady Elbassuoni","is_ca":false},{"name":"Sihem Amer-Yahia","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.0737925058077919,"gpt":0.2848671676757621,"spread":0.2110746618679702,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.005075653,0.0017487,0.00293746,0.002659282,0.001197609,0.002430967,0.002210197,0.002132922,0.003012106],"category_scores_gemma":[0.03364565,0.0009831894,0.0008236773,0.00235794,0.000817044,0.004934746,0.004065022,0.002015,0.004668569],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000592791,"about_ca_system_score_gemma":0.001141939,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005082074,"about_ca_topic_score_gemma":0.007056769,"domain_scores_codex":[0.9906346,0.002553579,0.0005063484,0.002741199,0.003073889,0.0004903472],"domain_scores_gemma":[0.9759454,0.009358608,0.002332093,0.005804904,0.005627154,0.0009318802],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002684848,0.0009180096,0.03993837,0.001519896,0.0005654275,0.001620684,0.002926859,0.04856645,0.07246398,0.01080092,0.06162312,0.7563714],"study_design_scores_gemma":[0.0002468418,0.0008719939,0.0232512,0.0002552427,0.0002398625,0.001533611,0.001703455,0.8305919,0.04032354,0.05437168,0.04628801,0.0003228131],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1876079,0.004435758,0.7713683,0.002931576,0.0006605514,0.0007009526,0.00617605,0.00568512,0.0204339],"genre_scores_gemma":[0.7037353,0.001110837,0.2778174,0.0008209937,0.0008564591,0.0003835202,0.006789055,0.0003962306,0.008090288],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9949244,"threshold_uncertainty_score":0.02684289,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4388332620","doi":"10.1609/hcomp.v11i1.27544","title":"Informing Users about Data Imputation: Exploring the Design Space for Dealing With Non-Responses","year":2023,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Innovative Human-Technology Interaction","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false},"ca_institutions":"","funders":"University of Toronto","keywords":"Imputation (statistics); Computer science; Software deployment; Autonomy; Missing data; Information retrieval; Data science; Machine learning","authors":[{"name":"Ananya Bhattacharjee","is_ca":false},{"name":"Haochen Song","is_ca":false},{"name":"Xuening Wu","is_ca":false},{"name":"Justice Tomlinson","is_ca":false},{"name":"Mohi Reza","is_ca":false},{"name":"Akmar Ehsan Chowdhury","is_ca":false},{"name":"Nina Deliu","is_ca":false},{"name":"Thomas Price","is_ca":false},{"name":"Joseph Jay Williams","is_ca":false}],"retraction":null,"screen_n_in":null,"score":{"opus":0.2356427033280684,"gpt":0.3595694917627186,"spread":0.1239267884346501,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.2994813,0.002495468,0.001977728,0.002567403,0.00359237,0.01135269,0.006433175,0.006041896,0.006823712],"category_scores_gemma":[0.5800729,0.002566186,0.002147849,0.001879813,0.007251276,0.01693324,0.009696989,0.006326079,0.002722841],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002683029,"about_ca_system_score_gemma":0.004824609,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008341832,"about_ca_topic_score_gemma":0.001021868,"domain_scores_codex":[0.5473022,0.4145248,0.01527742,0.01089002,0.009848967,0.002156672],"domain_scores_gemma":[0.2000506,0.6881288,0.01791067,0.07382762,0.01688666,0.003195579],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.009374337,0.002165166,0.04515202,0.005414869,0.0007008209,0.001642119,0.2333768,0.01458035,0.03024257,0.0774731,0.007498003,0.5723798],"study_design_scores_gemma":[0.004934172,0.007288795,0.01543963,0.005362612,0.001126239,0.002353941,0.0476301,0.4159665,0.06084306,0.3245945,0.1132373,0.001223263],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.05769232,0.0002800073,0.9272545,0.004993199,0.0001376761,0.003434415,0.0001731563,0.003442723,0.00259209],"genre_scores_gemma":[0.4031293,0.0001633815,0.5841725,0.001471716,0.0001483381,0.008502962,0.0002554321,0.0005612469,0.00159512],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.2994813,"threshold_uncertainty_score":0.8638643,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1487101415","doi":"10.1609/hcomp.v2i1.13137","title":"Phylo and Open-Phylo: A Human-Computing Platform for Comparative Genomics","year":2014,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"McGill University","funders":"","keywords":"Genomics; Comparative genomics; Computer science; Open science; Field (mathematics); Open source; Data science; Biology; Genetics; Genome; Gene","authors":[{"name":"Jérôme Waldispühl","is_ca":true},{"name":"Mathieu Blanchette","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.3266005297275615,"gpt":0.4420037016325501,"spread":0.1154031719049887,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["open_science"],"consensus_categories":[],"category_scores_codex":[0.004688039,0.0009375678,0.0008168385,0.002134824,0.001937193,0.002098995,0.003020457,0.001891006,0.0235695],"category_scores_gemma":[0.01192167,0.0006245429,0.001292829,0.001758221,0.002321372,0.005297833,0.00868574,0.002387088,0.006935478],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001353658,"about_ca_system_score_gemma":0.00260962,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005421313,"about_ca_topic_score_gemma":0.007323035,"domain_scores_codex":[0.9978531,0.0007665838,0.00009087789,0.0004192866,0.0005430634,0.0003270779],"domain_scores_gemma":[0.9940241,0.002670154,0.0003125198,0.0009885667,0.0005139227,0.001490828],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.005882374,0.0007723442,0.008611539,0.001197397,0.0002490614,0.0009837425,0.005711588,0.01189428,0.03304524,0.2255176,0.5070443,0.1990906],"study_design_scores_gemma":[0.0006120569,0.0003358967,0.006159514,0.0001986169,0.00006321849,0.0003277514,0.0008610343,0.08155851,0.007981208,0.2472814,0.6543911,0.0002297366],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"software","genre_scores_codex":[0.04906994,0.001373245,0.643304,0.009272547,0.001991776,0.002717717,0.04920848,0.1546543,0.08840799],"genre_scores_gemma":[0.2724117,0.0007979645,0.6268623,0.003127669,0.0003935056,0.005500685,0.04472136,0.0185024,0.02768233],"genre_candidate":"software","genre_consensus":null,"teacher_disagreement_score":0.9969795,"threshold_uncertainty_score":0.07884783,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W4403434723","doi":"10.1609/hcomp.v12i1.31600","title":"Unveiling the Inter-Related Preferences of Crowdworkers: Implications for Personalized and Flexible Platform Design","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Digital Marketing and Social Media","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Toronto","funders":"","keywords":"Human–computer interaction; Computer science; World Wide Web","authors":[{"name":"Senjuti Dutta","is_ca":false},{"name":"Rhema Linder","is_ca":false},{"name":"Alex C. Williams","is_ca":false},{"name":"Anastasia Kuzminykh","is_ca":true},{"name":"Scott Ruoti","is_ca":false}],"retraction":null,"screen_n_in":null,"score":{"opus":0.1322345241962908,"gpt":0.3694382266520061,"spread":0.2372037024557153,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006699172,0.0004691202,0.000346567,0.00136507,0.002772483,0.003739415,0.0008966386,0.0007946706,0.003381012],"category_scores_gemma":[0.02661564,0.0003422156,0.0002817038,0.0007456086,0.001763134,0.003103971,0.002984535,0.0009274423,0.0007183867],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009234524,"about_ca_system_score_gemma":0.001812747,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003897332,"about_ca_topic_score_gemma":0.008165504,"domain_scores_codex":[0.9961992,0.002028744,0.0001867902,0.0005427266,0.0006367952,0.0004057635],"domain_scores_gemma":[0.9873625,0.007871429,0.001512706,0.0009580295,0.001256377,0.001039004],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0009187619,0.000765778,0.396729,0.001262217,0.000118375,0.001273905,0.333616,0.003040086,0.01921966,0.006838493,0.006913317,0.2293044],"study_design_scores_gemma":[0.0001065579,0.0008286828,0.3590125,0.0007760804,0.000118892,0.000749306,0.5370134,0.01116851,0.005100984,0.02955195,0.05530079,0.0002724346],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9685423,0.0002910895,0.01916735,0.00185661,0.00005902065,0.0002509927,0.0001646208,0.0001221675,0.009545959],"genre_scores_gemma":[0.9898748,0.0001497364,0.007968239,0.0002840048,0.00002048533,0.0002673625,0.00007643857,0.00004238462,0.00131642],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.006699172,"threshold_uncertainty_score":0.03542906,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null},{"id":"W1926732392","doi":"10.1609/hcomp.v1i1.13110","title":"TrailView: Combining Gamification and Social Network Voting Mechanisms for Useful Data Collection","year":2013,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false},"ca_institutions":"University of Waterloo","funders":"","keywords":"Incentive; Voting; Data collection; Computer science; Point (geometry); Competition (biology); Scheme (mathematics); Social network (sociolinguistics); Social worlds; Data science; World Wide Web; Social media; Sociology; Political science; Economics","authors":[{"name":"Michael Weingert","is_ca":true},{"name":"Kate Larson","is_ca":true}],"retraction":null,"screen_n_in":null,"score":{"opus":0.08141775148202438,"gpt":0.2917844663459198,"spread":0.2103667148638954,"validation_status":"score_only:v0-immature-baseline"},"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.01417608,0.001447097,0.001529066,0.002932981,0.001316372,0.002464403,0.00382789,0.001644572,0.01119195],"category_scores_gemma":[0.03046191,0.00075667,0.001049395,0.002444363,0.001530987,0.006449749,0.008789231,0.001673963,0.002474907],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009822756,"about_ca_system_score_gemma":0.001881508,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002841364,"about_ca_topic_score_gemma":0.004291807,"domain_scores_codex":[0.9913962,0.00515297,0.0003767039,0.001174319,0.001402826,0.0004969813],"domain_scores_gemma":[0.9835618,0.009733905,0.0008293861,0.003553013,0.001317344,0.001004567],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.002727874,0.002088187,0.01003501,0.0005765886,0.0003975118,0.0003748148,0.001467799,0.07148071,0.007444379,0.08621157,0.02397491,0.7932206],"study_design_scores_gemma":[0.0005609895,0.0005604975,0.001977544,0.000107864,0.0001085554,0.0001434119,0.0003216345,0.8138507,0.004898157,0.1536824,0.02365965,0.0001287082],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.03679787,0.0003523778,0.9386528,0.001054567,0.0002460772,0.001408877,0.000596372,0.009977241,0.0109139],"genre_scores_gemma":[0.5492277,0.0002154091,0.4379631,0.0003836104,0.0001610794,0.001894943,0.0007756863,0.0005173876,0.00886107],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.01417608,"threshold_uncertainty_score":0.07497114,"prediction_status":"machine_predicted_unvalidated"},"labels":[],"label_agreement":null}]}