{"meta":{"query_hash":"0d7ccbc5d97c","filters":{"venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing"},"cohort_total":14,"direct_labels_cover":0,"predictions_cover":14,"exported":14,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/0d7ccbc5d97c","api":"https://metacan.xera.ac/api/v1/cohort?venue=Proceedings+of+the+AAAI+Conference+on+Human+Computation+and+Crowdsourcing"},"results":[{"id":"W1487101415","doi":"10.1609/hcomp.v2i1.13137","title":"Phylo and Open-Phylo: A Human-Computing Platform for Comparative Genomics","year":2014,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Genomics; Comparative genomics; Computer science; Open science; Field (mathematics); Open source; Data science; Biology; Genetics; Genome; Gene","score_opus":0.3266005297275615,"score_gpt":0.44200370163255015,"score_spread":0.11540317190498867,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1487101415","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9651324,0.000011167474,0.018909689,0.0005886302,0.00016592661,0.0005992447,0.0000070976766,0.00003508497,0.014550771],"genre_scores_gemma":[0.99559975,7.880431e-7,0.0035015754,0.0002099986,0.00007242779,0.00000793297,0.000004100268,0.000010275366,0.00059314986],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977631,0.00003014996,0.00065314485,0.0007517778,0.0005303355,0.00027152424],"domain_scores_gemma":[0.99778134,0.00056104106,0.0007418974,0.0002325715,0.00058265845,0.00010047905],"candidate_categories":["scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0039892597,0.00020132611,0.00041995198,0.00022747,0.0012905151,0.0021668745,0.0011972967,0.00004613593,0.000013278129],"category_scores_gemma":[0.00042682976,0.00014605209,0.000068485104,0.0003180458,0.0002408871,0.00035503777,0.0011680468,0.0001433101,0.000004563463],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011327856,0.00017613884,0.0036577524,0.00016935139,0.00007247081,1.4978514e-7,0.018357165,0.0026705188,0.02219819,0.8195386,0.009220793,0.12382562],"study_design_scores_gemma":[0.0019689337,0.00043460567,0.022942418,0.00033614188,0.000044249613,0.0000055865034,0.007554799,0.67434067,0.0032530474,0.28540978,0.0032650514,0.00044474308],"about_ca_topic_score_codex":0.000026849277,"about_ca_topic_score_gemma":0.000011312392,"teacher_disagreement_score":0.67167014,"about_ca_system_score_codex":0.000029869347,"about_ca_system_score_gemma":0.00002345699,"threshold_uncertainty_score":0.99886894},"labels":[],"label_agreement":null},{"id":"W1605921839","doi":"10.1609/hcomp.v1i1.13112","title":"Reducing Error in Context-Sensitive Crowdsourced Tasks","year":2013,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Luxmux Technology (Canada)","funders":"","keywords":"Computer science; Redundancy (engineering); Crowdsourcing; Task (project management); Workflow; Context (archaeology); Quality (philosophy); Hierarchy; Human–computer interaction; World Wide Web; Database; Engineering","score_opus":0.039345137894238796,"score_gpt":0.27670684857520716,"score_spread":0.23736171068096837,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1605921839","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.98473847,0.000027667811,0.00478095,0.0017875108,0.00014055926,0.0005442469,9.200373e-7,0.00014349748,0.007836189],"genre_scores_gemma":[0.99692804,0.000002807045,0.0021726764,0.0004439696,0.00005054079,0.0000285177,9.311542e-7,0.000024948211,0.00034756158],"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977354,0.000058686663,0.00063822186,0.0006814525,0.00042343134,0.00046284572],"domain_scores_gemma":[0.99846,0.00013539483,0.0004712686,0.00025550454,0.00053928286,0.00013857559],"candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0005677857,0.00033130473,0.0004383202,0.00033631606,0.0004894972,0.00069125695,0.000675181,0.00012665181,0.000018298539],"category_scores_gemma":[0.00014493153,0.0002758885,0.000111656846,0.0005560155,0.00029315715,0.00064977957,0.00038480616,0.0004608484,0.000017813969],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00006973859,0.00040460713,0.016145708,0.00048200603,0.00011437909,0.000008970163,0.059324823,0.0051877066,0.42912576,0.36239067,0.0021947152,0.124550946],"study_design_scores_gemma":[0.0034255534,0.0005244762,0.11559334,0.0034958078,0.000049113794,0.00013932378,0.013047906,0.71615314,0.107325755,0.038568903,0.00012410559,0.0015525775],"about_ca_topic_score_codex":0.00036826538,"about_ca_topic_score_gemma":0.000016552165,"teacher_disagreement_score":0.71096545,"about_ca_system_score_codex":0.000065118096,"about_ca_system_score_gemma":0.000053893436,"threshold_uncertainty_score":0.9999693},"labels":[],"label_agreement":null},{"id":"W1926732392","doi":"10.1609/hcomp.v1i1.13110","title":"TrailView: Combining Gamification and Social Network Voting Mechanisms for Useful Data Collection","year":2013,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Incentive; Voting; Data collection; Computer science; Point (geometry); Competition (biology); Scheme (mathematics); Social network (sociolinguistics); Social worlds; Data science; World Wide Web; Social media; Sociology; Political science; Economics","score_opus":0.08141775148202438,"score_gpt":0.2917844663459198,"score_spread":0.21036671486389544,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1926732392","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.6311432,0.0000413277,0.3647053,0.0023219232,0.00018833726,0.00081730157,0.0000021648816,0.00015836122,0.00062210666],"genre_scores_gemma":[0.98030347,0.000004226345,0.019201787,0.00024224028,0.00008338756,0.000034997985,0.0000071138024,0.000018223025,0.000104577804],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.998449,0.000032771735,0.0004085016,0.0005670849,0.00024919148,0.00029342106],"domain_scores_gemma":[0.9987201,0.00014187075,0.0004935642,0.00017060658,0.00041472414,0.000059117785],"candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.00079417735,0.00020137904,0.00027286282,0.000101191064,0.001356622,0.0008226905,0.0005957678,0.000089674606,0.000003279908],"category_scores_gemma":[0.00007621403,0.00017714813,0.000047798174,0.0002699627,0.00010984076,0.00063845055,0.0003910758,0.00018572806,0.0000014011407],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00003063311,0.0001065045,0.0022647288,0.00053440407,0.00008356368,1.7187193e-7,0.011013301,0.00071750936,0.0609935,0.8580435,0.0044727633,0.0617394],"study_design_scores_gemma":[0.00075049757,0.00015844812,0.008174521,0.00033988655,0.00003889187,0.000014494625,0.0010386809,0.9384454,0.0020132216,0.04864337,0.00007881552,0.00030378462],"about_ca_topic_score_codex":0.00002186618,"about_ca_topic_score_gemma":0.0000037465275,"teacher_disagreement_score":0.93772787,"about_ca_system_score_codex":0.00002817763,"about_ca_system_score_gemma":0.00003140053,"threshold_uncertainty_score":0.9999435},"labels":[],"label_agreement":null},{"id":"W2180780611","doi":"10.1609/hcomp.v3i1.13261","title":"Acquiring Reliable Ratings from the Crowd","year":2015,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Nautical Research Society","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Data science; Machine learning; World Wide Web","score_opus":0.0737925058077919,"score_gpt":0.28486716767576215,"score_spread":0.21107466186797025,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2180780611","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.96809274,0.000109787696,0.01631741,0.0033917383,0.0002956165,0.00029129666,0.000001559934,0.0001981667,0.011301655],"genre_scores_gemma":[0.99440175,0.000003899736,0.004389767,0.0006757743,0.00013557235,0.000010323945,0.0000012250164,0.000019452424,0.00036225788],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9981419,0.000036869096,0.0004412305,0.00051339256,0.0005423862,0.00032422418],"domain_scores_gemma":[0.9983847,0.0001677813,0.00046046893,0.00030370682,0.0005432197,0.00014012372],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000871977,0.00025401395,0.00028174158,0.00007827095,0.0007870893,0.0009746325,0.0010741467,0.00008391559,0.000006668188],"category_scores_gemma":[0.00020913138,0.00016924574,0.00008893319,0.00037841932,0.00022505934,0.00048077738,0.0005647236,0.00033596656,0.000010580942],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0001300345,0.00030319326,0.041887973,0.0002313701,0.0002061816,0.000005944374,0.08021287,0.010891346,0.13170905,0.65365547,0.030563716,0.05020282],"study_design_scores_gemma":[0.0037483547,0.0006593438,0.025831591,0.0026957083,0.0001352855,0.00008450397,0.011180315,0.7080093,0.080399185,0.16280897,0.0028925834,0.0015548285],"about_ca_topic_score_codex":0.00019075745,"about_ca_topic_score_gemma":0.0000067419037,"teacher_disagreement_score":0.697118,"about_ca_system_score_codex":0.00005180709,"about_ca_system_score_gemma":0.000079603094,"threshold_uncertainty_score":0.9398404},"labels":[],"label_agreement":null},{"id":"W2780153469","doi":"10.1609/hcomp.v5i1.13300","title":"Drafty: Enlisting Users To Be Editors Who Maintain Structured Data","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"Brown University","keywords":"Computer science; Construct (python library); Matching (statistics); World Wide Web; User modeling; Information retrieval; Data science; User interface","score_opus":0.0971098124215099,"score_gpt":0.3317737254023328,"score_spread":0.2346639129808229,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2780153469","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.96724784,0.000009778668,0.020982243,0.00593987,0.00080656155,0.00039711734,0.000010427296,0.00016074575,0.0044453857],"genre_scores_gemma":[0.99112463,0.0000017686485,0.0076735546,0.0004996529,0.0004401412,0.0000057272387,0.0000032843654,0.000023385586,0.00022787893],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9977656,0.0000225059,0.0004374582,0.00082767743,0.0005442461,0.00040251136],"domain_scores_gemma":[0.9976789,0.00009106446,0.0006886314,0.0009527871,0.00040004269,0.00018862613],"candidate_categories":["metaepi_narrow","sts","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.0008830107,0.0002936613,0.00035872668,0.00017000284,0.002090985,0.0020837972,0.003069931,0.00009865311,0.000006429315],"category_scores_gemma":[0.0005541915,0.00024591724,0.00006609516,0.00017972947,0.0002380382,0.0008606857,0.0021408638,0.00033264892,0.000002804024],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00011537483,0.00017990584,0.02480531,0.0006443438,0.00021997008,0.000011567942,0.0297318,0.0031778538,0.10563322,0.6976002,0.027741872,0.110138595],"study_design_scores_gemma":[0.0029177596,0.00066658726,0.0899237,0.0035229975,0.00015714427,0.0000956165,0.0053812154,0.7833179,0.039388463,0.068143524,0.0042048134,0.00228025],"about_ca_topic_score_codex":0.00011487022,"about_ca_topic_score_gemma":0.00002341558,"teacher_disagreement_score":0.7801401,"about_ca_system_score_codex":0.000049535058,"about_ca_system_score_gemma":0.000066484834,"threshold_uncertainty_score":0.9999993},"labels":[],"label_agreement":null},{"id":"W2793871201","doi":"10.1609/hcomp.v5i1.13309","title":"Lessons from an Online Massive Genomics Computer Game","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"Canadian Institutes of Health Research; Genome Canada","keywords":"Computer science; Crowdsourcing; Casual; Task (project management); Data science; Matching (statistics); Citizen science; Artificial intelligence; Human–computer interaction; World Wide Web; Engineering; Biology","score_opus":0.09804089691815192,"score_gpt":0.3325677333469798,"score_spread":0.23452683642882788,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2793871201","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.95835215,0.000015861528,0.036351595,0.0033196416,0.0002653085,0.00020412648,0.000010499064,0.00012313244,0.0013576872],"genre_scores_gemma":[0.9863022,0.000007808877,0.012934999,0.00038579138,0.0002257825,0.0000044162766,0.000005557623,0.000022705486,0.000110738314],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99824333,0.000029546636,0.00040367155,0.0006680756,0.00034373451,0.00031162112],"domain_scores_gemma":[0.99802285,0.000066760236,0.0008071286,0.00054581,0.00040384568,0.00015357495],"candidate_categories":["sts","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00030848515,0.00028210855,0.00035835302,0.00011528965,0.0013252816,0.0017572035,0.0016282285,0.00010963741,0.000007815699],"category_scores_gemma":[0.00005593614,0.00023536975,0.00010029032,0.00008737582,0.0003004663,0.00074286415,0.0007490575,0.00033598056,0.000004747174],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00008725408,0.0006986912,0.014319939,0.00020774025,0.0002138362,0.000011175563,0.026829671,0.0068808086,0.16328765,0.46227354,0.0009460203,0.32424366],"study_design_scores_gemma":[0.0011642555,0.0002944004,0.11505187,0.00062810554,0.000047462887,0.000017185197,0.0006985479,0.83223647,0.011673261,0.037408724,0.00019248122,0.0005872165],"about_ca_topic_score_codex":0.00015623061,"about_ca_topic_score_gemma":0.000024164663,"teacher_disagreement_score":0.8253557,"about_ca_system_score_codex":0.000048520393,"about_ca_system_score_gemma":0.000058017635,"threshold_uncertainty_score":0.99997485},"labels":[],"label_agreement":null},{"id":"W2794325666","doi":"10.1609/hcomp.v5i1.13307","title":"Deja Vu: Characterizing Worker Reliability Using Task Consistency","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada; National Science Foundation","keywords":"Consistency (knowledge bases); Task (project management); Reliability (semiconductor); Metric (unit); Computer science; Context (archaeology); Consistency model; Measure (data warehouse); Reliability engineering; Data mining; Artificial intelligence; Data consistency; Engineering; Geography; Operations management; Database","score_opus":0.06568918211076574,"score_gpt":0.3058742671703849,"score_spread":0.24018508505961916,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2794325666","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9813384,0.000025738922,0.009604414,0.0012582252,0.00028870095,0.00027722865,0.0000015403626,0.00012030287,0.00708545],"genre_scores_gemma":[0.99356836,0.0000050530275,0.005924912,0.00021292947,0.0000830273,0.00000604766,5.868292e-7,0.000020422178,0.00017867655],"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99798506,0.00003611805,0.00052461127,0.0006606278,0.00041504964,0.000378512],"domain_scores_gemma":[0.99769896,0.000079843754,0.0009906967,0.0005658284,0.0005364556,0.00012823415],"candidate_categories":["sts","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.00083054916,0.00029265063,0.0003927334,0.00012878647,0.0025186732,0.001629381,0.001184175,0.00011320816,0.000007448294],"category_scores_gemma":[0.00028767675,0.00024116712,0.00013394386,0.00014899512,0.0004462282,0.0008293572,0.0007230587,0.00034495743,0.0000039840143],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00009133115,0.0003345876,0.086614124,0.00064137275,0.00013609607,0.000007795934,0.012175172,0.0010602864,0.5687669,0.25200355,0.00042771347,0.07774106],"study_design_scores_gemma":[0.0023733897,0.00035578452,0.27128103,0.004209427,0.00015094686,0.00015973947,0.0014101743,0.6205763,0.0525928,0.04481362,0.00045877896,0.0016179492],"about_ca_topic_score_codex":0.00007851683,"about_ca_topic_score_gemma":0.0000031575048,"teacher_disagreement_score":0.6195161,"about_ca_system_score_codex":0.000062787716,"about_ca_system_score_gemma":0.00007633049,"threshold_uncertainty_score":0.99940705},"labels":[],"label_agreement":null},{"id":"W2811506820","doi":"10.1609/hcomp.v6i1.13323","title":"Towards Quantifying Behaviour in Social Crowdsourcing Communities","year":2018,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Crowdsourcing; Quality (philosophy); Proxy (statistics); Process (computing); Psychology; Computer science; Social psychology; Knowledge management; Applied psychology; Machine learning; World Wide Web","score_opus":0.1231108802447632,"score_gpt":0.3553618345602627,"score_spread":0.23225095431549952,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2811506820","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.97532904,0.000007498228,0.01472526,0.001433068,0.00012096731,0.00023302014,0.0000017546494,0.00013314684,0.00801623],"genre_scores_gemma":[0.9941709,0.0000011006615,0.005332465,0.0002830841,0.00009209984,0.000013382638,0.0000014425906,0.00001765645,0.0000878781],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.99845,0.0000426903,0.0005024121,0.00030596158,0.00039318783,0.00030573926],"domain_scores_gemma":[0.9986303,0.00006987826,0.0004099643,0.00016337504,0.000680406,0.00004608255],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0007233112,0.00021328023,0.00028143395,0.0003568226,0.0011644101,0.0006298592,0.0010335931,0.000092101945,0.000017101864],"category_scores_gemma":[0.00006393635,0.00019191069,0.000064962675,0.00080065604,0.00036820333,0.00059330044,0.0006275774,0.00037947926,0.0000057033953],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000024597699,0.00018397743,0.12649658,0.0001421785,0.000031479307,9.562332e-7,0.07895024,0.00010259409,0.010349687,0.76638013,0.00051256333,0.016825],"study_design_scores_gemma":[0.0026873413,0.0006829427,0.73245347,0.001973143,0.000054715543,0.00005061889,0.024880078,0.13566588,0.021879513,0.0780561,0.0002161289,0.0014000948],"about_ca_topic_score_codex":0.00022029446,"about_ca_topic_score_gemma":0.00006178843,"teacher_disagreement_score":0.68832403,"about_ca_system_score_codex":0.00006309063,"about_ca_system_score_gemma":0.00006187771,"threshold_uncertainty_score":0.89558196},"labels":[],"label_agreement":null},{"id":"W2999004052","doi":"10.1609/hcomp.v7i1.5274","title":"A Hybrid Approach to Identifying Unknown Unknowns of Predictive Models","year":2019,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Crowdsourcing; Computer science; Task (project management); Machine learning; Set (abstract data type); Artificial intelligence; Data mining; Engineering","score_opus":0.04897807387878888,"score_gpt":0.27072037477125627,"score_spread":0.22174230089246738,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2999004052","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.797131,0.000021757374,0.17783271,0.00013837873,0.0001232302,0.00047791572,0.0000021810026,0.000091137925,0.024181664],"genre_scores_gemma":[0.9917678,0.000002920156,0.00770943,0.0000983152,0.000029825476,0.0000133945905,0.0000011526103,0.000019729798,0.0003574003],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9980239,0.000030319718,0.00050521106,0.0006084478,0.00052561535,0.00030649698],"domain_scores_gemma":[0.9984873,0.0000712737,0.00046985532,0.0002723586,0.00057980075,0.00011939304],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00055494596,0.00025007373,0.00040800645,0.00027896703,0.00027534203,0.00030827278,0.000831425,0.000063141284,0.000003807334],"category_scores_gemma":[0.00004605139,0.00020759548,0.0001167881,0.00041640032,0.0001121103,0.000523778,0.00051353563,0.00024847192,0.0000050810236],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.000077313634,0.0003140203,0.002375832,0.0008608889,0.00011240324,5.681299e-7,0.021250818,0.08310567,0.12707734,0.75488573,0.00038343016,0.009555968],"study_design_scores_gemma":[0.00069036515,0.0003023078,0.0029251182,0.0010024848,0.00003147712,0.00002394307,0.0010663616,0.92493165,0.036427863,0.032217108,0.000025872152,0.00035544118],"about_ca_topic_score_codex":0.000028287774,"about_ca_topic_score_gemma":5.5509037e-7,"teacher_disagreement_score":0.84182596,"about_ca_system_score_codex":0.00004459666,"about_ca_system_score_gemma":0.00005393014,"threshold_uncertainty_score":0.8465498},"labels":[],"label_agreement":null},{"id":"W4306694236","doi":"10.1609/hcomp.v10i1.21986","title":"Eliciting and Learning with Soft Labels from Every Annotator","year":2022,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":26,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Engineering and Physical Sciences Research Council; Canadian Institute for Advanced Research","keywords":"Computer science; Categorical variable; Crowdsourcing; Artificial intelligence; Machine learning; Robustness (evolution); Generalization; Set (abstract data type); Soft skills; Natural language processing; World Wide Web","score_opus":0.02357610624059404,"score_gpt":0.24056210956220803,"score_spread":0.216986003321614,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4306694236","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9888698,0.00007159366,0.0083537195,0.0005863637,0.00006145684,0.00017424808,0.0000022634392,0.00014206479,0.0017385028],"genre_scores_gemma":[0.99569577,0.0000027507979,0.0037358454,0.0002904182,0.000040100767,0.000015734582,0.000001866767,0.000019704676,0.00019783316],"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9984179,0.000064192114,0.00029637304,0.0005142028,0.00044679362,0.00026053906],"domain_scores_gemma":[0.99899834,0.00017582708,0.00041216717,0.00012408342,0.00020519523,0.00008438285],"candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.00052811287,0.00021099724,0.00026013207,0.0001338691,0.0017509123,0.00048361908,0.00042064805,0.000036856047,0.000017420094],"category_scores_gemma":[0.000064798565,0.00017476703,0.00004204718,0.00032620982,0.00013343316,0.00030559034,0.00058553653,0.0005419081,0.0000011817214],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00029692295,0.00039675328,0.07686459,0.0005467107,0.00033793875,0.000021082966,0.10627192,0.038681407,0.3129994,0.18740264,0.001323573,0.27485704],"study_design_scores_gemma":[0.00374205,0.0017876747,0.040305242,0.0014723409,0.0001374853,0.00024862116,0.024649762,0.8725655,0.017339893,0.034865353,0.0011823636,0.0017037314],"about_ca_topic_score_codex":0.00008521218,"about_ca_topic_score_gemma":0.0000022613438,"teacher_disagreement_score":0.83388406,"about_ca_system_score_codex":0.000038747417,"about_ca_system_score_gemma":0.000042661344,"threshold_uncertainty_score":0.9995487},"labels":[],"label_agreement":null},{"id":"W4388332620","doi":"10.1609/hcomp.v11i1.27544","title":"Informing Users about Data Imputation: Exploring the Design Space for Dealing With Non-Responses","year":2023,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Innovative Human-Technology Interaction","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of Toronto","keywords":"Imputation (statistics); Computer science; Software deployment; Autonomy; Missing data; Information retrieval; Data science; Machine learning","score_opus":0.23564270332806844,"score_gpt":0.3595694917627186,"score_spread":0.12392678843465013,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4388332620","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.5819531,0.000005930747,0.4135877,0.0030692783,0.00013611198,0.0005725428,0.000002424449,0.00021573176,0.00045719903],"genre_scores_gemma":[0.9817005,0.000006406209,0.017895434,0.00014405466,0.000040130635,0.00007251481,0.0000037046632,0.00001637213,0.000120931036],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9987622,0.000016592941,0.00031168418,0.0003883617,0.00027655638,0.0002446077],"domain_scores_gemma":[0.9983287,0.00039029255,0.00043919627,0.0002574474,0.00055667234,0.000027706597],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0009810514,0.00018014737,0.00017310641,0.0002479744,0.0009491003,0.00044109102,0.0010953791,0.00004473283,0.0000019356576],"category_scores_gemma":[0.00020690524,0.00012050781,0.000029897892,0.0005300152,0.0001635185,0.0012145645,0.0004707079,0.00023005789,0.0000034141806],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00022317664,0.00003382747,0.0015354084,0.00022724293,0.00013017574,0.0000016718261,0.018515935,0.009756989,0.028478784,0.87935716,0.0008913186,0.060848325],"study_design_scores_gemma":[0.00089133385,0.00041218553,0.011067354,0.0007717104,0.000038119553,0.000030922856,0.0039862404,0.92277604,0.04044139,0.018970335,0.00025980704,0.00035453727],"about_ca_topic_score_codex":0.000014714379,"about_ca_topic_score_gemma":0.0000037969726,"teacher_disagreement_score":0.91301906,"about_ca_system_score_codex":0.000034004483,"about_ca_system_score_gemma":0.00005479786,"threshold_uncertainty_score":0.7299808},"labels":[],"label_agreement":null},{"id":"W4403434583","doi":"10.1609/hcomp.v12i1.31597","title":"Disclosures & Disclaimers: Investigating the Impact of Transparency Disclosures and Reliability Disclaimers on Learner-LLM Interactions","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Big Data and Business Intelligence","field":"Business, Management and Accounting","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Transparency (behavior); Reliability (semiconductor); Psychology; Political science; Law; Physics; Thermodynamics","score_opus":0.11561589765017749,"score_gpt":0.3606875602226767,"score_spread":0.24507166257249918,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403434583","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.99159765,0.000076800265,0.000378124,0.0024414388,0.0001368658,0.0002644578,0.000012104201,0.00005971202,0.0050328546],"genre_scores_gemma":[0.9995122,0.0000187415,0.000057632726,0.00007131219,0.00014717678,0.000012378651,0.000012034719,0.000020027685,0.00014849704],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.99869436,0.000013732734,0.00042845635,0.0003824699,0.00029112893,0.00018985356],"domain_scores_gemma":[0.99914646,0.00014937604,0.000328445,0.00013894377,0.00021433586,0.00002243706],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.00041676883,0.00024163313,0.00024761405,0.00017788418,0.00050069846,0.00068169774,0.00031673416,0.00004870429,0.000066837725],"category_scores_gemma":[0.00019559458,0.00013312911,0.00012797896,0.00043542008,0.00052650605,0.0008675594,0.00016206063,0.00033929222,0.0000049764417],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00035118725,0.0008074711,0.37176707,0.0058453167,0.00050265883,0.000001193577,0.017874774,0.012896624,0.075710975,0.43764573,0.0075869006,0.069010094],"study_design_scores_gemma":[0.0008350008,0.0003457498,0.68271226,0.009096797,0.0005715297,0.000016748703,0.0076827565,0.14671691,0.0037866228,0.14563154,0.0013823421,0.0012217198],"about_ca_topic_score_codex":0.0006788652,"about_ca_topic_score_gemma":0.000048528746,"teacher_disagreement_score":0.3109452,"about_ca_system_score_codex":0.000023831917,"about_ca_system_score_gemma":0.000026408321,"threshold_uncertainty_score":0.6573627},"labels":[],"label_agreement":null},{"id":"W4403434584","doi":"10.1609/hcomp.v12i1.31596","title":"“Hi. I’m Molly, Your Virtual Interviewer!” Exploring the Impact of Race and Gender in AI-Powered Virtual Interview Experiences","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"AI in Service Interactions","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Lehigh Hanson (Canada)","funders":"","keywords":"Interview; Race (biology); Psychology; Applied psychology; Gender studies; Sociology; Anthropology","score_opus":0.14233961263749645,"score_gpt":0.371991523183155,"score_spread":0.22965191054565853,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403434584","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9848084,0.00027195213,0.012526485,0.0011414733,0.0002873903,0.00022336804,0.0000015777658,0.00006183698,0.0006775466],"genre_scores_gemma":[0.999423,0.000048855523,0.00026063068,0.00011677023,0.000034459015,0.000045524623,4.3252388e-7,0.000013369692,0.000056941568],"study_design_codex":"qualitative","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9985285,0.000056649693,0.0005057772,0.00040862113,0.00029183045,0.00020866812],"domain_scores_gemma":[0.9991354,0.00016398789,0.00023798257,0.00015820212,0.00024587847,0.000058555277],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0006009336,0.00022086245,0.00029078766,0.00024807765,0.00019540908,0.00056605204,0.00079058873,0.000045257075,0.000025470803],"category_scores_gemma":[0.000062921565,0.00014078026,0.00013914212,0.00047840134,0.00018455724,0.0011248396,0.000586458,0.00035632576,0.0000026474463],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00005469171,0.00020235864,0.0032296919,0.00056402036,0.00022763974,0.000003788214,0.6672905,0.0019933577,0.022200665,0.19842078,0.00052104605,0.105291426],"study_design_scores_gemma":[0.0009451567,0.0014904191,0.040037293,0.005358444,0.00007134855,0.00013160975,0.08372224,0.8185149,0.010638577,0.03820734,0.00009665146,0.00078602106],"about_ca_topic_score_codex":0.00008482338,"about_ca_topic_score_gemma":0.0000066327634,"teacher_disagreement_score":0.8165215,"about_ca_system_score_codex":0.000060808474,"about_ca_system_score_gemma":0.000057299058,"threshold_uncertainty_score":0.57408524},"labels":[],"label_agreement":null},{"id":"W4403434723","doi":"10.1609/hcomp.v12i1.31600","title":"Unveiling the Inter-Related Preferences of Crowdworkers: Implications for Personalized and Flexible Platform Design","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Digital Marketing and Social Media","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Human–computer interaction; Computer science; World Wide Web","score_opus":0.13223452419629075,"score_gpt":0.3694382266520061,"score_spread":0.23720370245571534,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403434723","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"codex-gemma-dda1882f352a","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.96233034,0.00030434097,0.0026585879,0.0037635097,0.0001465742,0.00066582015,0.000006862075,0.00009616469,0.030027796],"genre_scores_gemma":[0.9984956,0.000036175992,0.0003557717,0.000028996094,0.000039840328,0.000027419857,0.0000010697153,0.000008242107,0.0010069268],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.9992655,0.000026186875,0.00023846023,0.00017805393,0.0001497126,0.00014207553],"domain_scores_gemma":[0.9988865,0.0006260447,0.00016426244,0.00003481015,0.00024571054,0.00004266817],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0010174459,0.00009227886,0.00014621069,0.00006545638,0.000652873,0.00035934153,0.00019345993,0.000060774782,0.000010721104],"category_scores_gemma":[0.00036526052,0.000061823295,0.000058500133,0.00021838465,0.0005570187,0.0001912853,0.000046595418,0.00011492421,4.3953102e-7],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00004945714,0.00002892873,0.0014519747,0.00024837913,0.000066033965,2.4102146e-8,0.070188396,0.000034851877,0.0023890196,0.87531483,0.00057068997,0.04965742],"study_design_scores_gemma":[0.0010359724,0.00049575523,0.010793538,0.004091253,0.00022973596,0.0000035984906,0.10032152,0.018648347,0.0025917073,0.858697,0.0025760294,0.0005155615],"about_ca_topic_score_codex":0.000060771155,"about_ca_topic_score_gemma":0.0000106830685,"teacher_disagreement_score":0.049141858,"about_ca_system_score_codex":0.000022526325,"about_ca_system_score_gemma":0.0000834265,"threshold_uncertainty_score":0.50214374},"labels":[],"label_agreement":null}]}