{"meta":{"query_hash":"0d7ccbc5d97c","filters":{"venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing"},"cohort_total":14,"direct_labels_cover":0,"predictions_cover":14,"exported":14,"export_cap":100000,"truncated":false,"label_status":"direct model label, unvalidated","prediction_status":"machine_predicted_unvalidated (Codex and Gemma teacher distillation)","score_status":"score_only:v0-immature-baseline","snapshot":{"source":"OpenAlex, pinned release, all 482 partitions","release":"2026-06-24","frame_built":"2026-07-12"},"permalink":"https://metacan.xera.ac/q/0d7ccbc5d97c","api":"https://metacan.xera.ac/api/v1/cohort?venue=Proceedings+of+the+AAAI+Conference+on+Human+Computation+and+Crowdsourcing"},"results":[{"id":"W1487101415","doi":"10.1609/hcomp.v2i1.13137","title":"Phylo and Open-Phylo: A Human-Computing Platform for Comparative Genomics","year":2014,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Scientific Computing and Data Management","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"","keywords":"Genomics; Comparative genomics; Computer science; Open science; Field (mathematics); Open source; Data science; Biology; Genetics; Genome; Gene","score_opus":0.3266005297275615,"score_gpt":0.44200370163255015,"score_spread":0.11540317190498867,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1487101415","genre_codex":"methods","genre_gemma":"software","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"software","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.049069937,0.001373245,0.643304,0.009272547,0.0019917756,0.002717717,0.049208477,0.1546543,0.08840799],"genre_scores_gemma":[0.2724117,0.00079796446,0.62686235,0.003127669,0.0003935056,0.0055006854,0.044721358,0.018502403,0.027682329],"study_design_codex":"not_applicable","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9978531,0.0007665838,0.00009087789,0.00041928663,0.0005430634,0.00032707793],"domain_scores_gemma":[0.9940241,0.0026701537,0.00031251975,0.0009885667,0.00051392266,0.0014908276],"candidate_categories":["open_science"],"consensus_categories":[],"category_scores_codex":[0.0046880394,0.00093756785,0.0008168385,0.002134824,0.0019371926,0.0020989948,0.0030204568,0.0018910061,0.023569496],"category_scores_gemma":[0.011921669,0.0006245429,0.0012928286,0.0017582212,0.0023213718,0.0052978327,0.00868574,0.0023870878,0.0069354777],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.005882374,0.00077234424,0.008611539,0.0011973972,0.00024906138,0.0009837425,0.005711588,0.011894281,0.03304524,0.22551757,0.5070443,0.19909061],"study_design_scores_gemma":[0.0006120569,0.00033589674,0.006159514,0.00019861692,0.00006321849,0.00032775136,0.00086103426,0.08155851,0.007981208,0.24728137,0.65439105,0.00022973659],"about_ca_topic_score_codex":0.0054213125,"about_ca_topic_score_gemma":0.007323035,"teacher_disagreement_score":0.99697953,"about_ca_system_score_codex":0.0013536583,"about_ca_system_score_gemma":0.00260962,"threshold_uncertainty_score":0.078847826},"labels":[],"label_agreement":null},{"id":"W1605921839","doi":"10.1609/hcomp.v1i1.13112","title":"Reducing Error in Context-Sensitive Crowdsourced Tasks","year":2013,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Luxmux Technology (Canada)","funders":"","keywords":"Computer science; Redundancy (engineering); Crowdsourcing; Task (project management); Workflow; Context (archaeology); Quality (philosophy); Hierarchy; Human–computer interaction; World Wide Web; Database; Engineering","score_opus":0.039345137894238796,"score_gpt":0.27670684857520716,"score_spread":0.23736171068096837,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1605921839","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.08351608,0.00059881865,0.91215,0.0005133609,0.00009471822,0.00018096429,0.000064587046,0.001211311,0.0016701096],"genre_scores_gemma":[0.8824542,0.00021185754,0.11540561,0.00021642724,0.00010621287,0.00014108703,0.00007315745,0.00019313669,0.0011982505],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99101126,0.0027800857,0.0004327221,0.002202208,0.0028790385,0.00069469836],"domain_scores_gemma":[0.97091067,0.014620722,0.0036909448,0.006201037,0.0033692773,0.0012073391],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009183922,0.0017484591,0.0019676145,0.0011451434,0.0017056584,0.0019152873,0.0036833494,0.001955168,0.0010807434],"category_scores_gemma":[0.046871793,0.0013432923,0.0008953311,0.001111486,0.0026887741,0.0035334108,0.006640292,0.0023763166,0.0005183287],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0010502646,0.0004668383,0.0080951,0.00035595993,0.00018331618,0.00032329053,0.0030463145,0.7798774,0.018390771,0.013201353,0.002743416,0.17226598],"study_design_scores_gemma":[0.00007103749,0.00024139008,0.00193468,0.00003856849,0.000044962566,0.00009962737,0.00025971522,0.9653032,0.0073446855,0.02292512,0.0016894009,0.00004757561],"about_ca_topic_score_codex":0.0074683735,"about_ca_topic_score_gemma":0.004448405,"teacher_disagreement_score":0.009183922,"about_ca_system_score_codex":0.001939114,"about_ca_system_score_gemma":0.0041247676,"threshold_uncertainty_score":0.048569858},"labels":[],"label_agreement":null},{"id":"W1926732392","doi":"10.1609/hcomp.v1i1.13110","title":"TrailView: Combining Gamification and Social Network Voting Mechanisms for Useful Data Collection","year":2013,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Incentive; Voting; Data collection; Computer science; Point (geometry); Competition (biology); Scheme (mathematics); Social network (sociolinguistics); Social worlds; Data science; World Wide Web; Social media; Sociology; Political science; Economics","score_opus":0.08141775148202438,"score_gpt":0.2917844663459198,"score_spread":0.21036671486389544,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W1926732392","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.03679787,0.00035237777,0.93865275,0.0010545673,0.00024607722,0.0014088769,0.000596372,0.009977241,0.010913896],"genre_scores_gemma":[0.54922765,0.00021540915,0.43796313,0.00038361043,0.0001610794,0.0018949427,0.0007756863,0.0005173876,0.00886107],"study_design_codex":"design_other","study_design_gemma":"not_applicable","domain_scores_codex":[0.99139625,0.0051529696,0.0003767039,0.0011743186,0.0014028264,0.0004969813],"domain_scores_gemma":[0.9835618,0.009733905,0.00082938606,0.0035530129,0.0013173444,0.0010045674],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.014176077,0.0014470974,0.0015290661,0.0029329807,0.0013163716,0.0024644027,0.00382789,0.0016445715,0.011191955],"category_scores_gemma":[0.030461911,0.00075667,0.0010493951,0.0024443632,0.001530987,0.006449749,0.008789231,0.0016739634,0.002474907],"study_design_candidate":"not_applicable","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0027278736,0.002088187,0.010035009,0.0005765886,0.00039751182,0.00037481482,0.0014677988,0.07148071,0.007444379,0.08621157,0.02397491,0.7932206],"study_design_scores_gemma":[0.0005609895,0.00056049746,0.0019775438,0.00010786405,0.000108555374,0.00014341189,0.0003216345,0.8138507,0.0048981565,0.15368237,0.023659648,0.00012870815],"about_ca_topic_score_codex":0.002841364,"about_ca_topic_score_gemma":0.0042918073,"teacher_disagreement_score":0.014176077,"about_ca_system_score_codex":0.0009822756,"about_ca_system_score_gemma":0.0018815077,"threshold_uncertainty_score":0.07497114},"labels":[],"label_agreement":null},{"id":"W2180780611","doi":"10.1609/hcomp.v3i1.13261","title":"Acquiring Reliable Ratings from the Crowd","year":2015,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Canadian Nautical Research Society","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Data science; Machine learning; World Wide Web","score_opus":0.0737925058077919,"score_gpt":0.28486716767576215,"score_spread":0.21107466186797025,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2180780611","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"methods","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"methods","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.18760791,0.004435758,0.77136827,0.002931576,0.00066055136,0.0007009526,0.00617605,0.00568512,0.0204339],"genre_scores_gemma":[0.7037353,0.0011108366,0.2778174,0.0008209937,0.0008564591,0.00038352018,0.006789055,0.0003962306,0.008090288],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.9906346,0.0025535792,0.0005063484,0.002741199,0.0030738893,0.0004903472],"domain_scores_gemma":[0.9759454,0.009358608,0.0023320934,0.0058049043,0.005627154,0.0009318802],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0050756526,0.0017487003,0.00293746,0.002659282,0.0011976092,0.002430967,0.0022101966,0.0021329224,0.0030121056],"category_scores_gemma":[0.033645652,0.0009831894,0.0008236773,0.0023579404,0.000817044,0.004934746,0.0040650223,0.0020149997,0.004668569],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0026848482,0.00091800955,0.039938368,0.0015198963,0.0005654275,0.0016206839,0.0029268588,0.048566446,0.07246398,0.010800918,0.061623123,0.7563714],"study_design_scores_gemma":[0.00024684175,0.0008719939,0.023251202,0.0002552427,0.0002398625,0.0015336112,0.0017034551,0.83059186,0.040323544,0.054371677,0.046288013,0.00032281305],"about_ca_topic_score_codex":0.005082074,"about_ca_topic_score_gemma":0.007056769,"teacher_disagreement_score":0.99492437,"about_ca_system_score_codex":0.000592791,"about_ca_system_score_gemma":0.0011419393,"threshold_uncertainty_score":0.026842892},"labels":[],"label_agreement":null},{"id":"W2780153469","doi":"10.1609/hcomp.v5i1.13300","title":"Drafty: Enlisting Users To Be Editors Who Maintain Structured Data","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"Brown University","keywords":"Computer science; Construct (python library); Matching (statistics); World Wide Web; User modeling; Information retrieval; Data science; User interface","score_opus":0.0971098124215099,"score_gpt":0.3317737254023328,"score_spread":0.2346639129808229,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2780153469","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.27081168,0.0027143245,0.41611373,0.014636034,0.0047035287,0.0071460158,0.022774955,0.22379416,0.037305597],"genre_scores_gemma":[0.30504817,0.0009659579,0.62483805,0.004336048,0.002391436,0.004928906,0.022475274,0.013118942,0.02189724],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.98070395,0.010557163,0.0019841732,0.002866028,0.0031704549,0.00071821327],"domain_scores_gemma":[0.7024991,0.15645364,0.018661918,0.09681355,0.015783504,0.0097883055],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.034133974,0.0015691321,0.0011276288,0.0033611548,0.0021193195,0.0049980474,0.0029259361,0.002112037,0.0126287],"category_scores_gemma":[0.22984643,0.0011814052,0.0009705288,0.002509744,0.0019936848,0.012196487,0.0107204,0.0019110315,0.009307933],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0028008765,0.0007994961,0.04564272,0.0040983963,0.00031402256,0.0010382918,0.029324345,0.0020464424,0.035635937,0.0066443807,0.40013877,0.47151637],"study_design_scores_gemma":[0.0009476078,0.0014037656,0.04258131,0.0007269387,0.00020221446,0.0014085295,0.007225887,0.021625854,0.025350686,0.017608454,0.88034755,0.00057122315],"about_ca_topic_score_codex":0.0015066859,"about_ca_topic_score_gemma":0.0038225313,"teacher_disagreement_score":0.034133974,"about_ca_system_score_codex":0.0006698364,"about_ca_system_score_gemma":0.004002551,"threshold_uncertainty_score":0.18051982},"labels":[],"label_agreement":null},{"id":"W2793871201","doi":"10.1609/hcomp.v5i1.13309","title":"Lessons from an Online Massive Genomics Computer Game","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":12,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"McGill University","funders":"Canadian Institutes of Health Research; Genome Canada","keywords":"Computer science; Crowdsourcing; Casual; Task (project management); Data science; Matching (statistics); Citizen science; Artificial intelligence; Human–computer interaction; World Wide Web; Engineering; Biology","score_opus":0.09804089691815192,"score_gpt":0.3325677333469798,"score_spread":0.23452683642882788,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2793871201","genre_codex":"other","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.35042578,0.0037428292,0.05539115,0.20129296,0.0021606055,0.00056961813,0.0020317025,0.0010352017,0.38335016],"genre_scores_gemma":[0.9399644,0.0020316758,0.0167256,0.009025782,0.00044661667,0.00026349735,0.000676852,0.00037022785,0.030495351],"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"observational","domain_scores_codex":[0.99830866,0.0009742006,0.00004825368,0.00018648474,0.00029898642,0.00018351094],"domain_scores_gemma":[0.99241793,0.0047769095,0.0002866594,0.00041270134,0.0008379792,0.0012679092],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002451192,0.00056849624,0.0003319514,0.00093183754,0.0026519,0.004913573,0.0018475463,0.0025574833,0.01208772],"category_scores_gemma":[0.020043952,0.00024436417,0.00036317421,0.0006924024,0.0027586285,0.0049665603,0.0027740188,0.0021745418,0.0028364963],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0008859328,0.0021499456,0.040688805,0.00082106073,0.00012301808,0.0030286727,0.026560854,0.013797779,0.0022976457,0.35350606,0.24838781,0.30775246],"study_design_scores_gemma":[0.00020377351,0.00068237906,0.016532067,0.0005617151,0.00004672795,0.0015230242,0.023948718,0.032261036,0.0019226187,0.4822639,0.4398724,0.00018163599],"about_ca_topic_score_codex":0.012781352,"about_ca_topic_score_gemma":0.016006954,"teacher_disagreement_score":0.012781352,"about_ca_system_score_codex":0.0015996143,"about_ca_system_score_gemma":0.0015933552,"threshold_uncertainty_score":0.04043752},"labels":[],"label_agreement":null},{"id":"W2794325666","doi":"10.1609/hcomp.v5i1.13307","title":"Deja Vu: Characterizing Worker Reliability Using Task Consistency","year":2017,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":15,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada; National Science Foundation","keywords":"Consistency (knowledge bases); Task (project management); Reliability (semiconductor); Metric (unit); Computer science; Context (archaeology); Consistency model; Measure (data warehouse); Reliability engineering; Data mining; Artificial intelligence; Data consistency; Engineering; Geography; Operations management; Database","score_opus":0.06568918211076574,"score_gpt":0.3058742671703849,"score_spread":0.24018508505961916,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2794325666","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":"methods","model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":"methods","domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.2668017,0.00085054984,0.72077173,0.0005402839,0.00018627319,0.000716035,0.00094268075,0.003086894,0.006103843],"genre_scores_gemma":[0.88268405,0.000113963586,0.11432421,0.00014192009,0.000071697796,0.00065805047,0.0005861067,0.00033107115,0.0010889452],"study_design_codex":"design_other","study_design_gemma":"observational","domain_scores_codex":[0.98131037,0.0070925625,0.0018783457,0.0035367517,0.005277734,0.00090423005],"domain_scores_gemma":[0.859104,0.07534956,0.019852422,0.028043423,0.015204653,0.002446028],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.018166753,0.0013313618,0.001265428,0.0029131642,0.0014006636,0.003067796,0.0026409344,0.0015700541,0.0013096207],"category_scores_gemma":[0.1581714,0.0008792727,0.00093011087,0.0021243333,0.0019416977,0.004787448,0.004089996,0.0019401135,0.0005855765],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0028458713,0.0010533927,0.2738426,0.0014825484,0.00075294,0.00049377885,0.0096296575,0.19471537,0.0479531,0.047822587,0.0112336,0.40817454],"study_design_scores_gemma":[0.00026009156,0.0020200615,0.10313266,0.00028302762,0.00028175645,0.00084778253,0.002160174,0.7385901,0.041598517,0.09697043,0.013372199,0.00048326366],"about_ca_topic_score_codex":0.003528017,"about_ca_topic_score_gemma":0.002074333,"teacher_disagreement_score":0.9818332,"about_ca_system_score_codex":0.0013807209,"about_ca_system_score_gemma":0.0021268213,"threshold_uncertainty_score":0.09607613},"labels":[],"label_agreement":null},{"id":"W2811506820","doi":"10.1609/hcomp.v6i1.13323","title":"Towards Quantifying Behaviour in Social Crowdsourcing Communities","year":2018,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Open Source Software Innovations","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Calgary","funders":"","keywords":"Crowdsourcing; Quality (philosophy); Proxy (statistics); Process (computing); Psychology; Computer science; Social psychology; Knowledge management; Applied psychology; Machine learning; World Wide Web","score_opus":0.1231108802447632,"score_gpt":0.3553618345602627,"score_spread":0.23225095431549952,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2811506820","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.44566214,0.0002710666,0.5482338,0.00041597727,0.000023218083,0.00048397513,0.0008478667,0.0005492707,0.0035127],"genre_scores_gemma":[0.8478249,0.00008927868,0.15055531,0.00004577245,0.00002228955,0.00025230538,0.00053325354,0.0000610784,0.00061581057],"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99420494,0.0027401687,0.00031474678,0.0011311343,0.0013125698,0.00029638532],"domain_scores_gemma":[0.9687996,0.016016794,0.0075440966,0.003138454,0.003333232,0.0011677089],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005713595,0.0010262658,0.0009127089,0.0069647916,0.0010993435,0.0029237526,0.0012136417,0.0012905992,0.000956139],"category_scores_gemma":[0.0408471,0.0007016199,0.0012483413,0.004232289,0.002548115,0.0034367628,0.0030826381,0.0013918463,0.00033160727],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00034475152,0.0005992206,0.43078437,0.0008611706,0.0007956657,0.00036059564,0.016612096,0.30190215,0.016412169,0.06016914,0.0019600266,0.1691987],"study_design_scores_gemma":[0.000029230152,0.00016782703,0.1050894,0.00012363525,0.00007326452,0.00014571322,0.0031985831,0.77128446,0.0036345266,0.1121151,0.003985293,0.00015291177],"about_ca_topic_score_codex":0.01664956,"about_ca_topic_score_gemma":0.011948882,"teacher_disagreement_score":0.01664956,"about_ca_system_score_codex":0.0024017913,"about_ca_system_score_gemma":0.0018512039,"threshold_uncertainty_score":0.033105254},"labels":[],"label_agreement":null},{"id":"W2999004052","doi":"10.1609/hcomp.v7i1.5274","title":"A Hybrid Approach to Identifying Unknown Unknowns of Predictive Models","year":2019,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Waterloo","funders":"","keywords":"Crowdsourcing; Computer science; Task (project management); Machine learning; Set (abstract data type); Artificial intelligence; Data mining; Engineering","score_opus":0.04897807387878888,"score_gpt":0.27072037477125627,"score_spread":0.22174230089246738,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W2999004052","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.010931425,0.00037713666,0.9851692,0.00079487165,0.000040368064,0.00014207608,0.0001956201,0.0009317782,0.0014175416],"genre_scores_gemma":[0.40887144,0.00039504387,0.58381957,0.0009624808,0.00039144218,0.00046846384,0.0011094288,0.00021280666,0.0037693032],"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.99290997,0.0025002097,0.0004458264,0.00202149,0.0017736277,0.0003487235],"domain_scores_gemma":[0.97323495,0.018714318,0.001958644,0.0037010773,0.0019341201,0.00045688957],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006632691,0.002531257,0.0032499293,0.0045759054,0.0019869986,0.005271351,0.0053534633,0.004033286,0.002343828],"category_scores_gemma":[0.031030627,0.0015457938,0.0023542843,0.0034057817,0.0029750934,0.005578817,0.005159665,0.0055452506,0.0011864167],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.00062119216,0.0005451645,0.009186961,0.00055818853,0.0006004997,0.00084718433,0.0015537477,0.50600606,0.0056617665,0.05553769,0.007979921,0.41090173],"study_design_scores_gemma":[0.00002781004,0.000051593324,0.0004198454,0.000051463987,0.000052215135,0.00014369554,0.000090497226,0.9437753,0.0014551999,0.05164784,0.0022482383,0.000036284102],"about_ca_topic_score_codex":0.006949726,"about_ca_topic_score_gemma":0.008463151,"teacher_disagreement_score":0.006949726,"about_ca_system_score_codex":0.001831484,"about_ca_system_score_gemma":0.0026989498,"threshold_uncertainty_score":0.035077393},"labels":[],"label_agreement":null},{"id":"W4306694236","doi":"10.1609/hcomp.v10i1.21986","title":"Eliciting and Learning with Soft Labels from Every Annotator","year":2022,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":26,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"Engineering and Physical Sciences Research Council; Canadian Institute for Advanced Research","keywords":"Computer science; Categorical variable; Crowdsourcing; Artificial intelligence; Machine learning; Robustness (evolution); Generalization; Set (abstract data type); Soft skills; Natural language processing; World Wide Web","score_opus":0.02357610624059404,"score_gpt":0.24056210956220803,"score_spread":0.216986003321614,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4306694236","genre_codex":"methods","genre_gemma":"methods","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"methods","genre_consensus":"methods","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.031140754,0.00051006844,0.93917567,0.0040179016,0.00050235074,0.0009666719,0.0033899203,0.003347327,0.016949423],"genre_scores_gemma":[0.22018263,0.00039005137,0.7511899,0.00258629,0.0004230202,0.004173973,0.007790813,0.0014338321,0.011829556],"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","domain_scores_codex":[0.948999,0.03356538,0.002291614,0.008119404,0.0058267824,0.001197835],"domain_scores_gemma":[0.8619913,0.06826486,0.007511863,0.041619428,0.017484428,0.0031280522],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.041469835,0.002886866,0.002105475,0.0029489712,0.0046104584,0.0041030375,0.003499218,0.0037801852,0.008652726],"category_scores_gemma":[0.11064955,0.0016264237,0.0018228143,0.0036416536,0.004149523,0.007918087,0.011557616,0.006739689,0.008014644],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.002411801,0.0009170792,0.030555815,0.0034151692,0.0004273346,0.0010574834,0.017174637,0.04485076,0.07421295,0.10574216,0.14507945,0.57415533],"study_design_scores_gemma":[0.00050057383,0.0005651163,0.0086469725,0.0010401154,0.0002817837,0.0009642007,0.0076034563,0.2619488,0.049119443,0.41204727,0.25682965,0.0004526556],"about_ca_topic_score_codex":0.0048381723,"about_ca_topic_score_gemma":0.018887948,"teacher_disagreement_score":0.041469835,"about_ca_system_score_codex":0.0030285546,"about_ca_system_score_gemma":0.0085644135,"threshold_uncertainty_score":0.219316},"labels":[],"label_agreement":null},{"id":"W4388332620","doi":"10.1609/hcomp.v11i1.27544","title":"Informing Users about Data Imputation: Exploring the Design Space for Dealing With Non-Responses","year":2023,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Innovative Human-Technology Interaction","field":"Computer Science","cited_by":1,"is_retracted":false,"has_abstract":true,"route_ca_aff":false,"route_ca_fund":true,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"","funders":"University of Toronto","keywords":"Imputation (statistics); Computer science; Software deployment; Autonomy; Missing data; Information retrieval; Data science; Machine learning","score_opus":0.23564270332806844,"score_gpt":0.3595694917627186,"score_spread":0.12392678843465013,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4388332620","genre_codex":"methods","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":null,"domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.057692323,0.00028000734,0.9272545,0.0049931994,0.00013767606,0.0034344145,0.00017315628,0.0034427226,0.0025920896],"genre_scores_gemma":[0.40312934,0.00016338147,0.5841725,0.0014717162,0.00014833806,0.008502962,0.0002554321,0.0005612469,0.0015951198],"study_design_codex":"design_other","study_design_gemma":"theoretical_or_conceptual","domain_scores_codex":[0.5473022,0.41452476,0.015277418,0.010890024,0.009848967,0.002156672],"domain_scores_gemma":[0.20005056,0.6881288,0.017910667,0.07382762,0.016886655,0.0031955792],"candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.29948127,0.0024954681,0.001977728,0.0025674028,0.0035923698,0.011352687,0.0064331754,0.0060418956,0.0068237125],"category_scores_gemma":[0.5800729,0.0025661855,0.0021478494,0.0018798134,0.0072512757,0.01693324,0.009696989,0.0063260794,0.0027228415],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.009374337,0.0021651657,0.04515202,0.005414869,0.00070082094,0.0016421194,0.23337685,0.014580348,0.030242568,0.077473104,0.007498003,0.5723798],"study_design_scores_gemma":[0.004934172,0.007288795,0.015439631,0.005362612,0.001126239,0.0023539406,0.0476301,0.41596645,0.06084306,0.32459447,0.11323729,0.0012232631],"about_ca_topic_score_codex":0.00083418324,"about_ca_topic_score_gemma":0.0010218677,"teacher_disagreement_score":0.29948127,"about_ca_system_score_codex":0.0026830288,"about_ca_system_score_gemma":0.004824609,"threshold_uncertainty_score":0.8638643},"labels":[],"label_agreement":null},{"id":"W4403434583","doi":"10.1609/hcomp.v12i1.31597","title":"Disclosures & Disclaimers: Investigating the Impact of Transparency Disclosures and Reliability Disclaimers on Learner-LLM Interactions","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Big Data and Business Intelligence","field":"Business, Management and Accounting","cited_by":9,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Transparency (behavior); Reliability (semiconductor); Psychology; Political science; Law; Physics; Thermodynamics","score_opus":0.11561589765017749,"score_gpt":0.3606875602226767,"score_spread":0.24507166257249918,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403434583","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.9775914,0.0002075791,0.01167886,0.0013081161,0.00009773936,0.00043794152,0.00016263528,0.00046765245,0.00804802],"genre_scores_gemma":[0.9911179,0.00007228222,0.006429297,0.00029505786,0.000039604252,0.00034734016,0.000079641126,0.000078221026,0.001540658],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9302277,0.04836952,0.004829378,0.003648866,0.0113584045,0.0015662255],"domain_scores_gemma":[0.4130424,0.4647113,0.07535342,0.0237022,0.01733529,0.0058553326],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.041468326,0.0006335202,0.00059456326,0.0009505257,0.0018716716,0.005190299,0.0017901575,0.0019078463,0.0059742527],"category_scores_gemma":[0.3869017,0.00063106994,0.00047027762,0.0005740681,0.0020220408,0.00579478,0.0040487773,0.002768978,0.0010813758],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.006734579,0.00664391,0.37393248,0.004095502,0.00045375997,0.0015590715,0.15314925,0.012422985,0.04642516,0.017133716,0.011808725,0.36564082],"study_design_scores_gemma":[0.0023206077,0.02122117,0.5335383,0.0030169499,0.0014865301,0.0023165876,0.097731866,0.11522778,0.09337744,0.047299113,0.08100658,0.0014570933],"about_ca_topic_score_codex":0.0010204391,"about_ca_topic_score_gemma":0.0011949104,"teacher_disagreement_score":0.041468326,"about_ca_system_score_codex":0.0018222913,"about_ca_system_score_gemma":0.0027783,"threshold_uncertainty_score":0.21930808},"labels":[],"label_agreement":null},{"id":"W4403434584","doi":"10.1609/hcomp.v12i1.31596","title":"“Hi. I’m Molly, Your Virtual Interviewer!” Exploring the Impact of Race and Gender in AI-Powered Virtual Interview Experiences","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"AI in Service Interactions","field":"Computer Science","cited_by":4,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"Lehigh Hanson (Canada)","funders":"","keywords":"Interview; Race (biology); Psychology; Applied psychology; Gender studies; Sociology; Anthropology","score_opus":0.14233961263749645,"score_gpt":0.371991523183155,"score_spread":0.22965191054565853,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403434584","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.92908806,0.0005861123,0.022849644,0.005300431,0.00046095627,0.00045009,0.00021214102,0.00012203548,0.040930487],"genre_scores_gemma":[0.97995937,0.00023907726,0.008496124,0.002156808,0.00007509618,0.0006299559,0.000064218155,0.000043972803,0.008335418],"study_design_codex":"qualitative","study_design_gemma":"observational","domain_scores_codex":[0.99115914,0.0075554205,0.00013058717,0.0003701967,0.0004011934,0.00038353784],"domain_scores_gemma":[0.99028134,0.0063809985,0.0011732251,0.0007878643,0.0007173436,0.0006592056],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.009901873,0.000269333,0.00022627962,0.0004116734,0.004171847,0.0026778039,0.0005398606,0.0005912759,0.007910112],"category_scores_gemma":[0.021463275,0.00021344995,0.00027041754,0.00030509607,0.00304527,0.0019245005,0.0024258983,0.0010424347,0.0016488824],"study_design_candidate":"observational","study_design_consensus":null,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0010806235,0.00033243524,0.049621623,0.0006171468,0.000058156682,0.0010848916,0.77245367,0.00039513689,0.014164557,0.017166562,0.021464575,0.12156063],"study_design_scores_gemma":[0.0001246121,0.001131649,0.039701544,0.0005121321,0.00007483542,0.0016331078,0.72214395,0.0021389488,0.006641889,0.011653116,0.21407004,0.0001742435],"about_ca_topic_score_codex":0.0012615968,"about_ca_topic_score_gemma":0.0020180196,"teacher_disagreement_score":0.009901873,"about_ca_system_score_codex":0.00065214926,"about_ca_system_score_gemma":0.0007220973,"threshold_uncertainty_score":0.052366674},"labels":[],"label_agreement":null},{"id":"W4403434723","doi":"10.1609/hcomp.v12i1.31600","title":"Unveiling the Inter-Related Preferences of Crowdworkers: Implications for Personalized and Flexible Platform Design","year":2024,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Digital Marketing and Social Media","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"route_ca_aff":true,"route_ca_fund":false,"route_ca_venue":false,"route_about_ca":false,"ca_institutions":"University of Toronto","funders":"","keywords":"Human–computer interaction; Computer science; World Wide Web","score_opus":0.13223452419629075,"score_gpt":0.3694382266520061,"score_spread":0.23720370245571534,"validation_status":"score_only:v0-immature-baseline","prediction":{"id":"W4403434723","genre_codex":"empirical","genre_gemma":"empirical","domain_codex":null,"domain_gemma":null,"model_version":"metacan-v3-hybrid-931329e0061c","genre_candidate":"empirical","genre_consensus":"empirical","domain_candidate":null,"domain_consensus":null,"prediction_status":"machine_predicted_unvalidated","genre_scores_codex":[0.96854234,0.00029108947,0.019167354,0.0018566098,0.000059020655,0.00025099274,0.00016462075,0.0001221675,0.009545959],"genre_scores_gemma":[0.98987484,0.0001497364,0.007968239,0.0002840048,0.000020485333,0.0002673625,0.00007643857,0.000042384625,0.0013164199],"study_design_codex":"observational","study_design_gemma":"observational","domain_scores_codex":[0.9961992,0.002028744,0.00018679017,0.0005427266,0.00063679524,0.00040576345],"domain_scores_gemma":[0.9873625,0.0078714285,0.0015127057,0.0009580295,0.0012563772,0.0010390038],"candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006699172,0.00046912025,0.00034656702,0.0013650696,0.0027724826,0.003739415,0.00089663855,0.0007946706,0.0033810122],"category_scores_gemma":[0.026615644,0.00034221562,0.00028170377,0.0007456086,0.0017631341,0.0031039715,0.0029845354,0.00092744234,0.0007183867],"study_design_candidate":"observational","study_design_consensus":"observational","about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_system_candidate":false,"about_ca_system_consensus":false,"study_design_scores_codex":[0.0009187619,0.000765778,0.39672902,0.0012622174,0.000118374985,0.0012739047,0.333616,0.0030400865,0.01921966,0.006838493,0.0069133174,0.22930443],"study_design_scores_gemma":[0.00010655795,0.0008286828,0.35901245,0.00077608036,0.00011889201,0.000749306,0.5370134,0.01116851,0.005100984,0.029551955,0.05530079,0.00027243464],"about_ca_topic_score_codex":0.0038973321,"about_ca_topic_score_gemma":0.008165504,"teacher_disagreement_score":0.006699172,"about_ca_system_score_codex":0.00092345243,"about_ca_system_score_gemma":0.0018127467,"threshold_uncertainty_score":0.03542906},"labels":[],"label_agreement":null}]}