{"id":"W2740478029","doi":"10.24963/ijcai.2017/413","title":"Obtaining High-Quality Label by Distinguishing between Easy and Hard Items in Crowdsourcing","year":2017,"lang":"en","type":"article","venue":"","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":16,"is_retracted":false,"has_abstract":true,"ca_institutions":"Novelis (Canada)","funders":"National Key Research and Development Program of China; National Natural Science Foundation of China","keywords":"Crowdsourcing; Quality (philosophy); Computer science; Set (abstract data type); Payment; Construct (python library); Scale (ratio); Machine learning; Artificial intelligence; Data science; Data mining; World Wide Web","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.002057279,0.0002465347,0.0004359375,0.00009678104,0.001067448,0.002333302,0.00105535,0.0001255372,0.000003873414],"category_scores_gemma":[0.001109055,0.0002398936,0.00004266727,0.0001293727,0.0001403192,0.0008933142,0.0007841908,0.0003662699,0.000009316033],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00006252135,"about_ca_system_score_gemma":0.00003769497,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003882777,"about_ca_topic_score_gemma":0.0001237075,"domain_scores_codex":[0.9976964,0.000156057,0.0004869466,0.0007508353,0.0003283322,0.0005814427],"domain_scores_gemma":[0.9977832,0.0004433413,0.0003264126,0.001177713,0.00006925468,0.0002000874],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00000676893,0.00005254555,0.8323616,0.00009026299,0.00003186005,0.00004827699,0.00241967,0.00005095587,0.009802856,0.0167938,0.0007802714,0.1375612],"study_design_scores_gemma":[0.002180214,0.00008651926,0.9539407,0.0004781925,0.00001852093,0.00002015138,0.0003661494,0.02583259,0.007814371,0.0044616,0.003628593,0.001172354],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8951638,0.0001084592,0.09731594,0.0009078073,0.0002501072,0.00012735,0.000004545162,0.0002384228,0.005883581],"genre_scores_gemma":[0.9797866,0.0000048075,0.01899432,0.0001682406,0.0001796319,0.000006236956,0.000003833351,0.000019942,0.0008364053],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1363888,"threshold_uncertainty_score":0.9987023,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04632613660003589,"score_gpt":0.3053481823572383,"score_spread":0.2590220457572024,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}