{"id":"W4306694236","doi":"10.1609/hcomp.v10i1.21986","title":"Eliciting and Learning with Soft Labels from Every Annotator","year":2022,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":26,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Engineering and Physical Sciences Research Council; Canadian Institute for Advanced Research","keywords":"Computer science; Categorical variable; Crowdsourcing; Artificial intelligence; Machine learning; Robustness (evolution); Generalization; Set (abstract data type); Soft skills; Natural language processing; World Wide Web","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["sts"],"consensus_categories":[],"category_scores_codex":[0.0005281129,0.0002109972,0.0002601321,0.0001338691,0.001750912,0.0004836191,0.0004206481,0.00003685605,0.00001742009],"category_scores_gemma":[0.00006479857,0.000174767,0.00004204718,0.0003262098,0.0001334332,0.0003055903,0.0005855365,0.0005419081,0.000001181721],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003874742,"about_ca_system_score_gemma":0.00004266134,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00008521218,"about_ca_topic_score_gemma":0.000002261344,"domain_scores_codex":[0.9984179,0.00006419211,0.000296373,0.0005142028,0.0004467936,0.0002605391],"domain_scores_gemma":[0.9989983,0.0001758271,0.0004121672,0.0001240834,0.0002051952,0.00008438285],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000296923,0.0003967533,0.07686459,0.0005467107,0.0003379387,0.00002108297,0.1062719,0.03868141,0.3129994,0.1874026,0.001323573,0.274857],"study_design_scores_gemma":[0.00374205,0.001787675,0.04030524,0.001472341,0.0001374853,0.0002486212,0.02464976,0.8725655,0.01733989,0.03486535,0.001182364,0.001703731],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9888698,0.00007159366,0.008353719,0.0005863637,0.00006145684,0.0001742481,0.000002263439,0.0001420648,0.001738503],"genre_scores_gemma":[0.9956958,0.000002750798,0.003735845,0.0002904182,0.00004010077,0.00001573458,0.000001866767,0.00001970468,0.0001978332],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8338841,"threshold_uncertainty_score":0.9995487,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02357610624059404,"score_gpt":0.240562109562208,"score_spread":0.216986003321614,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}