{"id":"W2180780611","doi":"10.1609/hcomp.v3i1.13261","title":"Acquiring Reliable Ratings from the Crowd","year":2015,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Canadian Nautical Research Society","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Data science; Machine learning; World Wide Web","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.000871977,0.000254014,0.0002817416,0.00007827095,0.0007870893,0.0009746325,0.001074147,0.00008391559,0.000006668188],"category_scores_gemma":[0.0002091314,0.0001692457,0.00008893319,0.0003784193,0.0002250593,0.0004807774,0.0005647236,0.0003359666,0.00001058094],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005180709,"about_ca_system_score_gemma":0.00007960309,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0001907574,"about_ca_topic_score_gemma":0.000006741904,"domain_scores_codex":[0.9981419,0.0000368691,0.0004412305,0.0005133926,0.0005423862,0.0003242242],"domain_scores_gemma":[0.9983847,0.0001677813,0.0004604689,0.0003037068,0.0005432197,0.0001401237],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001300345,0.0003031933,0.04188797,0.0002313701,0.0002061816,0.000005944374,0.08021287,0.01089135,0.1317091,0.6536555,0.03056372,0.05020282],"study_design_scores_gemma":[0.003748355,0.0006593438,0.02583159,0.002695708,0.0001352855,0.00008450397,0.01118032,0.7080093,0.08039919,0.162809,0.002892583,0.001554828],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9680927,0.0001097877,0.01631741,0.003391738,0.0002956165,0.0002912967,0.000001559934,0.0001981667,0.01130166],"genre_scores_gemma":[0.9944018,0.000003899736,0.004389767,0.0006757743,0.0001355724,0.00001032395,0.000001225016,0.00001945242,0.0003622579],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.697118,"threshold_uncertainty_score":0.9398404,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0737925058077919,"score_gpt":0.2848671676757621,"score_spread":0.2110746618679702,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}