{"id":"W2180780611","doi":"10.1609/hcomp.v3i1.13261","title":"Acquiring Reliable Ratings from the Crowd","year":2015,"lang":"en","type":"article","venue":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Canadian Nautical Research Society","funders":"","keywords":"Crowdsourcing; Computer science; Artificial intelligence; Data science; Machine learning; World Wide Web","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.005075653,0.0017487,0.00293746,0.002659282,0.001197609,0.002430967,0.002210197,0.002132922,0.003012106],"category_scores_gemma":[0.03364565,0.0009831894,0.0008236773,0.00235794,0.000817044,0.004934746,0.004065022,0.002015,0.004668569],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000592791,"about_ca_system_score_gemma":0.001141939,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005082074,"about_ca_topic_score_gemma":0.007056769,"domain_scores_codex":[0.9906346,0.002553579,0.0005063484,0.002741199,0.003073889,0.0004903472],"domain_scores_gemma":[0.9759454,0.009358608,0.002332093,0.005804904,0.005627154,0.0009318802],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.002684848,0.0009180096,0.03993837,0.001519896,0.0005654275,0.001620684,0.002926859,0.04856645,0.07246398,0.01080092,0.06162312,0.7563714],"study_design_scores_gemma":[0.0002468418,0.0008719939,0.0232512,0.0002552427,0.0002398625,0.001533611,0.001703455,0.8305919,0.04032354,0.05437168,0.04628801,0.0003228131],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1876079,0.004435758,0.7713683,0.002931576,0.0006605514,0.0007009526,0.00617605,0.00568512,0.0204339],"genre_scores_gemma":[0.7037353,0.001110837,0.2778174,0.0008209937,0.0008564591,0.0003835202,0.006789055,0.0003962306,0.008090288],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9949244,"threshold_uncertainty_score":0.02684289,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0737925058077919,"score_gpt":0.2848671676757621,"score_spread":0.2110746618679702,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}