{"id":"W2741691725","doi":"10.18653/v1/p17-2074","title":"Best-Worst Scaling More Reliable than Rating Scales: A Case Study on Sentiment Intensity Annotation","year":2017,"lang":"en","type":"preprint","venue":"","topic":"Sentiment Analysis and Opinion Mining","field":"Computer Science","cited_by":30,"is_retracted":false,"has_abstract":true,"ca_institutions":"National Research Council Canada","funders":"","keywords":"Annotation; Consistency (knowledge bases); Rating scale; Computer science; Set (abstract data type); Scaling; Scale (ratio); Quality (philosophy); Data mining; Data set; Information retrieval; Artificial intelligence; Natural language processing; Machine learning; Statistics; Mathematics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","scholarly_communication"],"consensus_categories":[],"category_scores_codex":[0.001256242,0.0004608735,0.0006523706,0.0003655597,0.0009515879,0.001832426,0.001193502,0.0001793462,0.00001826034],"category_scores_gemma":[0.0000878371,0.0004064762,0.0003060046,0.0002086855,0.00005285925,0.0003948367,0.002553917,0.0006189909,0.00008983937],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00014576,"about_ca_system_score_gemma":0.0000878353,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002393219,"about_ca_topic_score_gemma":0.0001704231,"domain_scores_codex":[0.9965354,0.0001258586,0.0007106768,0.001406849,0.0008127499,0.0004084323],"domain_scores_gemma":[0.9963903,0.00009350817,0.000768581,0.002194004,0.0003931753,0.0001604564],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001732997,0.009472206,0.5628152,0.000621738,0.004184113,0.02121716,0.141636,0.1517254,0.0005171314,0.003365927,0.009844774,0.09442698],"study_design_scores_gemma":[0.0008038781,0.0002666271,0.00490901,0.0008420335,0.0002028049,0.0001911051,0.02171054,0.9686735,0.001076935,0.0003270252,0.0000564962,0.0009400453],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9129547,0.00007215883,0.08222537,0.0007164526,0.001463759,0.0007811369,0.000002436748,0.0002081803,0.001575769],"genre_scores_gemma":[0.9767469,0.000009847551,0.02156484,0.0001571056,0.0002445083,0.00004760713,0.0000288565,0.00002257685,0.001177742],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.8169481,"threshold_uncertainty_score":0.9998387,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.0678353615902395,"score_gpt":0.3486248772234626,"score_spread":0.2807895156332231,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}