{"id":"W2773532513","doi":"10.26615/978-954-452-049-6_015","title":"Inter-Annotator Agreement in Sentiment Analysis: Machine Learning Perspective","year":2017,"lang":"en","type":"article","venue":"","topic":"Sentiment Analysis and Opinion Mining","field":"Computer Science","cited_by":69,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Ottawa; Dalhousie University","funders":"","keywords":"Annotation; Computer science; Perspective (graphical); Artificial intelligence; Sentiment analysis; Reliability (semiconductor); Agreement; Natural language processing; Machine learning; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005388678,0.0001457091,0.0002866752,0.0004744246,0.0003219262,0.0006395442,0.001003563,0.00002997142,0.0004551056],"category_scores_gemma":[0.00005004896,0.0001237374,0.0002342659,0.0004196054,0.0000315819,0.0005017627,0.0006042543,0.0001529847,0.0001313804],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001229523,"about_ca_system_score_gemma":0.00001901108,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001197276,"about_ca_topic_score_gemma":0.0006795838,"domain_scores_codex":[0.9985428,0.00007959088,0.0002916467,0.0005148821,0.0003080838,0.0002630276],"domain_scores_gemma":[0.9987485,0.00002825165,0.0002338088,0.0008270996,0.00008442376,0.0000779585],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001076987,0.0002771348,0.9157192,0.00000415552,0.001706774,0.0000582175,0.004798945,0.001970793,0.0002801128,0.05983972,0.000349863,0.01498437],"study_design_scores_gemma":[0.0006653339,0.00007324352,0.1127763,0.00002052711,0.0001570586,0.0000010868,0.001400868,0.8805507,0.001193901,0.0003564609,0.002495868,0.0003086347],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1186216,0.0004461149,0.8327725,0.006666225,0.0004442141,0.0002467629,0.000001355779,0.0001429069,0.04065833],"genre_scores_gemma":[0.9863483,0.00001701312,0.007078104,0.0001133886,0.0000328081,0.000008108155,0.000003201202,0.00000418857,0.006394855],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8785799,"threshold_uncertainty_score":0.6167139,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02254186839848775,"score_gpt":0.3058372293485844,"score_spread":0.2832953609500967,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}