{"id":"W2988314392","doi":"10.22148/16.049","title":"A Shared Task for the Digital Humanities Chapter 2: Evaluating Annotation Guidelines","year":2019,"lang":"en","type":"article","venue":"Journal of Cultural Analytics","topic":"Natural Language Processing Techniques","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Guideline; Annotation; Task (project management); Section (typography); Computer science; Data science; Artificial intelligence; Political science; Engineering; Systems engineering","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0004400817,0.00009930227,0.0001543452,0.00006475738,0.0001074268,0.0006875288,0.0006813385,0.00003307668,0.00000864346],"category_scores_gemma":[0.0005134908,0.00004937507,0.0001491836,0.0001213658,0.0000252599,0.001411406,0.00008158749,0.0001351314,0.000004578614],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00003419332,"about_ca_system_score_gemma":0.00003553005,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000002071346,"about_ca_topic_score_gemma":0.000001214916,"domain_scores_codex":[0.9989566,0.00001101795,0.0004312768,0.00009892352,0.000384575,0.0001176599],"domain_scores_gemma":[0.9974825,0.0001571689,0.0005021181,0.000149214,0.001684718,0.00002428416],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002308572,0.0001212296,0.001131758,0.0002820738,0.0007565171,0.00003825226,0.01477267,0.005629469,0.0456671,0.2856811,0.0304919,0.6151971],"study_design_scores_gemma":[0.001233227,0.001251709,0.0004563506,0.0005597718,0.0001914391,0.0004245721,0.002257594,0.8769921,0.005303404,0.09174637,0.01900338,0.0005801012],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.09434935,0.008749652,0.8791858,0.01499478,0.001134822,0.0006680347,0.00002481141,0.0001922852,0.0007004633],"genre_scores_gemma":[0.7944111,0.00002047229,0.2037668,0.00055717,0.0003017345,0.00000187977,0.000002655195,0.0000064561,0.0009317147],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.8713626,"threshold_uncertainty_score":0.6629856,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1018420387922206,"score_gpt":0.3715093349101505,"score_spread":0.2696672961179299,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}