{"id":"W2534147738","doi":"10.1145/2983323.2983694","title":"Optimizing Nugget Annotations with Active Learning","year":2016,"lang":"en","type":"article","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"Natural Sciences and Engineering Research Council of Canada; University of Waterloo; Google","keywords":"Automatic summarization; Computer science; Annotation; Sentence; Process (computing); Sequence (biology); Precision and recall; Recall; Natural language processing; Artificial intelligence; Information retrieval; Programming language","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.007440864,0.002638204,0.002217495,0.003399201,0.001647589,0.002940308,0.00471112,0.00392733,0.00451604],"category_scores_gemma":[0.02740388,0.001002692,0.001344981,0.002672799,0.001202243,0.007225139,0.003312095,0.004097027,0.002696728],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001670321,"about_ca_system_score_gemma":0.001881991,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00803819,"about_ca_topic_score_gemma":0.01456024,"domain_scores_codex":[0.9949781,0.002107012,0.0002987205,0.001361817,0.0008960312,0.0003583767],"domain_scores_gemma":[0.9774914,0.01656082,0.000930752,0.002254979,0.002254357,0.0005077461],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001299678,0.001014508,0.00480748,0.0004083791,0.0002201116,0.0001953161,0.0009697989,0.183381,0.01395104,0.007847095,0.0222865,0.7636191],"study_design_scores_gemma":[0.00007486174,0.0001497515,0.0004279532,0.00003217568,0.00006530353,0.00006191583,0.000135726,0.979728,0.006281693,0.009799549,0.003214474,0.00002854223],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04505194,0.001335438,0.9361083,0.0005588782,0.000212182,0.0003370139,0.0006176841,0.01149429,0.004284277],"genre_scores_gemma":[0.5086246,0.0004678518,0.4710999,0.0007542155,0.0004243244,0.0006737875,0.003899856,0.001547321,0.01250811],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.00803819,"threshold_uncertainty_score":0.03935152,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01746247009185028,"score_gpt":0.236603102277309,"score_spread":0.2191406321854587,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}