{"id":"W3158225217","doi":"10.48550/arxiv.2104.12690","title":"Towards Good Practices for Efficiently Annotating Large-Scale Image Classification Datasets","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Advanced Image and Video Retrieval Techniques","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada; Defense Advanced Research Projects Agency; Canadian Institute for Advanced Research","keywords":"Computer science; Annotation; Exploit; Crowdsourcing; Key (lock); Probabilistic logic; Machine learning; Ranking (information retrieval); Artificial intelligence; Automatic image annotation; Scale (ratio); Information retrieval; Supervised learning; Aggregate (composite); Image (mathematics); Data mining; Image retrieval; World Wide Web","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.03785955,0.002225596,0.001773755,0.003848511,0.002327292,0.00701601,0.009978939,0.004436723,0.00254743],"category_scores_gemma":[0.1205642,0.002608271,0.00182686,0.004283383,0.003917329,0.01436582,0.008918326,0.007933561,0.006576678],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004175295,"about_ca_system_score_gemma":0.005690346,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.008898018,"about_ca_topic_score_gemma":0.01706318,"domain_scores_codex":[0.9593822,0.02326243,0.002554213,0.006198317,0.007716001,0.0008868439],"domain_scores_gemma":[0.8976616,0.03490587,0.003595356,0.04593291,0.01604191,0.001862393],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0007744351,0.001473727,0.01173002,0.001225652,0.0003478518,0.0002635716,0.002613657,0.1705921,0.02577463,0.08936772,0.06514791,0.6306888],"study_design_scores_gemma":[0.0002022775,0.0001573823,0.001494132,0.0002463906,0.00005173714,0.0002209632,0.0005637539,0.7835125,0.02178897,0.1594425,0.03221469,0.0001046673],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.003398904,0.0003902873,0.9852447,0.00120296,0.00003702219,0.0002833833,0.0003284596,0.008195868,0.0009183292],"genre_scores_gemma":[0.02754677,0.0002259513,0.9684399,0.000330635,0.00003360719,0.0006221462,0.001416492,0.0008908277,0.0004936775],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.03785955,"threshold_uncertainty_score":0.2002228,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1064398660439206,"score_gpt":0.2820409161173054,"score_spread":0.1756010500733848,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}