{"id":"W3158225217","doi":"10.48550/arxiv.2104.12690","title":"Towards Good Practices for Efficiently Annotating Large-Scale Image Classification Datasets","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Advanced Image and Video Retrieval Techniques","field":"Computer Science","cited_by":3,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Toronto","funders":"Natural Sciences and Engineering Research Council of Canada; Defense Advanced Research Projects Agency; Canadian Institute for Advanced Research","keywords":"Computer science; Annotation; Exploit; Crowdsourcing; Key (lock); Probabilistic logic; Machine learning; Ranking (information retrieval); Artificial intelligence; Automatic image annotation; Scale (ratio); Information retrieval; Supervised learning; Aggregate (composite); Image (mathematics); Data mining; Image retrieval; World Wide Web","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0006159322,0.0003171921,0.0003447001,0.0001907797,0.0002916178,0.000473312,0.001776312,0.0002449451,0.000009706812],"category_scores_gemma":[0.0005235879,0.0003709278,0.0002126166,0.0007053224,0.00007588307,0.001810592,0.002118203,0.0005250297,0.00001326922],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001961319,"about_ca_system_score_gemma":0.0003170705,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006551711,"about_ca_topic_score_gemma":0.00004986878,"domain_scores_codex":[0.9974738,0.0001615717,0.0002837102,0.001492939,0.000145808,0.000442223],"domain_scores_gemma":[0.9965667,0.0002469393,0.0009984222,0.001579015,0.0004719631,0.0001369204],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0009573335,0.005937027,0.005938129,0.006478268,0.00148388,0.003934912,0.009787994,0.03267043,0.0583877,0.6135003,0.02376601,0.237158],"study_design_scores_gemma":[0.0008471546,0.0001638494,0.0008459014,0.0002529727,0.0001591517,0.0000122412,0.001339053,0.9437445,0.02417631,0.01103998,0.0164653,0.0009536264],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.01750576,0.000125934,0.9793137,0.0001890696,0.0003057755,0.0006116933,0.0003783382,0.0004349411,0.001134843],"genre_scores_gemma":[0.7028728,0.0003097223,0.2952287,0.0001737682,0.0001009004,0.000009150689,0.0009199489,0.00003050174,0.0003546061],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.911074,"threshold_uncertainty_score":0.9998743,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1064398660439206,"score_gpt":0.2820409161173054,"score_spread":0.1756010500733848,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}