{"id":"W4386631162","doi":"10.1109/oceanslimerick52467.2023.10244502","title":"Better quantifying inter-annotator variability: A step towards citizen science in underwater passive acoustics","year":2023,"lang":"en","type":"preprint","venue":"","topic":"Marine animal studies overview","field":"Environmental Science","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Carleton University","funders":"Terres Australes et Antarctiques Françaises","keywords":"Annotation; Computer science; Process (computing); Quality (philosophy); Citizen science; Artificial intelligence; Bioacoustics; Ground truth; Natural language processing","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1305742,0.001707128,0.003454743,0.005197478,0.003851415,0.009332214,0.004174869,0.004546086,0.001798413],"category_scores_gemma":[0.3034919,0.00113402,0.001700605,0.004924573,0.004522353,0.009772619,0.01502858,0.005632997,0.001589366],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002090589,"about_ca_system_score_gemma":0.005611673,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003663514,"about_ca_topic_score_gemma":0.005743759,"domain_scores_codex":[0.8016784,0.1260166,0.01091844,0.02775539,0.03117385,0.00245744],"domain_scores_gemma":[0.5175284,0.2905718,0.0286704,0.07792031,0.08066632,0.004642737],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002129415,0.001183646,0.1469508,0.005614322,0.002188195,0.0009506778,0.0649555,0.01711334,0.08010123,0.01832964,0.03020394,0.6302794],"study_design_scores_gemma":[0.0002911397,0.001220159,0.2025221,0.004653338,0.001245966,0.001724696,0.04395143,0.2049493,0.06815626,0.2357724,0.234363,0.00115031],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1332423,0.004169677,0.8399725,0.007427289,0.001148771,0.0008524163,0.001611258,0.002952773,0.008623016],"genre_scores_gemma":[0.4968209,0.0008558663,0.4889658,0.002629495,0.0008335988,0.001988164,0.00329028,0.001981651,0.002634351],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.1305742,"threshold_uncertainty_score":0.6905504,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06676903191768715,"score_gpt":0.3073647139253792,"score_spread":0.240595682007692,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}