{"id":"W4388524631","doi":"10.1136/bmjresp-2023-001942","title":"Performance evaluation of human cough annotators: optimal metrics and sex differences","year":2023,"lang":"en","type":"article","venue":"BMJ Open Respiratory Research","topic":"Respiratory and Cough-Related Research","field":"Medicine","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"Generalitat de Catalunya; Ministerio de Ciencia e Innovación; Centres de Recerca de Catalunya; Patrick J. McGovern Foundation","keywords":"Medicine; Gold standard (test); Limits of agreement; Annotation; Correlation; Linear correlation; Pearson product-moment correlation coefficient; Audiology; Artificial intelligence; Statistics; Computer science; Nuclear medicine; Mathematics; Internal medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02867744,0.0002178669,0.0005887615,0.001418466,0.0006909588,0.0002143776,0.0007740286,0.000264642,0.0004905402],"category_scores_gemma":[0.001805907,0.0001788205,0.00007320039,0.003961109,0.0007212266,0.0004472261,0.001048089,0.001003052,0.0001804079],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001599096,"about_ca_system_score_gemma":0.001585161,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00007832412,"about_ca_topic_score_gemma":0.00001027109,"domain_scores_codex":[0.991475,0.001819279,0.0007166195,0.0006783985,0.004516796,0.0007939463],"domain_scores_gemma":[0.9960656,0.000574202,0.000151044,0.0008887244,0.001947563,0.0003728445],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001324207,0.0005547795,0.7446621,0.001107996,0.0003844789,0.0002479482,0.001703661,0.00018751,0.04858563,0.0006449369,0.01998264,0.1806141],"study_design_scores_gemma":[0.007515783,0.006956671,0.8961987,0.0009371912,0.0001541709,0.00003137377,0.004924568,0.02928139,0.0387981,0.0002499406,0.01437197,0.0005801666],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9722261,0.0006146013,0.0000015262,0.0002038281,0.00005484726,0.004335951,0.00001266065,0.00005169595,0.02249875],"genre_scores_gemma":[0.9956999,0.0001483513,0.00003791735,0.00007610726,0.0001607838,0.0008433542,0.00001781861,0.00004178878,0.002974016],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1800339,"threshold_uncertainty_score":0.9939087,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5294249097745025,"score_gpt":0.5562476579354084,"score_spread":0.02682274816090591,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}