{"id":"W4401502016","doi":"10.1101/2024.08.11.24311830","title":"The accuracy of large language models in labelling neurosurgical ‘case-control studies’ and risk of bias assessment: protocol for a study of interrater agreement with human reviewers","year":2024,"lang":"en","type":"preprint","venue":"medRxiv","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Inter-rater reliability; Ranking (information retrieval); Psychology; Critical appraisal; Checklist; Confidence interval; Medicine; Applied psychology; Computer science; Pathology; Alternative medicine; Rating scale; Artificial intelligence; Cognitive psychology; Internal medicine","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4722511,0.007790633,0.01268333,0.0131199,0.007403823,0.006817717,0.007752763,0.01267816,0.0410481],"category_scores_gemma":[0.5894653,0.00753958,0.01225771,0.01208853,0.009931658,0.008628835,0.007562163,0.0100689,0.01783583],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01405198,"about_ca_system_score_gemma":0.05068317,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002783478,"about_ca_topic_score_gemma":0.006037785,"domain_scores_codex":[0.5154321,0.3100679,0.117714,0.01839136,0.03032821,0.008066363],"domain_scores_gemma":[0.3460934,0.2729102,0.09398311,0.1026521,0.1764373,0.007923819],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"not_applicable","study_design_scores_codex":[0.1434201,0.01089811,0.008090394,0.3013736,0.005974067,0.002413848,0.03237174,0.01138939,0.02091642,0.03138829,0.1434421,0.288322],"study_design_scores_gemma":[0.1648516,0.03225687,0.02684758,0.1528089,0.004216068,0.001597576,0.008093643,0.03430412,0.02931166,0.06865043,0.4732672,0.003794384],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"protocol","genre_gemma":"protocol","genre_scores_codex":[0.0009341145,0.0002145804,0.009631799,0.0002434707,0.0003037868,0.987122,0.000686496,0.0002250445,0.0006386961],"genre_scores_gemma":[0.000569955,0.00006121951,0.01104698,0.0000633392,0.00002174059,0.9880137,0.00009374195,0.00002016908,0.0001091371],"genre_candidate":"protocol","genre_consensus":"protocol","teacher_disagreement_score":0.5277489,"threshold_uncertainty_score":0.6508083,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2700434374823152,"score_gpt":0.5237714989013509,"score_spread":0.2537280614190357,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}