{"id":"W4220967648","doi":"10.1097/bpo.0000000000002136","title":"Interobserver and Intraobserver Agreement are Unsatisfactory When Determining Abstract Study Design and Level of Evidence","year":2022,"lang":"en","type":"article","venue":"Journal of Pediatric Orthopaedics","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"BC Children's Hospital","funders":"","keywords":"Medicine; Medical physics; Nuclear medicine","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"evaluation","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"methods","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.1011383,0.0002376115,0.002091512,0.000626428,0.0001667422,0.0002464595,0.0009510632,0.00004218292,0.002175275],"category_scores_gemma":[0.01452284,0.0001233442,0.0006149845,0.0006518312,0.00004635283,0.0005266336,0.0004255784,0.0004224823,0.000006542012],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00005232917,"about_ca_system_score_gemma":0.0001750483,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001030846,"about_ca_topic_score_gemma":0.0000254892,"domain_scores_codex":[0.9798998,0.005690583,0.008000407,0.0004746632,0.005736724,0.0001977815],"domain_scores_gemma":[0.9811372,0.005409378,0.01142208,0.0008320425,0.0009874759,0.0002118757],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00003051305,0.00009736859,0.9704108,0.00006739886,0.000169394,0.00008736545,0.002460211,0.0003824812,0.00001340583,0.000002751802,0.008705458,0.01757285],"study_design_scores_gemma":[0.0005169729,0.0008303245,0.9896944,0.00004079404,0.0009076062,0.00008735423,0.006115331,0.0005942215,0.000002526167,0.0002833008,0.000778311,0.0001488287],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9860373,0.005321052,0.007604611,0.0001630469,0.000395291,0.0004522029,0.00001435951,0.00000108335,0.00001110893],"genre_scores_gemma":[0.9909825,0.0004611288,0.007969304,0.00007167673,0.0002068396,0.000006950149,2.296916e-7,0.000009915958,0.0002914176],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.08661543,"threshold_uncertainty_score":0.9987369,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9015335398561041,"score_gpt":0.5068814397572236,"score_spread":0.3946521000988805,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}