{"id":"W2582439330","doi":"10.1136/bmj.j14","title":"Inter-rater agreement in evaluation of disability: systematic review of reproducibility studies","year":2017,"lang":"en","type":"review","venue":"BMJ","topic":"Musculoskeletal pain and rehabilitation","field":"Medicine","cited_by":94,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"PsycINFO; Inter-rater reliability; Observational study; Inclusion (mineral); Normative; Research design; MEDLINE; Psychology; Descriptive statistics; Systematic review; Medicine; Applied psychology; Rating scale; Medical education; Family medicine; Social psychology; Statistics; Pathology; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"methods","study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.3996411,0.002982165,0.01755823,0.02282524,0.002325428,0.009736182,0.006669188,0.004266652,0.003635011],"category_scores_gemma":[0.6989886,0.003740054,0.01963245,0.02410226,0.006647165,0.01459585,0.008035194,0.003384769,0.0005777734],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01137621,"about_ca_system_score_gemma":0.03035618,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005903973,"about_ca_topic_score_gemma":0.01091846,"domain_scores_codex":[0.4235241,0.3304674,0.1741185,0.01776191,0.05211391,0.002014095],"domain_scores_gemma":[0.2037501,0.5993376,0.120259,0.02541788,0.05010008,0.001135266],"domain_codex":null,"domain_gemma":"reproducibility","domain_candidate":"reproducibility","domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[0.0005674746,0.00004400118,0.003957817,0.9185814,0.02480437,0.00007507522,0.002534724,0.0001933157,0.0001169276,0.000706615,0.001359785,0.04705862],"study_design_scores_gemma":[0.0006266912,0.0003728245,0.007397822,0.9274241,0.05190693,0.0002473147,0.001918176,0.0002807916,0.0004238323,0.001300472,0.007971431,0.0001295566],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.007091135,0.9730911,0.007233392,0.001078592,0.000681583,0.008513878,0.001141743,0.00006440503,0.001104068],"genre_scores_gemma":[0.2183422,0.6784146,0.04015258,0.002562698,0.0009449862,0.05662736,0.002343092,0.0001771716,0.0004352414],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.6003589,"threshold_uncertainty_score":0.7403494,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3008854814152268,"score_gpt":0.5367544582384001,"score_spread":0.2358689768231733,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}