{"id":"W2582439330","doi":"10.1136/bmj.j14","title":"Inter-rater agreement in evaluation of disability: systematic review of reproducibility studies","year":2017,"lang":"en","type":"review","venue":"BMJ","topic":"Musculoskeletal pain and rehabilitation","field":"Medicine","cited_by":94,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"","keywords":"PsycINFO; Inter-rater reliability; Observational study; Inclusion (mineral); Normative; Research design; MEDLINE; Psychology; Descriptive statistics; Systematic review; Medicine; Applied psychology; Rating scale; Medical education; Family medicine; Social psychology; Statistics; Pathology; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"methods","study_design":"systematic_review","genre":"review","about_ca_system":false,"about_ca_topic":false,"confidence":"high","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.04552368,0.0002544433,0.004821733,0.0001025079,0.00001618903,0.000003103562,0.0001665551,0.0001237892,0.00006745619],"category_scores_gemma":[0.05730479,0.0001472281,0.0007755532,0.0001635806,0.0003088816,0.0000458292,0.000101228,0.0001634417,0.00001381491],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005247862,"about_ca_system_score_gemma":0.0003030256,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002639623,"about_ca_topic_score_gemma":0.0000277877,"domain_scores_codex":[0.9920651,0.002997415,0.003248488,0.0007360971,0.0008317804,0.000121083],"domain_scores_gemma":[0.9920228,0.0004867997,0.002088695,0.004498582,0.000867909,0.00003520185],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"systematic_review","study_design_gemma":"systematic_review","study_design_scores_codex":[0.000004352233,0.0001935208,0.0001274306,0.8407837,0.0003034183,2.723651e-7,0.0001316511,4.634254e-8,3.225402e-7,0.000006649725,0.000189227,0.1582595],"study_design_scores_gemma":[0.0002379543,0.0001581233,0.000619528,0.9891784,0.005293078,0.000002596617,0.0001097144,0.000005605154,7.86485e-7,0.00008030228,0.004201122,0.0001127626],"study_design_candidate":"systematic_review","study_design_consensus":"systematic_review","genre_codex":"review","genre_gemma":"review","genre_scores_codex":[0.0001278629,0.9812889,0.000001089232,0.0005263133,0.000224814,0.01721743,0.000008756543,0.000005712117,0.0005991275],"genre_scores_gemma":[0.0008699474,0.997295,0.00007373074,0.00002212835,0.00007378888,0.001410741,0.00005458956,0.00001390145,0.0001862148],"genre_candidate":"review","genre_consensus":"review","teacher_disagreement_score":0.1581467,"threshold_uncertainty_score":0.9828342,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3008854814152268,"score_gpt":0.5367544582384001,"score_spread":0.2358689768231733,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}