{"id":"W2289170923","doi":"10.1097/acm.0000000000000786","title":"Appraising the Quality of Medical Education Research Methods","year":2015,"lang":"en","type":"article","venue":"Academic Medicine","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":789,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Inter-rater reliability; Intraclass correlation; Normative; Reliability (semiconductor); Psychology; Raw score; Scale (ratio); Medicine; Clinical psychology; Statistics; Psychometrics; Rating scale; Mathematics; Developmental psychology","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"evaluation","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"methods","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.6750579,0.002062983,0.006755837,0.04361462,0.003018034,0.01472346,0.006161185,0.004350153,0.0035353],"category_scores_gemma":[0.8597586,0.002198755,0.006896793,0.02697488,0.01004382,0.007781552,0.0106334,0.004259038,0.000654458],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.01421111,"about_ca_system_score_gemma":0.03059375,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003070319,"about_ca_topic_score_gemma":0.003356052,"domain_scores_codex":[0.1636343,0.4641084,0.2424867,0.01225115,0.1160477,0.001471693],"domain_scores_gemma":[0.04469439,0.7281938,0.1001139,0.03287549,0.09191638,0.002206047],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"design_other","study_design_gemma":"not_applicable","study_design_scores_codex":[0.001515362,0.0002290909,0.1061199,0.3230039,0.02342363,0.0003495441,0.01918137,0.002682865,0.001235584,0.02033456,0.01826374,0.4836605],"study_design_scores_gemma":[0.001780709,0.001510385,0.1124894,0.6316888,0.01608996,0.001065184,0.01180225,0.006871567,0.00329339,0.04704335,0.1655803,0.0007845836],"study_design_candidate":"not_applicable","study_design_consensus":null,"genre_codex":"review","genre_gemma":"methods","genre_scores_codex":[0.05508148,0.5506725,0.2474321,0.04779259,0.009322525,0.04945177,0.01109661,0.0007585008,0.02839194],"genre_scores_gemma":[0.5091414,0.09122162,0.3000223,0.01021977,0.003792435,0.07867832,0.005264235,0.0003835451,0.001276394],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.3249421,"threshold_uncertainty_score":0.4007115,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9786162941917345,"score_gpt":0.8122788227648552,"score_spread":0.1663374714268793,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}