{"id":"W4400485729","doi":"10.1097/acm.0000000000005802","title":"The Mean Delta Method: Quantifying Assessor Stringency and Leniency and Identifying Outliers in Workplace-Based Assessments","year":2024,"lang":"en","type":"article","venue":"Academic Medicine","topic":"Occupational Health and Safety Research","field":"Health Professions","cited_by":4,"is_retracted":false,"has_abstract":true,"ca_institutions":"Canadian Network for Innovation in Education; Newborn Screening Ontario; Medical Council of Canada; University of Ottawa","funders":"","keywords":"Outlier; Statistics; Econometrics; Delta; Psychology; Computer science; Mathematics; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":true,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.0376995,0.001084808,0.001077966,0.006739631,0.001141171,0.002276637,0.001769789,0.001045963,0.001927215],"category_scores_gemma":[0.1295391,0.0005181234,0.001141585,0.00453263,0.001181089,0.001540078,0.003006592,0.001663963,0.0006842033],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001388808,"about_ca_system_score_gemma":0.002346335,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004562022,"about_ca_topic_score_gemma":0.009578696,"domain_scores_codex":[0.9604178,0.01627113,0.004589709,0.003647909,0.01450856,0.0005648441],"domain_scores_gemma":[0.8631182,0.07717974,0.019279,0.009952189,0.02889746,0.001573382],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.001211386,0.0003296663,0.5927737,0.0009281987,0.001004928,0.0002193217,0.00405802,0.005948134,0.008570464,0.0037284,0.008804199,0.3724236],"study_design_scores_gemma":[0.0002336503,0.002217349,0.8369299,0.0007493541,0.0005272239,0.001206081,0.004021145,0.1004899,0.02462373,0.01034153,0.01812991,0.0005303396],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.4388825,0.001222447,0.5421346,0.0006690782,0.0004637412,0.001268369,0.002414291,0.002410981,0.01053386],"genre_scores_gemma":[0.7704724,0.0002531739,0.2242765,0.0002490129,0.00008912049,0.001310764,0.0008442528,0.0002804032,0.002224361],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.9623005,"threshold_uncertainty_score":0.1993764,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2884083444783235,"score_gpt":0.6051797454383355,"score_spread":0.3167714009600121,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}