{"id":"W4287448176","doi":"10.1080/15366367.2021.1991742","title":"Using Think-aloud Interviews to Examine a Clinically Oriented Performance Assessment Rubric","year":2022,"lang":"en","type":"article","venue":"Measurement Interdisciplinary Research and Perspectives","topic":"Innovations in Medical Education","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Rubric; Operationalization; Psychology; Context (archaeology); Peer assessment; Think aloud protocol; Rating scale; Applied psychology; Educational measurement; Medical education; Computer science; Mathematics education; Pedagogy; Medicine; Developmental psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.009070431,0.0001871348,0.0003314597,0.0007630373,0.001265309,0.00005586487,0.000269126,0.00004840921,0.0009344326],"category_scores_gemma":[0.0007050541,0.0001628501,0.00007202646,0.001506171,0.0002798191,0.0001404973,0.001631795,0.001347707,0.00001819627],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002252137,"about_ca_system_score_gemma":0.0005209459,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002070411,"about_ca_topic_score_gemma":0.000006513344,"domain_scores_codex":[0.9954105,0.0005060927,0.0005899314,0.0006453763,0.002323684,0.0005244337],"domain_scores_gemma":[0.997591,0.00008182307,0.00009436766,0.0005077932,0.001503037,0.0002220057],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"qualitative","study_design_scores_codex":[0.0102982,0.02062306,0.0913158,0.001710862,0.001332165,0.0002530879,0.5222696,0.0002553358,0.06891526,0.008637837,0.06621014,0.2081786],"study_design_scores_gemma":[0.003602046,0.02648965,0.2587608,0.001182751,0.0001086124,0.0004173334,0.6760862,0.01970279,0.0004203658,0.001389297,0.01117034,0.0006698173],"study_design_candidate":"qualitative","study_design_consensus":"qualitative","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9768699,0.0004741328,0.002313796,0.01105021,0.0004350871,0.001263196,0.000007269942,0.00005079074,0.007535556],"genre_scores_gemma":[0.9916106,0.00005358947,0.006417019,0.0003104226,0.0003360462,0.0001772558,0.00002412853,0.00002941015,0.001041453],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.2075088,"threshold_uncertainty_score":0.9999788,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3522250709098728,"score_gpt":0.5102631214747934,"score_spread":0.1580380505649206,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}