{"id":"W4388697936","doi":"10.1186/s12909-023-04837-6","title":"Collecting evidence of validity for an assessment tool for Norwegian medical students’ non-technical skills (NorMS-NTS): usability and reliability when used by novice raters","year":2023,"lang":"en","type":"article","venue":"BMC Medical Education","topic":"Innovations in Medical Education","field":"Medicine","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"McMaster University","funders":"Helse Nord RHF; Universitetet i Tromsø","keywords":"Usability; Norwegian; Medical education; Reliability (semiconductor); Psychology; Educational measurement; Validity; Psychometrics; Applied psychology; Computer science; Medicine; Clinical psychology; Pedagogy; Curriculum","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.1102007,0.0007105796,0.001033939,0.003606051,0.001067757,0.001690025,0.001152275,0.0007650535,0.001274412],"category_scores_gemma":[0.2113212,0.0005492294,0.00181581,0.001590091,0.002204909,0.001724578,0.002671244,0.0007212648,0.0005568463],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001652537,"about_ca_system_score_gemma":0.003981544,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002299325,"about_ca_topic_score_gemma":0.004443456,"domain_scores_codex":[0.8852841,0.06347964,0.0147601,0.004906307,0.03034697,0.001222823],"domain_scores_gemma":[0.7213277,0.1539368,0.02037303,0.01647681,0.08524562,0.002640097],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001571632,0.001106483,0.7150825,0.002209681,0.0008063272,0.0003168916,0.03618207,0.001659311,0.01012481,0.001197011,0.004283035,0.2254602],"study_design_scores_gemma":[0.0004115187,0.004325987,0.9358429,0.001863258,0.0004465284,0.0008634359,0.01657148,0.0120811,0.01362387,0.001428185,0.01227643,0.0002652764],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9596711,0.0006844499,0.02907179,0.0002616886,0.000298189,0.002849084,0.0005491785,0.0001589473,0.006455424],"genre_scores_gemma":[0.9470867,0.0003474575,0.04631107,0.00009864497,0.00008019956,0.004630302,0.0007736019,0.00008212124,0.0005900753],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.1102007,"threshold_uncertainty_score":0.5828042,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.05014147639249875,"score_gpt":0.4683742688775581,"score_spread":0.4182327924850593,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}