{"id":"W2335254851","doi":"10.1016/j.annemergmed.2016.02.018","title":"Examining Reliability and Validity of an Online Score (ALiEM AIR) for Rating Free Open Access Medical Education Resources","year":2016,"lang":"en","type":"article","venue":"Annals of Emergency Medicine","topic":"Innovations in Medical Education","field":"Medicine","cited_by":106,"is_retracted":false,"has_abstract":false,"ca_institutions":"The Wilson Centre; University Health Network; University of Toronto; McMaster University","funders":"","keywords":"Generalizability theory; Inter-rater reliability; Reliability (semiconductor); Rating scale; Medicine; Gestalt psychology; Medical education; Applied psychology; Clinical psychology; Psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02542328,0.0005819926,0.0006066016,0.003288398,0.001013059,0.001874223,0.00101858,0.001155988,0.001791413],"category_scores_gemma":[0.07976956,0.0006911749,0.001995737,0.001645268,0.001458999,0.001889528,0.002338202,0.001435177,0.0007452357],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001251142,"about_ca_system_score_gemma":0.001410397,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005628997,"about_ca_topic_score_gemma":0.009899041,"domain_scores_codex":[0.9816526,0.006800048,0.002638065,0.001880914,0.006150023,0.000878416],"domain_scores_gemma":[0.8683972,0.07282152,0.01757086,0.007844311,0.03058961,0.002776511],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0003753417,0.0005503033,0.9834518,0.00004971628,0.0003235435,0.0000479936,0.002205168,0.0002923385,0.0005589203,0.0003680614,0.0005997939,0.0111769],"study_design_scores_gemma":[0.0001206527,0.0007747755,0.9889323,0.00008709103,0.0001624708,0.0001437243,0.002830005,0.003807544,0.0009578365,0.0003817623,0.001760358,0.00004160672],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.993641,0.000124988,0.001684997,0.0001539824,0.0001168363,0.0002057142,0.0001982883,0.0000169324,0.003857367],"genre_scores_gemma":[0.9959364,0.00009403489,0.00225176,0.0001162106,0.00005605273,0.0002892145,0.0003375854,0.00002299151,0.0008956706],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02542328,"threshold_uncertainty_score":0.1344528,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4208905800458987,"score_gpt":0.5295119252164578,"score_spread":0.1086213451705592,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}