{"id":"W2335254851","doi":"10.1016/j.annemergmed.2016.02.018","title":"Examining Reliability and Validity of an Online Score (ALiEM AIR) for Rating Free Open Access Medical Education Resources","year":2016,"lang":"en","type":"article","venue":"Annals of Emergency Medicine","topic":"Innovations in Medical Education","field":"Medicine","cited_by":106,"is_retracted":false,"has_abstract":false,"ca_institutions":"The Wilson Centre; University Health Network; University of Toronto; McMaster University","funders":"","keywords":"Generalizability theory; Inter-rater reliability; Reliability (semiconductor); Rating scale; Medicine; Gestalt psychology; Medical education; Applied psychology; Clinical psychology; Psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.004521162,0.0001827657,0.0006007087,0.0002539988,0.00009629283,0.000004008517,0.0008442487,0.0001742637,0.001749741],"category_scores_gemma":[0.05638032,0.0001180625,0.0000520025,0.0006093371,0.0004217307,0.0005759099,0.0003237265,0.0001685008,4.541346e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00002423802,"about_ca_system_score_gemma":0.0006615868,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0004544449,"about_ca_topic_score_gemma":0.00007711691,"domain_scores_codex":[0.9969212,0.0001553664,0.00141661,0.0004472576,0.0008099099,0.000249608],"domain_scores_gemma":[0.9958184,0.0003972137,0.0007004114,0.0008868593,0.001937293,0.0002598102],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0008078982,0.003276171,0.2719939,0.002411456,0.0001463997,0.000001559641,0.003137014,0.000001674497,0.01012789,0.002744513,0.2666746,0.4386769],"study_design_scores_gemma":[0.006520137,0.006159927,0.8924274,0.009149008,0.0003974919,0.00002190927,0.005097983,0.0009002119,0.01365975,0.02505833,0.04007754,0.0005302774],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9439669,0.000304718,0.0007713664,0.05192048,0.001181979,0.0008545448,0.00003118407,0.00002336251,0.0009454114],"genre_scores_gemma":[0.9896674,0.0008003181,0.005228922,0.001882871,0.00149958,0.0001048662,0.0001988115,0.0000315954,0.0005857049],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.6204335,"threshold_uncertainty_score":0.9991628,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.4208905800458987,"score_gpt":0.5295119252164578,"score_spread":0.1086213451705592,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}