{"id":"W2793261989","doi":"10.1080/10401334.2017.1414609","title":"Quality Evaluation Scores are no more Reliable than Gestalt in Evaluating the Quality of Emergency Medicine Blogs: A METRIQ Study","year":2018,"lang":"en","type":"article","venue":"Teaching and Learning in Medicine","topic":"Social Media in Health Education","field":"Social Sciences","cited_by":29,"is_retracted":false,"has_abstract":true,"ca_institutions":"Queen's University; McMaster University; Western University; University of Alberta; University of Saskatchewan","funders":"Royal College of Physicians and Surgeons of Canada; Canadian Association of Emergency Physicians","keywords":"Gestalt psychology; Intraclass correlation; Reliability (semiconductor); Quality (philosophy); Metric (unit); Medicine; Emergency department; Psychology; Medical education; Clinical psychology; Nursing; Psychometrics","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.03446251,0.0003867012,0.0006745009,0.003160689,0.0005124364,0.00139937,0.000720484,0.0006776723,0.001515674],"category_scores_gemma":[0.1155299,0.0003564559,0.001231588,0.002476928,0.001266951,0.001990016,0.001848384,0.0007426997,0.0002866515],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0009023104,"about_ca_system_score_gemma":0.0007849072,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001043386,"about_ca_topic_score_gemma":0.001603818,"domain_scores_codex":[0.9802701,0.009035693,0.002925124,0.001341455,0.005802676,0.000624864],"domain_scores_gemma":[0.8042149,0.1099395,0.04849695,0.00785004,0.02636792,0.003130617],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0002310186,0.0001307172,0.9863557,0.0001003775,0.0002472443,0.00002133202,0.001127081,0.0001685502,0.0001974879,0.0001271165,0.0002681095,0.01102538],"study_design_scores_gemma":[0.00004243673,0.0008013637,0.9934366,0.00009146293,0.0001164187,0.0001521838,0.001312145,0.002373246,0.0004278578,0.0002214403,0.0009965559,0.00002830324],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9937808,0.0003448051,0.002943228,0.0001694807,0.00003233757,0.0002330171,0.0002513197,0.00002082743,0.002224162],"genre_scores_gemma":[0.9976216,0.00006144175,0.001703993,0.00005369327,0.00001989501,0.0001898624,0.0001788305,0.0000107596,0.0001599181],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9655375,"threshold_uncertainty_score":0.1822574,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3946124905160667,"score_gpt":0.5938683588428553,"score_spread":0.1992558683267886,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}