{"id":"W4367175790","doi":"10.1080/02763869.2023.2193122","title":"Librarian and Researcher Assessments of Search Result Relevance: How Well Do They Align?","year":2023,"lang":"en","type":"article","venue":"Medical Reference Services Quarterly","topic":"Health Sciences Research and Education","field":"Health Professions","cited_by":2,"is_retracted":false,"has_abstract":true,"ca_institutions":"Western University","funders":"","keywords":"Rubric; Relevance (law); Interview; Medical education; Information retrieval; Computer science; Psychology; MEDLINE; Medicine; Mathematics education; Sociology; Political science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.5321029,0.0008842748,0.003878442,0.03349515,0.003358801,0.01265122,0.00251821,0.002670738,0.001785736],"category_scores_gemma":[0.8300281,0.001660675,0.003068662,0.03169183,0.005561247,0.01771987,0.01197617,0.002679563,0.0009849445],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006064706,"about_ca_system_score_gemma":0.01138613,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003070885,"about_ca_topic_score_gemma":0.005565474,"domain_scores_codex":[0.2455606,0.5114639,0.1326111,0.009711382,0.09692928,0.003723756],"domain_scores_gemma":[0.1783042,0.5376245,0.0898546,0.03863572,0.1524402,0.003140819],"domain_codex":"methods","domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.002206756,0.0004823744,0.32739,0.01364392,0.00446103,0.0002823149,0.1473831,0.0008018864,0.001619472,0.007927293,0.008704951,0.485097],"study_design_scores_gemma":[0.00147115,0.004805696,0.677802,0.01984338,0.004103121,0.001514363,0.1805165,0.01119345,0.005455081,0.04270585,0.04937029,0.001219321],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6469847,0.0408559,0.1621225,0.0236268,0.001876899,0.02401529,0.003024623,0.001883587,0.09560972],"genre_scores_gemma":[0.9116893,0.002962634,0.06872749,0.002347715,0.0004132403,0.01148562,0.0010082,0.0002486941,0.001117162],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.4678971,"threshold_uncertainty_score":0.5770005,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1681793684858725,"score_gpt":0.5108772863021858,"score_spread":0.3426979178163133,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}