{"id":"W4417312644","doi":"10.1142/9789819824755_0026","title":"Automated Evaluation of Large Language Model Response Concordance with Human Specialist Responses on Physician-to-Physician eConsult Cases","year":2025,"lang":"en","type":"article","venue":"","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Concordance; Meaningful use; Verifiable secret sharing; Primary care; Kappa; Cohen's kappa","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.03922243,0.001303778,0.001002474,0.002209045,0.000727619,0.002090524,0.001817856,0.001912796,0.002951598],"category_scores_gemma":[0.1487624,0.0003670618,0.0008911722,0.001064082,0.001187361,0.001462969,0.003025059,0.001317578,0.001764963],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0012672,"about_ca_system_score_gemma":0.00185424,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002962063,"about_ca_topic_score_gemma":0.004268818,"domain_scores_codex":[0.9386337,0.04674311,0.0030927,0.005974215,0.004867197,0.0006889983],"domain_scores_gemma":[0.797846,0.1669313,0.008637967,0.01143377,0.01308292,0.002067984],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.009439521,0.003556972,0.2129054,0.004367919,0.001479481,0.002139533,0.01970408,0.0972925,0.04571049,0.004946726,0.04012319,0.5583342],"study_design_scores_gemma":[0.001345268,0.00275645,0.0903139,0.0004320453,0.0003480995,0.001462961,0.004093321,0.8197596,0.05316241,0.006655165,0.01931556,0.0003552339],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8879781,0.0009931177,0.09479369,0.0009749075,0.0003429343,0.001157498,0.002745096,0.006075263,0.004939206],"genre_scores_gemma":[0.9255593,0.0001462703,0.06725373,0.0003748441,0.0001014887,0.0005913402,0.004437634,0.0002436013,0.001291838],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.03922243,"threshold_uncertainty_score":0.2074305,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.142789947074574,"score_gpt":0.494918692853436,"score_spread":0.352128745778862,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}