{"id":"W4417152224","doi":"10.64898/2025.11.29.25341091","title":"The accuracy and repeatability of OpenEvidence on complex medical subspecialty scenarios: a pilot study","year":2025,"lang":"","type":"article","venue":"medRxiv","topic":"Artificial Intelligence in Healthcare and Education","field":"Medicine","cited_by":1,"is_retracted":false,"has_abstract":true,"ca_institutions":"Centennial College","funders":"","keywords":"Repeatability; Subspecialty; Concordance; Sample (material); Specialty; Reliability (semiconductor)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.02136948,0.0007954388,0.000717775,0.001467876,0.000556468,0.001567888,0.001399139,0.001414327,0.001608842],"category_scores_gemma":[0.1202141,0.0002387408,0.000973304,0.0008757213,0.001007254,0.001952011,0.002100555,0.001273938,0.001127917],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0006972728,"about_ca_system_score_gemma":0.0008580969,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.004203978,"about_ca_topic_score_gemma":0.004514293,"domain_scores_codex":[0.9774645,0.01362267,0.001779226,0.004052012,0.002622486,0.0004591533],"domain_scores_gemma":[0.8741528,0.09386705,0.004967987,0.009917258,0.01503504,0.002059909],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.00748787,0.002800378,0.6580133,0.002229938,0.001948205,0.0007669604,0.007181839,0.0271084,0.01386558,0.001443134,0.03475006,0.2424043],"study_design_scores_gemma":[0.00158549,0.005989015,0.6664833,0.0008149736,0.001118463,0.002667073,0.00537284,0.2361675,0.03474481,0.00446471,0.04016165,0.0004300856],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9798355,0.001114288,0.009512861,0.000425899,0.0002145376,0.0003094714,0.00409283,0.001322416,0.003172038],"genre_scores_gemma":[0.9845196,0.0001328092,0.007183221,0.0002112779,0.0000840444,0.000174734,0.006833528,0.0001427168,0.0007180584],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9786305,"threshold_uncertainty_score":0.113014,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.258008141723206,"score_gpt":0.4817616256259034,"score_spread":0.2237534839026974,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}