{"id":"W2089954256","doi":"10.1080/15305058.2010.501536","title":"Evaluating the Bookmark Standard Setting Method: The Impact of Random Item Ordering","year":2011,"lang":"en","type":"article","venue":"International Journal of Testing","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"Dalhousie University","funders":"","keywords":"Licensure; Assertion; Test (biology); Computer science; Psychology; Process (computing); Mathematics education; Statistics; Medical education; Mathematics; Medicine; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.0506505,0.000156414,0.0004118926,0.0004451472,0.0002394274,0.0002392276,0.002447673,0.0000425364,0.0002829545],"category_scores_gemma":[0.4542646,0.00006730375,0.0004149899,0.001075511,0.0001106196,0.0003514488,0.0003382165,0.0004685088,0.000003091925],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00009901107,"about_ca_system_score_gemma":0.0002704283,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002117521,"about_ca_topic_score_gemma":0.000001902563,"domain_scores_codex":[0.9937204,0.001542182,0.001806524,0.0002083034,0.002471321,0.0002513139],"domain_scores_gemma":[0.7766666,0.2104888,0.005999786,0.0005824589,0.006142882,0.0001194781],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.0005028751,0.00002647702,0.2080978,0.000002548917,0.0002816883,0.00002421933,0.002053581,0.01270621,0.005844319,0.0001080362,0.000483008,0.7698693],"study_design_scores_gemma":[0.004446464,0.00233301,0.5941211,0.0008088346,0.000159693,0.002877492,0.006850333,0.2586156,0.004439106,0.1245557,0.0003651883,0.0004274721],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.8057243,0.0004516366,0.1824968,0.0004695417,0.001360765,0.000138867,0.000007998587,0.00001459468,0.009335478],"genre_scores_gemma":[0.6995415,0.000005330368,0.3000032,0.00005446354,0.0003431174,0.000001524026,1.078831e-7,0.000009296701,0.00004138342],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7694418,"threshold_uncertainty_score":0.9775551,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.7272170666147368,"score_gpt":0.606127092720526,"score_spread":0.1210899738942108,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}