{"id":"W2089954256","doi":"10.1080/15305058.2010.501536","title":"Evaluating the Bookmark Standard Setting Method: The Impact of Random Item Ordering","year":2011,"lang":"en","type":"article","venue":"International Journal of Testing","topic":"Psychometric Methodologies and Testing","field":"Decision Sciences","cited_by":19,"is_retracted":false,"has_abstract":true,"ca_institutions":"Dalhousie University","funders":"","keywords":"Licensure; Assertion; Test (biology); Computer science; Psychology; Process (computing); Mathematics education; Statistics; Medical education; Mathematics; Medicine; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.357548,0.002143259,0.002252157,0.003234189,0.002042145,0.004618255,0.003781863,0.002741862,0.003008903],"category_scores_gemma":[0.7159848,0.001073983,0.001776584,0.004570171,0.003652348,0.005118246,0.004330048,0.004077772,0.001013491],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003286357,"about_ca_system_score_gemma":0.003632868,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003825306,"about_ca_topic_score_gemma":0.006205426,"domain_scores_codex":[0.4238163,0.4982771,0.01247504,0.01312155,0.05130249,0.001007639],"domain_scores_gemma":[0.116538,0.8171533,0.02342523,0.02558533,0.01583722,0.001461011],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.01476602,0.002900686,0.162186,0.001813332,0.002425641,0.0003402984,0.009942658,0.01753962,0.005427731,0.02629081,0.01457524,0.741792],"study_design_scores_gemma":[0.006122467,0.02896084,0.2682913,0.004636158,0.00390276,0.001994533,0.007465896,0.4410382,0.04193072,0.1272738,0.06640621,0.001977104],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.3924102,0.006477902,0.5602587,0.003932684,0.001482885,0.007143352,0.0005401916,0.001030811,0.02672324],"genre_scores_gemma":[0.6447042,0.0009092847,0.3450716,0.002226205,0.000329016,0.003098569,0.0005271231,0.0004081761,0.00272578],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.642452,"threshold_uncertainty_score":0.7922577,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.7272170666147368,"score_gpt":0.606127092720526,"score_spread":0.1210899738942108,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}