{"id":"W2580371235","doi":"10.1177/0146621616684584","title":"An Evaluation of Interrater Reliability Measures on Binary Tasks Using <i>d-Prime</i>","year":2016,"lang":"en","type":"article","venue":"Applied Psychological Measurement","topic":"Reliability and Agreement in Measurement","field":"Decision Sciences","cited_by":35,"is_retracted":false,"has_abstract":true,"ca_institutions":"Memorial University of Newfoundland","funders":"","keywords":"Inter-rater reliability; Kappa; Prime (order theory); Psychology; Statistics; Reliability (semiconductor); Cohen's kappa; Binary number; Agreement; Psychometrics; Social psychology; Mathematics; Combinatorics; Arithmetic; Rating scale; Linguistics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","insufficient_payload"],"consensus_categories":[],"category_scores_codex":[0.05134014,0.000394202,0.0006055025,0.0002234392,0.0001880145,0.00009159016,0.001356576,0.0002438929,0.001702357],"category_scores_gemma":[0.003415193,0.0001996039,0.000243475,0.0005670022,0.0003950023,0.0002262965,0.000107899,0.0002344962,0.0003053813],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005671026,"about_ca_system_score_gemma":0.00009329298,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00001081787,"about_ca_topic_score_gemma":0.000007664737,"domain_scores_codex":[0.9826584,0.002180463,0.001717181,0.001610274,0.01130955,0.0005240708],"domain_scores_gemma":[0.9938155,0.0004484059,0.0006040013,0.002404244,0.002462943,0.0002648791],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"bench_or_experimental","study_design_gemma":"bench_or_experimental","study_design_scores_codex":[0.0008011095,0.002245128,0.003600555,0.000005222885,0.00003785496,7.912633e-7,0.0001377712,0.001315764,0.7722567,0.0006300569,0.002136969,0.216832],"study_design_scores_gemma":[0.01000882,0.005190865,0.3183182,0.000424425,0.0003987158,0.000007052219,0.0008105951,0.002037558,0.3831044,0.2652449,0.01266921,0.001785318],"study_design_candidate":"bench_or_experimental","study_design_consensus":"bench_or_experimental","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9672068,0.00007349334,0.01767798,0.001277419,0.0008208781,0.001464181,0.00001615485,0.00007133694,0.01139171],"genre_scores_gemma":[0.9975315,0.000009064229,0.001461287,0.0006964535,0.0001240196,0.0001432953,0.000001591808,0.00001743879,0.00001539191],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.3891523,"threshold_uncertainty_score":0.9992102,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.5204254440661797,"score_gpt":0.4693431790011876,"score_spread":0.05108226506499214,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}