{"id":"W3114083521","doi":"10.20982/tqmp.16.5.p467","title":"Inter-Rater Agreement, Data Reliability, and The Crisis of Confidence in Psychological Research","year":2020,"lang":"en","type":"article","venue":"The Quantitative Methods for Psychology","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":6,"is_retracted":false,"has_abstract":true,"ca_institutions":"Memorial University of Newfoundland","funders":"","keywords":"Inter-rater reliability; Psychology; Reliability (semiconductor); Clinical psychology; Developmental psychology; Physics; Thermodynamics; Rating scale","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[{"model":"gemma","categories":["metaresearch"],"domain":"reproducibility","study_design":"theoretical_or_conceptual","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"},{"model":"gpt","categories":["metaresearch"],"domain":"reproducibility","study_design":"observational","genre":"empirical","about_ca_system":false,"about_ca_topic":false,"confidence":"low","status":"direct model label, unvalidated"}],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","open_science","insufficient_payload"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.4766948,0.0001938607,0.00235924,0.0002053746,0.0001395437,0.0001721098,0.005467533,0.0000916398,0.001498198],"category_scores_gemma":[0.2032961,0.00006468471,0.0004252645,0.001715106,0.001874356,0.0001421928,0.0008557093,0.0004554358,0.0002137846],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.000006477338,"about_ca_system_score_gemma":0.00001690203,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00004167516,"about_ca_topic_score_gemma":0.00003803071,"domain_scores_codex":[0.807177,0.1802807,0.007774979,0.00236586,0.001917409,0.0004840819],"domain_scores_gemma":[0.8661276,0.1205034,0.002665389,0.008789059,0.001788817,0.0001258257],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.002915314,0.0003153165,0.006240443,0.0001476211,0.0004978797,0.000003497842,0.03167936,0.00001869592,0.002457265,0.5946577,0.2412259,0.119841],"study_design_scores_gemma":[0.001514085,0.0008454628,0.006990911,0.00002206176,0.0001229156,0.000008371046,0.01448863,0.01564967,0.00006458771,0.859484,0.1006685,0.0001408089],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":"theoretical_or_conceptual","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.02020426,0.002004296,0.8749604,0.09772167,0.0002219912,0.001818974,0.0000444686,0.000002236193,0.003021654],"genre_scores_gemma":[0.4615953,0.0002267001,0.5324017,0.005180654,0.00004967423,0.0001920299,0.000006266784,0.00001176563,0.0003359084],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.441391,"threshold_uncertainty_score":0.9999133,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.9696765528338096,"score_gpt":0.7788110470329636,"score_spread":0.190865505800846,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}