{"id":"W1521272648","doi":"","title":"Should we pursue inter-rater reliability or diversity? An empirical study of pilot performance assessment","year":2014,"lang":"en","type":"article","venue":"Griffith Research Online (Griffith University, Queensland, Australia)","topic":"Air Traffic Management and Optimization","field":"Engineering","cited_by":11,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Victoria","funders":"Griffith University","keywords":"Officer; Reliability (semiconductor); Humanities; Inter-rater reliability; Operations research; Computer science; Philosophy; Political science; Physics; Mathematics; Statistics; Law; Thermodynamics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.581938,0.001238858,0.002827536,0.00500204,0.002600584,0.007038616,0.003779391,0.003264399,0.001195138],"category_scores_gemma":[0.7614483,0.00116196,0.002816927,0.00793302,0.01686746,0.01888155,0.006787823,0.005905913,0.0008167778],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.003847298,"about_ca_system_score_gemma":0.005243246,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003595403,"about_ca_topic_score_gemma":0.003687257,"domain_scores_codex":[0.3080323,0.5903865,0.02294255,0.02337907,0.05220955,0.00305005],"domain_scores_gemma":[0.1159253,0.7279206,0.04681935,0.04981789,0.05728149,0.002235446],"domain_codex":"methods","domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.001329525,0.0007397055,0.5572425,0.003387872,0.007670491,0.0004196668,0.08426365,0.007081814,0.001620354,0.06329314,0.008766482,0.2641848],"study_design_scores_gemma":[0.0007196997,0.00728399,0.5303665,0.009453235,0.002006796,0.002048719,0.07156719,0.04310126,0.005586992,0.273835,0.05272741,0.001303182],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.4858604,0.02486171,0.4049007,0.04864218,0.003585435,0.001953356,0.0007181896,0.0003729484,0.02910505],"genre_scores_gemma":[0.9604678,0.001261257,0.03371982,0.002407433,0.0005503272,0.0008873501,0.0001384591,0.00008155053,0.0004860914],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.581938,"threshold_uncertainty_score":0.515545,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2301109010364429,"score_gpt":0.4022583749856182,"score_spread":0.1721474739491753,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}