{"id":"W4244516975","doi":"10.31222/osf.io/2pczv","title":"Predicting reliability through structured expert elicitation with repliCATS (Collaborative Assessments for Trustworthy Science)","year":2021,"lang":"en","type":"preprint","venue":"","topic":"Meta-analysis and systematic reviews","field":"Decision Sciences","cited_by":11,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University; University of British Columbia","funders":"Advanced Research Projects Agency; Defense Advanced Research Projects Agency","keywords":"Replication (statistics); Generalizability theory; Expert elicitation; Computer science; Process (computing); Resource (disambiguation); Reliability (semiconductor); Underpinning; Trustworthiness; Protocol (science); Data science; Management science; Knowledge management; Psychology; Engineering; Computer security","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":["metaresearch"],"category_scores_codex":[0.5818765,0.003807794,0.004259262,0.01310409,0.003562412,0.007237947,0.005010912,0.004573727,0.006051579],"category_scores_gemma":[0.8628894,0.003155045,0.007967123,0.009949299,0.007998366,0.009619373,0.01043602,0.005806277,0.002373791],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.004083039,"about_ca_system_score_gemma":0.01039167,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002174206,"about_ca_topic_score_gemma":0.002661047,"domain_scores_codex":[0.289549,0.5799714,0.05465934,0.02467363,0.04968933,0.001457303],"domain_scores_gemma":[0.05041112,0.7682554,0.05116013,0.08948523,0.03993861,0.0007496114],"domain_codex":"methods","domain_gemma":"methods","domain_candidate":"methods","domain_consensus":"methods","study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00449623,0.001463252,0.05958262,0.0287979,0.01085103,0.0009785562,0.06014998,0.02918986,0.01037239,0.1093865,0.01685484,0.6678768],"study_design_scores_gemma":[0.004399783,0.00710853,0.04996078,0.01030378,0.006396875,0.001522469,0.006270412,0.194942,0.02787236,0.6378375,0.05162136,0.001764033],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.02341473,0.00109328,0.9504358,0.0007737923,0.0003890632,0.01806889,0.0007461712,0.0008290499,0.004249278],"genre_scores_gemma":[0.1342656,0.0005512085,0.830123,0.0003817524,0.0001990797,0.0331429,0.0005914749,0.0002311064,0.0005138615],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.4181235,"threshold_uncertainty_score":0.5156207,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.604102953090157,"score_gpt":0.5768386905679156,"score_spread":0.02726426252224146,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}