{"id":"W3193068792","doi":"10.1162/coli_a_00422","title":"Probing Classifiers: Promises, Shortcomings, and Advances","year":2021,"lang":"en","type":"preprint","venue":"Computational Linguistics","topic":"Topic Modeling","field":"Computer Science","cited_by":21,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"Azrieli Foundation; Israel Science Foundation","keywords":"Computer science; Artificial intelligence; Variety (cybernetics); Classifier (UML); Machine learning; Property (philosophy); Artificial neural network; Deep neural networks; Natural language processing; Epistemology","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0002415431,0.0002736971,0.0003386974,0.0001119495,0.0001617901,0.000681244,0.0006678646,0.0001810798,0.000003448753],"category_scores_gemma":[0.001126551,0.0003054516,0.00007448009,0.0001146423,0.00006978245,0.00008416878,0.001789385,0.0005543225,0.00000236914],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.00008920155,"about_ca_system_score_gemma":0.0006261425,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000008811446,"about_ca_topic_score_gemma":0.000003833994,"domain_scores_codex":[0.9979324,0.00006127217,0.0004656321,0.0008673725,0.0004219654,0.0002514019],"domain_scores_gemma":[0.9981336,0.0002827734,0.0002468131,0.0004676806,0.0007436311,0.0001255024],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000005089625,0.00009434447,0.001114312,0.0009144395,0.00009535552,0.0001550596,0.002002074,0.5675159,0.000008241533,0.385594,0.0003102965,0.04219098],"study_design_scores_gemma":[0.0001427802,0.00001284882,0.000251031,0.0003267005,0.00002005197,0.00001640964,0.00003940128,0.8806025,0.00002245966,0.1106846,0.007538482,0.0003426272],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.003136722,0.003456555,0.9872974,0.0002413801,0.002736717,0.0002748164,0.00001163538,0.0002075238,0.002637291],"genre_scores_gemma":[0.4142305,0.00007310679,0.5847118,0.0001722496,0.0006440337,0.00002055756,0.00007635282,0.00001537944,0.0000559677],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.4110938,"threshold_uncertainty_score":0.9999397,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03569677429684543,"score_gpt":0.2887908698049079,"score_spread":0.2530940955080625,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}