{"id":"W3028930880","doi":"10.48550/arxiv.2003.04642","title":"A Framework for Evaluation of Machine Reading Comprehension Gold\\n Standards","year":2020,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":20,"is_retracted":false,"has_abstract":true,"ca_institutions":"Open Text (Canada)","funders":"","keywords":"Computer science; Correctness; Paragraph; Reading comprehension; Artificial intelligence; Natural language processing; Schema (genetic algorithms); Comprehension; Ambiguity; Popularity; Set (abstract data type); Reading (process); Machine learning; Linguistics; Psychology; World Wide Web; Programming language","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0009356891,0.0002112574,0.000361199,0.0001612686,0.0000704932,0.00005098716,0.001136814,0.0002503479,0.00001409387],"category_scores_gemma":[0.0002979,0.0002494504,0.0001930129,0.0003310199,0.00003778276,0.0001817563,0.001272597,0.0004033664,0.000003336075],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003556,"about_ca_system_score_gemma":0.0004286196,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00006891008,"about_ca_topic_score_gemma":0.000009027274,"domain_scores_codex":[0.9981332,0.0001782697,0.0002514849,0.0008999045,0.0003313346,0.0002058075],"domain_scores_gemma":[0.9975628,0.0002131311,0.0003418283,0.0009990934,0.0007792257,0.0001038826],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005085042,0.00003171677,0.0002704321,0.0001951121,0.00006953194,0.00001027446,0.0004466474,0.4640314,0.0001104687,0.5287281,0.0001141666,0.005941314],"study_design_scores_gemma":[0.0004546568,0.00003629783,0.00008552882,0.0001802925,0.00009365038,5.461159e-7,0.00002291452,0.703489,0.0001611913,0.2951668,0.0001519916,0.0001571792],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.07544892,0.00009712072,0.9224457,0.000224262,0.0005305463,0.0005792404,0.00006227008,0.0001259113,0.0004860434],"genre_scores_gemma":[0.9265327,0.00003422135,0.07323091,0.00006069457,0.00007048415,0.000001766731,0.00001825931,0.00001358986,0.00003735392],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8510838,"threshold_uncertainty_score":0.9999958,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1712827149806367,"score_gpt":0.2700363482322421,"score_spread":0.09875363325160538,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}