{"id":"W6947871790","doi":"10.48448/tnqn-bc19","title":"A linguistically-motivated evaluation methodology for unraveling model's abilities in reading comprehension tasks","year":2024,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"","keywords":"Reading comprehension; Comprehension; Benchmark (surveying); Intuition; Annotation; Task (project management); Task analysis","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaresearch","metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.01328355,0.0005810092,0.0008719779,0.003498674,0.0001740961,0.000211827,0.0008270337,0.0005492425,0.0002744048],"category_scores_gemma":[0.01478098,0.0005460439,0.0001192332,0.002075138,0.001559007,0.0001326713,0.000356023,0.0006833426,0.0005633491],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001328652,"about_ca_system_score_gemma":0.001955545,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0006909929,"about_ca_topic_score_gemma":0.000961744,"domain_scores_codex":[0.9941741,0.0005273992,0.0009379893,0.001843203,0.001476381,0.001040949],"domain_scores_gemma":[0.9962661,0.001267177,0.0004047167,0.0008231529,0.001046165,0.0001926858],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0005631104,0.0008111955,0.0001281343,0.00340313,0.000333756,0.00005953992,0.008808245,0.2873099,0.2535403,0.2957981,0.1126554,0.0365891],"study_design_scores_gemma":[0.0006296007,0.0000972906,0.000006089627,0.00117508,0.0001691392,0.000008641107,0.0003848538,0.8930126,0.0005693262,0.09608447,0.007336718,0.0005261888],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"methods","genre_scores_codex":[0.003004897,0.003280828,0.6212803,0.0007007435,0.006651241,0.01040056,0.001243157,0.003238433,0.3501999],"genre_scores_gemma":[0.1324195,0.00003251231,0.8057668,0.0002227036,0.001000678,0.0004032993,0.0005689869,0.002008771,0.05757681],"genre_candidate":"methods","genre_consensus":"methods","teacher_disagreement_score":0.6057027,"threshold_uncertainty_score":0.9996991,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2127492182627536,"score_gpt":0.4438704265680997,"score_spread":0.2311212083053461,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}