{"id":"W3143565436","doi":"10.1007/s11145-021-10153-6","title":"Validity of automated text evaluation tools for written-expression curriculum-based measurement: a comparison study","year":2021,"lang":"en","type":"article","venue":"Reading and Writing","topic":"Writing and Handwriting Education","field":"Social Sciences","cited_by":13,"is_retracted":false,"has_abstract":false,"ca_institutions":"University of British Columbia","funders":"Institute of Education Sciences","keywords":"Computer science; Metric (unit); Natural language processing; Curriculum-based measurement; Artificial intelligence; Expression (computer science); Measure (data warehouse); Machine learning; Curriculum; Data mining; Information retrieval; Psychology; Curriculum development; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.006484513,0.0001018183,0.0002489678,0.00007510724,0.0008431462,0.000241991,0.00007707124,0.00007962956,0.00001402557],"category_scores_gemma":[0.004900229,0.0001086857,0.00006051275,0.0002684186,0.00005462921,0.0001878253,0.00001885015,0.000096851,9.999493e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001332501,"about_ca_system_score_gemma":0.0003561338,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0003455438,"about_ca_topic_score_gemma":0.0001659328,"domain_scores_codex":[0.9976251,0.0006985838,0.0004052205,0.0002921024,0.000724047,0.0002549838],"domain_scores_gemma":[0.9976576,0.0007190673,0.0002394601,0.0001282256,0.001176829,0.00007883731],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"qualitative","study_design_scores_codex":[0.00005151809,0.003484631,0.7009283,0.000925407,0.00009001532,0.000004360014,0.0423706,0.0005251153,0.07411737,0.002858424,0.00276682,0.1718775],"study_design_scores_gemma":[0.006824487,0.0006793135,0.114322,0.006494785,0.0007824855,0.000005736646,0.5846316,0.2160673,0.0649913,0.0009083059,0.003142601,0.001150067],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9954491,0.0002519067,0.0007172202,0.0003691871,0.0001443825,0.0005884816,0.000006288872,0.0001983494,0.00227508],"genre_scores_gemma":[0.9977107,0.000009108584,0.001873543,0.00001859366,0.0002449628,0.00008222734,0.00003113177,0.00001162706,0.0000180844],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.5866063,"threshold_uncertainty_score":0.6484884,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1788067842855857,"score_gpt":0.4282953256649054,"score_spread":0.2494885413793197,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}