{"id":"W4244295316","doi":"10.31219/osf.io/gcetv","title":"Validity of Automated Text Evaluation Tools for Written-Expression Curriculum-Based Measurement: A Comparison Study","year":2021,"lang":"en","type":"preprint","venue":"","topic":"Writing and Handwriting Education","field":"Social Sciences","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of British Columbia","funders":"Institute of Education Sciences; University of Houston; U.S. Department of Education","keywords":"Metric (unit); Computer science; Measure (data warehouse); Expression (computer science); Natural language processing; Curriculum-based measurement; Artificial intelligence; Machine learning; Curriculum; Sample (material); Data mining; Medical physics; Psychology; Medicine; Engineering; Curriculum development","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.04988489,0.0006157861,0.0006777234,0.003163353,0.0006077364,0.001740135,0.00106894,0.0008179425,0.0008676887],"category_scores_gemma":[0.1718892,0.0003594055,0.001068477,0.001705037,0.001191325,0.002006967,0.001933283,0.0006470016,0.0004854678],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001070255,"about_ca_system_score_gemma":0.001197299,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001590522,"about_ca_topic_score_gemma":0.001954259,"domain_scores_codex":[0.9412947,0.03439063,0.004486795,0.00374549,0.01523141,0.0008510129],"domain_scores_gemma":[0.7433321,0.1766038,0.02014826,0.01697864,0.04132428,0.001612945],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.002518817,0.001340494,0.7952867,0.0002849723,0.0008051617,0.00009200584,0.004329695,0.001793809,0.002921837,0.0009849262,0.000951487,0.1886902],"study_design_scores_gemma":[0.0004056529,0.00585877,0.952534,0.0002734173,0.0002794917,0.0003906873,0.001936718,0.02620339,0.00680325,0.001185321,0.004036308,0.00009300265],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9877598,0.0002860893,0.008175151,0.00007035864,0.00006721247,0.0004194712,0.0001780039,0.0000839986,0.002959921],"genre_scores_gemma":[0.9939826,0.00007830914,0.004936584,0.00004642969,0.00003251114,0.0003160153,0.000195557,0.00003041341,0.000381659],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.04988489,"threshold_uncertainty_score":0.2638196,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2613056123284385,"score_gpt":0.4539135859038707,"score_spread":0.1926079735754322,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}