{"id":"W3121886268","doi":"10.5539/ijel.v11n2p68","title":"Unveiling the Scoring Validity of Two Chinese Automated Writing Evaluation Systems: A Quantitative Study","year":2021,"lang":"en","type":"article","venue":"International Journal of English Linguistics","topic":"Second Language Acquisition and Learning","field":"Psychology","cited_by":8,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"","keywords":"Formative assessment; Computer science; Artificial intelligence; Natural language processing; Workload; Writing assessment; Construct (python library); Point (geometry); F1 score; Scoring system; Test (biology); Kappa; Psychology; Mathematics education; Linguistics; Mathematics","routes":{"ca_aff":false,"ca_fund":false,"ca_venue":true,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.04242465,0.0006402715,0.0007704789,0.003915125,0.001482025,0.001746104,0.0009970024,0.0007004057,0.001267939],"category_scores_gemma":[0.08573293,0.0003973654,0.0006126401,0.0024834,0.002805564,0.001819078,0.002255752,0.0008435256,0.0003403008],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001642462,"about_ca_system_score_gemma":0.001456029,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002279671,"about_ca_topic_score_gemma":0.002533086,"domain_scores_codex":[0.9732268,0.01157013,0.003508667,0.002807682,0.00797784,0.0009087922],"domain_scores_gemma":[0.890383,0.06298827,0.007998653,0.006602058,0.02976295,0.002265132],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"observational","study_design_gemma":"observational","study_design_scores_codex":[0.0009505206,0.001656545,0.6723593,0.0007206722,0.0002793504,0.0005869717,0.1775468,0.0007905432,0.01797364,0.001396444,0.0009172272,0.124822],"study_design_scores_gemma":[0.000133886,0.003865118,0.9021797,0.0001518528,0.0001780415,0.000404551,0.06273512,0.01207657,0.01364185,0.0007080184,0.003738897,0.0001864252],"study_design_candidate":"observational","study_design_consensus":"observational","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9980039,0.00002437405,0.001088302,0.00002626922,0.00000604082,0.0001547811,0.00004084254,0.000007667486,0.0006477866],"genre_scores_gemma":[0.9965736,0.00002795016,0.002500164,0.00002230716,0.00001217705,0.0003369079,0.0001117099,0.000008706981,0.000406625],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.04242465,"threshold_uncertainty_score":0.2243657,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06693725572005338,"score_gpt":0.4425607206956705,"score_spread":0.3756234649756172,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}