{"id":"W4405618562","doi":"10.1111/emip.12663","title":"Instruction‐Tuned Large‐Language Models for Quality Control in Automatic Item Generation: A Feasibility Study","year":2024,"lang":"en","type":"article","venue":"Educational Measurement Issues and Practice","topic":"Topic Modeling","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"","keywords":"Quality (philosophy); Computer science; Control (management); Item response theory; Language proficiency; Mathematics education; Natural language processing; Psychology; Artificial intelligence; Psychometrics; Developmental psychology","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.005749509,0.000120679,0.0001613308,0.00009274339,0.0001460665,0.0003856867,0.0001815943,0.00003596849,0.00002230146],"category_scores_gemma":[0.001928048,0.0001137154,0.00003452355,0.000202767,0.000009175192,0.00147966,0.00003713127,0.0001145597,0.000004883827],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0002008802,"about_ca_system_score_gemma":0.0003292847,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00028818,"about_ca_topic_score_gemma":0.0001914033,"domain_scores_codex":[0.9978186,0.0005365051,0.0004124921,0.0004844852,0.0005806974,0.0001671958],"domain_scores_gemma":[0.9982303,0.0009441695,0.00009201572,0.0003467821,0.0003262642,0.00006044625],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0002264336,0.005907715,0.00945171,0.0009870771,0.0005332706,0.00001373641,0.1164344,0.004923196,0.003104149,0.7764305,0.003272068,0.0787157],"study_design_scores_gemma":[0.00132691,0.000116066,0.008536451,0.00006225905,0.00004774348,0.00001959335,0.003598936,0.969852,0.00003303837,0.0134019,0.002790081,0.0002149741],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.1889016,0.0071505,0.7537353,0.04604129,0.001803602,0.001851264,0.00001110202,0.000123558,0.0003818061],"genre_scores_gemma":[0.9584287,0.00001324042,0.04036526,0.0004096206,0.0004125712,0.0002524418,0.00000526028,0.000006703417,0.0001061687],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.9649289,"threshold_uncertainty_score":0.4637178,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.1967005621322491,"score_gpt":0.4278503410420334,"score_spread":0.2311497789097843,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}