{"id":"W3126939008","doi":"10.1080/08957347.2020.1835911","title":"Rethinking Think-Alouds: The Often-Problematic Collection of Response Process Data","year":2021,"lang":"en","type":"article","venue":"Applied Measurement in Education","topic":"Behavioral and Psychological Studies","field":"Psychology","cited_by":14,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"Social Sciences and Humanities Research Council of Canada","keywords":"Think aloud protocol; Psychology; Unobservable; Process (computing); Test (biology); Cognitive psychology; Data collection; Applied psychology; Computer science; Epistemology; Social science; Sociology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1472099,0.001614656,0.001324352,0.003825041,0.009103992,0.0138958,0.00649594,0.01104781,0.001256904],"category_scores_gemma":[0.4806538,0.001197504,0.001447118,0.003664316,0.01833821,0.008456752,0.005026215,0.02071768,0.001545724],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.007317377,"about_ca_system_score_gemma":0.00878235,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.006561346,"about_ca_topic_score_gemma":0.01082567,"domain_scores_codex":[0.7038115,0.2108245,0.01391123,0.009264038,0.0597176,0.002471147],"domain_scores_gemma":[0.2938145,0.6005114,0.02355027,0.01231515,0.06754475,0.002263907],"domain_codex":null,"domain_gemma":"methods","domain_candidate":"methods","domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"theoretical_or_conceptual","study_design_scores_codex":[0.0004513697,0.0001036779,0.002787854,0.004687654,0.0002761153,0.00237412,0.3513232,0.0006992815,0.01013589,0.04844603,0.4677808,0.1109341],"study_design_scores_gemma":[0.0001173547,0.0003072708,0.005415726,0.007354957,0.0001542927,0.001929349,0.153758,0.003259834,0.01449705,0.06869075,0.7440034,0.0005120587],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"commentary","genre_gemma":"empirical","genre_scores_codex":[0.01964463,0.008497101,0.1119097,0.7036604,0.1483109,0.0007628314,0.0003295812,0.0007216305,0.00616328],"genre_scores_gemma":[0.2758163,0.009954286,0.1020823,0.4939908,0.1000808,0.003894889,0.0003769848,0.003308508,0.01049505],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.8527901,"threshold_uncertainty_score":0.7785296,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3773982575143485,"score_gpt":0.3923418996521501,"score_spread":0.01494364213780164,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}