{"id":"W3126939008","doi":"10.1080/08957347.2020.1835911","title":"Rethinking Think-Alouds: The Often-Problematic Collection of Response Process Data","year":2021,"lang":"en","type":"article","venue":"Applied Measurement in Education","topic":"Behavioral and Psychological Studies","field":"Psychology","cited_by":14,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Alberta","funders":"Social Sciences and Humanities Research Council of Canada","keywords":"Think aloud protocol; Psychology; Unobservable; Process (computing); Test (biology); Cognitive psychology; Data collection; Applied psychology; Computer science; Epistemology; Social science; Sociology","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002978272,0.0001198234,0.0001854807,0.00005499141,0.0001407699,0.00002034048,0.0003806291,0.00009705694,0.0002317394],"category_scores_gemma":[0.0002538009,0.0000846366,0.00002534584,0.0006832059,0.00006446467,0.00005208069,0.00006774547,0.0002306027,0.00003396815],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001092911,"about_ca_system_score_gemma":0.0002416774,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00009469057,"about_ca_topic_score_gemma":0.0001375913,"domain_scores_codex":[0.9982158,0.0003397097,0.0004227982,0.0003947374,0.0004459008,0.0001810672],"domain_scores_gemma":[0.9986216,0.0001302344,0.0002013665,0.0007540135,0.0002664219,0.00002638732],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"qualitative","study_design_gemma":"observational","study_design_scores_codex":[0.005655857,0.01734253,0.03960573,0.0008273451,0.0005530186,0.000007481457,0.339992,0.0000638454,0.1304496,0.03493101,0.178391,0.2521806],"study_design_scores_gemma":[0.001742882,0.0001973437,0.7917764,0.0004510158,0.0002350818,0.00002467244,0.05766438,0.00001303398,0.00487742,0.1349367,0.007600782,0.0004803173],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9648556,0.002447868,0.0001586743,0.006747357,0.001572462,0.001078025,0.000006278628,0.00004942071,0.02308437],"genre_scores_gemma":[0.9982759,0.00003687787,0.000399093,0.0006398546,0.00006975587,0.0002760873,0.00003167002,0.00001029364,0.0002604208],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.7521707,"threshold_uncertainty_score":0.345138,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.3773982575143485,"score_gpt":0.3923418996521501,"score_spread":0.01494364213780164,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}