{"id":"W4391833165","doi":"10.1145/3640543.3645200","title":"Why and When LLM-Based Assistants Can Go Wrong: Investigating the Effectiveness of Prompt-Based Interactions for Software Help-Seeking","year":2024,"lang":"en","type":"preprint","venue":"","topic":"Topic Modeling","field":"Computer Science","cited_by":61,"is_retracted":false,"has_abstract":true,"ca_institutions":"Simon Fraser University","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Context (archaeology); Task (project management); Domain (mathematical analysis); Software; Baseline (sea); Relevance (law); Perception; Software engineering; Psychology; Engineering; Programming language","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.001417774,0.0003226459,0.0004093922,0.0001964753,0.0001962281,0.0004503686,0.0009267317,0.0001454174,0.00000540167],"category_scores_gemma":[0.0005646739,0.0002424833,0.0001721443,0.0001729972,0.0001087751,0.0001014622,0.001181529,0.0006202594,8.938723e-7],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001848746,"about_ca_system_score_gemma":0.0007435121,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0008592881,"about_ca_topic_score_gemma":0.0005773797,"domain_scores_codex":[0.9976589,0.0003688158,0.0004610916,0.0008928125,0.0003223322,0.0002960111],"domain_scores_gemma":[0.996269,0.002184743,0.0002636483,0.0009490031,0.0002349827,0.00009858399],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0003861348,0.001103423,0.04318464,0.09119856,0.002079435,0.00006977859,0.0142137,0.4859492,0.04719759,0.1116789,0.003762844,0.1991758],"study_design_scores_gemma":[0.0003351593,0.00005861393,0.0008081323,0.003251519,0.00006463618,0.000002610369,0.00003917823,0.9337057,0.01075421,0.0504871,0.0001870067,0.0003061303],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.04532005,0.0001495717,0.9494879,0.002254507,0.00100887,0.001301964,0.00004787768,0.0002912367,0.0001380431],"genre_scores_gemma":[0.7189347,4.083333e-7,0.2799147,0.0005175113,0.00007262308,0.0004598857,0.00001807029,0.00003216436,0.00004993598],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.6736146,"threshold_uncertainty_score":0.9888183,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.04492530613574788,"score_gpt":0.2926778040444654,"score_spread":0.2477524979087175,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}