{"id":"W4393213239","doi":"10.1145/3597503.3639194","title":"ChatGPT Incorrectness Detection in Software Reviews","year":2024,"lang":"en","type":"preprint","venue":"","topic":"Software Engineering Research","field":"Computer Science","cited_by":10,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University; University of Calgary","funders":"Natural Sciences and Engineering Research Council of Canada","keywords":"Computer science; Suite; Benchmark (surveying); Selection (genetic algorithm); Generative grammar; Artificial intelligence; Software; Machine learning; Natural language processing; Information retrieval; Programming language","routes":{"ca_aff":true,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.001131081,0.000291621,0.0004039591,0.0007053915,0.00002624358,0.0003713158,0.001589845,0.0003002572,0.00002971521],"category_scores_gemma":[0.001424047,0.0002623665,0.0001430775,0.0009818847,0.00001879646,0.0001200738,0.005212533,0.001711587,0.0007604425],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003730779,"about_ca_system_score_gemma":0.0001892277,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.0002888135,"about_ca_topic_score_gemma":0.0002148328,"domain_scores_codex":[0.9977651,0.000112928,0.000411995,0.0009332034,0.0003920033,0.0003847522],"domain_scores_gemma":[0.9980872,0.0003890086,0.00005425027,0.001306016,0.00006586155,0.00009762971],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.000002198709,0.00003624207,0.0009215376,0.001824624,0.00001649041,0.0001087868,0.0006847201,0.002070632,0.00006450974,0.0004552205,0.002291177,0.9915239],"study_design_scores_gemma":[0.000874309,0.0002795721,0.03117755,0.01204135,0.00004768786,0.0002568359,0.00003648056,0.6822848,0.01848558,0.1205224,0.1289598,0.005033673],"study_design_candidate":"design_other","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.008760555,0.006521329,0.9758427,0.0002945534,0.005674262,0.0007988674,0.000001898236,0.00178883,0.0003170408],"genre_scores_gemma":[0.7832527,0.002221689,0.2045603,0.0002695649,0.0009232387,0.002215022,0.00001824979,0.0001750639,0.006364197],"genre_candidate":"methods","genre_consensus":null,"teacher_disagreement_score":0.9864902,"threshold_uncertainty_score":0.9999828,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02675092483946025,"score_gpt":0.299219897492254,"score_spread":0.2724689726527938,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}