{"id":"W4321022177","doi":"10.2139/ssrn.4361607","title":"Auditing Large Language Models: A Three-Layered Approach","year":2023,"lang":"en","type":"article","venue":"SSRN Electronic Journal","topic":"Software Engineering Research","field":"Computer Science","cited_by":72,"is_retracted":false,"has_abstract":false,"ca_institutions":"Institute on Governance","funders":"","keywords":"Audit; Corporate governance; Blueprint; Engineering ethics; Business; Engineering; Accounting; Finance","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.003396912,0.0001523723,0.0001597982,0.0003339814,0.000253325,0.0002107885,0.001301386,0.0000696445,0.000004860491],"category_scores_gemma":[0.0003154929,0.0001402585,0.0001044268,0.001146115,0.00001507341,0.0005469766,0.000331163,0.001960804,0.0001398316],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0005396765,"about_ca_system_score_gemma":0.001090548,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002485483,"about_ca_topic_score_gemma":0.00005731927,"domain_scores_codex":[0.9953739,0.00006491624,0.0002078362,0.0003194827,0.0006059452,0.003427897],"domain_scores_gemma":[0.9990923,0.0002134887,0.00006791457,0.0004254011,0.00007182726,0.0001290593],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0000141804,0.0001302734,0.0024978,0.00003945445,0.0002687084,0.0001402141,0.002840445,0.03210032,0.0006512895,0.8861357,0.001220888,0.07396072],"study_design_scores_gemma":[0.0006133607,0.0001001872,0.001077126,0.00001628871,0.000004514689,0.000570946,0.0008393899,0.8425889,0.00005659953,0.1536885,0.0001913603,0.0002527633],"study_design_candidate":"theoretical_or_conceptual","study_design_consensus":null,"genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.08257446,0.001168238,0.9144627,0.0003308292,0.0001523968,0.00009764137,0.000001503457,0.000660674,0.0005515184],"genre_scores_gemma":[0.9936175,0.0002743819,0.004875114,0.00003623284,0.0002649153,0.00001625171,0.00000393017,0.00003284261,0.0008788397],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.911043,"threshold_uncertainty_score":0.8518825,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01955062911801558,"score_gpt":0.2626686702789555,"score_spread":0.2431180411609399,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}