OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: accept UTF-8 BOMs in scan inputs and calibration datasets (#2151)
Michael D'Angelo607038e13988
diff --git a/evals/triage-finding/README.md b/evals/triage-finding/README.mdindex 4ed04a1cc..2a11cef76 100644--- a/evals/triage-finding/README.md+++ b/evals/triage-finding/README.md@@ -94,6 +94,8 @@ pnpm --dir evals/triage-finding run calibration:generate --output artifacts/caliThe optional exporter retains `--dataset`, `--output`, `--repo-root`, `--case`, and `--variant`. Its default output remains `tests/calibration-oss.yaml`; generated YAML is not needed to run the suite.+The exporter and hydrator accept UTF-8 JSON datasets with or without a leading byte-order mark.+Hydrate the local OSS checkouts:```bashdiff --git a/evals/triage-finding/scripts/generate-calibration-tests.mts b/evals/triage-finding/scripts/generate-calibration-tests.mtsindex f74f90acd..c396d72cf 100644--- a/evals/triage-finding/scripts/generate-calibration-tests.mts+++ b/evals/triage-finding/scripts/generate-calibration-tests.mts@@ -158,6 +158,12 @@ export function selectedVariants(return variants;}+export function readCalibrationDataset(filename: string): {+ cases: CalibrationCase[];+} {+ return JSON.parse(fs.readFileSync(filename, "utf8").replace(/^\uFEFF/u, ""));+}+export function generateTests(options: {datasetPath?: string;@@ -166,8 +172,8 @@ export function generateTests(variantId?: string | null;} = {},) {- const dataset = JSON.parse(- fs.readFileSync(options.datasetPath ?? DEFAULT_DATASET, "utf8"),+ const dataset = readCalibrationDataset(+ options.datasetPath ?? DEFAULT_DATASET,);return selectedVariants(dataset, options).map(({ testCase, variant }) =>buildPromptfooTest(diff --git a/evals/triage-finding/scripts/hydrate-calibration-repos.mts b/evals/triage-finding/scripts/hydrate-calibration-repos.mtsindex adb000535..de8bbf0c7 100644--- a/evals/triage-finding/scripts/hydrate-calibration-repos.mts+++ b/evals/triage-finding/scripts/hydrate-calibration-repos.mts@@ -3,6 +3,7 @@ import { runGit } from "../sastbench/scripts/hydrate-sastbench-repos.mts";import {DEFAULT_DATASET,+ readCalibrationDataset,selectedVariants,variantCaseId,} from "./generate-calibration-tests.mts";@@ -117,7 +118,7 @@ if (import.meta.filename === fs.realpathSync(process.argv[1])) {const args = parseArgs(process.argv.slice(2));- const dataset = JSON.parse(fs.readFileSync(args.dataset, "utf8"));+ const dataset = readCalibrationDataset(args.dataset);const jobs = plannedJobs(dataset, args);const variantWord = jobs.length === 1 ? "variant" : "variants";console.log(diff --git a/evals/triage-finding/scripts/test-generate-calibration-tests.mts b/evals/triage-finding/scripts/test-generate-calibration-tests.mtsindex f47015d15..982626c6e 100644--- a/evals/triage-finding/scripts/test-generate-calibration-tests.mts+++ b/evals/triage-finding/scripts/test-generate-calibration-tests.mts@@ -28,6 +28,8 @@ const tmpDir = fs.realpathSync();const generated = path.join(tmpDir, "calibration-oss.yaml");const generatedSmoke = path.join(tmpDir, "calibration-smoke.yaml");+const markedDataset = path.join(tmpDir, "calibration.json");+fs.writeFileSync(markedDataset, "\uFEFF" + fs.readFileSync(dataset, "utf8"));execFileSync(process.execPath,@@ -119,7 +121,7 @@ execFileSync("--experimental-strip-types",generator,"--dataset",- dataset,+ markedDataset,"--output",generatedSmoke,"--case",@@ -158,6 +160,8 @@ const hydration = execFileSync(["--experimental-strip-types",path.join(evalDir, "scripts", "hydrate-calibration-repos.mts"),+ "--dataset",+ markedDataset,"--case","oss-dompurify-ghsa-v8jm-5vwx-cfxm","--variant",diff --git a/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts b/plugins/codex-security/mcp-app/src/deep-scan/project-config.tsindex 0e08c4d1e..31a997fff 100644--- a/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts+++ b/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts@@ -42,7 +42,7 @@ export async function readDeepScanProjectConfig(const text = await readFile(path, "utf8");let value: unknown;try {- if (extension === ".json") value = JSON.parse(text);+ if (extension === ".json") value = JSON.parse(text.replace(/^\uFEFF/u, ""));else {const document = parseDocument(text, { prettyErrors: false });if (document.errors.length > 0) throw document.errors[0];diff --git a/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts b/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.tsindex 19833c35e..a0a03864b 100644--- a/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts+++ b/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts@@ -125,7 +125,9 @@ function readAssessment(path: string): unknown {const text =path === "-"? decodePosixBytes(contents)- : decodeUtf8(contents).replace(/\r\n?/gu, "\n");+ : decodeUtf8(contents)+ .replace(/^\uFEFF/u, "")+ .replace(/\r\n?/gu, "\n");try {return parseJson(text);} catch (error) {diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.tsindex 2d7783f00..a45dfe77a 100644--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts@@ -23,7 +23,7 @@ await test("explicit YAML and JSON selection use the CLI project format and minu],["security.json",- JSON.stringify({+ `\uFEFF${JSON.stringify({scan: {mode: "deep",max_time_minutes: 30,@@ -34,7 +34,7 @@ await test("explicit YAML and JSON selection use the CLI project format and minumax_discovery_runs: 4,},},- }),+ })}`,],]) {await writeFile(join(root, name), content);diff --git a/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py b/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.pyindex 324ccb35a..8b60340ec 100644--- a/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py+++ b/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py@@ -519,7 +519,7 @@ def main():parser.add_argument("--output", required=True, type=Path)args = parser.parse_args()try:- model = json.loads(args.model.read_text(encoding="utf-8"))+ model = json.loads(args.model.read_text(encoding="utf-8-sig"))assessment = args.assessment.read_bytes().decode("utf-8")if args.output.exists():require(diff --git a/plugins/codex-security/tests/test_threat_model_visualize.py b/plugins/codex-security/tests/test_threat_model_visualize.pyindex a3f807794..45af98637 100644--- a/plugins/codex-security/tests/test_threat_model_visualize.py+++ b/plugins/codex-security/tests/test_threat_model_visualize.py@@ -113,7 +113,7 @@ class ControlValidation(unittest.TestCase):def effect(self):return self.model["scenarios"][0]["controls"][0]["effect"]- def build_html(self, assessment, *, error=None, existing_html=None):+ def build_html(self, assessment, *, error=None, existing_html=None, model_encoding="utf-8"):with tempfile.TemporaryDirectory() as folder:folder = Path(folder)model_file, source, output = (@@ -121,7 +121,9 @@ class ControlValidation(unittest.TestCase):folder / "assessment.md",folder / "output.html",)- model_file.write_text(json.dumps(self.model, ensure_ascii=False), encoding="utf-8")+ model_file.write_text(+ json.dumps(self.model, ensure_ascii=False), encoding=model_encoding+ )source.write_bytes(assessment.encode("utf-8"))if existing_html is not None:output.write_text(existing_html, encoding="utf-8")@@ -651,12 +653,14 @@ class ControlValidation(unittest.TestCase):def test_template_tokens_in_input_are_preserved_without_resubstitution(self):tokens = "__MODEL_JSON__ __ASSESSMENT_JSON__ __EXPLORER_JS__ __EXPLORER_CSS__"assessment = (- "policy denies this caller\n" + tokens + "\n</script><script>unsafe()</script> & café"+ "\ufeffpolicy denies this caller\n"+ + tokens+ + "\n</script><script>unsafe()</script> & café")self.model["title"] = tokensself.model["metadata"] = [{"label": "Scope", "value": "Quoted tokens"}]self.model["evidence"][0]["quote"] = assessment- html = self.build_html(assessment)+ html = self.build_html(assessment, model_encoding="utf-8-sig")self.assertEqual(self.embedded_data(html, "model-data"), self.model)self.assertEqual(self.embedded_data(html, "assessment-data"), assessment)self.assertNotIn("</script><script>unsafe()</script>", html)diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.mdindex 131911b04..3ed321e07 100644--- a/sdk/typescript/docs/cli.md+++ b/sdk/typescript/docs/cli.md@@ -644,6 +644,9 @@ Explicit `-c` wins. Without either, no file is discovered. `scan`, `bulk-scan`,`scan-components`, and `info` accept project files. SDK calls and saved rerunsdo not load them automatically.+JSON project files and component plans (`--components-file`) accept UTF-8 with+or without a leading byte-order mark.+Treat the selected file as trusted operator configuration: native settings canstart MCP processes and choose model-service destinations. Keep CI scannerconfiguration outside the untrusted checkout being assessed.@@ -2686,6 +2689,9 @@ native structured output. The CLI also checks the completed JSON before writingstdout; it does not convert a prose assessment into JSON afterward. Invalid JSONor a schema mismatch fails with exit code 2 and no assessment on stdout.+Save the output schema as UTF-8 without a byte-order mark so both the CLI and+Codex can read it.+Use a self-contained schema in the subset supported by the selected Codex modeland provider. The local checker uses JSON Schema 2020-12, but accepting a schemalocally does not guarantee provider support. Keep the schema small and use it onlydiff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 5de5409e9..6215c33bb 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -5118,10 +5118,12 @@ export async function main(? options.component.map((path) => ({ name: path, paths: [path] })): componentPlanSchema.parse(JSON.parse(- await readRegularInputFile(- resolveCliPath(directory, options.componentsFile),- repository,- ),+ (+ await readRegularInputFile(+ resolveCliPath(directory, options.componentsFile),+ repository,+ )+ ).replace(/^\uFEFF/u, ""),),).components;if (diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.tsindex 2989df6c1..eb263f07f 100644--- a/sdk/typescript/tests-ts/component-scan.test.ts+++ b/sdk/typescript/tests-ts/component-scan.test.ts@@ -1982,7 +1982,10 @@ test("CLI requires an explicitly selected API key before automatic planning", astest("CLI forwards scan settings and accepts legacy scope notes", async () => {const paths = await fixture();const planFile = join(paths.root, "plan.json");- await writeFile(planFile, JSON.stringify({ components: [components[0]] }));+ await writeFile(+ planFile,+ `\uFEFF${JSON.stringify({ components: [components[0]] })}`,+ );const signals = new FakeSignals();const createSecurity = mock((_value: unknown) => {return client(run)();diff --git a/sdk/typescript/tests-ts/patch-risk-contract.test.ts b/sdk/typescript/tests-ts/patch-risk-contract.test.tsindex 7cfff394d..86177e5e5 100644--- a/sdk/typescript/tests-ts/patch-risk-contract.test.ts+++ b/sdk/typescript/tests-ts/patch-risk-contract.test.ts@@ -709,16 +709,17 @@ describe("patch risk assessment contract", () => {try {const original =JSON.stringify(assessment(), null, 2).replaceAll("\n", "\r\n") + "\r\n";+ const marked = "\uFEFF" + original;for (const name of ["assessment with spaces.json", "-1", "- item", "-"]) {const path = join(outside, name);- await writeFile(path, original);+ await writeFile(path, marked);const result = validateText("not stdin", outside, [name.startsWith("-") ? `./${name}` : name,]);expect(result.status, result.stderr).toBe(0);expect(result.stdout).toBe("");expect(result.stderr).toBe("");- expect(await readFile(path, "utf8")).toBe(original);+ expect(await readFile(path, "utf8")).toBe(marked);expect(validateText("", outside, ["--", name]).status).toBe(name === "-" ? 1 : 0,);