Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: accept UTF-8 BOMs in scan inputs and calibration datasets (#2151)

Michael D'Angelo607038e13988
diff --git a/evals/triage-finding/README.md b/evals/triage-finding/README.md
index 4ed04a1cc..2a11cef76 100644
--- a/evals/triage-finding/README.md
+++ b/evals/triage-finding/README.md
@@ -94,6 +94,8 @@ pnpm --dir evals/triage-finding run calibration:generate --output artifacts/cali
The optional exporter retains `--dataset`, `--output`, `--repo-root`, `--case`, and `--variant`. Its default output remains `tests/calibration-oss.yaml`; generated YAML is not needed to run the suite.
+The exporter and hydrator accept UTF-8 JSON datasets with or without a leading byte-order mark.
+
Hydrate the local OSS checkouts:
```bash
diff --git a/evals/triage-finding/scripts/generate-calibration-tests.mts b/evals/triage-finding/scripts/generate-calibration-tests.mts
index f74f90acd..c396d72cf 100644
--- a/evals/triage-finding/scripts/generate-calibration-tests.mts
+++ b/evals/triage-finding/scripts/generate-calibration-tests.mts
@@ -158,6 +158,12 @@ export function selectedVariants(
return variants;
}
+export function readCalibrationDataset(filename: string): {
+ cases: CalibrationCase[];
+} {
+ return JSON.parse(fs.readFileSync(filename, "utf8").replace(/^\uFEFF/u, ""));
+}
+
export function generateTests(
options: {
datasetPath?: string;
@@ -166,8 +172,8 @@ export function generateTests(
variantId?: string | null;
} = {},
) {
- const dataset = JSON.parse(
- fs.readFileSync(options.datasetPath ?? DEFAULT_DATASET, "utf8"),
+ const dataset = readCalibrationDataset(
+ options.datasetPath ?? DEFAULT_DATASET,
);
return selectedVariants(dataset, options).map(({ testCase, variant }) =>
buildPromptfooTest(
diff --git a/evals/triage-finding/scripts/hydrate-calibration-repos.mts b/evals/triage-finding/scripts/hydrate-calibration-repos.mts
index adb000535..de8bbf0c7 100644
--- a/evals/triage-finding/scripts/hydrate-calibration-repos.mts
+++ b/evals/triage-finding/scripts/hydrate-calibration-repos.mts
@@ -3,6 +3,7 @@ import { runGit } from "../sastbench/scripts/hydrate-sastbench-repos.mts";
import {
DEFAULT_DATASET,
+ readCalibrationDataset,
selectedVariants,
variantCaseId,
} from "./generate-calibration-tests.mts";
@@ -117,7 +118,7 @@ if (
import.meta.filename === fs.realpathSync(process.argv[1])
) {
const args = parseArgs(process.argv.slice(2));
- const dataset = JSON.parse(fs.readFileSync(args.dataset, "utf8"));
+ const dataset = readCalibrationDataset(args.dataset);
const jobs = plannedJobs(dataset, args);
const variantWord = jobs.length === 1 ? "variant" : "variants";
console.log(
diff --git a/evals/triage-finding/scripts/test-generate-calibration-tests.mts b/evals/triage-finding/scripts/test-generate-calibration-tests.mts
index f47015d15..982626c6e 100644
--- a/evals/triage-finding/scripts/test-generate-calibration-tests.mts
+++ b/evals/triage-finding/scripts/test-generate-calibration-tests.mts
@@ -28,6 +28,8 @@ const tmpDir = fs.realpathSync(
);
const generated = path.join(tmpDir, "calibration-oss.yaml");
const generatedSmoke = path.join(tmpDir, "calibration-smoke.yaml");
+const markedDataset = path.join(tmpDir, "calibration.json");
+fs.writeFileSync(markedDataset, "\uFEFF" + fs.readFileSync(dataset, "utf8"));
execFileSync(
process.execPath,
@@ -119,7 +121,7 @@ execFileSync(
"--experimental-strip-types",
generator,
"--dataset",
- dataset,
+ markedDataset,
"--output",
generatedSmoke,
"--case",
@@ -158,6 +160,8 @@ const hydration = execFileSync(
[
"--experimental-strip-types",
path.join(evalDir, "scripts", "hydrate-calibration-repos.mts"),
+ "--dataset",
+ markedDataset,
"--case",
"oss-dompurify-ghsa-v8jm-5vwx-cfxm",
"--variant",
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts b/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts
index 0e08c4d1e..31a997fff 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/project-config.ts
@@ -42,7 +42,7 @@ export async function readDeepScanProjectConfig(
const text = await readFile(path, "utf8");
let value: unknown;
try {
- if (extension === ".json") value = JSON.parse(text);
+ if (extension === ".json") value = JSON.parse(text.replace(/^\uFEFF/u, ""));
else {
const document = parseDocument(text, { prettyErrors: false });
if (document.errors.length > 0) throw document.errors[0];
diff --git a/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts b/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts
index 19833c35e..a0a03864b 100644
--- a/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts
@@ -125,7 +125,9 @@ function readAssessment(path: string): unknown {
const text =
path === "-"
? decodePosixBytes(contents)
- : decodeUtf8(contents).replace(/\r\n?/gu, "\n");
+ : decodeUtf8(contents)
+ .replace(/^\uFEFF/u, "")
+ .replace(/\r\n?/gu, "\n");
try {
return parseJson(text);
} catch (error) {
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts
index 2d7783f00..a45dfe77a 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_project_config.ts
@@ -23,7 +23,7 @@ await test("explicit YAML and JSON selection use the CLI project format and minu
],
[
"security.json",
- JSON.stringify({
+ `\uFEFF${JSON.stringify({
scan: {
mode: "deep",
max_time_minutes: 30,
@@ -34,7 +34,7 @@ await test("explicit YAML and JSON selection use the CLI project format and minu
max_discovery_runs: 4,
},
},
- }),
+ })}`,
],
]) {
await writeFile(join(root, name), content);
diff --git a/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py b/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py
index 324ccb35a..8b60340ec 100644
--- a/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py
+++ b/plugins/codex-security/skills/threat-model-visualize/scripts/build_explorer.py
@@ -519,7 +519,7 @@ def main():
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
try:
- model = json.loads(args.model.read_text(encoding="utf-8"))
+ model = json.loads(args.model.read_text(encoding="utf-8-sig"))
assessment = args.assessment.read_bytes().decode("utf-8")
if args.output.exists():
require(
diff --git a/plugins/codex-security/tests/test_threat_model_visualize.py b/plugins/codex-security/tests/test_threat_model_visualize.py
index a3f807794..45af98637 100644
--- a/plugins/codex-security/tests/test_threat_model_visualize.py
+++ b/plugins/codex-security/tests/test_threat_model_visualize.py
@@ -113,7 +113,7 @@ class ControlValidation(unittest.TestCase):
def effect(self):
return self.model["scenarios"][0]["controls"][0]["effect"]
- def build_html(self, assessment, *, error=None, existing_html=None):
+ def build_html(self, assessment, *, error=None, existing_html=None, model_encoding="utf-8"):
with tempfile.TemporaryDirectory() as folder:
folder = Path(folder)
model_file, source, output = (
@@ -121,7 +121,9 @@ class ControlValidation(unittest.TestCase):
folder / "assessment.md",
folder / "output.html",
)
- model_file.write_text(json.dumps(self.model, ensure_ascii=False), encoding="utf-8")
+ model_file.write_text(
+ json.dumps(self.model, ensure_ascii=False), encoding=model_encoding
+ )
source.write_bytes(assessment.encode("utf-8"))
if existing_html is not None:
output.write_text(existing_html, encoding="utf-8")
@@ -651,12 +653,14 @@ class ControlValidation(unittest.TestCase):
def test_template_tokens_in_input_are_preserved_without_resubstitution(self):
tokens = "__MODEL_JSON__ __ASSESSMENT_JSON__ __EXPLORER_JS__ __EXPLORER_CSS__"
assessment = (
- "policy denies this caller\n" + tokens + "\n</script><script>unsafe()</script> & café"
+ "\ufeffpolicy denies this caller\n"
+ + tokens
+ + "\n</script><script>unsafe()</script> & café"
)
self.model["title"] = tokens
self.model["metadata"] = [{"label": "Scope", "value": "Quoted tokens"}]
self.model["evidence"][0]["quote"] = assessment
- html = self.build_html(assessment)
+ html = self.build_html(assessment, model_encoding="utf-8-sig")
self.assertEqual(self.embedded_data(html, "model-data"), self.model)
self.assertEqual(self.embedded_data(html, "assessment-data"), assessment)
self.assertNotIn("</script><script>unsafe()</script>", html)
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 131911b04..3ed321e07 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -644,6 +644,9 @@ Explicit `-c` wins. Without either, no file is discovered. `scan`, `bulk-scan`,
`scan-components`, and `info` accept project files. SDK calls and saved reruns
do not load them automatically.
+JSON project files and component plans (`--components-file`) accept UTF-8 with
+or without a leading byte-order mark.
+
Treat the selected file as trusted operator configuration: native settings can
start MCP processes and choose model-service destinations. Keep CI scanner
configuration outside the untrusted checkout being assessed.
@@ -2686,6 +2689,9 @@ native structured output. The CLI also checks the completed JSON before writing
stdout; it does not convert a prose assessment into JSON afterward. Invalid JSON
or a schema mismatch fails with exit code 2 and no assessment on stdout.
+Save the output schema as UTF-8 without a byte-order mark so both the CLI and
+Codex can read it.
+
Use a self-contained schema in the subset supported by the selected Codex model
and provider. The local checker uses JSON Schema 2020-12, but accepting a schema
locally does not guarantee provider support. Keep the schema small and use it only
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 5de5409e9..6215c33bb 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -5118,10 +5118,12 @@ export async function main(
? options.component.map((path) => ({ name: path, paths: [path] }))
: componentPlanSchema.parse(
JSON.parse(
- await readRegularInputFile(
- resolveCliPath(directory, options.componentsFile),
- repository,
- ),
+ (
+ await readRegularInputFile(
+ resolveCliPath(directory, options.componentsFile),
+ repository,
+ )
+ ).replace(/^\uFEFF/u, ""),
),
).components;
if (
diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.ts
index 2989df6c1..eb263f07f 100644
--- a/sdk/typescript/tests-ts/component-scan.test.ts
+++ b/sdk/typescript/tests-ts/component-scan.test.ts
@@ -1982,7 +1982,10 @@ test("CLI requires an explicitly selected API key before automatic planning", as
test("CLI forwards scan settings and accepts legacy scope notes", async () => {
const paths = await fixture();
const planFile = join(paths.root, "plan.json");
- await writeFile(planFile, JSON.stringify({ components: [components[0]] }));
+ await writeFile(
+ planFile,
+ `\uFEFF${JSON.stringify({ components: [components[0]] })}`,
+ );
const signals = new FakeSignals();
const createSecurity = mock((_value: unknown) => {
return client(run)();
diff --git a/sdk/typescript/tests-ts/patch-risk-contract.test.ts b/sdk/typescript/tests-ts/patch-risk-contract.test.ts
index 7cfff394d..86177e5e5 100644
--- a/sdk/typescript/tests-ts/patch-risk-contract.test.ts
+++ b/sdk/typescript/tests-ts/patch-risk-contract.test.ts
@@ -709,16 +709,17 @@ describe("patch risk assessment contract", () => {
try {
const original =
JSON.stringify(assessment(), null, 2).replaceAll("\n", "\r\n") + "\r\n";
+ const marked = "\uFEFF" + original;
for (const name of ["assessment with spaces.json", "-1", "- item", "-"]) {
const path = join(outside, name);
- await writeFile(path, original);
+ await writeFile(path, marked);
const result = validateText("not stdin", outside, [
name.startsWith("-") ? `./${name}` : name,
]);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toBe("");
expect(result.stderr).toBe("");
- expect(await readFile(path, "utf8")).toBe(original);
+ expect(await readFile(path, "utf8")).toBe(marked);
expect(validateText("", outside, ["--", name]).status).toBe(
name === "-" ? 1 : 0,
);