Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: retain replay, SARIF, and credential coverage (#1994)

Michael D'Angelo30fd1851b126
diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mts
index efae11302..35520e993 100644
--- a/evals/triage-finding/scripts/test-triage-provider.mts
+++ b/evals/triage-finding/scripts/test-triage-provider.mts
@@ -4,7 +4,7 @@ import fs from "node:fs";
import os from "node:os";
import { createServer } from "node:net";
import path from "node:path";
-import test, { after } from "node:test";
+import test, { after, type TestContext } from "node:test";
import { DatabaseSync } from "node:sqlite";
import { parse } from "yaml";
import { beforeAll } from "./triage-provider.mts";
@@ -289,31 +289,44 @@ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_t
},
);
-// Exercise the pinned provider and Codex SDK subprocess without a model request.
-// The normal setup command supplies the host helper build used by the runner.
-test(
- "provider preserves proxies and rebinds saved evaluations to fresh runtimes",
- { skip: process.platform === "win32", timeout: 120000 },
- async (t) => {
- const root = fs.realpathSync(
- fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-")),
+// Reuse the immutable Node binaries; captures and Promptfoo databases belong to each test.
+function providerNode(...parts: string[]) {
+ const executable = path.join(fixtureRoot, ...parts);
+ if (!fs.existsSync(executable)) {
+ fs.mkdirSync(path.dirname(executable), { recursive: true });
+ fs.copyFileSync(
+ process.execPath,
+ executable,
+ fs.constants.COPYFILE_FICLONE,
);
- t.after(() => fs.rmSync(root, { recursive: true, force: true }));
- const capture = path.join(root, "captures.jsonl");
- const fakeCodex = path.join(root, "codex");
- const fail = path.join(root, "fail");
- function writeCaptureCodex(executable: string, checkPolicy: boolean) {
- fs.writeFileSync(
- executable,
- `#!${process.execPath}
+ }
+ return executable;
+}
+
+function captureProvider(t: TestContext, mode: "launcher" | "replay") {
+ const root = fs.realpathSync(
+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-")),
+ );
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ const capture = path.join(root, "captures.jsonl");
+ const fakeCodex = path.join(root, "codex");
+ const fail = path.join(root, "fail");
+ fs.writeFileSync(
+ fakeCodex,
+ `#!${process.execPath}
const fs = require('node:fs');
const path = require('node:path');
const cwd = process.argv[process.argv.indexOf('--cd') + 1];
fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));
-${checkPolicy ? probeFixturePolicy : "const policy = undefined;"}
+// Each replay has a fresh cwd. One concurrent caller claims its helper probe.
+let capturedPolicy = '';
+if (${mode === "launcher"} || (!fs.existsSync(${JSON.stringify(fail)}) && fs.mkdirSync(path.join(cwd, '.policy-probe'), {recursive: true}))) {
+ ${probeFixturePolicy}
+ capturedPolicy = policy;
+}
const directories = process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []);
const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);
-fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');
+fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy:capturedPolicy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');
console.log(JSON.stringify({type:'thread.started', thread_id:'synthetic-thread'}));
if (fs.existsSync(${JSON.stringify(fail)})) {
console.log(JSON.stringify({type:'turn.failed',error:{message:'synthetic retryable failure'}}));
@@ -322,26 +335,52 @@ if (fs.existsSync(${JSON.stringify(fail)})) {
console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));
}
`,
- { mode: 0o755 },
- );
- }
- writeCaptureCodex(fakeCodex, true);
- const ambientNode = path.join(root, "ambient-node", "bin", "node");
- fs.mkdirSync(path.dirname(ambientNode), { recursive: true });
- fs.copyFileSync(
- process.execPath,
+ { mode: 0o755 },
+ );
+ const ambientNode = providerNode("ambient-node", "bin", "node");
+ const customNode = providerNode("custom-node", "bin", "node");
+ const selectedNode = providerNode("selected-node", "bin", "synthetic-node");
+ const environment = {
+ ...proxies,
+ CODEX_MCP_NODE_PATH: ambientNode,
+ OPENAI_API_KEY: "synthetic-test-key",
+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),
+ PROMPTFOO_DISABLE_WAL_MODE: "true",
+ PROMPTFOO_DISABLE_TELEMETRY: "1",
+ PROMPTFOO_DISABLE_UPDATE: "1",
+ };
+ const readRows = (): Capture[] =>
+ fs
+ .readFileSync(capture, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line) as Capture);
+ return {
+ root,
+ fakeCodex,
+ fail,
+ ambientNode,
+ customNode,
+ selectedNode,
+ environment,
+ readRows,
+ };
+}
+
+// Exercise the pinned provider and Codex SDK subprocess without a model request.
+test(
+ "provider preserves proxies and selects each Node launcher",
+ { skip: process.platform === "win32", timeout: 120000 },
+ async (t) => {
+ const {
+ root,
+ fakeCodex,
ambientNode,
- fs.constants.COPYFILE_FICLONE,
- );
- const environment = {
- ...proxies,
- CODEX_MCP_NODE_PATH: ambientNode,
- OPENAI_API_KEY: "synthetic-test-key",
- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),
- PROMPTFOO_DISABLE_WAL_MODE: "true",
- PROMPTFOO_DISABLE_TELEMETRY: "1",
- PROMPTFOO_DISABLE_UPDATE: "1",
- };
+ customNode,
+ selectedNode,
+ environment,
+ readRows,
+ } = captureProvider(t, "launcher");
const configPath = path.join(root, "config.json");
const provider = providerConfig("triage");
provider.config.codex_path_override = fakeCodex;
@@ -356,26 +395,7 @@ if (fs.existsSync(${JSON.stringify(fail)})) {
const calibration = providerConfig("calibration");
calibration.id = provider.id;
calibration.config.codex_path_override = fakeCodex;
- const customNode = path.join(root, "custom-node", "bin", "node");
- fs.mkdirSync(path.dirname(customNode), { recursive: true });
- fs.copyFileSync(
- process.execPath,
- customNode,
- fs.constants.COPYFILE_FICLONE,
- );
calibration.config.cli_env.CODEX_MCP_NODE_PATH = customNode;
- const selectedNode = path.join(
- root,
- "selected-node",
- "bin",
- "synthetic-node",
- );
- fs.mkdirSync(path.dirname(selectedNode), { recursive: true });
- fs.copyFileSync(
- process.execPath,
- selectedNode,
- fs.constants.COPYFILE_FICLONE,
- );
const nodeAlias = path.join(root, "node-alias");
const nodeDispatcher = path.join(root, "node-runtime", "dispatcher");
fs.mkdirSync(path.dirname(nodeDispatcher));
@@ -447,11 +467,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
);
for (const result of simultaneous)
assert.equal(result.code, 0, result.output);
- const rows = fs
- .readFileSync(capture, "utf8")
- .trim()
- .split("\n")
- .map((line) => JSON.parse(line) as Capture);
+ const rows = readRows();
assert.equal(rows.length, 8);
assert.equal(new Set(rows.map((row) => row.cwd)).size, 3);
for (const row of rows) {
@@ -480,13 +496,23 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
.length,
1,
);
+ },
+);
- // Prompt-specific settings merge and render separately for each concurrent
- // case, including the original configuration saved for retry and resume.
- // The initial matrix already ran the helper through each selected Node
- // launcher; these calls capture the configuration saved for replay.
- const templatedCodex = path.join(root, "templated-codex");
- writeCaptureCodex(templatedCodex, false);
+test(
+ "provider rebinds saved templated settings for retry and resume",
+ { skip: process.platform === "win32", timeout: 120000 },
+ async (t) => {
+ const {
+ root,
+ fakeCodex,
+ fail,
+ customNode,
+ selectedNode,
+ environment,
+ readRows,
+ } = captureProvider(t, "replay");
+ const provider = providerConfig("triage");
const templatedConfig = path.join(root, "templated.json");
fs.writeFileSync(
templatedConfig,
@@ -496,7 +522,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
...provider,
config: {
...provider.config,
- codex_path_override: templatedCodex,
+ codex_path_override: fakeCodex,
cli_env: {
...provider.config.cli_env,
CODEX_MCP_NODE_PATH: "{{node_path}}",
@@ -533,11 +559,6 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
})),
}),
);
- const templatedEnvironment = {
- ...environment,
- PROMPTFOO_CONFIG_DIR: path.join(root, "templated-state"),
- };
- const startingRows = rows.length;
fs.writeFileSync(fail, "");
const templatedInitial = await invoke(
[
@@ -550,7 +571,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
"--no-share",
"--no-progress-bar",
],
- templatedEnvironment,
+ environment,
);
assert.notEqual(templatedInitial.code, 0, templatedInitial.output);
assert.match(templatedInitial.output, /synthetic retryable failure/);
@@ -563,28 +584,28 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
"--no-share",
"--no-progress-bar",
],
- templatedEnvironment,
+ environment,
);
assert.equal(templatedRetry.code, 0, templatedRetry.output);
const templatedDatabase = new DatabaseSync(
- path.join(templatedEnvironment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),
+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),
{ enableForeignKeyConstraints: false },
);
templatedDatabase.prepare("DELETE FROM eval_results").run();
templatedDatabase.close();
const templatedResume = await invoke(
["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],
- templatedEnvironment,
+ environment,
);
assert.equal(templatedResume.code, 0, templatedResume.output);
- const templatedRows = fs
- .readFileSync(capture, "utf8")
- .trim()
- .split("\n")
- .map((line) => JSON.parse(line) as Capture)
- .slice(startingRows);
+ const templatedRows = readRows();
assert.equal(templatedRows.length, 12);
assert.equal(new Set(templatedRows.map((row) => row.cwd)).size, 3);
+ const replayProbes = templatedRows.filter((row) => row.policy);
+ assert.equal(replayProbes.length, 2);
+ assert.equal(new Set(replayProbes.map((row) => row.cwd)).size, 2);
+ for (const row of replayProbes)
+ assert.match(row.policy, /Synthetic policy/);
assert.equal(
templatedRows.filter((row) => row.nodePath === customNode).length,
6,
@@ -600,7 +621,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "
path.dirname(row.nodePath),
path.dirname(row.nodePath),
path.dirname(row.nodePath),
- path.dirname(templatedCodex),
+ path.dirname(fakeCodex),
]);
}
},
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index 27d52ad9b..da4ebd529 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -2148,7 +2148,6 @@ The extraction root is not enforced.
second = FINALIZER._sarif_rule(sibling["ruleId"], [sibling])
for key in ("id", "name", "shortDescription"):
self.assertEqual(first[key], second[key])
- self.assertNotEqual(first["name"], self.finding["ruleId"])
self.assertEqual(first["properties"]["security-severity"], "8.1")
self.assertEqual(second["properties"]["security-severity"], "9.4")
diff --git a/plugins/codex-security/tests/test_sarif_projection.py b/plugins/codex-security/tests/test_sarif_projection.py
index a4032c3d9..e45b11ff8 100644
--- a/plugins/codex-security/tests/test_sarif_projection.py
+++ b/plugins/codex-security/tests/test_sarif_projection.py
@@ -12,6 +12,7 @@ from contextlib import redirect_stderr
from pathlib import Path
from unittest import mock
+import pytest
from workbench_test_support import load_script
PLUGIN_ROOT = Path(__file__).resolve().parent.parent
@@ -25,12 +26,14 @@ class SarifProjectionTest(unittest.TestCase):
self.findings = json.loads((EXAMPLE_DIR / "findings.json").read_text())
self.finding = self.findings["findings"][0]
+ @pytest.mark.cross_platform
def test_sarif_preserves_alert_identity_when_finding_text_changes(self) -> None:
finding = self.finding
finding["extensions"] = {"candidateId": "candidate-example"}
run = FINALIZER.build_sarif(self.manifest, self.findings)["runs"][0]
rule = run["tool"]["driver"]["rules"][0]
result = run["results"][0]
+ self.assertNotEqual(rule["name"], finding["ruleId"])
self.assertEqual(rule["shortDescription"]["text"], rule["name"])
self.assertIn(finding["remediation"], rule["help"]["text"])
for field in ("findingId", "occurrenceId"):
@@ -52,6 +55,7 @@ class SarifProjectionTest(unittest.TestCase):
for field in ("ruleId", "ruleIndex", "locations", "partialFingerprints", "properties"):
self.assertEqual(changed["results"][0][field], result[field])
+ @pytest.mark.cross_platform
def test_sarif_merges_shared_rule_metadata_without_mixing_results(self) -> None:
first = copy.deepcopy(self.finding)
first["severity"] = {"level": "low"}
@@ -95,6 +99,7 @@ class SarifProjectionTest(unittest.TestCase):
self.assertEqual(result["properties"]["severity"], own["severity"]["level"])
self.assertEqual(result["ruleIndex"], 0)
+ @pytest.mark.cross_platform
def test_sarif_security_severity_mapping(self) -> None:
for level, score, expected in (
("critical", None, "9.5"),
diff --git a/sdk/typescript/tests-ts/cli-authentication.test.ts b/sdk/typescript/tests-ts/cli-authentication.test.ts
index 0314343a4..a0555996b 100644
--- a/sdk/typescript/tests-ts/cli-authentication.test.ts
+++ b/sdk/typescript/tests-ts/cli-authentication.test.ts
@@ -928,9 +928,10 @@ describe("CLI authentication", () => {
expect(run(["login", "--with-api-key"], "synthetic-key\n")).toBe(0);
expect(await stat(join(credentialHome, "auth.json"))).toBeDefined();
await expect(stat(join(repository, "auth.json"))).rejects.toThrow();
- if (configuredHome === mountedHome) {
+ if (expectedHome === relativeHome || expectedHome === mountedHome) {
expect(run(["login", "status"])).toBe(0);
expect(run(["logout"])).toBe(0);
+ expect(existsSync(join(credentialHome, "auth.json"))).toBe(false);
} else {
// Empty and blank settings share the default. Each login must create its own file.
await rm(join(credentialHome, "auth.json"));