OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test: retain replay, SARIF, and credential coverage (#1994)
Michael D'Angelo30fd1851b126
diff --git a/evals/triage-finding/scripts/test-triage-provider.mts b/evals/triage-finding/scripts/test-triage-provider.mtsindex efae11302..35520e993 100644--- a/evals/triage-finding/scripts/test-triage-provider.mts+++ b/evals/triage-finding/scripts/test-triage-provider.mts@@ -4,7 +4,7 @@ import fs from "node:fs";import os from "node:os";import { createServer } from "node:net";import path from "node:path";-import test, { after } from "node:test";+import test, { after, type TestContext } from "node:test";import { DatabaseSync } from "node:sqlite";import { parse } from "yaml";import { beforeAll } from "./triage-provider.mts";@@ -289,31 +289,44 @@ console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,output_t},);-// Exercise the pinned provider and Codex SDK subprocess without a model request.-// The normal setup command supplies the host helper build used by the runner.-test(- "provider preserves proxies and rebinds saved evaluations to fresh runtimes",- { skip: process.platform === "win32", timeout: 120000 },- async (t) => {- const root = fs.realpathSync(- fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-")),+// Reuse the immutable Node binaries; captures and Promptfoo databases belong to each test.+function providerNode(...parts: string[]) {+ const executable = path.join(fixtureRoot, ...parts);+ if (!fs.existsSync(executable)) {+ fs.mkdirSync(path.dirname(executable), { recursive: true });+ fs.copyFileSync(+ process.execPath,+ executable,+ fs.constants.COPYFILE_FICLONE,);- t.after(() => fs.rmSync(root, { recursive: true, force: true }));- const capture = path.join(root, "captures.jsonl");- const fakeCodex = path.join(root, "codex");- const fail = path.join(root, "fail");- function writeCaptureCodex(executable: string, checkPolicy: boolean) {- fs.writeFileSync(- executable,- `#!${process.execPath}+ }+ return executable;+}++function captureProvider(t: TestContext, mode: "launcher" | "replay") {+ const root = fs.realpathSync(+ fs.mkdtempSync(path.join(os.tmpdir(), "triage-provider-")),+ );+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const capture = path.join(root, "captures.jsonl");+ const fakeCodex = path.join(root, "codex");+ const fail = path.join(root, "fail");+ fs.writeFileSync(+ fakeCodex,+ `#!${process.execPath}const fs = require('node:fs');const path = require('node:path');const cwd = process.argv[process.argv.indexOf('--cd') + 1];fs.readFileSync(path.join(cwd, 'plugins/codex-security/skills/triage-finding/SKILL.md'));-${checkPolicy ? probeFixturePolicy : "const policy = undefined;"}+// Each replay has a fresh cwd. One concurrent caller claims its helper probe.+let capturedPolicy = '';+if (${mode === "launcher"} || (!fs.existsSync(${JSON.stringify(fail)}) && fs.mkdirSync(path.join(cwd, '.policy-probe'), {recursive: true}))) {+ ${probeFixturePolicy}+ capturedPolicy = policy;+}const directories = process.argv.flatMap((arg, index) => arg === '--add-dir' ? [process.argv[index + 1]] : []);const overrides = process.argv.flatMap((arg, index) => arg === '--config' ? [process.argv[index + 1]] : []);-fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');+fs.appendFileSync(${JSON.stringify(capture)}, JSON.stringify({cwd,policy:capturedPolicy,directories,overrides,nodePath:process.env.CODEX_MCP_NODE_PATH,nodeTarget:fs.realpathSync.native(process.env.CODEX_MCP_NODE_PATH), proxies: Object.fromEntries(${JSON.stringify(Object.keys(proxies))}.map(key => [key,process.env[key]]))}) + '\\n');console.log(JSON.stringify({type:'thread.started', thread_id:'synthetic-thread'}));if (fs.existsSync(${JSON.stringify(fail)})) {console.log(JSON.stringify({type:'turn.failed',error:{message:'synthetic retryable failure'}}));@@ -322,26 +335,52 @@ if (fs.existsSync(${JSON.stringify(fail)})) {console.log(JSON.stringify({type:'turn.completed',usage:{input_tokens:1,cached_input_tokens:0,output_tokens:1}}));}`,- { mode: 0o755 },- );- }- writeCaptureCodex(fakeCodex, true);- const ambientNode = path.join(root, "ambient-node", "bin", "node");- fs.mkdirSync(path.dirname(ambientNode), { recursive: true });- fs.copyFileSync(- process.execPath,+ { mode: 0o755 },+ );+ const ambientNode = providerNode("ambient-node", "bin", "node");+ const customNode = providerNode("custom-node", "bin", "node");+ const selectedNode = providerNode("selected-node", "bin", "synthetic-node");+ const environment = {+ ...proxies,+ CODEX_MCP_NODE_PATH: ambientNode,+ OPENAI_API_KEY: "synthetic-test-key",+ PROMPTFOO_CONFIG_DIR: path.join(root, "state"),+ PROMPTFOO_DISABLE_WAL_MODE: "true",+ PROMPTFOO_DISABLE_TELEMETRY: "1",+ PROMPTFOO_DISABLE_UPDATE: "1",+ };+ const readRows = (): Capture[] =>+ fs+ .readFileSync(capture, "utf8")+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line) as Capture);+ return {+ root,+ fakeCodex,+ fail,+ ambientNode,+ customNode,+ selectedNode,+ environment,+ readRows,+ };+}++// Exercise the pinned provider and Codex SDK subprocess without a model request.+test(+ "provider preserves proxies and selects each Node launcher",+ { skip: process.platform === "win32", timeout: 120000 },+ async (t) => {+ const {+ root,+ fakeCodex,ambientNode,- fs.constants.COPYFILE_FICLONE,- );- const environment = {- ...proxies,- CODEX_MCP_NODE_PATH: ambientNode,- OPENAI_API_KEY: "synthetic-test-key",- PROMPTFOO_CONFIG_DIR: path.join(root, "state"),- PROMPTFOO_DISABLE_WAL_MODE: "true",- PROMPTFOO_DISABLE_TELEMETRY: "1",- PROMPTFOO_DISABLE_UPDATE: "1",- };+ customNode,+ selectedNode,+ environment,+ readRows,+ } = captureProvider(t, "launcher");const configPath = path.join(root, "config.json");const provider = providerConfig("triage");provider.config.codex_path_override = fakeCodex;@@ -356,26 +395,7 @@ if (fs.existsSync(${JSON.stringify(fail)})) {const calibration = providerConfig("calibration");calibration.id = provider.id;calibration.config.codex_path_override = fakeCodex;- const customNode = path.join(root, "custom-node", "bin", "node");- fs.mkdirSync(path.dirname(customNode), { recursive: true });- fs.copyFileSync(- process.execPath,- customNode,- fs.constants.COPYFILE_FICLONE,- );calibration.config.cli_env.CODEX_MCP_NODE_PATH = customNode;- const selectedNode = path.join(- root,- "selected-node",- "bin",- "synthetic-node",- );- fs.mkdirSync(path.dirname(selectedNode), { recursive: true });- fs.copyFileSync(- process.execPath,- selectedNode,- fs.constants.COPYFILE_FICLONE,- );const nodeAlias = path.join(root, "node-alias");const nodeDispatcher = path.join(root, "node-runtime", "dispatcher");fs.mkdirSync(path.dirname(nodeDispatcher));@@ -447,11 +467,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' ");for (const result of simultaneous)assert.equal(result.code, 0, result.output);- const rows = fs- .readFileSync(capture, "utf8")- .trim()- .split("\n")- .map((line) => JSON.parse(line) as Capture);+ const rows = readRows();assert.equal(rows.length, 8);assert.equal(new Set(rows.map((row) => row.cwd)).size, 3);for (const row of rows) {@@ -480,13 +496,23 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' ".length,1,);+ },+);- // Prompt-specific settings merge and render separately for each concurrent- // case, including the original configuration saved for retry and resume.- // The initial matrix already ran the helper through each selected Node- // launcher; these calls capture the configuration saved for replay.- const templatedCodex = path.join(root, "templated-codex");- writeCaptureCodex(templatedCodex, false);+test(+ "provider rebinds saved templated settings for retry and resume",+ { skip: process.platform === "win32", timeout: 120000 },+ async (t) => {+ const {+ root,+ fakeCodex,+ fail,+ customNode,+ selectedNode,+ environment,+ readRows,+ } = captureProvider(t, "replay");+ const provider = providerConfig("triage");const templatedConfig = path.join(root, "templated.json");fs.writeFileSync(templatedConfig,@@ -496,7 +522,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "...provider,config: {...provider.config,- codex_path_override: templatedCodex,+ codex_path_override: fakeCodex,cli_env: {...provider.config.cli_env,CODEX_MCP_NODE_PATH: "{{node_path}}",@@ -533,11 +559,6 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "})),}),);- const templatedEnvironment = {- ...environment,- PROMPTFOO_CONFIG_DIR: path.join(root, "templated-state"),- };- const startingRows = rows.length;fs.writeFileSync(fail, "");const templatedInitial = await invoke([@@ -550,7 +571,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' ""--no-share","--no-progress-bar",],- templatedEnvironment,+ environment,);assert.notEqual(templatedInitial.code, 0, templatedInitial.output);assert.match(templatedInitial.output, /synthetic retryable failure/);@@ -563,28 +584,28 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' ""--no-share","--no-progress-bar",],- templatedEnvironment,+ environment,);assert.equal(templatedRetry.code, 0, templatedRetry.output);const templatedDatabase = new DatabaseSync(- path.join(templatedEnvironment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),+ path.join(environment.PROMPTFOO_CONFIG_DIR, "promptfoo.db"),{ enableForeignKeyConstraints: false },);templatedDatabase.prepare("DELETE FROM eval_results").run();templatedDatabase.close();const templatedResume = await invoke(["eval", "--resume", "--no-cache", "--no-share", "--no-progress-bar"],- templatedEnvironment,+ environment,);assert.equal(templatedResume.code, 0, templatedResume.output);- const templatedRows = fs- .readFileSync(capture, "utf8")- .trim()- .split("\n")- .map((line) => JSON.parse(line) as Capture)- .slice(startingRows);+ const templatedRows = readRows();assert.equal(templatedRows.length, 12);assert.equal(new Set(templatedRows.map((row) => row.cwd)).size, 3);+ const replayProbes = templatedRows.filter((row) => row.policy);+ assert.equal(replayProbes.length, 2);+ assert.equal(new Set(replayProbes.map((row) => row.cwd)).size, 2);+ for (const row of replayProbes)+ assert.match(row.policy, /Synthetic policy/);assert.equal(templatedRows.filter((row) => row.nodePath === customNode).length,6,@@ -600,7 +621,7 @@ case "$0" in */node-alias) exec '${process.execPath.replaceAll("'", "'\\''")}' "path.dirname(row.nodePath),path.dirname(row.nodePath),path.dirname(row.nodePath),- path.dirname(templatedCodex),+ path.dirname(fakeCodex),]);}},diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.pyindex 27d52ad9b..da4ebd529 100644--- a/plugins/codex-security/tests/test_finalize_scan_contract.py+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py@@ -2148,7 +2148,6 @@ The extraction root is not enforced.second = FINALIZER._sarif_rule(sibling["ruleId"], [sibling])for key in ("id", "name", "shortDescription"):self.assertEqual(first[key], second[key])- self.assertNotEqual(first["name"], self.finding["ruleId"])self.assertEqual(first["properties"]["security-severity"], "8.1")self.assertEqual(second["properties"]["security-severity"], "9.4")diff --git a/plugins/codex-security/tests/test_sarif_projection.py b/plugins/codex-security/tests/test_sarif_projection.pyindex a4032c3d9..e45b11ff8 100644--- a/plugins/codex-security/tests/test_sarif_projection.py+++ b/plugins/codex-security/tests/test_sarif_projection.py@@ -12,6 +12,7 @@ from contextlib import redirect_stderrfrom pathlib import Pathfrom unittest import mock+import pytestfrom workbench_test_support import load_scriptPLUGIN_ROOT = Path(__file__).resolve().parent.parent@@ -25,12 +26,14 @@ class SarifProjectionTest(unittest.TestCase):self.findings = json.loads((EXAMPLE_DIR / "findings.json").read_text())self.finding = self.findings["findings"][0]+ @pytest.mark.cross_platformdef test_sarif_preserves_alert_identity_when_finding_text_changes(self) -> None:finding = self.findingfinding["extensions"] = {"candidateId": "candidate-example"}run = FINALIZER.build_sarif(self.manifest, self.findings)["runs"][0]rule = run["tool"]["driver"]["rules"][0]result = run["results"][0]+ self.assertNotEqual(rule["name"], finding["ruleId"])self.assertEqual(rule["shortDescription"]["text"], rule["name"])self.assertIn(finding["remediation"], rule["help"]["text"])for field in ("findingId", "occurrenceId"):@@ -52,6 +55,7 @@ class SarifProjectionTest(unittest.TestCase):for field in ("ruleId", "ruleIndex", "locations", "partialFingerprints", "properties"):self.assertEqual(changed["results"][0][field], result[field])+ @pytest.mark.cross_platformdef test_sarif_merges_shared_rule_metadata_without_mixing_results(self) -> None:first = copy.deepcopy(self.finding)first["severity"] = {"level": "low"}@@ -95,6 +99,7 @@ class SarifProjectionTest(unittest.TestCase):self.assertEqual(result["properties"]["severity"], own["severity"]["level"])self.assertEqual(result["ruleIndex"], 0)+ @pytest.mark.cross_platformdef test_sarif_security_severity_mapping(self) -> None:for level, score, expected in (("critical", None, "9.5"),diff --git a/sdk/typescript/tests-ts/cli-authentication.test.ts b/sdk/typescript/tests-ts/cli-authentication.test.tsindex 0314343a4..a0555996b 100644--- a/sdk/typescript/tests-ts/cli-authentication.test.ts+++ b/sdk/typescript/tests-ts/cli-authentication.test.ts@@ -928,9 +928,10 @@ describe("CLI authentication", () => {expect(run(["login", "--with-api-key"], "synthetic-key\n")).toBe(0);expect(await stat(join(credentialHome, "auth.json"))).toBeDefined();await expect(stat(join(repository, "auth.json"))).rejects.toThrow();- if (configuredHome === mountedHome) {+ if (expectedHome === relativeHome || expectedHome === mountedHome) {expect(run(["login", "status"])).toBe(0);expect(run(["logout"])).toBe(0);+ expect(existsSync(join(credentialHome, "auth.json"))).toBe(false);} else {// Empty and blank settings share the default. Each login must create its own file.await rm(join(credentialHome, "auth.json"));