OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: stabilize randomized SDK tests (#476)
mldangelo-oai37bf87a692fc
diff --git a/sdk/typescript/AGENTS.md b/sdk/typescript/AGENTS.mdindex fe67f6708..6fdcaa3f2 100644--- a/sdk/typescript/AGENTS.md+++ b/sdk/typescript/AGENTS.md@@ -41,13 +41,15 @@ accepted and rejected inputs, and each real bug or security boundary.From the SDK directory, run a focused test while iterating, then run the package checks:```bash-bun test tests-ts/<module>.test.ts-bun test --randomize --seed 12345+bun test --timeout 30000 tests-ts/<module>.test.ts+pnpm run test --seed 12345pnpm run typespnpm run formatpnpm run test```+Tests run in random order by default. To reproduce a failure, use the seed printed in Bun's test summary.+After the implementation is verified, keep the test in the final change onlywhen it provides meaningful, durable regression coverage. If it is merelydisposable implementation scaffolding, duplicates existing coverage, or woulddiff --git a/sdk/typescript/bunfig.toml b/sdk/typescript/bunfig.tomlnew file mode 100644index 000000000..d1639b19a--- /dev/null+++ b/sdk/typescript/bunfig.toml@@ -0,0 +1,2 @@+[test]+randomize = truediff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.tsindex 9b72f890c..85d1b6577 100644--- a/sdk/typescript/src/api.ts+++ b/sdk/typescript/src/api.ts@@ -2235,6 +2235,7 @@ interface ScanEventRunOptions {onObserverError?: (observer: ScanObserverName, error: unknown) => void;}+/** @internal */export async function runScanEvents(options: ScanEventRunOptions,): Promise<ScanResult> {diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 001efa4b4..5f7c03fd8 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -7,7 +7,7 @@ import {existsSync,lstatSync,realpathSync,- type Stats,+ type BigIntStats,writeSync,} from "node:fs";import {@@ -603,9 +603,9 @@ async function readPromptFiles(async function readRegularInputFile(path: string,repository: string,- metadata?: Pick<Stats, "isFile" | "dev" | "ino">,+ metadata?: Pick<BigIntStats, "isFile" | "dev" | "ino">,): Promise<string> {- const selected = metadata ?? (await lstat(path));+ const selected = metadata ?? (await lstat(path, { bigint: true }));if (!selected.isFile()) {throw new CodexSecurityError("Input files must be regular files.");}@@ -632,7 +632,7 @@ async function readRegularInputFile((constants.O_NONBLOCK ?? 0),);try {- const opened = await file.stat();+ const opened = await file.stat({ bigint: true });if (!opened.isFile() ||opened.dev !== selected.dev ||@@ -3273,22 +3273,24 @@ async function runSkill(localDeviceRoot !== normalizedDeviceRoot);if (!windowsNetworkPath) {const path = resolve(directory, input);- const metadata = await lstat(path).catch((error: unknown) => {- if (- typeof error === "object" &&- error !== null &&- "code" in error &&- (error.code === "ENOENT" ||- error.code === "ENOTDIR" ||- error.code === "ENAMETOOLONG" ||- error.code === "EINVAL")- ) {- return undefined;- }- throw new CodexSecurityError(- "Could not read the finding or issue input.",- );- });+ const metadata = await lstat(path, { bigint: true }).catch(+ (error: unknown) => {+ if (+ typeof error === "object" &&+ error !== null &&+ "code" in error &&+ (error.code === "ENOENT" ||+ error.code === "ENOTDIR" ||+ error.code === "ENAMETOOLONG" ||+ error.code === "EINVAL")+ ) {+ return undefined;+ }+ throw new CodexSecurityError(+ "Could not read the finding or issue input.",+ );+ },+ );if (metadata !== undefined) {if (!metadata.isFile()) {throw new CodexSecurityError(diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.tsindex cc25c2457..d198909fa 100644--- a/sdk/typescript/tests-ts/api.test.ts+++ b/sdk/typescript/tests-ts/api.test.ts@@ -4507,7 +4507,7 @@ describe("CodexSecurity orchestration", () => {try {const results = await Promise.allSettled(- clients.map((client) => client.run(repository)),+ clients.map((client) => client.run(repository).finally(releaseScans)),);for (const result of results) {expect(result).toMatchObject({@@ -4519,6 +4519,7 @@ describe("CodexSecurity orchestration", () => {}expect(scansStarted).toBe(2);} finally {+ releaseScans();await Promise.all(clients.map(async (client) => await client.close()));}});@@ -4610,7 +4611,9 @@ describe("CodexSecurity orchestration", () => {try {const results = await Promise.allSettled(clients.map((client, index) =>- client.run(repository, { mode: "deep", workers: index + 2 }),+ client+ .run(repository, { mode: "deep", workers: index + 2 })+ .finally(releaseScans),),);for (const result of results) {@@ -4633,6 +4636,7 @@ describe("CodexSecurity orchestration", () => {),).toBe(true);} finally {+ releaseScans();await Promise.all(clients.map(async (client) => await client.close()));}});@@ -6521,20 +6525,17 @@ setInterval(() => {}, 1000););const login = client.loginApiKey("secret-key");void login.catch(() => undefined);- for (let attempt = 0; attempt < 100; attempt += 1) {- const started = await import("node:fs/promises").then(({ stat }) =>- stat(ready).catch(() => null),- );- if (started !== null) break;- await new Promise((resolve) => setTimeout(resolve, 10));+ try {+ const deadline = Date.now() + 10_000;+ while (!existsSync(ready) && Date.now() < deadline) {+ await new Promise((resolve) => setTimeout(resolve, 25));+ }+ expect(existsSync(ready), "the fake login process started").toBe(true);+ await client.close();+ await expect(login).rejects.toThrow();+ await expect(stat(codexHome)).resolves.toBeDefined();+ } finally {+ await client.close();}- await expect(- import("node:fs/promises").then(({ stat }) => stat(ready)),- ).resolves.toBeDefined();- await client.close();- await expect(login).rejects.toThrow();- await expect(- import("node:fs/promises").then(({ stat }) => stat(codexHome)),- ).resolves.toBeDefined();- });+ }, 30_000);});diff --git a/sdk/typescript/tests-ts/cli-skills.test.ts b/sdk/typescript/tests-ts/cli-skills.test.tsindex aa6375f8b..60c778245 100644--- a/sdk/typescript/tests-ts/cli-skills.test.ts+++ b/sdk/typescript/tests-ts/cli-skills.test.ts@@ -13,6 +13,7 @@ import {} from "../src/cli.js";import type { LinearClientFactory } from "../src/linear.js";import { capture, dependencies } from "./cli-fixtures.js";+import { runMockInSubprocess } from "./support/isolated-mock.js";function linearIssue(identifier: string) {return {@@ -459,11 +460,102 @@ describe("CLI skill commands", () => {}});+ test("rejects input replacements whose numeric file IDs collide", async () => {+ if (+ runMockInSubprocess(+ import.meta.path,+ "rejects input replacements whose numeric file IDs collide",+ )+ ) {+ return;+ }+ const root = await mkdtemp(join(tmpdir(), "codex-security-file-identity-"));+ const selected = join(root, "finding.txt");+ const replacement = join(root, "replacement.txt");+ const selectedInode = 2n ** 60n;+ const replacementInode = selectedInode + 1n;+ expect(Number(selectedInode)).toBe(Number(replacementInode));+ await writeFile(selected, "ordinary finding\n");+ await writeFile(replacement, "SYNTHETIC_REPLACEMENT_FINDING\n");+ const canonicalSelected = await filesystem.realpath(selected);+ const originalLstat = filesystem.lstat;+ const originalOpen = filesystem.open;+ let restoreOpenedStat: (() => void) | undefined;+ let replaced = false;+ const reading = spyOn(filesystem, "lstat").mockImplementation((async (+ ...args: Parameters<typeof filesystem.lstat>+ ) => {+ const metadata = await originalLstat(...args);+ if (String(args[0]) === selected) {+ metadata.ino =+ typeof metadata.ino === "bigint"+ ? selectedInode+ : Number(selectedInode);+ }+ return metadata;+ }) as typeof filesystem.lstat);+ const opening = spyOn(filesystem, "open").mockImplementation(+ async (...args: Parameters<typeof filesystem.open>) => {+ if (String(args[0]) !== canonicalSelected) {+ return await originalOpen(...args);+ }+ replaced = true;+ const file = await originalOpen(replacement, args[1], args[2]);+ const originalStat = file.stat.bind(file);+ const openedStat = spyOn(file, "stat").mockImplementation((async (+ ...statArgs: Parameters<typeof file.stat>+ ) => {+ const metadata = await originalStat(...statArgs);+ metadata.ino =+ typeof metadata.ino === "bigint"+ ? replacementInode+ : Number(replacementInode);+ return metadata;+ }) as typeof file.stat);+ restoreOpenedStat = () => openedStat.mockRestore();+ return file;+ },+ );+ try {+ let started = false;+ const stderr = capture();+ const status = await main(+ ["validate", "finding.txt"],+ capture().stream,+ stderr.stream,+ dependencies({+ currentDirectory: root,+ onCodex: () => {+ started = true;+ return 0;+ },+ }),+ );+ expect(replaced).toBe(true);+ expect(status).toBe(2);+ expect(stderr.text()).not.toContain("SYNTHETIC_REPLACEMENT_FINDING");+ expect(started).toBe(false);+ } finally {+ restoreOpenedStat?.();+ opening.mockRestore();+ reading.mockRestore();+ await rm(root, { recursive: true, force: true });+ }+ });+test.each(process.platform === "win32"? ["symbolic link"]: ["symbolic link", "FIFO"],)("rejects finding files replaced with a %s", async (replacement) => {+ if (+ runMockInSubprocess(+ import.meta.path,+ `rejects finding files replaced with a ${replacement}`,+ )+ ) {+ return;+ }const root = await mkdtemp(join(tmpdir(), "codex-security-skill-inputs-"));try {const repository = join(root, "repository");@@ -475,6 +567,7 @@ describe("CLI skill commands", () => {const canonicalSelected = await filesystem.realpath(selected);const originalOpen = filesystem.open;+ let replaced = false;const opening = spyOn(filesystem, "open").mockImplementation(async (...args: Parameters<typeof filesystem.open>) => {if (String(args[0]) === canonicalSelected) {@@ -482,6 +575,7 @@ describe("CLI skill commands", () => {await rm(selected);if (replacement === "FIFO") execFileSync("mkfifo", [selected]);else await symlink(external, selected);+ replaced = true;}return await originalOpen(...args);},@@ -490,20 +584,20 @@ describe("CLI skill commands", () => {try {let started = false;const stderr = capture();- expect(- await main(- ["validate", "finding.txt"],- capture().stream,- stderr.stream,- dependencies({- currentDirectory: repository,- onCodex: () => {- started = true;- return 0;- },- }),- ),- ).toBe(2);+ const status = await main(+ ["validate", "finding.txt"],+ capture().stream,+ stderr.stream,+ dependencies({+ currentDirectory: repository,+ onCodex: () => {+ started = true;+ return 0;+ },+ }),+ );+ expect(replaced, "the file-open replacement hook ran").toBe(true);+ expect(status).toBe(2);expect(stderr.text()).not.toContain("SYNTHETIC_EXTERNAL_FINDING");expect(started).toBe(false);} finally {diff --git a/sdk/typescript/tests-ts/config.test.ts b/sdk/typescript/tests-ts/config.test.tsindex 8951c6d62..6cc2c4765 100644--- a/sdk/typescript/tests-ts/config.test.ts+++ b/sdk/typescript/tests-ts/config.test.ts@@ -62,6 +62,45 @@ function runPinnedCodex(codexHome: string, arguments_: readonly string[]) {);}+function macOsSandboxUnavailable(): boolean {+ if (process.platform !== "darwin") return false;++ // Check the host independently of the generated scan permission profile.+ const result = Bun.spawnSync(+ [+ "/usr/bin/sandbox-exec",+ "-p",+ "(version 1) (allow default)",+ "/usr/bin/true",+ ],+ { stdout: "pipe", stderr: "pipe" },+ );+ return (+ result.exitCode !== 0 &&+ new TextDecoder().decode(result.stderr).trim() ===+ "sandbox-exec: sandbox_apply: Operation not permitted"+ );+}++async function scanSandboxFixture() {+ const root = await temporaryDirectory();+ const codexHome = join(root, "codex-home");+ const workspace = join(root, "workspace");+ const stateDirectory = join(root, "state");+ await Promise.all(+ [codexHome, workspace, stateDirectory].map((path) => mkdir(path)),+ );+ await writeCodexConfig(+ join(codexHome, "config.toml"),+ scanRuntimeCodexConfig(+ await mergedCodexConfig({}),+ stateDirectory,+ codexHome,+ ),+ );+ return { root, codexHome, workspace };+}+describe("Codex configuration", () => {test("automatically reviews scan execution approvals by default", async () => {expect(await mergedCodexConfig({})).toMatchObject({@@ -333,63 +372,64 @@ describe("Codex configuration", () => {});});- test("denies writes outside the scan workspace and state directory", async () => {- const root = await temporaryDirectory();- const codexHome = join(root, "codex-home");- const workspace = join(root, "workspace");- const stateDirectory = join(root, "state");- await Promise.all(- [codexHome, workspace, stateDirectory].map((path) => mkdir(path)),- );- await writeCodexConfig(- join(codexHome, "config.toml"),- scanRuntimeCodexConfig(- await mergedCodexConfig({}),- stateDirectory,- codexHome,- ),- );- const node = Bun.which("node");- expect(node).not.toBeNull();- const attemptWrite = (path: string) =>- runPinnedCodex(codexHome, [- "sandbox",- "--config",- "permissions.codex_security_scan.network.enabled=true",- "--permission-profile",- "codex_security_scan",- "--cd",- workspace,- node!,- "-e",- "require('node:fs').writeFileSync(process.argv[1], 'probe')",- path,- ]);-- const allowed = join(workspace, "inside.txt");- const permitted = attemptWrite(allowed);- const outside = join(root, "outside.txt");- expect(attemptWrite(outside).exitCode).not.toBe(0);- await expect(stat(outside)).rejects.toMatchObject({ code: "ENOENT" });- if (permitted.exitCode !== 0) {- const details = new TextDecoder().decode(permitted.stderr);- if (- process.platform === "linux" &&- /bwrap: (?:setting up uid map: Permission denied|loopback: Failed RTM_NEWADDR: Operation not permitted)/u.test(- details,- )- ) {- expect(runPinnedCodex(codexHome, ["features", "list"]).exitCode).toBe(- 0,+ test("writes scan permissions accepted by the pinned Codex CLI", async () => {+ const { codexHome, workspace } = await scanSandboxFixture();+ const result = runPinnedCodex(codexHome, [+ "--cd",+ workspace,+ "features",+ "list",+ ]);+ expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);+ expect(result.stdout.length).toBeGreaterThan(0);+ });++ test.skipIf(macOsSandboxUnavailable())(+ "denies writes outside the scan workspace and state directory",+ async () => {+ const { root, codexHome, workspace } = await scanSandboxFixture();+ const node = Bun.which("node");+ expect(node).not.toBeNull();+ const attemptWrite = (path: string) =>+ runPinnedCodex(codexHome, [+ "sandbox",+ "--config",+ "permissions.codex_security_scan.network.enabled=true",+ "--permission-profile",+ "codex_security_scan",+ "--cd",+ workspace,+ node!,+ "-e",+ "require('node:fs').writeFileSync(process.argv[1], 'probe')",+ path,+ ]);++ const allowed = join(workspace, "inside.txt");+ const permitted = attemptWrite(allowed);+ const outside = join(root, "outside.txt");+ expect(attemptWrite(outside).exitCode).not.toBe(0);+ await expect(stat(outside)).rejects.toMatchObject({ code: "ENOENT" });+ if (permitted.exitCode !== 0) {+ const details = new TextDecoder().decode(permitted.stderr);+ if (+ process.platform === "linux" &&+ /bwrap: (?:setting up uid map: Permission denied|loopback: Failed RTM_NEWADDR: Operation not permitted)/u.test(+ details,+ )+ ) {+ expect(runPinnedCodex(codexHome, ["features", "list"]).exitCode).toBe(+ 0,+ );+ return;+ }+ throw new Error(+ `The pinned Codex CLI rejected an allowed scan write: ${details}`,);- return;}- throw new Error(- `The pinned Codex CLI rejected an allowed scan write: ${details}`,- );- }- expect(await readFile(allowed, "utf8")).toBe("probe");- });+ expect(await readFile(allowed, "utf8")).toBe("probe");+ },+ );test("writes Windows sandbox settings accepted by the pinned Codex CLI", async () => {const root = await temporaryDirectory();diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.tsindex dc72f52fe..807ecc753 100644--- a/sdk/typescript/tests-ts/skeleton.test.ts+++ b/sdk/typescript/tests-ts/skeleton.test.ts@@ -1,5 +1,6 @@import { readFile } from "node:fs/promises";import { describe, expect, test } from "bun:test";+import { parse } from "smol-toml";import { CodexSecurity, CodexSecurityError, VERSION } from "../src/index.js";import { main } from "../src/cli.js";@@ -62,7 +63,7 @@ describe("TypeScript package skeleton", () => {}});- test("uses the default test timeout consistently across CI platforms", async () => {+ test("randomizes tests with the default timeout across CI platforms", async () => {const packageJson = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8"),);@@ -70,10 +71,14 @@ describe("TypeScript package skeleton", () => {new URL("../../../.github/workflows/node-ci.yml", import.meta.url),"utf8",);+ const bunConfig = parse(+ await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"),+ );expect(packageJson.scripts.test).toBe("bun test --timeout 30000 ./tests-ts",);+ expect(bunConfig).toMatchObject({ test: { randomize: true } });expect(ciWorkflow).toContain("run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",);diff --git a/sdk/typescript/tsconfig.build.json b/sdk/typescript/tsconfig.build.jsonindex ccf7d2ecb..cf223fc5e 100644--- a/sdk/typescript/tsconfig.build.json+++ b/sdk/typescript/tsconfig.build.json@@ -7,6 +7,7 @@"outDir": "dist","declaration": true,"declarationMap": true,+ "stripInternal": true,"sourceMap": true,"inlineSources": true,"noEmit": false,