Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: preserve Windows CI timeouts in isolated tests (#608)

faizan-oaif4adc8549e39
diff --git a/sdk/typescript/scripts/run-ci-tests.mjs b/sdk/typescript/scripts/run-ci-tests.mjs
index 477be4ad5..f530fa555 100644
--- a/sdk/typescript/scripts/run-ci-tests.mjs
+++ b/sdk/typescript/scripts/run-ci-tests.mjs
@@ -1,6 +1,7 @@
import { spawn } from "node:child_process";
import { mkdir, readFile, readdir } from "node:fs/promises";
import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
import { shardTestFiles } from "./test-shards.mjs";
const selection = /^([1-9]\d*)\/([1-9]\d*)$/.exec(process.argv[2] ?? "");
@@ -24,6 +25,7 @@ const timings = JSON.parse(
await readFile(new URL("./ci-test-durations.json", import.meta.url), "utf8"),
);
const windows = process.platform === "win32";
+const defaultTestTimeoutMs = windows ? "120000" : "30000";
const assignments = shardTestFiles(
tests,
count,
@@ -41,19 +43,111 @@ await mkdir(new URL("../reports/", import.meta.url), { recursive: true }).catch(
// Bun warns about report write failures without changing the test result.
},
);
-const child = spawn(
- "bun",
- [
- "test",
- "--timeout",
- windows ? "120000" : "30000",
- "--reporter=junit",
- `--reporter-outfile=reports/junit-${shard}.xml`,
- ...process.argv.slice(3),
- ...selected.files.map((file) => `./tests-ts/${file}`),
- ],
- { cwd: packageDirectory, stdio: "inherit", windowsHide: true },
-);
+const testArguments = [
+ "test",
+ "--timeout",
+ defaultTestTimeoutMs,
+ "--reporter=junit",
+ `--reporter-outfile=reports/junit-${shard}.xml`,
+ ...process.argv.slice(3),
+ ...selected.files.map((file) => `./tests-ts/${file}`),
+];
+// Keep arities aligned with the pinned Bun's test_params:
+// https://github.com/oven-sh/bun/blob/bun-v1.3.14/src/cli/Arguments.zig
+const optionalBunOptions = {
+ bail: { type: "string" },
+ changed: { type: "string" },
+ parallel: { type: "string" },
+ inspect: { type: "string" },
+ "inspect-wait": { type: "string" },
+ "inspect-brk": { type: "string" },
+ config: { type: "string", short: "c" },
+};
+const parsingArguments = [...testArguments];
+let parsed;
+let bareOptional;
+do {
+ parsed = parseArgs({
+ args: parsingArguments,
+ allowPositionals: true,
+ strict: false,
+ tokens: true,
+ // Bun options that consume values must not turn those values into timeouts.
+ options: {
+ ...optionalBunOptions,
+ timeout: { type: "string" },
+ "test-name-pattern": { type: "string", short: "t" },
+ grep: { type: "string" },
+ "rerun-each": { type: "string" },
+ retry: { type: "string" },
+ seed: { type: "string" },
+ "coverage-reporter": { type: "string" },
+ "coverage-dir": { type: "string" },
+ reporter: { type: "string" },
+ "reporter-outfile": { type: "string" },
+ "max-concurrency": { type: "string" },
+ "path-ignore-patterns": { type: "string" },
+ "parallel-delay": { type: "string" },
+ shard: { type: "string" },
+ preload: { type: "string", short: "r" },
+ require: { type: "string" },
+ import: { type: "string" },
+ "cpu-prof-name": { type: "string" },
+ "cpu-prof-dir": { type: "string" },
+ "cpu-prof-interval": { type: "string" },
+ "heap-prof-name": { type: "string" },
+ "heap-prof-dir": { type: "string" },
+ install: { type: "string" },
+ eval: { type: "string", short: "e" },
+ print: { type: "string", short: "p" },
+ port: { type: "string" },
+ origin: { type: "string" },
+ conditions: { type: "string" },
+ "fetch-preconnect": { type: "string" },
+ "max-http-header-size": { type: "string" },
+ "dns-result-order": { type: "string" },
+ title: { type: "string" },
+ "unhandled-rejections": { type: "string" },
+ "console-depth": { type: "string" },
+ "user-agent": { type: "string" },
+ "cron-title": { type: "string" },
+ "cron-period": { type: "string" },
+ "main-fields": { type: "string" },
+ "extension-order": { type: "string" },
+ "tsconfig-override": { type: "string" },
+ define: { type: "string", short: "d" },
+ drop: { type: "string" },
+ feature: { type: "string" },
+ loader: { type: "string", short: "l" },
+ "jsx-factory": { type: "string" },
+ "jsx-fragment": { type: "string" },
+ "jsx-import-source": { type: "string" },
+ "jsx-runtime": { type: "string" },
+ "env-file": { type: "string" },
+ cwd: { type: "string" },
+ },
+ });
+ // Bun leaves a following flag outside an optional value; parseArgs does not.
+ bareOptional = parsed.tokens.find(
+ (token) =>
+ token.kind === "option" &&
+ Object.hasOwn(optionalBunOptions, token.name) &&
+ !token.inlineValue &&
+ token.value?.startsWith("-"),
+ );
+ if (bareOptional) {
+ parsingArguments[bareOptional.index] = `--${bareOptional.name}=`;
+ }
+} while (bareOptional);
+const child = spawn("bun", testArguments, {
+ cwd: packageDirectory,
+ env: {
+ ...process.env,
+ CODEX_SECURITY_TEST_TIMEOUT_MS: parsed.values.timeout,
+ },
+ stdio: "inherit",
+ windowsHide: true,
+});
child.once("error", (error) => {
console.error(error);
process.exitCode = 1;
diff --git a/sdk/typescript/tests-ts/support/test-subprocess.ts b/sdk/typescript/tests-ts/support/test-subprocess.ts
index dfe4588f5..c840aa5ee 100644
--- a/sdk/typescript/tests-ts/support/test-subprocess.ts
+++ b/sdk/typescript/tests-ts/support/test-subprocess.ts
@@ -5,11 +5,12 @@ import { expect } from "bun:test";
export function runTestInSubprocess(file: string, name: string): boolean {
const identity = `${file}::${name}`;
if (process.env["CODEX_SECURITY_ISOLATED_TEST"] === identity) return false;
+ const timeout = process.env["CODEX_SECURITY_TEST_TIMEOUT_MS"] ?? "30000";
const pattern = `${name.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")}$`;
const result = spawnSync(
process.execPath,
- ["test", "--timeout", "30000", "--test-name-pattern", pattern, file],
+ ["test", "--timeout", timeout, "--test-name-pattern", pattern, file],
{
cwd: fileURLToPath(new URL("../../", import.meta.url)),
encoding: "utf8",
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index 95b5289eb..a416db464 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -1,9 +1,11 @@
+import { spawnSync } from "node:child_process";
import {
copyFile,
mkdir,
mkdtemp,
readdir,
readFile,
+ realpath,
rm,
writeFile,
} from "node:fs/promises";
@@ -68,14 +70,35 @@ test.each([
expect(shards.flatMap(({ files }) => files).sort()).toEqual(tests);
});
+const defaultTimeoutMs = process.platform === "win32" ? "120000" : "30000";
+
test.each([
- ["available", false],
- ["available", true],
- ["blocked directory", false],
- ["blocked directory", true],
+ [defaultTimeoutMs, [], "available", false],
+ [defaultTimeoutMs, [], "available", true],
+ [defaultTimeoutMs, [], "blocked directory", false],
+ [defaultTimeoutMs, [], "blocked directory", true],
+ ["5000", ["--timeout", "5000"], "available", false],
+ ["5000", ["--timeout=5000"], "available", false],
+ ["5000", ["--timeout", "9000", "--timeout=5000"], "available", false],
+ ["5000", ["--timeout=9000", "--timeout", "5000"], "available", false],
+ [defaultTimeoutMs, ["--", "--timeout", "5000"], "available", false],
+ [defaultTimeoutMs, ["-t", "--timeout=5000"], "available", false],
+ [defaultTimeoutMs, ["--grep", "--timeout=5000"], "available", false],
+ [
+ defaultTimeoutMs,
+ ["--test-name-pattern", "--timeout=5000"],
+ "available",
+ false,
+ ],
+ ["7000", ["--timeout", "7000", "-t", "--timeout=5000"], "available", false],
+ [defaultTimeoutMs, ["--coverage-dir", "--timeout=5000"], "available", false],
+ [defaultTimeoutMs, ["--title", "--timeout=5000"], "available", false],
+ ["7000", ["--config", "--timeout=7000"], "available", false],
+ ["7000", ["-c", "--timeout=7000"], "available", false],
+ ["7000", ["--bail", "--timeout=7000"], "available", false],
] as const)(
- "preserves the test result with %s reports and failure=%p",
- async (report, fail) => {
+ "uses timeout %s with %p, %s reports and failure=%p",
+ async (timeout, options, report, fail) => {
const node = Bun.which("node");
expect(node).not.toBeNull();
const root = await mkdtemp(join(tmpdir(), "codex-security-shard-report-"));
@@ -94,15 +117,25 @@ test.each([
}
await writeFile(
join(root, "tests-ts", "probe.test.ts"),
- `import { expect, test } from "bun:test"; test("synthetic report probe", () => expect(true).toBe(${!fail}));\n`,
+ `import { expect, test } from "bun:test";
+test("synthetic report probe --timeout=5000", () => {
+ expect(process.env["CODEX_SECURITY_TEST_TIMEOUT_MS"]).toBe(${JSON.stringify(timeout)});
+ expect(true).toBe(${!fail});
+});\n`,
);
if (report === "blocked directory") {
await writeFile(join(root, "reports"), "synthetic blocker");
}
const child = Bun.spawn({
- cmd: [node!, join(root, "scripts", "run-ci-tests.mjs"), "1/1"],
+ cmd: [
+ node!,
+ join(root, "scripts", "run-ci-tests.mjs"),
+ "1/1",
+ ...options,
+ ],
env: {
...process.env,
+ CODEX_SECURITY_TEST_TIMEOUT_MS: "1",
PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,
},
stdin: "ignore",
@@ -136,3 +169,38 @@ test.each([
}
},
);
+
+test("preserves the configured timeout in isolated test subprocesses", async () => {
+ const directory = await realpath(
+ await mkdtemp(join(tmpdir(), "codex-security-test-timeout-")),
+ );
+ const fixture = join(directory, "isolated.test.ts");
+ const helper = new URL("./support/test-subprocess.ts", import.meta.url).href;
+ await writeFile(
+ fixture,
+ `import { test } from "bun:test";
+import { runTestInSubprocess } from ${JSON.stringify(helper)};
+test("isolated timeout", async () => {
+ if (runTestInSubprocess(import.meta.path, "isolated timeout")) return;
+ await Bun.sleep(1_000);
+});
+`,
+ );
+
+ try {
+ const result = spawnSync(
+ process.execPath,
+ ["test", "--timeout", "30000", fixture],
+ {
+ encoding: "utf8",
+ env: { ...process.env, CODEX_SECURITY_TEST_TIMEOUT_MS: "100" },
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect(result.status, result.stderr || result.error?.message).toBe(1);
+ expect(result.stderr).toContain("this test timed out after 100ms");
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});