Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(sdk): consolidate scan runtime coverage (#1982)

Michael D'Angelo7dcfe0d1dbd8
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index a19ecbb83..658b75085 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -203,6 +203,8 @@ jobs:
TMP: ${{ runner.temp }}
TMPDIR: ${{ runner.temp }}
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
+ # The separate MCP job covers Linux worker tests only when it is scheduled.
+ CODEX_SECURITY_TEST_MCP_WORKERS_SEPARATE: ${{ needs.classify-changes.outputs.plugin-tests-required || needs.classify-changes.outputs.source-required }}
run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ matrix.shard }}/6 --seed=${{ github.run_number }} ${{ matrix.os == 'ubuntu-latest' && '--coverage --coverage-reporter=text --coverage-reporter=lcov' || '' }}
- name: Upload test reports
if: always()
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index 9a392a6bf..ac19c872f 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -47,6 +47,9 @@ The SDK shards in `node-ci` use the workflow run number as their seed, so
rerunning a job repeats its test order. To replay a shard locally, use its
platform, shard selector, and seed, for example
`node --experimental-strip-types scripts/run-ci-tests.mts 2/6 --seed=12345`.
+On Linux, if the original workflow ran the separate MCP tests, prefix that
+command with `CODEX_SECURITY_TEST_MCP_WORKERS_SEPARATE=true` to skip the four
+worker groups they cover. Leave it unset to replay a run that skipped MCP.
The local test commands pass a 30-second per-test timeout explicitly. Windows
CI and the Windows runner diagnostics allow 120 seconds for slower native
@@ -184,8 +187,9 @@ Refresh timing hints from successful CI JUnit artifacts when shard runtimes
become uneven. From the SDK directory, pass reports for one platform to
`node --experimental-strip-types scripts/update-test-timings.mts win32 <junit.xml> ...`
(use `linux` or `darwin` for the other platforms). The updater sums testcase
-durations per file and uses their median across reports, preserving the other
-platforms. Review and commit the generated snapshot. The initial hints use runs
+durations for executed cases per file and uses their median across reports. A
+report that skipped every case in a file leaves its timing hint unchanged. Other
+platforms retain their hints. Review and commit the generated snapshot. The initial hints use runs
`39521539826`, `39694506447`, and `39694510011`. Keep runner versions, coverage,
timeouts, seeds, and platform-specific checks unchanged when comparing scheduling
changes; compare complete JUnit inventories and outcomes as well as wall time.
diff --git a/sdk/typescript/scripts/update-test-timings.mts b/sdk/typescript/scripts/update-test-timings.mts
index ee30cf0bb..556e0177f 100644
--- a/sdk/typescript/scripts/update-test-timings.mts
+++ b/sdk/typescript/scripts/update-test-timings.mts
@@ -12,6 +12,8 @@ export function updateTestTimings(
const samples = new Map<string, number[]>();
for (const report of reports) {
const totals = new Map<string, number>();
+ let sawTestcase = false;
+ let current: { file: string; seconds: number } | undefined;
const parser = new SaxesParser({ xmlns: false });
parser.on("opentag", ({ name, attributes }) => {
if (
@@ -24,7 +26,9 @@ export function updateTestTimings(
"Use successful JUnit reports to refresh test timings.",
);
}
+ if (name === "skipped") current = undefined;
if (name !== "testcase") return;
+ sawTestcase = true;
const file = attributes["file"]?.replaceAll("\\", "/").split("/").at(-1);
const seconds = Number(attributes["time"] ?? 0);
if (!file || !Number.isFinite(seconds) || seconds < 0) {
@@ -32,10 +36,16 @@ export function updateTestTimings(
"JUnit testcase needs a file and a nonnegative duration.",
);
}
+ current = { file, seconds };
+ });
+ parser.on("closetag", ({ name }) => {
+ if (name !== "testcase" || !current) return;
+ const { file, seconds } = current;
totals.set(file, (totals.get(file) ?? 0) + seconds);
+ current = undefined;
});
parser.write(report).close();
- if (!totals.size) throw new Error("JUnit report contains no test cases.");
+ if (!sawTestcase) throw new Error("JUnit report contains no test cases.");
for (const [file, seconds] of totals) {
const values = samples.get(file) ?? [];
values.push(seconds);
diff --git a/sdk/typescript/tests-ts/compact-diff-scan.test.ts b/sdk/typescript/tests-ts/compact-diff-scan.test.ts
index 44248a083..b337a7812 100644
--- a/sdk/typescript/tests-ts/compact-diff-scan.test.ts
+++ b/sdk/typescript/tests-ts/compact-diff-scan.test.ts
@@ -366,12 +366,18 @@ describe("compact diff scan", () => {
userContext,
);
- const started = await client.call(
- "start_codex_security_prompt_only_scan",
- selection,
- "user-context-stdin-owner",
- );
- const scan = started["scan"] as JsonObject;
+ const started = await client.request("tools/call", {
+ name: "start_codex_security_prompt_only_scan",
+ arguments: selection,
+ _meta: { "openai/threadId": "user-context-stdin-owner" },
+ });
+ expect(started["isError"], JSON.stringify(started)).not.toBe(true);
+ const instructions = (started["content"] as { text: string }[])[0]?.text;
+ expect(instructions).toContain("complete_codex_security_scan");
+ expect(instructions).not.toContain("get_codex_security_completed_scan");
+ const scan = (started["structuredContent"] as JsonObject)[
+ "scan"
+ ] as JsonObject;
expect(scan["userContext"]).toBe(userContext);
const updated = await client.call(
"update_codex_security_scan_context",
diff --git a/sdk/typescript/tests-ts/completed-scan-handoff.test.ts b/sdk/typescript/tests-ts/completed-scan-handoff.test.ts
deleted file mode 100644
index dd0526af5..000000000
--- a/sdk/typescript/tests-ts/completed-scan-handoff.test.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import { expect, test } from "bun:test";
-import { loadBundledRuntime } from "./plugin-root.js";
-
-test("does not request completed findings after a prompt-only scan", async () => {
- const runtime = await loadBundledRuntime();
- const source = /function promptOnlyScanResult\([^\n]*\) \{[\s\S]*?\n\}/u.exec(
- runtime,
- )?.[0];
- expect(source).toBeDefined();
- const actionSource =
- /function scanActionResult\([^\n]*\) \{[\s\S]*?\n\}/u.exec(runtime)?.[0];
- expect(actionSource).toBeDefined();
-
- const promptOnlyScanResult = new Function(
- "isRecord",
- "string2",
- "toolErrorResult",
- `${source}\n${actionSource}\nreturn promptOnlyScanResult;`,
- )(
- (value: unknown) => value !== null && typeof value === "object",
- () => ({ uuid: () => ({ safeParse: () => ({ success: true }) }) }),
- (message: string) => ({ content: [{ text: message }], isError: true }),
- ) as (input: {
- startDisposition: string;
- scan: { scanId: string; scanDir: string; handoffStatus: string };
- workspace: { results: { scanId: string } };
- }) => { content: { text: string }[]; isError?: boolean };
-
- const scanId = "00000000-0000-4000-8000-000000000000";
- const result = promptOnlyScanResult({
- startDisposition: "created",
- scan: { scanId, scanDir: "/tmp/scan", handoffStatus: "delivered" },
- workspace: { results: { scanId } },
- });
-
- expect(result.isError).toBeUndefined();
- expect(result.content[0]?.text).toContain("complete_codex_security_scan");
- expect(result.content[0]?.text).not.toContain(
- "get_codex_security_completed_scan",
- );
- expect(runtime).toContain('name: "get_codex_security_completed_scan"');
-});
diff --git a/sdk/typescript/tests-ts/deep-scan-windows-executable.test.ts b/sdk/typescript/tests-ts/deep-scan-windows-executable.test.ts
index 1cd11d404..f576f887d 100644
--- a/sdk/typescript/tests-ts/deep-scan-windows-executable.test.ts
+++ b/sdk/typescript/tests-ts/deep-scan-windows-executable.test.ts
@@ -165,27 +165,3 @@ test.each([
}
},
);
-
-test("keeps operating-system worker failures retryable", async () => {
- const runtime = await loadBundledRuntime();
- const source =
- /function classifyCodexWorkerError\([^\n]*\) \{[\s\S]*?\n\}/u.exec(
- runtime,
- )?.[0];
- expect(source).toBeDefined();
- const classify = new Function(
- `${source}\nreturn classifyCodexWorkerError;`,
- )() as (error: Error) => Error;
- for (const original of [
- Object.assign(new Error("spawn codex EPERM"), { code: "EPERM" }),
- ...[
- "Error: No such file or directory (os error 2)",
- "Error: The system cannot find the file specified. (os error 2)",
- ].map(
- (diagnostic) =>
- new Error(`Codex Exec exited with code 1:\n${diagnostic}`),
- ),
- ]) {
- expect(classify(original)).toBe(original);
- }
-});
diff --git a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts
index 495d2c7ce..296936b3f 100644
--- a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts
+++ b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts
@@ -3,13 +3,19 @@ import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { test } from "bun:test";
+// Always keep this file in the same shard for local replay and JUnit reporting.
+const it = test.skipIf(
+ process.platform === "linux" &&
+ process.env["CODEX_SECURITY_TEST_MCP_WORKERS_SEPARATE"] === "true",
+);
+
for (const boundary of [
"lifecycle",
"runtime settings",
"executable selection",
"diagnostics",
]) {
- test(`Deep worker ${boundary} at the child-process boundary`, async () => {
+ it(`Deep worker ${boundary} at the child-process boundary`, async () => {
// Keep each portable boundary in the SDK's macOS and Windows test shards.
const { stdout, stderr } = await promisify(execFile)(
"node",
diff --git a/sdk/typescript/tests-ts/hosted-scan.test.ts b/sdk/typescript/tests-ts/hosted-scan.test.ts
index 96c06e303..52548a92e 100644
--- a/sdk/typescript/tests-ts/hosted-scan.test.ts
+++ b/sdk/typescript/tests-ts/hosted-scan.test.ts
@@ -48,9 +48,12 @@ test("hosted owner artifact is opt-in and included in the completed receipt", as
for (const enabled of [false, true]) {
const input = await fixture();
const executor = draftExecutor();
+ const startThread = mock(() => {
+ throw new Error("No additional owner turn");
+ });
const result = await runHostedScan(
{ ...input, ownerSuggestions: enabled },
- { executor },
+ enabled ? { executor, ownerCodex: { startThread } } : { executor },
);
expect(result.status).toBe("completed");
expect(executor.run).toHaveBeenCalledTimes(1);
@@ -62,7 +65,9 @@ test("hosted owner artifact is opt-in and included in the completed receipt", as
join(input.outputDirectory, "scan-manifest.json"),
);
expect(JSON.stringify(manifest)).not.toContain("owner-suggestions.json");
- if (enabled)
+ if (enabled) {
+ expect(startThread).not.toHaveBeenCalled();
+ expect(result).not.toHaveProperty("ownerSuggestionsError");
expect(
await readJson(join(input.outputDirectory, "owner-suggestions.json")),
).toMatchObject({
@@ -70,6 +75,7 @@ test("hosted owner artifact is opt-in and included in the completed receipt", as
revision: input.revision,
results: [],
});
+ }
}
});
@@ -656,7 +662,10 @@ test.each([false, true])(
"CLI pipes retain reasoning effort, progress and execution receipts, including cancellation (%p)",
async (cancel) => {
const input = await fixture();
- if (!cancel) input.reasoningEffort = "synthetic-future-effort";
+ if (!cancel) {
+ input.reasoningEffort = "synthetic-future-effort";
+ input.ownerSuggestions = true;
+ }
const child = spawn(
process.execPath,
[join(import.meta.dirname, "../src/cli.ts"), "scan", "--host"],
@@ -731,13 +740,27 @@ test.each([false, true])(
method: "scan.progress",
params: { id: "run", event },
});
- expect(messages.at(-1)).toMatchObject({
+ const response = messages.at(-1);
+ expect(response).toMatchObject({
id: "run",
result: {
status: cancel ? "canceled" : "completed",
execution: { sessionId: "fake-host", usage },
},
});
+ if (!cancel) {
+ expect(response.result).not.toHaveProperty("ownerSuggestionsError");
+ expect(response.result.artifacts).toContainEqual(
+ expect.objectContaining({ path: "owner-suggestions.json" }),
+ );
+ expect(
+ await readJson(join(input.outputDirectory, "owner-suggestions.json")),
+ ).toMatchObject({
+ schemaVersion: 2,
+ revision: input.revision,
+ results: [],
+ });
+ }
} finally {
child.kill();
}
@@ -1253,31 +1276,3 @@ test("ordinary SDK rejects the removed hosted option instead of using local infe
await client.close();
}
});
-
-test.each([true, false])(
- "hosted owner reporting never invokes the legacy callback (supplied=%p)",
- async (supplied) => {
- const input = await fixture();
- const startThread = mock(() => {
- throw new Error("No additional owner turn");
- });
- const result = await runHostedScan(
- { ...input, ownerSuggestions: true },
- {
- executor: draftExecutor(),
- ...(supplied ? { ownerCodex: { startThread } } : {}),
- },
- );
- expect(result.status).toBe("completed");
- expect(result).not.toHaveProperty("ownerSuggestionsError");
- expect(startThread).not.toHaveBeenCalled();
- expect(
- JSON.parse(
- await readFile(
- join(input.outputDirectory, "owner-suggestions.json"),
- "utf8",
- ),
- ),
- ).toHaveProperty("results");
- },
-);
diff --git a/sdk/typescript/tests-ts/patch-risk-contract.test.ts b/sdk/typescript/tests-ts/patch-risk-contract.test.ts
index e24535596..7cfff394d 100644
--- a/sdk/typescript/tests-ts/patch-risk-contract.test.ts
+++ b/sdk/typescript/tests-ts/patch-risk-contract.test.ts
@@ -410,7 +410,12 @@ describe("patch risk assessment contract", () => {
expect(validate(payload).status).not.toBe(0);
});
- test("preserves the ordered auto-merge gates", () => {
+ test("preserves the ordered auto-merge gates", async () => {
+ const helperSource = new URL(
+ "../../../plugins/codex-security/mcp-app/src/helpers/validate-patch-risk-assessment.ts",
+ import.meta.url,
+ );
+ const { validatePatchRiskAssessment } = await import(helperSource.href);
const cases: Array<[string, (value: Assessment) => void]> = [
[
"impact.rating",
@@ -490,18 +495,19 @@ describe("patch risk assessment contract", () => {
const payload = assessment();
payload.workflowLabel = "auto_merge_candidate";
payload.impact.rating = "low";
- expect(validate(payload).status).toBe(0);
+ expect(validatePatchRiskAssessment(payload)).toEqual([]);
+ const combined = structuredClone(payload);
for (const [field, mutate] of cases) {
const changed = structuredClone(payload);
mutate(changed);
- const result = validate(changed);
- expect(result.status).toBe(1);
- expect(result.stderr).toContain(
+ expect(validatePatchRiskAssessment(changed)).toContain(
`auto_merge_candidate gate failed: ${field}`,
);
- mutate(payload);
+ mutate(combined);
}
- expect(validate(payload).stderr.trim().split("\n")).toEqual([
+ const result = validate(combined);
+ expect(result.status).toBe(1);
+ expect(result.stderr.trim().split("\n")).toEqual([
"merge requires confirmed applicability",
...cases.map(([field]) => `auto_merge_candidate gate failed: ${field}`),
]);
diff --git a/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts b/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts
index 79d4527bf..67e958266 100644
--- a/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts
+++ b/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts
@@ -1172,90 +1172,81 @@ connection.close()
});
}
-test.each([
- ["offsets", "2026-08-01T03:00:00+02:00", "2026-08-01T01:30:00Z"],
- ["fractional seconds", "2026-08-01T01:00:00Z", "2026-08-01T01:00:00.100000Z"],
- [
- "microseconds",
- "2026-08-01T01:00:00.000001Z",
- "2026-08-01T01:00:00.000002Z",
- ],
- [
- "millisecond boundaries",
- "2026-08-01T01:00:00.481998Z",
- "2026-08-01T01:00:00.482012Z",
- ],
- [
- "offset microseconds",
- "2026-08-01T03:00:00.000001+02:00",
- "2026-08-01T01:00:00.000002Z",
- ],
-])(
- "latest orders timestamps chronologically with %s",
- async (_, older, newer) => {
- const f = await fixture();
- const db = new Database(
- join(f.environment.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"),
- );
- try {
- // The older scan wins an ID tie, so lost timestamp precision cannot pass.
- const [olderScan, newerScan] =
- f.first.scanId < f.second.scanId
- ? ([f.first, f.second] as const)
- : ([f.second, f.first] as const);
- for (const [scan, timestamp] of [
- [olderScan, older],
- [newerScan, newer],
- ] as const) {
- db.query(
- "UPDATE scans SET updated_at = ?, started_at = '2000-01-01T00:00:00Z' WHERE id = ?",
- ).run(timestamp, scan.scanId);
- db.query(
- "UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?",
- ).run(older, scan.scanId);
- }
- async function latest() {
- const workbench = await savedScanWorkbench("latest", {
- environment: { ...f.environment, PYTHON: f.python },
- pluginRoot: PLUGIN_ROOT,
- currentDirectory: f.repository,
- });
- return (
- await resolveCompletedScan("latest", {
- currentDirectory: () => f.repository,
- runWorkbench: workbench,
- })
- ).scanId;
- }
- expect(await latest()).toBe(newerScan.scanId);
- // Either the scan or its progress may hold the latest update.
- db.query("UPDATE scans SET updated_at = ?").run(older);
- db.query("UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?").run(
- newer,
- newerScan.scanId,
- );
- expect(await latest()).toBe(newerScan.scanId);
- // Equal updates fall back to the scans' chronological start times.
- db.query("UPDATE scans SET updated_at = '2099-01-01T00:00:00Z'").run();
+test("latest considers scan and progress updates, start times, and scan IDs", async () => {
+ const older = "2026-08-01T03:00:00.000001+02:00";
+ const newer = "2026-08-01T01:00:00.000002Z";
+ const f = await fixture();
+ const db = new Database(
+ join(f.environment.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"),
+ );
+ try {
+ // The older scan wins an ID tie, so lost timestamp precision cannot pass.
+ const [olderScan, newerScan] =
+ f.first.scanId < f.second.scanId
+ ? ([f.first, f.second] as const)
+ : ([f.second, f.first] as const);
+ for (const [scan, timestamp] of [
+ [olderScan, older],
+ [newerScan, newer],
+ ] as const) {
db.query(
- "UPDATE scan_progress SET updated_at = '2099-01-01T00:00:00Z'",
- ).run();
- db.query("UPDATE scans SET started_at = ? WHERE id = ?").run(
+ "UPDATE scans SET updated_at = ?, started_at = '2000-01-01T00:00:00Z' WHERE id = ?",
+ ).run(timestamp, scan.scanId);
+ db.query("UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?").run(
older,
- olderScan.scanId,
- );
- db.query("UPDATE scans SET started_at = ? WHERE id = ?").run(
- newer,
- newerScan.scanId,
+ scan.scanId,
);
- expect(await latest()).toBe(newerScan.scanId);
- db.query("UPDATE scans SET started_at = ?").run(older);
- expect(await latest()).toBe(olderScan.scanId);
- } finally {
- db.close();
}
- },
-);
+ async function latest() {
+ const workbench = await savedScanWorkbench("latest", {
+ environment: { ...f.environment, PYTHON: f.python },
+ pluginRoot: PLUGIN_ROOT,
+ currentDirectory: f.repository,
+ });
+ return (
+ await resolveCompletedScan("latest", {
+ currentDirectory: () => f.repository,
+ runWorkbench: workbench,
+ })
+ ).scanId;
+ }
+ expect(await latest()).toBe(newerScan.scanId);
+ // Either the scan or its progress may hold the latest update.
+ db.query("UPDATE scans SET updated_at = ?").run(older);
+ db.query("UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?").run(
+ newer,
+ newerScan.scanId,
+ );
+ expect(await latest()).toBe(newerScan.scanId);
+ // Progress can outrank both the other scan's update and its later start.
+ db.query(
+ "UPDATE scans SET updated_at = ?, started_at = ? WHERE id = ?",
+ ).run(newer, newer, newerScan.scanId);
+ db.query("UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?").run(
+ "2026-08-01T01:00:00.000003Z",
+ olderScan.scanId,
+ );
+ expect(await latest()).toBe(olderScan.scanId);
+ // Equal updates fall back to the scans' chronological start times.
+ db.query("UPDATE scans SET updated_at = '2099-01-01T00:00:00Z'").run();
+ db.query(
+ "UPDATE scan_progress SET updated_at = '2099-01-01T00:00:00Z'",
+ ).run();
+ db.query("UPDATE scans SET started_at = ? WHERE id = ?").run(
+ older,
+ olderScan.scanId,
+ );
+ db.query("UPDATE scans SET started_at = ? WHERE id = ?").run(
+ newer,
+ newerScan.scanId,
+ );
+ expect(await latest()).toBe(newerScan.scanId);
+ db.query("UPDATE scans SET started_at = ?").run(older);
+ expect(await latest()).toBe(olderScan.scanId);
+ } finally {
+ db.close();
+ }
+});
test("latest follows progress timestamps and excludes canceled scans", async () => {
const f = await fixture();
diff --git a/sdk/typescript/tests-ts/scan-recipe.test.ts b/sdk/typescript/tests-ts/scan-recipe.test.ts
index 8ebca1172..9a03e5f94 100644
--- a/sdk/typescript/tests-ts/scan-recipe.test.ts
+++ b/sdk/typescript/tests-ts/scan-recipe.test.ts
@@ -3,6 +3,7 @@ import { workbenchCommand } from "./support/workbench-command.js";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, expect, test } from "bun:test";
+import { scanPreflightCodexConfig } from "../src/api.js";
import { main } from "../src/cli.js";
import { runWorkbench } from "../src/runtime.js";
import { resolveCodexProfile } from "../src/config.js";
@@ -14,26 +15,51 @@ import { createApiTestFixtures } from "./support/temporary-directories.js";
const { temporaryDirectory, cleanup } = createApiTestFixtures();
afterEach(cleanup);
-test.each(
- ["standard", "deep"].flatMap((mode) =>
- ["review", "review.v2", "review mode", "分析"].map(
- (profile) => [mode, profile] as const,
- ),
- ),
-)(
- "%s scans save prompts and replay the selected %s profile",
- async (mode, profile) => {
- const selected = {
- model: "synthetic-selected-model",
- model_reasoning_effort: "high",
- model_provider: "amazon-bedrock",
- features: { goals: false },
- };
- const selectedProviders = {
+const selected = {
+ model: "synthetic-selected-model",
+ model_reasoning_effort: "high",
+ model_provider: "amazon-bedrock",
+ features: { goals: false },
+};
+const selectedProviders = {
+ "amazon-bedrock": {
+ aws: { region: "us-east-2", profile: "scan-profile" },
+ },
+};
+function profileConfig(profile: string) {
+ return {
+ model: "synthetic-root-model",
+ model_reasoning_effort: "low",
+ model_provider: "openai",
+ model_providers: {
"amazon-bedrock": {
- aws: { region: "us-east-2", profile: "scan-profile" },
+ aws: { region: "us-west-2", profile: "ambient-profile" },
},
- };
+ },
+ profile,
+ profiles: {
+ [profile]: { ...selected, model_providers: selectedProviders },
+ },
+ };
+}
+
+test.each(["review", "review.v2", "review mode", "分析"])(
+ "preserves effective settings for profile %s in the recipe configuration",
+ (profile) => {
+ const recipeConfig = scanPreflightCodexConfig(profileConfig(profile));
+ expect(resolveCodexProfile(recipeConfig)).toMatchObject({
+ ...selected,
+ model_providers: selectedProviders,
+ });
+ },
+);
+
+test.each([
+ ["standard", "review"],
+ ["deep", "分析"],
+])(
+ "%s scans save prompts and replay the selected %s profile",
+ async (mode, profile) => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
const codexHome = join(root, "state", "codex-home");
@@ -80,23 +106,7 @@ test.each(
new TestClient(
{
...config,
- codexOverrides: {
- model: "synthetic-root-model",
- model_reasoning_effort: "low",
- model_provider: "openai",
- model_providers: {
- "amazon-bedrock": {
- aws: { region: "us-west-2", profile: "ambient-profile" },
- },
- },
- profile,
- profiles: {
- [profile]: {
- ...selected,
- model_providers: selectedProviders,
- },
- },
- },
+ codexOverrides: profileConfig(profile),
},
{
environment,
diff --git a/sdk/typescript/tests-ts/scan-recovery.test.ts b/sdk/typescript/tests-ts/scan-recovery.test.ts
index d388ac29a..b23ee0e8d 100644
--- a/sdk/typescript/tests-ts/scan-recovery.test.ts
+++ b/sdk/typescript/tests-ts/scan-recovery.test.ts
@@ -770,7 +770,7 @@ describe("scan publication and artifact recovery", () => {
expect((await completeScan(fixture)).findingCount).toBe(1);
});
- test("keeps verified coverage receipts and records invalid references", async () => {
+ test("seals verified receipts and discards unsafe references without losing findings", async () => {
const fixture = await startDraftScan();
const path = join(fixture.scanDir, "coverage.json");
const document = await readJson<CoverageDocument>(path);
@@ -788,17 +788,29 @@ describe("scan publication and artifact recovery", () => {
null,
];
await writeJson(path, document);
+ const manifestPath = join(fixture.scanDir, "scan-manifest.json");
+ const draft = await readJson<{ scan: { hardening?: unknown } }>(
+ manifestPath,
+ );
+ draft.scan.hardening = { portfolioPath: "../outside.md" };
+ await writeJson(manifestPath, draft);
const completed = await completeScan(fixture);
expect(completed.progress.status).toBe("complete");
- expect(completed.warnings).toHaveLength(4);
+ expect(completed.findingCount).toBe(1);
+ expect(completed.warnings).toHaveLength(5);
expect(
- completed.warnings.every((warning) =>
+ completed.warnings.filter((warning) =>
warning.startsWith("Skipped malformed coverage receipt"),
),
+ ).toHaveLength(4);
+ expect(
+ completed.warnings.some((warning) =>
+ warning.startsWith("Skipped malformed hardening portfolio:"),
+ ),
).toBe(true);
- expect(completed.warnings.join("\n")).not.toContain("../outside.json");
+ expect(completed.warnings.join("\n")).not.toContain("../outside");
const recovered = await readJson<CoverageDocument>(path);
expect((recovered.surfaces as CoverageSurface[])[0]).toMatchObject({
disposition: "needs_follow_up",
@@ -806,61 +818,63 @@ describe("scan publication and artifact recovery", () => {
});
const manifest = await readJson<{
scan: { artifacts: Array<{ path: string }> };
- }>(join(fixture.scanDir, "scan-manifest.json"));
+ }>(manifestPath);
+ expect(manifest.scan).not.toHaveProperty("hardening");
expect(manifest.scan.artifacts.map((artifact) => artifact.path)).toContain(
receipt,
);
});
- test("recovers malformed coverage collections", async () => {
- const fixture = await startDraftScan();
- const path = join(fixture.scanDir, "coverage.json");
- const document = await readJson<CoverageDocument>(path);
- document.surfaces = { id: "not-an-array" };
- document.explicitExclusions = null;
- document.deferred = "later";
- await writeJson(path, document);
-
- const completed = await completeScan(fixture);
-
- expect(completed.progress.status).toBe("complete");
- expect(completed.warnings).toHaveLength(3);
- const recovered = await readJson<CoverageDocument>(path);
- expect(recovered).toMatchObject({
- surfaces: [],
- explicitExclusions: [],
- deferred: [],
- });
- });
-
- test("discards unsafe hardening portfolios without discarding findings", async () => {
- for (const hardening of [
- "not an object",
- { portfolioPath: "../outside.md" },
- { portfolioPath: "hardening/hardening.md" },
- ]) {
+ test.each([
+ ["invalid object", "not an object"],
+ ["missing file", { portfolioPath: "hardening/hardening.md" }],
+ ] as const)(
+ "recovers malformed coverage collections and omits a hardening portfolio (%s)",
+ async (_case, hardening) => {
const fixture = await startDraftScan();
- const path = join(fixture.scanDir, "scan-manifest.json");
+ const coveragePath = join(fixture.scanDir, "coverage.json");
+ const manifestPath = join(fixture.scanDir, "scan-manifest.json");
+ const findingsPath = join(fixture.scanDir, "findings.json");
+ const { findings } = await readJson<FindingsDocument>(findingsPath);
+ const coverage = await readJson<CoverageDocument>(coveragePath);
+ coverage.surfaces = { id: "not-an-array" };
+ coverage.explicitExclusions = null;
+ coverage.deferred = "later";
+ await writeJson(coveragePath, coverage);
const manifest = await readJson<{
- scan: { hardening?: unknown };
- }>(path);
+ scan: { hardening?: unknown; sealedAt?: string };
+ }>(manifestPath);
manifest.scan.hardening = hardening;
- await writeJson(path, manifest);
+ await writeJson(manifestPath, manifest);
const completed = await completeScan(fixture);
- expect(completed.progress.status).toBe("complete");
- expect(completed.findingCount).toBe(1);
- expect(completed.warnings).toHaveLength(1);
- expect(completed.warnings[0]).toContain(
- "Skipped malformed hardening portfolio:",
- );
- expect(completed.warnings[0]).not.toContain("../outside.md");
+ expect(completed).toMatchObject({
+ findingCount: 1,
+ progress: { status: "complete" },
+ });
+ expect(completed.warnings).toHaveLength(4);
expect(
- (await readJson<{ scan: { hardening?: unknown } }>(path)).scan,
- ).not.toHaveProperty("hardening");
- }
- });
+ completed.warnings.filter((warning) =>
+ warning.endsWith("expected an array."),
+ ),
+ ).toHaveLength(3);
+ expect(completed.warnings).toContainEqual(
+ expect.stringMatching(/^Skipped malformed hardening portfolio:/),
+ );
+ expect(await readJson<CoverageDocument>(coveragePath)).toMatchObject({
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [],
+ });
+ expect(await readJson<FindingsDocument>(findingsPath)).toMatchObject({
+ findings: [{ summary: findings[0]!.summary }],
+ });
+ const sealed = await readJson<typeof manifest>(manifestPath);
+ expect(sealed.scan.sealedAt).toBeTruthy();
+ expect(sealed.scan).not.toHaveProperty("hardening");
+ },
+ );
test("refuses to repair scan-wide coverage contract violations", async () => {
const fixture = await startDraftScan();
diff --git a/sdk/typescript/tests-ts/targets.test.ts b/sdk/typescript/tests-ts/targets.test.ts
index f32c681c4..efa32a061 100644
--- a/sdk/typescript/tests-ts/targets.test.ts
+++ b/sdk/typescript/tests-ts/targets.test.ts
@@ -224,11 +224,6 @@ function environmentWithPath(entries: readonly string[]): NodeJS.ProcessEnv {
}
describe("scan target normalization", () => {
- test("tolerates a temporary repository removed before cleanup", async () => {
- const repo = await repository();
- await rm(join(repo, ".."), { recursive: true, force: true });
- });
-
test("normalizes repository and path targets", async () => {
const repo = await repository();
const alias = join(repo, "source-link");
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index 034a89b6f..41df1f2f2 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -157,29 +157,50 @@ test(${JSON.stringify(file)}, () => {
test("rejects an empty shard while allowing a nonempty shard with the same count", async () => {
const node = Bun.which("node");
expect(node).not.toBeNull();
- const root = await temporaryDirectory("codex-security-empty-shard-");
+ const fixture = await temporaryDirectory("codex-security-empty-shard-");
+ const root = join(fixture, "sdk", "typescript");
try {
- await mkdir(join(root, "scripts"));
+ await mkdir(join(root, "scripts"), { recursive: true });
await mkdir(join(root, "tests-ts"));
await copyRunner(root);
+ const worker = "deep-scan-worker-shutdown.test.ts";
+ await copyFile(
+ new URL(worker, import.meta.url),
+ join(root, "tests-ts", worker),
+ );
+ const mcpApp = join(fixture, "plugins", "codex-security", "mcp-app");
+ const executor = join(mcpApp, "tests", "test_deep_scan_executor.ts");
+ const executed = join(root, "worker-ran");
+ await mkdir(dirname(executor), { recursive: true });
+ await writeFile(join(mcpApp, "package.json"), '{"type":"module"}\n');
await writeFile(
- join(root, "tests-ts", "probe.test.ts"),
- 'import { test } from "bun:test"; test("synthetic shard probe", () => {});\n',
+ executor,
+ `import { writeFileSync } from "node:fs";
+import { test } from "node:test";
+test("Deep worker lifecycle", () => writeFileSync(${JSON.stringify(executed)}, "ran"));\n`,
);
await writeFile(
join(root, "tests-ts", "windows-machine-policy.test.ts"),
'throw new Error("Machine policy test must run separately");\n',
);
- for (const shard of [1, 2]) {
+ for (const [shard, separateMcp, runsWorker] of [
+ [1, false, true],
+ [2, false, false],
+ [1, true, process.platform !== "linux"],
+ ] as const) {
+ await rm(executed, { force: true });
const child = Bun.spawn({
cmd: [
node!,
"--experimental-strip-types",
join(root, "scripts", "run-ci-tests.mts"),
`${shard}/2`,
+ "-t",
+ "Deep worker lifecycle",
],
env: {
...process.env,
+ CODEX_SECURITY_TEST_MCP_WORKERS_SEPARATE: String(separateMcp),
PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,
},
stdin: "ignore",
@@ -190,15 +211,15 @@ test("rejects an empty shard while allowing a nonempty shard with the same count
});
const { status, stdout, stderr } = await readSubprocess(child);
expect(status, stderr).toBe(shard === 1 ? 0 : 1);
+ expect(await Bun.file(executed).exists()).toBe(runsWorker);
if (shard === 1) {
- expect(stdout).toContain("Test shard 1/2: probe.test.ts");
- expect(stderr).toContain("synthetic shard probe");
+ expect(stdout).toContain(`Test shard 1/2: ${worker}`);
} else {
- expect(stderr).toContain("Test shard 2/2 is empty.");
+ expect(stderr).toContain(`Test shard ${shard}/2 is empty.`);
}
}
} finally {
- await rm(root, { recursive: true, force: true });
+ await rm(fixture, { recursive: true, force: true });
}
});
diff --git a/sdk/typescript/tests-ts/test-timings.test.ts b/sdk/typescript/tests-ts/test-timings.test.ts
index 02f253da1..723cd19b4 100644
--- a/sdk/typescript/tests-ts/test-timings.test.ts
+++ b/sdk/typescript/tests-ts/test-timings.test.ts
@@ -32,18 +32,26 @@ test("refreshes per-file medians while retaining other platforms and undiscovere
]);
});
-test("handles XML-escaped and Windows file paths and zero-duration skipped cases", () => {
- const updated = updateTestTimings({}, "win32", [
+test("updates executed Windows cases and retains estimates for skipped files", () => {
+ const previous = { "skipped.test.ts": { win32: 34.5 } };
+ const updated = updateTestTimings(previous, "win32", [
report(String.raw`
<testcase file="C:\checkout\tests-ts\names&amp;paths.test.ts" time="1.2" />
<testcase file="tests-ts/names&amp;paths.test.ts" time="0.5" />
- <testcase file="tests-ts/skipped.test.ts"><skipped /></testcase>
+ <testcase file="tests-ts/names&amp;paths.test.ts" time="8"><skipped /></testcase>
+ <testcase file="tests-ts/skipped.test.ts" time="0"><skipped /></testcase>
+ <testcase file="tests-ts/new-skipped.test.ts"><skipped /></testcase>
+ <testcase file="tests-ts/quick.test.ts" time="0" />
`),
report('<testcase file="tests-ts/names&amp;paths.test.ts" time="3.1" />'),
+ report(
+ '<testcase file="tests-ts/names&amp;paths.test.ts"><skipped /></testcase>',
+ ),
]);
expect(updated).toEqual({
"names&paths.test.ts": { win32: 2.4 },
- "skipped.test.ts": { win32: 0.1 },
+ "quick.test.ts": { win32: 0.1 },
+ "skipped.test.ts": { win32: 34.5 },
});
});
diff --git a/sdk/typescript/tests-ts/timestamps.test.ts b/sdk/typescript/tests-ts/timestamps.test.ts
new file mode 100644
index 000000000..39b0b46d4
--- /dev/null
+++ b/sdk/typescript/tests-ts/timestamps.test.ts
@@ -0,0 +1,25 @@
+import { expect, test } from "bun:test";
+import { compareTimestamps } from "../src/timestamps.js";
+
+test.each([
+ ["offsets", "2026-08-01T03:00:00+02:00", "2026-08-01T01:30:00Z"],
+ ["fractional seconds", "2026-08-01T01:00:00Z", "2026-08-01T01:00:00.100000Z"],
+ [
+ "microseconds",
+ "2026-08-01T01:00:00.000001Z",
+ "2026-08-01T01:00:00.000002Z",
+ ],
+ [
+ "millisecond boundaries",
+ "2026-08-01T01:00:00.481998Z",
+ "2026-08-01T01:00:00.482012Z",
+ ],
+ [
+ "offset microseconds",
+ "2026-08-01T03:00:00.000001+02:00",
+ "2026-08-01T01:00:00.000002Z",
+ ],
+])("compares %s chronologically in both directions", (_, older, newer) => {
+ expect(compareTimestamps(older, newer)).toBeLessThan(0);
+ expect(compareTimestamps(newer, older)).toBeGreaterThan(0);
+});