Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: make MCP and worker failures easier to diagnose (#1903)

Michael D'Angeloc3a1e5b868cb
diff --git a/plugins/codex-security/mcp-app/tests/import-module.ts b/plugins/codex-security/mcp-app/tests/import-module.ts
index efdb75f1f..c15cbc6fd 100644
--- a/plugins/codex-security/mcp-app/tests/import-module.ts
+++ b/plugins/codex-security/mcp-app/tests/import-module.ts
@@ -1,4 +1,17 @@
import { build, type BuildOptions } from "esbuild";
+import { createHash } from "node:crypto";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+const moduleDirectory = mkdtempSync(
+ join(tmpdir(), "codex-security-test-modules-"),
+);
+const writtenModules = new Set<string>();
+process.once("exit", () =>
+ rmSync(moduleDirectory, { recursive: true, force: true }),
+);
export async function importModule(options: BuildOptions) {
const result = await build({
@@ -8,9 +21,15 @@ export async function importModule(options: BuildOptions) {
write: false,
...options,
});
- return import(
- `data:text/javascript;base64,${Buffer.from(result.outputFiles![0].contents).toString("base64")}`
- );
+ const contents = result.outputFiles![0].contents;
+ // Keep content-based module identity without embedding the bundle in stacks.
+ const digest = createHash("sha256").update(contents).digest("hex");
+ const modulePath = join(moduleDirectory, `${digest}.mjs`);
+ if (!writtenModules.has(modulePath)) {
+ writeFileSync(modulePath, contents);
+ writtenModules.add(modulePath);
+ }
+ return import(pathToFileURL(modulePath).href);
}
export function importSource(entryPoint: string, options: BuildOptions = {}) {
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
index 5c0005a83..9feb2ff04 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
@@ -16,7 +16,7 @@ import {
nativeConfigOverrides,
} from "./assertions.ts";
import { createTemporaryDirectories } from "./support/temporary-directories.ts";
-import { mock } from "node:test";
+import { mock, test, type TestContext } from "node:test";
import { parse as parseToml, stringify as stringifyToml } from "smol-toml";
import assert from "node:assert/strict";
import childProcess, {
@@ -102,66 +102,149 @@ const ipcFrameError =
"code-mode delegate response exceeds the IPC frame limit: code-mode IPC frame length 76008279 exceeds 67108864 bytes";
try {
- testCodeModeFrameDiagnosticBoundaries();
- await testLargeScopeTransport();
- await testOpenAiCredentialsReachWorker();
- await testWorkerCancellation();
- testUsageLimitClassification();
- await testWorkerTerminalEvents();
- await testWorkerRuntimeSettings();
- await testUnsupportedProviderSnapshotFailsBeforeLaunch();
- await testWorkerCyberAccessSettings({
- CodexSdkWorkerExecutor,
- fakeCodexFixture,
- parentSandbox: trustedParentSandbox,
- workerRequest,
- assertConfigOverrides,
- assertReadOnlyWorkerPolicy,
- assertWorkerSubagentPolicy,
- restoreEnv,
+ await test("Deep worker lifecycle", async (t) => {
+ await check(t, testWorkerCancellation);
+ await check(t, testWorkerTerminalEvents);
+ await check(t, testCompletedWorkerSettlesWithoutWaitingForProcessExit);
+ if (process.platform !== "win32") {
+ await check(t, testStreamTerminationWithoutTerminalEventFails);
+ await check(t, testAbortPropagation);
+ }
});
- await testCompletedWorkerSettlesWithoutWaitingForProcessExit();
- if (process.platform !== "win32") {
- await testDeepScanContextSnapshots({
- CodexSdkWorkerExecutor,
- fakeCodexFixture,
- parentSandbox: trustedParentSandbox,
- });
- await testMissingParentSandboxFailsBeforeWorkerLaunch();
- await testDisallowedWorkerProfileFailsBeforeWorkerLaunch();
- await testRuntimePermissionProfileFallbackStopsAndDiscards();
- await testWorkerLaunchesWithoutGlobalCodex();
- await testPreflightBindsExecutableAndHomeBeforeChangingCwd();
- await testConcurrentWorkerExecutableAliases({
- CodexSdkWorkerExecutor,
- fakeCodexFixture,
- parentSandbox: trustedParentSandbox,
- bundledCodexSdkEnvironment,
- assertReadOnlyWorkerPolicy,
- });
- await testSdkInvocationAndThreadCapture();
- await testBedrockCredentialsReachWorker();
- await testBedrockCredentialsReachWorker("oidc");
- await testExternalProviderCredentialsReachWorker();
- await testArtifactServerUsesExtendedStartupTimeout();
- await testZeroSubagentsPreservesHostRestrictions();
- await testSdkResumesExistingThread();
- await testRetryNotificationDoesNotInterruptTurn();
- await testSandboxNamespaceDiagnosticIsPreserved();
- await testOwnedArtifactToolFailureDiagnosticIsPreserved();
- for (const reference of ["discovery:\0" + "0", "😀".repeat(1300)]) {
- await testArtifactDiagnosticSurvivesWorkerRetries(reference);
+
+ await test("Deep worker runtime settings", async (t) => {
+ await check(t, testLargeScopeTransport);
+ await check(t, testOpenAiCredentialsReachWorker);
+ await check(t, testWorkerRuntimeSettings);
+ await check(t, testUnsupportedProviderSnapshotFailsBeforeLaunch);
+ await check(
+ t,
+ () =>
+ testWorkerCyberAccessSettings({
+ CodexSdkWorkerExecutor,
+ fakeCodexFixture,
+ parentSandbox: trustedParentSandbox,
+ workerRequest,
+ assertConfigOverrides,
+ assertReadOnlyWorkerPolicy,
+ assertWorkerSubagentPolicy,
+ restoreEnv,
+ }),
+ "cyber access settings",
+ );
+ if (process.platform !== "win32") {
+ await check(
+ t,
+ () =>
+ testDeepScanContextSnapshots({
+ CodexSdkWorkerExecutor,
+ fakeCodexFixture,
+ parentSandbox: trustedParentSandbox,
+ }),
+ "isolated context snapshots",
+ );
+ await check(t, testMissingParentSandboxFailsBeforeWorkerLaunch);
+ await check(t, testDisallowedWorkerProfileFailsBeforeWorkerLaunch);
+ await check(t, testRuntimePermissionProfileFallbackStopsAndDiscards);
+ await check(t, testSdkInvocationAndThreadCapture);
+ await check(
+ t,
+ () => testBedrockCredentialsReachWorker(),
+ "Bedrock credentials",
+ );
+ await check(
+ t,
+ () => testBedrockCredentialsReachWorker("oidc"),
+ "Bedrock OIDC credentials",
+ );
+ await check(t, testExternalProviderCredentialsReachWorker);
+ await check(t, testArtifactServerUsesExtendedStartupTimeout);
+ await check(t, testZeroSubagentsPreservesHostRestrictions);
+ await check(t, testSdkResumesExistingThread);
}
- await testCodeModeFrameDiagnosticSurvivesSuccessfulTurn();
- await testStreamTerminationWithoutTerminalEventFails();
- await testAbortPropagation();
- await testUnstructuredConfigurationFailureRemainsRetryable();
- await testUnstructuredThreadStartFailureRemainsRetryable();
- await testPolicyFailuresRemainWorkerErrors();
- await testMalformedCommandEventsRemainRetryable();
- await testRateLimitPolicyFailureRemainsRetryable();
- await testArtifactStartupTimeoutClassification();
- }
+ });
+
+ await test("Deep worker executable selection", async (t) => {
+ await check(t, testConfiguredExecutablePaths);
+ if (process.platform !== "win32") {
+ await check(t, testWorkerLaunchesWithoutGlobalCodex);
+ await check(t, testPreflightBindsExecutableAndHomeBeforeChangingCwd);
+ await check(
+ t,
+ () =>
+ testConcurrentWorkerExecutableAliases({
+ CodexSdkWorkerExecutor,
+ fakeCodexFixture,
+ parentSandbox: trustedParentSandbox,
+ bundledCodexSdkEnvironment,
+ assertReadOnlyWorkerPolicy,
+ }),
+ "concurrent executable aliases",
+ );
+ }
+ await check(t, testWindowsAppsCodexFallsBackToRelocatedBinary);
+ await check(
+ t,
+ () => testWindowsNpmPackageResolution(),
+ "global Windows npm package resolution",
+ );
+ await check(
+ t,
+ () => testWindowsNpmPackageResolution("managed"),
+ "managed Windows npm package resolution",
+ );
+ if (process.platform === "win32") {
+ await check(t, testWindowsLongExecutableLaunches);
+ await check(t, testWindowsRootRelativePathsStayBoundToOriginalDrive);
+ await check(t, testWindowsWorkerEnvironmentPreservesMixedCaseKeys);
+ await check(t, testWindowsLauncherSkipsExtensionlessNpmShim);
+ }
+ });
+
+ await test("Deep worker diagnostics", async (t) => {
+ await check(t, testCodeModeFrameDiagnosticBoundaries);
+ await check(t, testUsageLimitClassification);
+ await check(t, testOsErrorCodesRemainRetryable);
+ await check(t, testTextualMissingPathErrorsRemainRetryable);
+ await check(t, testWorkerErrorClassificationPreservesExplicitFailures);
+ if (process.platform !== "win32") {
+ await check(t, testRetryNotificationDoesNotInterruptTurn);
+ await check(t, testSandboxNamespaceDiagnosticIsPreserved);
+ await check(t, testOwnedArtifactToolFailureDiagnosticIsPreserved);
+ await check(
+ t,
+ () => testArtifactDiagnosticSurvivesWorkerRetries("discovery:\0" + "0"),
+ "artifact retry diagnostic: controls",
+ );
+ await check(
+ t,
+ () => testArtifactDiagnosticSurvivesWorkerRetries("😀".repeat(1300)),
+ "artifact retry diagnostic: Unicode",
+ );
+ await check(t, testCodeModeFrameDiagnosticSurvivesSuccessfulTurn);
+ await check(t, testUnstructuredConfigurationFailureRemainsRetryable);
+ await check(t, testUnstructuredThreadStartFailureRemainsRetryable);
+ await check(t, testPolicyFailuresRemainWorkerErrors);
+ await check(t, testMalformedCommandEventsRemainRetryable);
+ await check(t, testRateLimitPolicyFailureRemainsRetryable);
+ await check(t, testArtifactStartupTimeoutClassification);
+ }
+ });
+} finally {
+ restoreEnv("FAKE_CODEX_MARKER", previousMarker);
+ await temporaryDirectories.cleanup();
+}
+
+async function check(
+ t: TestContext,
+ run: () => void | Promise<void>,
+ name = run.name,
+) {
+ process.stdout.write(`Running ${t.name} / ${name}\n`);
+ await t.test(name, run);
+}
+
+function testConfiguredExecutablePaths() {
assert.equal(
resolveCodexPath({}, "darwin"),
path.join(process.cwd(), "codex"),
@@ -220,21 +303,6 @@ try {
),
path.join(originalCwd, "codex"),
);
- testOsErrorCodesRemainRetryable();
- testTextualMissingPathErrorsRemainRetryable();
- testWorkerErrorClassificationPreservesExplicitFailures();
- await testWindowsAppsCodexFallsBackToRelocatedBinary();
- await testWindowsNpmPackageResolution();
- await testWindowsNpmPackageResolution("managed");
- if (process.platform === "win32") {
- await testWindowsLongExecutableLaunches();
- await testWindowsRootRelativePathsStayBoundToOriginalDrive();
- await testWindowsWorkerEnvironmentPreservesMixedCaseKeys();
- await testWindowsLauncherSkipsExtensionlessNpmShim();
- }
-} finally {
- restoreEnv("FAKE_CODEX_MARKER", previousMarker);
- await temporaryDirectories.cleanup();
}
function testOsErrorCodesRemainRetryable() {
diff --git a/plugins/codex-security/mcp-app/tests/test_import_module.ts b/plugins/codex-security/mcp-app/tests/test_import_module.ts
new file mode 100644
index 000000000..79907d0e7
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_import_module.ts
@@ -0,0 +1,58 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { importModule } from "./import-module.ts";
+
+test("bundled test errors preserve diagnostics without copying the module into stacks", async () => {
+ const payload = "large bundled fixture ".repeat(10_000);
+ const module = await importModule({
+ stdin: {
+ contents: `
+ export const payload = ${JSON.stringify(payload)};
+ export function fail() {
+ throw new Error("Synthetic failure token=SYNTHETIC_DIAGNOSTIC_VALUE");
+ }
+ `,
+ },
+ });
+ assert.equal(module.payload, payload);
+ assert.throws(module.fail, (error: Error) => {
+ assert.equal(
+ error.message,
+ "Synthetic failure token=SYNTHETIC_DIAGNOSTIC_VALUE",
+ );
+ assert.match(error.stack!, /file:\/\/.*\.mjs:\d+:\d+/);
+ assert.doesNotMatch(error.stack!, /data:text\/javascript;base64,/);
+ assert.ok(!error.stack!.includes(payload));
+ return true;
+ });
+});
+
+test("identical test bundles retain their module identity", async () => {
+ const options = {
+ stdin: { contents: "export const state = { count: 0 };" },
+ };
+ const [first, second] = await Promise.all([
+ importModule(options),
+ importModule(options),
+ ]);
+ first.state.count++;
+ assert.equal(first, second);
+ assert.equal(second.state.count, 1);
+});
+
+test("different build options keep test module state isolated", async () => {
+ const options = {
+ stdin: { contents: "export const state = { value: FIXTURE_VALUE };" },
+ };
+ const first = await importModule({
+ ...options,
+ define: { FIXTURE_VALUE: '"first"' },
+ });
+ const second = await importModule({
+ ...options,
+ define: { FIXTURE_VALUE: '"second"' },
+ });
+ assert.equal(first.state.value, "first");
+ assert.equal(second.state.value, "second");
+ assert.notEqual(first, second);
+});
diff --git a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts
index f01acc52b..495d2c7ce 100644
--- a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts
+++ b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts
@@ -3,19 +3,39 @@ import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { test } from "bun:test";
-test("Deep worker cancellation at the child-process boundary", async () => {
- // Keep the portable child fixture in the SDK's macOS and Windows test shards.
- await promisify(execFile)(
- "node",
- [
- "--experimental-strip-types",
- fileURLToPath(
- new URL(
- "../../../plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts",
- import.meta.url,
+for (const boundary of [
+ "lifecycle",
+ "runtime settings",
+ "executable selection",
+ "diagnostics",
+]) {
+ test(`Deep worker ${boundary} at the child-process boundary`, async () => {
+ // Keep each portable boundary in the SDK's macOS and Windows test shards.
+ const { stdout, stderr } = await promisify(execFile)(
+ "node",
+ [
+ "--experimental-strip-types",
+ "--test-reporter=tap",
+ "--test-name-pattern",
+ `^Deep worker ${boundary}`,
+ fileURLToPath(
+ new URL(
+ "../../../plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts",
+ import.meta.url,
+ ),
),
- ),
- ],
- { timeout: 110_000 },
- );
-}, 120_000);
+ ],
+ { timeout: 110_000 },
+ );
+ const passed = new RegExp(`^ok [0-9]+ - Deep worker ${boundary}\\r?$`, "m");
+ if (!passed.test(stdout)) {
+ throw Object.assign(
+ new Error(`No passing Deep worker ${boundary} group.`),
+ {
+ stdout,
+ stderr,
+ },
+ );
+ }
+ }, 120_000);
+}