Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

8.1 KB
Raw Blame
import { chmod, mkdir, readFile, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, describe, expect, test, spyOn } from "bun:test";
import { SandboxUnavailableError } from "../src/index.js";
import { probeCodexSandbox } from "../src/runtime.js";
import { withProtectedExecutableRoots } from "../src/trusted-executable.js";
import { TestClient } from "./support/api-client.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";

const SANDBOX_DENIAL =
  process.platform === "darwin"
    ? "sandbox-exec: sandbox_apply: Operation not permitted"
    : "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted";

const { temporaryDirectory, cleanup } = createApiTestFixtures(
  "codex-security-sandbox-",
);

afterEach(cleanup);

async function syntheticPlugin(root: string): Promise<string> {
  const path = join(root, "plugin");
  await mkdir(join(path, ".codex-plugin"), { recursive: true });
  await writeFile(
    join(path, ".codex-plugin", "plugin.json"),
    JSON.stringify({ name: "codex-security", version: "1.2.3" }),
  );
  return path;
}

async function syntheticCodex(root: string, exitCode: number): Promise<string> {
  const command = join(root, "codex");
  await writeFile(
    command,
    [
      "#!/bin/sh",
      'printf \'%s\\n\' "$@" >> "$0.args"',
      'printf \'%s\\n\' "$PATH" "$CODEX_HOME" "$PROBE_SETTING" > "$0.environment"',
      'if [ "$1" != sandbox ]; then exit 0; fi',
      `if [ ${exitCode} -ne 0 ]; then echo "${SANDBOX_DENIAL}" >&2; fi`,
      `exit ${exitCode}`,
      "",
    ].join("\n"),
  );
  await chmod(command, 0o700);
  return command;
}

describe("Codex sandbox probe", () => {
  test.skipIf(process.platform === "win32")(
    "reports its timeout without sandbox-denial advice",
    async () => {
      const timeout = spyOn(AbortSignal, "timeout").mockReturnValue(
        AbortSignal.abort(
          new DOMException("Synthetic timeout", "TimeoutError"),
        ),
      );
      try {
        const root = await temporaryDirectory();
        const command = await syntheticCodex(root, 0);
        const failure = await probeCodexSandbox({ command }, {}).then(
          () => null,
          (error: Error) => error,
        );
        expect(failure).toBeInstanceOf(SandboxUnavailableError);
        expect(failure?.cause).toBeDefined();
        expect(failure?.message).toContain("timed out");
        expect(failure?.message).not.toContain("user namespaces");
      } finally {
        timeout.mockRestore();
      }
    },
  );

  test.skipIf(process.platform === "win32")(
    "reports what the sandbox said and how to reproduce it",
    async () => {
      const root = await temporaryDirectory();
      const command = await syntheticCodex(root, 1);
      const failure = await probeCodexSandbox({ command }, {}).then(
        () => null,
        (error: Error) => error,
      );
      expect(failure).toBeInstanceOf(SandboxUnavailableError);
      expect(failure?.message).toContain(SANDBOX_DENIAL);
      expect(failure?.message).toContain(
        `${command} sandbox -- ${command} --version`,
      );
      expect(failure?.message.indexOf(SANDBOX_DENIAL)).toBeLessThan(
        failure?.message.indexOf("reproduce") ?? -1,
      );
      expect(failure?.message.includes("AppArmor")).toBe(
        process.platform === "linux",
      );
      expect(failure?.message.includes("user namespaces")).toBe(
        process.platform === "linux",
      );
    },
  );

  test.skipIf(process.platform === "win32")(
    "uses the resolved executable and preserves the scan environment",
    async () => {
      const root = await temporaryDirectory();
      const commandDirectory = join(root, "installed tools");
      await mkdir(commandDirectory);
      const command = await syntheticCodex(commandDirectory, 0);
      const environment = {
        PATH: join(root, "repository", "bin"),
        CODEX_HOME: join(root, "codex-home"),
        PROBE_SETTING: "scan-specific-setting",
      };
      await expect(
        probeCodexSandbox({ command }, environment),
      ).resolves.toBeUndefined();
      const args = ["sandbox", "--", command, "--version", ""].join("\n");
      expect(await readFile(`${command}.args`, "utf8")).toBe(args);
      expect(await readFile(`${command}.environment`, "utf8")).toBe(
        [
          environment.PATH,
          environment.CODEX_HOME,
          environment.PROBE_SETTING,
          "",
        ].join("\n"),
      );
      const source = join(root, "repository");
      await mkdir(source);
      const alias = join(source, "codex");
      await symlink(command, alias);
      await withProtectedExecutableRoots([source], () =>
        probeCodexSandbox({ command: alias }, environment),
      );
      expect(await readFile(`${command}.args`, "utf8")).toBe(args + args);
    },
  );

  test.skipIf(process.platform === "win32")(
    "preserves cancellation before the probe starts",
    async () => {
      const root = await temporaryDirectory();
      const reason = new Error("scan canceled");
      const signal = AbortSignal.abort(reason);
      await expect(
        probeCodexSandbox(
          { command: await syntheticCodex(root, 0) },
          {},
          signal,
        ),
      ).rejects.toBe(reason);
    },
  );

  test.skipIf(process.platform === "win32")(
    "does not launch a probe canceled during campaign tool selection",
    async () => {
      const tools = await temporaryDirectory();
      const command = await syntheticCodex(tools, 0);
      const controller = new AbortController();
      const source = await temporaryDirectory();
      await withProtectedExecutableRoots([source], async () => {
        using timeout = spyOn(AbortSignal, "timeout");
        const probe = probeCodexSandbox(
          { command },
          { PATH: tools },
          controller.signal,
        );
        controller.abort(new Error("scan canceled"));
        await expect(probe).rejects.toBe(controller.signal.reason);
        expect(timeout).not.toHaveBeenCalled();
      });
    },
  );

  test.skipIf(process.platform === "win32")(
    "reports an executable that cannot be started",
    async () => {
      const root = await temporaryDirectory();
      await expect(
        probeCodexSandbox({ command: join(root, "missing-codex") }, {}),
      ).rejects.toMatchObject({
        name: SandboxUnavailableError.name,
        message: expect.stringContaining("ENOENT"),
      });
    },
  );

  test.skipIf(process.platform !== "win32")(
    "does not probe on Windows",
    async () => {
      const root = await temporaryDirectory();
      await expect(
        probeCodexSandbox({ command: join(root, "missing-codex.exe") }, {}),
      ).resolves.toBeUndefined();
    },
  );

  test.skipIf(process.platform === "win32")(
    "stops a scan before it starts a billed Codex thread",
    async () => {
      const root = await temporaryDirectory();
      const repository = join(root, "repository");
      const state = join(root, "state");
      const scanDir = join(root, "scan");
      await mkdir(repository, { mode: 0o700 });
      await mkdir(state, { mode: 0o700 });
      await mkdir(scanDir, { mode: 0o700 });
      await writeFile(join(repository, "app.py"), "print('synthetic')\n");
      const command = await syntheticCodex(root, 1);
      const client = new TestClient(
        { pluginPath: await syntheticPlugin(root) },
        {
          environment: {
            OPENAI_API_KEY: "synthetic-key",
            CODEX_SECURITY_STATE_DIR: state,
          },
          resolveCodexCommand: () => ({ command }),
          probeCodexSandbox,
          resolvePluginPython: async () => "/managed/python",
          prepareOutputDir: async () => scanDir,
          repositoryRevision: async () => "deadbeef",
        },
      );
      try {
        await expect(client.run(repository)).rejects.toMatchObject({
          name: SandboxUnavailableError.name,
          message: expect.stringContaining(SANDBOX_DENIAL),
        });
        expect(await readFile(`${command}.args`, "utf8")).toBe(
          ["sandbox", "--", command, "--version", ""].join("\n"),
        );
      } finally {
        await client.close();
      }
    },
  );
});