Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

20.3 KB
Raw Blame
import { hash } from "node:crypto";
import {
  chmod,
  cp,
  mkdir,
  open,
  readFile,
  rename,
  rm,
  stat,
  symlink,
  writeFile,
} from "node:fs/promises";
import { dirname, join } from "node:path";
import type { ThreadEvent, TurnOptions } from "@openai/codex-sdk";
import { afterEach, describe, expect, spyOn, test } from "bun:test";

import {
  prepareScanArtifactRestorer,
  type ScanArtifactRestorer,
} from "../src/runtime.js";
import * as runtime from "../src/runtime.js";
import { writeThreatModel } from "../src/artifact-export.js";
import { copyCompletedScan, PLUGIN_ROOT } from "./plugin-root.js";
import { TestClient } from "./support/api-client.js";
import {
  codexFactory,
  completedEvents,
  failedPostScanEvents,
  preparedRuntime,
} from "./support/api-events.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";

const { cleanup, temporaryDirectory } = createApiTestFixtures();

afterEach(cleanup);

type PostScanContext = Pick<
  Awaited<ReturnType<typeof startPostScan>>,
  "artifactPath" | "outside" | "scanDir"
>;

interface PostScanScenario {
  artifact: string;
  initialContents?: string | Uint8Array;
  threatModel?: { summary: string };
  selectedPluginFinalizer?: string;
  mutate(context: PostScanContext): Promise<void>;
  followUpEvents?(): AsyncGenerator<ThreadEvent>;
  wrapRestorer?(
    restorer: ScanArtifactRestorer,
    context: PostScanContext,
  ): ScanArtifactRestorer;
}

async function startPostScan(scenario: PostScanScenario) {
  const root = await temporaryDirectory();
  const repository = join(root, "repository");
  const codexHome = join(root, "codex-home");
  const scanDir = join(root, "scan");
  const outside = join(root, "outside");
  const artifactPath = join(scanDir, scenario.artifact);
  const context = { artifactPath, outside, scanDir };
  const python = Bun.which("python3") ?? Bun.which("python");
  expect(python).not.toBeNull();
  await mkdir(repository);
  await mkdir(codexHome);
  await mkdir(scanDir, { mode: 0o700 });
  const runtime = preparedRuntime(codexHome);
  if (scenario.selectedPluginFinalizer !== undefined) {
    const selectedPluginRoot = join(root, "selected-plugin");
    await cp(PLUGIN_ROOT, selectedPluginRoot, { recursive: true });
    await writeFile(
      join(selectedPluginRoot, "scripts", "finalize_scan_contract.py"),
      scenario.selectedPluginFinalizer,
    );
    runtime.plugin = {
      ...runtime.plugin,
      pluginRoot: selectedPluginRoot,
      installedRoot: selectedPluginRoot,
    };
  }
  let turns = 0;
  let original = Buffer.alloc(0);
  const client = TestClient.withDependencies({
    prepareRuntime: async () => runtime,
    resolvePluginPython: async () => python!,
    prepareOutputDir: async () => scanDir,
    repositoryRevision: async () => "deadbeef",
    prepareScanArtifactRestorer: async (...args) => {
      const restorer = await prepareScanArtifactRestorer(...args);
      return scenario.wrapRestorer?.(restorer, context) ?? restorer;
    },
    createCodex: codexFactory(async (_input: string, options: TurnOptions) => {
      expect(options.cyberAccessProgram).toBe("daybreak_blue");
      turns += 1;
      if (turns === 1) {
        await copyCompletedScan(root);
        if (scenario.initialContents !== undefined) {
          await mkdir(dirname(artifactPath), { recursive: true });
          await writeFile(artifactPath, scenario.initialContents);
          const manifestPath = join(scanDir, "scan-manifest.json");
          const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
          if (scenario.threatModel !== undefined)
            manifest.scan.threatModel = scenario.threatModel;
          if (scenario.artifact !== "threatmodel.md")
            manifest.scan.artifacts.push({
              path: scenario.artifact,
              sha256: hash("sha256", await readFile(artifactPath)),
              mediaType: scenario.artifact.endsWith(".bin")
                ? "application/octet-stream"
                : "application/json",
            });
          await writeFile(manifestPath, JSON.stringify(manifest));
          if (scenario.threatModel !== undefined)
            await writeThreatModel(scanDir, {
              pluginRoot: runtime.plugin.installedRoot,
              pythonPath: python!,
              protectedRoot: repository,
            });
        }
        original = await readFile(artifactPath);
        return { events: completedEvents() };
      }
      if (scenario.artifact === "threatmodel.md")
        original = await readFile(artifactPath);
      await scenario.mutate(context);
      return {
        events: scenario.followUpEvents?.() ?? failedPostScanEvents(),
      };
    }, "thread-1"),
  });
  const scan = client.run(repository, {
    postScanPrompt: "Draft confirmed fixes.",
    cyberAccessProgram: "daybreak_blue",
  });
  return {
    client,
    scan,
    scanDir,
    repository,
    artifactPath,
    outside,
    python: python!,
    get turns() {
      return turns;
    },
    get original() {
      return original;
    },
  };
}

const ordinaryRestorationCases: ReadonlyArray<
  readonly [string, PostScanScenario]
> = [
  [
    "missing report",
    { artifact: "report.md", mutate: ({ artifactPath }) => rm(artifactPath) },
  ],
  [
    "derived threat model",
    {
      artifact: "threatmodel.md",
      initialContents: "# Saved model\n",
      threatModel: { summary: "Saved model." },
      mutate: ({ artifactPath }) =>
        writeFile(artifactPath, "# Incorrect replacement\n"),
    },
  ],
  [
    "partial report",
    {
      artifact: "report.md",
      mutate: async ({ artifactPath }) => {
        await writeFile(artifactPath, "# Incomplete draft\n");
      },
    },
  ],
  [
    "replaced report",
    {
      artifact: "report.md",
      mutate: async ({ artifactPath }) => {
        await rm(artifactPath);
        await writeFile(artifactPath, "# Replacement\n");
      },
    },
  ],
  [
    "invalid findings",
    {
      artifact: "findings.json",
      mutate: async ({ artifactPath }) => {
        await writeFile(artifactPath, "{invalid");
      },
    },
  ],
  [
    "sealed nested artifact",
    {
      artifact: "artifacts/worker.json",
      initialContents: '{"complete":true}\n',
      mutate: async ({ artifactPath }) => {
        await writeFile(artifactPath, '{"partial":true}');
      },
    },
  ],
  [
    "binary artifact",
    {
      artifact: "artifacts/worker.bin",
      initialContents: Buffer.from([0, 255, 10, 1]),
      mutate: async ({ artifactPath }) => {
        await writeFile(artifactPath, Buffer.from([9, 0, 8]));
      },
    },
  ],
  [
    "selected custom plugin",
    {
      artifact: "report.md",
      selectedPluginFinalizer:
        "raise RuntimeError('selected plugin helper must not run')\n",
      mutate: ({ artifactPath }) =>
        writeFile(artifactPath, "# Incomplete draft\n"),
    },
  ],
  [
    "nested artifact with a missing parent",
    {
      artifact: "artifacts/worker.json",
      initialContents: '{"complete":true}\n',
      mutate: ({ artifactPath }) =>
        rm(dirname(artifactPath), { recursive: true }),
    },
  ],
];

describe("completed scan follow-up instructions", () => {
  test.each(["removed", "rewritten", "updated model", "unchanged"])(
    "refreshes the model path after a successful follow-up leaves the document %s",
    async (change) => {
      const fixture = await startPostScan({
        artifact: "threatmodel.md",
        initialContents: "# Saved model\n",
        threatModel: { summary: "Saved component boundaries." },
        mutate: async ({ artifactPath, scanDir }) => {
          if (change === "removed") await rm(artifactPath);
          else if (change === "rewritten")
            await writeFile(artifactPath, "# Unrelated replacement\n");
          else if (change === "updated model") {
            const manifestPath = join(scanDir, "scan-manifest.json");
            const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
            manifest.scan.threatModel = {
              summary: "Updated component boundaries.",
            };
            await writeFile(manifestPath, JSON.stringify(manifest));
            await writeThreatModel(scanDir, { pluginRoot: PLUGIN_ROOT });
            expect(await readFile(artifactPath, "utf8")).toContain(
              "Updated component boundaries.",
            );
          }
        },
        async *followUpEvents() {
          yield {
            type: "item.completed",
            item: {
              id: "follow-up-message",
              type: "agent_message",
              text: "Follow-up complete.",
            },
          };
          yield {
            type: "turn.completed",
            usage: {
              input_tokens: 1000,
              cached_input_tokens: 0,
              cache_write_input_tokens: 0,
              output_tokens: 100,
              reasoning_output_tokens: 0,
            },
          };
        },
      });
      try {
        const result = await fixture.scan;
        expect(fixture.turns).toBe(2);
        expect(result.threatModelPath).toBe(
          change === "unchanged" ? fixture.artifactPath : null,
        );
        expect(result.threatModel).toEqual({
          summary: "Saved component boundaries.",
        });
        expect(result.turnResult.finalResponse).toBe("scan complete");
        expect(result.turnResult.usage).toMatchObject({ input_tokens: 10 });
        expect(result.cost?.inputTokens).toBe(10);
      } finally {
        await fixture.client.close();
      }
    },
  );

  test.each(ordinaryRestorationCases)(
    "restores completed scan artifacts after failed post-scan instructions: %s",
    async (_name, scenario) => {
      const python = spyOn(runtime, "resolvePluginPython");
      try {
        const fixture = await startPostScan(scenario);
        const result = await fixture.scan;
        expect(result).toMatchObject({ scanDir: fixture.scanDir });
        if (scenario.artifact === "threatmodel.md")
          expect(result.threatModelPath).toBe(fixture.artifactPath);
        expect(fixture.turns).toBe(2);
        expect(await readFile(fixture.artifactPath)).toEqual(fixture.original);
        await fixture.client.close();
        expect(python).toHaveBeenCalled();
        for (const [options] of python.mock.calls)
          expect(options).toMatchObject({
            configuredPath: fixture.python,
            protectedRoot: fixture.repository,
          });
      } finally {
        python.mockRestore();
      }
    },
  );

  test("does not rewrite artifacts unchanged by a failed follow-up", async () => {
    let before: { dev: number; ino: number; mtimeMs: number } | null = null;
    const fixture = await startPostScan({
      artifact: "report.md",
      mutate: async ({ artifactPath }) => {
        const metadata = await stat(artifactPath);
        before = {
          dev: Number(metadata.dev),
          ino: Number(metadata.ino),
          mtimeMs: Number(metadata.mtimeMs),
        };
      },
    });

    expect(await fixture.scan).toMatchObject({ scanDir: fixture.scanDir });
    const after = await stat(fixture.artifactPath);
    expect(before).not.toBeNull();
    expect(Number(after.dev)).toBe(before!.dev);
    expect(Number(after.ino)).toBe(before!.ino);
    expect(Number(after.mtimeMs)).toBe(before!.mtimeMs);
    await fixture.client.close();
  });

  test.skipIf(process.platform === "win32")(
    "ordinary identical writes retain private replacement semantics",
    async () => {
      const root = await temporaryDirectory();
      const scanDir = join(root, "scan");
      const artifactPath = join(scanDir, "artifact.bin");
      const payload = Buffer.from("unchanged\n");
      const python = Bun.which("python3") ?? Bun.which("python");
      expect(python).not.toBeNull();
      await mkdir(scanDir, { mode: 0o700 });
      await writeFile(artifactPath, payload);
      await chmod(artifactPath, 0o644);
      const before = await stat(artifactPath);
      const script = [
        "from pathlib import Path",
        "from runpy import run_path",
        "import sys",
        "module = run_path(sys.argv[1])",
        "scan_dir = Path(sys.argv[2])",
        "module['write_scan_local_bytes'](scan_dir, 'artifact.bin', b'unchanged\\n')",
      ].join("\n");
      const execution = Bun.spawnSync([
        python!,
        "-I",
        "-B",
        "-c",
        script,
        join(PLUGIN_ROOT, "scripts", "finalize_scan_contract.py"),
        scanDir,
      ]);

      expect(
        execution.exitCode,
        new TextDecoder().decode(execution.stderr),
      ).toBe(0);
      const after = await stat(artifactPath);
      expect(after.mode & 0o777).toBe(0o600);
      expect(after.ino).not.toBe(before.ino);
      expect(await readFile(artifactPath)).toEqual(payload);
    },
  );

  test.skipIf(process.platform !== "linux")(
    "replaces a large sparse artifact within bounded comparison memory",
    async () => {
      const root = await temporaryDirectory();
      const scanDir = join(root, "scan");
      const artifactPath = join(scanDir, "artifact.bin");
      const artifactSize = 32 * 1024 * 1024;
      const python = Bun.which("python3") ?? Bun.which("python");
      expect(python).not.toBeNull();
      await mkdir(scanDir, { mode: 0o700 });
      const script = [
        "from pathlib import Path",
        "from runpy import run_path",
        "import os",
        "import resource",
        "import sys",
        "module = run_path(sys.argv[1])",
        "scan_dir = Path(sys.argv[2])",
        "artifact = scan_dir / 'artifact.bin'",
        "size = 32 * 1024 * 1024",
        "payload = b'x' * size",
        "with artifact.open('wb') as stream:",
        "    stream.truncate(size)",
        "canonical, identity = module['scan_root_identity'](scan_dir)",
        "pages = int(Path('/proc/self/statm').read_text().split()[0])",
        "current_vms = pages * os.sysconf('SC_PAGE_SIZE')",
        "_, hard_limit = resource.getrlimit(resource.RLIMIT_AS)",
        "resource.setrlimit(resource.RLIMIT_AS, (current_vms + 8 * 1024 * 1024, hard_limit))",
        "module['write_scan_local_bytes'](canonical, 'artifact.bin', payload, expected_root_identity=identity)",
      ].join("\n");
      const execution = Bun.spawnSync([
        python!,
        "-I",
        "-B",
        "-c",
        script,
        join(PLUGIN_ROOT, "scripts", "finalize_scan_contract.py"),
        scanDir,
      ]);

      expect(
        execution.exitCode,
        new TextDecoder().decode(execution.stderr),
      ).toBe(0);
      expect((await stat(artifactPath)).size).toBe(artifactSize);
      const artifact = await open(artifactPath, "r");
      try {
        const first = Buffer.alloc(1);
        const last = Buffer.alloc(1);
        await artifact.read(first, 0, 1, 0);
        await artifact.read(last, 0, 1, artifactSize - 1);
        expect(first).toEqual(Buffer.from("x"));
        expect(last).toEqual(Buffer.from("x"));
      } finally {
        await artifact.close();
      }
    },
  );

  test.skipIf(process.platform === "win32")(
    "restores a changed artifact that cannot be read for comparison",
    async () => {
      const fixture = await startPostScan({
        artifact: "report.md",
        mutate: async ({ artifactPath }) => {
          await writeFile(artifactPath, "# Incomplete draft\n");
          await chmod(artifactPath, 0);
        },
      });

      expect(await fixture.scan).toMatchObject({ scanDir: fixture.scanDir });
      expect(await readFile(fixture.artifactPath)).toEqual(fixture.original);
      await fixture.client.close();
    },
  );

  test("rejects a replaced artifact parent without writing through it", async () => {
    const fixture = await startPostScan({
      artifact: "artifacts/worker.json",
      initialContents: '{"complete":true}\n',
      mutate: async ({ artifactPath, outside }) => {
        await mkdir(outside);
        await writeFile(join(outside, "worker.json"), "untouched\n");
        await rm(dirname(artifactPath), { recursive: true });
        await symlink(
          outside,
          dirname(artifactPath),
          process.platform === "win32" ? "junction" : "dir",
        );
      },
    });

    await expect(fixture.scan).rejects.toThrow("scan directory");
    expect(await readFile(join(fixture.outside, "worker.json"), "utf8")).toBe(
      "untouched\n",
    );
    await fixture.client.close();
  });

  test("rejects an artifact parent swapped immediately before the bound write", async () => {
    let swapped = false;
    const artifact = "artifacts/worker.json";
    const fixture = await startPostScan({
      artifact,
      initialContents: '{"complete":true}\n',
      mutate: async ({ artifactPath, outside }) => {
        await mkdir(outside);
        await writeFile(join(outside, "worker.json"), "untouched\n");
        await writeFile(artifactPath, '{"partial":true}');
      },
      wrapRestorer: (restorer, { outside, scanDir }) => ({
        ...restorer,
        async restore(relativePath, contents) {
          if (!swapped && relativePath === artifact) {
            const parent = dirname(join(scanDir, relativePath));
            await rename(parent, `${parent}.original`);
            await symlink(
              outside,
              parent,
              process.platform === "win32" ? "junction" : "dir",
            );
            swapped = true;
          }
          await restorer.restore(relativePath, contents);
        },
      }),
    });

    await expect(fixture.scan).rejects.toThrow("scan directory");
    expect(await readFile(join(fixture.outside, "worker.json"), "utf8")).toBe(
      "untouched\n",
    );
    await fixture.client.close();
  });

  test("rejects a scan root replaced after restoration setup", async () => {
    const fixture = await startPostScan({
      artifact: "report.md",
      mutate: async ({ scanDir }) => {
        await rename(scanDir, `${scanDir}.original`);
        await mkdir(scanDir, { mode: 0o700 });
        await writeFile(join(scanDir, "scan-manifest.json"), "untouched\n");
      },
    });

    await expect(fixture.scan).rejects.toThrow("scan directory");
    expect(
      await readFile(join(fixture.scanDir, "scan-manifest.json"), "utf8"),
    ).toBe("untouched\n");
    await fixture.client.close();
  });

  test.skipIf(process.platform === "win32")(
    "keeps the final rename bound to the validated parent when its path is replaced",
    async () => {
      const root = await temporaryDirectory();
      const scanDir = join(root, "scan");
      const parent = join(scanDir, "artifacts");
      const movedParent = join(root, "moved-artifacts");
      const outside = join(root, "outside");
      const python = Bun.which("python3") ?? Bun.which("python");
      expect(python).not.toBeNull();
      await mkdir(scanDir, { mode: 0o700 });
      await mkdir(parent);
      await mkdir(outside);
      await writeFile(join(parent, "worker.bin"), Buffer.from([1]));
      await writeFile(join(outside, "worker.bin"), "untouched\n");
      const script = [
        "from pathlib import Path",
        "from runpy import run_path",
        "import sys",
        "module = run_path(sys.argv[1])",
        "scan_dir = Path(sys.argv[2])",
        "parent = scan_dir / 'artifacts'",
        "moved_parent = Path(sys.argv[4])",
        "outside = Path(sys.argv[3])",
        "canonical, identity = module['scan_root_identity'](scan_dir)",
        "original_replace = module['os'].replace",
        "swapped = False",
        "def replace(source, destination, *, src_dir_fd=None, dst_dir_fd=None):",
        "    global swapped",
        "    if not swapped:",
        "        parent.rename(moved_parent)",
        "        parent.symlink_to(outside, target_is_directory=True)",
        "        swapped = True",
        "    return original_replace(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd)",
        "module['os'].replace = replace",
        "module['write_scan_local_bytes'](canonical, 'artifacts/worker.bin', bytes([0, 255, 10, 1]), expected_root_identity=identity)",
      ].join("\n");
      const execution = Bun.spawnSync([
        python!,
        "-I",
        "-B",
        "-c",
        script,
        join(PLUGIN_ROOT, "scripts", "finalize_scan_contract.py"),
        scanDir,
        outside,
        movedParent,
      ]);

      expect(
        execution.exitCode,
        new TextDecoder().decode(execution.stderr),
      ).toBe(0);
      expect(await readFile(join(outside, "worker.bin"), "utf8")).toBe(
        "untouched\n",
      );
      expect(await readFile(join(movedParent, "worker.bin"))).toEqual(
        Buffer.from([0, 255, 10, 1]),
      );
    },
  );
});