Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

20.8 KB
Raw Blame
import { afterEach, expect, spyOn, test } from "bun:test";
import * as files from "node:fs/promises";
import {
  cp,
  mkdir,
  readFile,
  readdir,
  rename,
  rm,
  symlink,
  writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import * as timers from "node:timers/promises";
import * as executable from "../src/codex-sdk-environment.js";
import { PluginBootstrapError } from "../src/errors.js";
import { bootstrapPlugin, createMarketplace } from "../src/runtime.js";
import { nodeCommand } from "./support/shell.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";

const { temporaryDirectory, cleanup } = createApiTestFixtures();
afterEach(cleanup);

const denied =
  "Error: failed to activate updated plugin cache entry: Access is denied. (os error 5)";
const backup = denied.replace("activate updated", "back up");
const rollback =
  "Error: failed to activate updated plugin cache entry at target: Access is denied. (os error 5); failed to restore previous cache entry";
const unrelated = "Error: could not enable the plugin";

type Step = {
  stderr?: string;
  stdout?: string;
  exit?: number;
  stale?: boolean;
  replaceMarketplace?: boolean;
  extraSkill?: "marketplace" | "installed";
  invalidMigrationFile?: boolean;
};

// Exercise the real subprocess/error boundary. A successful native install can
// add files of its own when it migrates a plugin's commands.
const fakeCli = `
  const fs = require("node:fs");
  const { join } = require("node:path");
  const home = process.env.CODEX_HOME;
  const args = process.argv.slice(1);
  if (args.join(" ") !== "plugin add --json codex-security@codex-security-sdk")
    throw new Error("Unexpected fixture command " + JSON.stringify(args));
  const plan = join(home, "steps.json");
  const steps = JSON.parse(fs.readFileSync(plan, "utf8"));
  const step = steps.shift();
  if (!step) throw new Error("Unexpected extra install");
  fs.writeFileSync(plan, JSON.stringify(steps));
  const staged = join(home, "sdk-marketplace", "plugins", "codex-security");
  const addSkill = (root, name) => {
    const path = join(root, name);
    fs.mkdirSync(path, { recursive: true });
    fs.writeFileSync(join(path, "SKILL.md"), "synthetic skill");
  };
  if (step.extraSkill === "marketplace") addSkill(staged, "skills/unselected");
  if (step.replaceMarketplace)
    fs.writeFileSync(join(home, "sdk-marketplace", "plugins", "codex-security", "marker.txt"), "different selection");
  if (step.exit) {
    process.stderr.write(step.stderr ?? "");
    process.stdout.write(step.stdout ?? "");
    process.exit(step.exit);
  }
  const installedPath = join(home, "installed");
  fs.rmSync(installedPath, { recursive: true, force: true });
  fs.cpSync(staged, installedPath, { recursive: true });
  if (step.stale) fs.writeFileSync(join(installedPath, "marker.txt"), "previous");
  if (step.extraSkill === "installed") addSkill(installedPath, "skills/unselected");
  const migrated = join(installedPath, ".codex-plugin", "migrated-command-skills");
  fs.rmSync(migrated, { recursive: true, force: true });
  if (step.invalidMigrationFile) fs.writeFileSync(migrated, "unexpected file");
  else if (fs.existsSync(join(staged, "commands", "sample.md"))) addSkill(migrated, "source-command-sample");
  process.stdout.write(JSON.stringify({ installedPath, version: "1.2.3" }));
`;

async function fixture(
  steps: Step[],
  staleMigration?: "regenerated" | "removed" | "case-preserved",
) {
  const root = await temporaryDirectory();
  const selected = join(root, "selected");
  const home = join(root, "home");
  await mkdir(join(selected, ".codex-plugin"), { recursive: true });
  await writeFile(
    join(selected, ".codex-plugin", "plugin.json"),
    JSON.stringify({ name: "codex-security", version: "1.2.3" }),
  );
  await writeFile(join(selected, "marker.txt"), "selected");
  if (staleMigration !== "removed") {
    await mkdir(join(selected, "commands"));
    await writeFile(
      join(selected, "commands", "sample.md"),
      "---\ndescription: local test\n---\nSample.",
    );
  }
  if (staleMigration) {
    const retired = join(
      selected,
      ".codex-plugin",
      staleMigration === "case-preserved"
        ? "MIGRATED-COMMAND-SKILLS"
        : "migrated-command-skills",
      "source-command-retired",
    );
    await mkdir(retired, { recursive: true });
    await writeFile(join(retired, "SKILL.md"), "retired");
  }
  const marketplace = await createMarketplace(home, selected);
  await writeFile(
    join(home, "config.toml"),
    `[marketplaces.codex-security-sdk]\nsource_type="local"\nsource=${JSON.stringify(marketplace)}\n[plugins."codex-security@codex-security-sdk"]\nenabled=true\n`,
  );
  const previous = join(home, "previous");
  await cp(selected, previous, { recursive: true });
  await writeFile(join(previous, "marker.txt"), "previous");
  const record = join(marketplace, "installed-plugin.json");
  const previousReceipt = JSON.stringify({
    installedPath: previous,
    version: "1.2.3",
  });
  await writeFile(record, previousReceipt);
  const plan = join(home, "steps.json");
  await writeFile(plan, JSON.stringify(steps));
  const command = nodeCommand().command;
  return {
    command,
    selected,
    previous,
    record,
    previousReceipt,
    remaining: async () => JSON.parse(await readFile(plan, "utf8")) as Step[],
    install: (signal?: AbortSignal) =>
      bootstrapPlugin(home, selected, {
        codexCommand: { command, args: ["--eval", fakeCli, "--"] },
        environment: { SystemRoot: process.env["SystemRoot"] },
        signal,
      }),
  };
}

test("only Windows retries a rolled-back bundled plugin activation and records the selected bytes", async () => {
  const f = await fixture([{ exit: 1, stderr: denied }, {}]);
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.command);
  if (process.platform !== "win32") {
    await expect(f.install()).rejects.toThrow(denied);
    expect(resolver).not.toHaveBeenCalled();
    expect(await f.remaining()).toHaveLength(1);
    expect(await readFile(f.record, "utf8")).toBe(f.previousReceipt);
    return;
  }
  const installed = await f.install();
  expect(resolver).toHaveBeenCalled();
  expect(await f.remaining()).toHaveLength(0);
  expect(
    await readFile(join(installed.installedRoot, "marker.txt"), "utf8"),
  ).toBe("selected");
  expect(
    await readFile(
      join(
        installed.installedRoot,
        ".codex-plugin",
        "migrated-command-skills",
        "source-command-sample",
        "SKILL.md",
      ),
      "utf8",
    ),
  ).toBe("synthetic skill");
  expect(JSON.parse(await readFile(f.record, "utf8"))).toEqual({
    installedPath: installed.installedRoot,
    version: "1.2.3",
  });
  expect((await f.install()).installedRoot).toBe(installed.installedRoot);
});

test
  .skipIf(process.platform !== "win32")
  .each(["regenerated", "removed", "case-preserved"] as const)(
  "accepts stale native command output that is %s during recovery",
  async (mode) => {
    const f = await fixture([{ exit: 1, stderr: denied }, {}], mode);
    using resolver = spyOn(
      executable,
      "resolveBundledCodexExecutable",
    ).mockReturnValue(f.command);
    const result = await f.install();
    const migration = join(".codex-plugin", "migrated-command-skills");
    expect(
      await readFile(
        join(
          result.pluginRoot,
          mode === "case-preserved"
            ? join(".codex-plugin", "MIGRATED-COMMAND-SKILLS")
            : migration,
          "source-command-retired",
          "SKILL.md",
        ),
        "utf8",
      ),
    ).toBe("retired");
    const installedEntries = await readdir(
      join(result.installedRoot, migration),
    ).catch((error: NodeJS.ErrnoException) => error.code);
    expect(installedEntries).toEqual(
      mode === "removed" ? "ENOENT" : ["source-command-sample"],
    );
    expect(JSON.parse(await readFile(f.record, "utf8")).installedPath).toBe(
      result.installedRoot,
    );
    expect(await f.remaining()).toHaveLength(0);
    expect((await f.install()).installedRoot).toBe(result.installedRoot);
    expect(resolver).toHaveBeenCalled();
  },
);

test("only the bundled installer reuses its regenerated command output", async () => {
  const f = await fixture([{}, {}, {}], "regenerated");
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.previous);
  await f.install();
  expect(await f.remaining()).toHaveLength(2);

  resolver.mockReturnValue(f.command);
  const installed = await f.install();
  expect(await f.remaining()).toHaveLength(1);
  expect((await f.install()).installedRoot).toBe(installed.installedRoot);
  expect(await f.remaining()).toHaveLength(1);

  resolver.mockReturnValue(f.previous);
  await f.install();
  expect(await f.remaining()).toHaveLength(0);
});

test("rechecks the full marketplace selection and the recovered installed files", async () => {
  const f = await fixture([{}, {}, {}], "regenerated");
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.command);
  const { installedRoot } = await f.install();
  const staleSkill = join(
    f.selected,
    ".codex-plugin",
    "migrated-command-skills",
    "source-command-retired",
    "SKILL.md",
  );
  await writeFile(staleSkill, "revised stale source");
  await f.install();
  expect(await f.remaining()).toHaveLength(1);

  await mkdir(join(installedRoot, "skills", "other-selection"), {
    recursive: true,
  });
  await writeFile(
    join(installedRoot, "skills", "other-selection", "SKILL.md"),
    "other",
  );
  await f.install();
  expect(await f.remaining()).toHaveLength(0);
  expect(resolver).toHaveBeenCalled();
});

test("preserves runtime files when the selection already contains the native command output", async () => {
  const f = await fixture([{}, {}, {}, {}]);
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.command);
  const { installedRoot } = await f.install();
  const migration = join(".codex-plugin", "migrated-command-skills");
  const generated = join(installedRoot, migration);
  await cp(generated, join(f.selected, migration), { recursive: true });
  await f.install();
  const receipt = await readFile(f.record, "utf8");
  expect(JSON.parse(receipt).nativeCommandMigration).toMatch(/^sha256:/u);
  const bytecode = join(installedRoot, "scripts", "__pycache__");
  await mkdir(bytecode, { recursive: true });
  const cached = join(bytecode, "helper.cpython-312.pyc");
  await writeFile(cached, "runtime-generated");

  expect((await f.install()).installedRoot).toBe(installedRoot);
  expect(await f.remaining()).toHaveLength(2);
  expect(await readFile(cached, "utf8")).toBe("runtime-generated");
  expect(await readFile(f.record, "utf8")).toBe(receipt);

  const skill = join(generated, "source-command-sample", "SKILL.md");
  await rm(skill);
  await f.install();
  expect(await f.remaining()).toHaveLength(1);
  expect(await readFile(skill, "utf8")).toBe("synthetic skill");
  await mkdir(join(generated, "foreign-command"));
  await writeFile(join(generated, "foreign-command", "SKILL.md"), "foreign");
  await f.install();
  expect(await f.remaining()).toHaveLength(0);
  expect(await readdir(generated)).toEqual(["source-command-sample"]);
  expect(resolver).toHaveBeenCalled();
});

test("a link replacing the installed command migration directory requires reinstall", async () => {
  const f = await fixture([{}, {}], "regenerated");
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.command);
  const { installedRoot } = await f.install();
  const migration = join(
    installedRoot,
    ".codex-plugin",
    "migrated-command-skills",
  );
  const outside = await temporaryDirectory();
  await writeFile(join(outside, "marker.txt"), "unrelated");
  await rm(migration, { recursive: true });
  await symlink(
    outside,
    migration,
    process.platform === "win32" ? "junction" : "dir",
  );
  await f.install();
  expect(await f.remaining()).toHaveLength(0);
  expect(await readFile(join(outside, "marker.txt"), "utf8")).toBe("unrelated");
  expect(resolver).toHaveBeenCalled();
});

test.each([
  "missing file",
  "changed bytes",
  "renamed skill",
  "changed file type",
  "restored source with changed output",
  "extra output",
  "missing directory",
  "nested link",
  "previously absent",
] as const)(
  "repairs %s in a recorded native command migration",
  async (damage) => {
    const absent = damage === "previously absent";
    const f = await fixture([{}, {}], absent ? "removed" : "regenerated");
    using resolver = spyOn(
      executable,
      "resolveBundledCodexExecutable",
    ).mockReturnValue(f.command);
    const { installedRoot } = await f.install();
    const migration = join(
      installedRoot,
      ".codex-plugin",
      "migrated-command-skills",
    );
    const skill = join(migration, "source-command-sample");
    const file = join(skill, "SKILL.md");
    const outside = await temporaryDirectory();
    const sentinel = join(outside, "SKILL.md");
    await writeFile(sentinel, "outside");
    switch (damage) {
      case "missing file":
        await rm(file);
        break;
      case "restored source with changed output":
        await cp(
          join(f.selected, ".codex-plugin", "migrated-command-skills"),
          migration,
          { recursive: true },
        );
        await writeFile(file, "different skill");
        break;
      case "changed bytes":
        await writeFile(file, "different skill");
        break;
      case "renamed skill":
        await rename(skill, join(migration, "other-command-name"));
        break;
      case "changed file type":
        await rm(file);
        await mkdir(file);
        break;
      case "extra output":
        await writeFile(join(skill, "extra.md"), "unselected");
        break;
      case "missing directory":
        await rm(migration, { recursive: true });
        break;
      case "nested link":
        await rm(skill, { recursive: true });
        await symlink(
          outside,
          skill,
          process.platform === "win32" ? "junction" : "dir",
        );
        break;
      case "previously absent":
        await mkdir(migration);
        break;
    }
    await f.install();
    expect(await f.remaining()).toHaveLength(0);
    if (absent) {
      await expect(readdir(migration)).rejects.toMatchObject({
        code: "ENOENT",
      });
    } else {
      expect(await readdir(migration)).toEqual(["source-command-sample"]);
      expect(await readdir(skill)).toEqual(["SKILL.md"]);
      expect(await readFile(file, "utf8")).toBe("synthetic skill");
    }
    expect(await readFile(sentinel, "utf8")).toBe("outside");
    expect(resolver).toHaveBeenCalled();
  },
);

test("canceling a generated skill read leaves its native receipt and caller reason intact", async () => {
  const f = await fixture([{}], "regenerated");
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.command);
  const { installedRoot } = await f.install();
  const generated = join(
    installedRoot,
    ".codex-plugin",
    "migrated-command-skills",
    "source-command-sample",
    "SKILL.md",
  );
  const receipt = await readFile(f.record);
  const controller = new AbortController();
  const reason = new Error(
    "synthetic cancellation while reading generated skill",
  );
  const prior = files.open;
  using opening = spyOn(files, "open").mockImplementation(async (...args) => {
    const result = await prior(...args);
    if (String(args[0]) === generated) controller.abort(reason);
    return result;
  });
  await expect(f.install(controller.signal)).rejects.toBe(reason);
  expect(await f.remaining()).toHaveLength(0);
  expect(await readFile(f.record)).toEqual(receipt);
  expect(opening).toHaveBeenCalled();
  expect(resolver).toHaveBeenCalled();
});

test("checks the generated output and selected files in the same installation", async () => {
  const f = await fixture([{}, {}], "regenerated");
  using resolver = spyOn(
    executable,
    "resolveBundledCodexExecutable",
  ).mockReturnValue(f.command);
  const { installedRoot } = await f.install();
  const previous = join(await temporaryDirectory(), "previous");
  const original = files.realpath;
  let replaced = false;
  using resolving = spyOn(files, "realpath").mockImplementation((async (
    ...args: Parameters<typeof original>
  ) => {
    if (!replaced && args[0] === installedRoot) {
      replaced = true;
      await rename(installedRoot, previous);
      await cp(previous, installedRoot, { recursive: true });
      await writeFile(join(installedRoot, "marker.txt"), "other selection");
    }
    return original(...args);
  }) as typeof original);
  await f.install();
  expect(replaced).toBe(true);
  expect(await f.remaining()).toHaveLength(0);
  expect(await readFile(join(installedRoot, "marker.txt"), "utf8")).toBe(
    "selected",
  );
  expect(resolving).toHaveBeenCalled();
  expect(resolver).toHaveBeenCalled();
});

test.skipIf(process.platform !== "win32").each([
  [
    "persistent denial",
    [
      { exit: 1, stderr: denied + "\nfirst detail" },
      {
        exit: 1,
        stderr:
          denied.replace("Access is denied.", "Zugriff verweigert.") +
          "\nsecond detail",
      },
    ],
    "first detail",
    0,
  ],
  [
    "different retry failure",
    [
      { exit: 1, stderr: denied },
      { exit: 1, stderr: unrelated },
    ],
    unrelated,
    0,
  ],
  [
    "stale same-version result",
    [{ exit: 1, stderr: denied }, { stale: true }],
    "selected plugin contents",
    0,
  ],
  [
    "marketplace replaced before retry",
    [{ exit: 1, stderr: denied, replaceMarketplace: true }, {}],
    "changed before the native installation could be retried",
    1,
  ],
  [
    "marketplace replaced during retry",
    [{ exit: 1, stderr: denied }, { replaceMarketplace: true }],
    "selected plugin contents",
    0,
  ],
  [
    "additional marketplace skill during retry",
    [{ exit: 1, stderr: denied }, { extraSkill: "marketplace" }],
    "selected plugin contents",
    0,
  ],
  [
    "additional installed skill after the original marketplace is restored",
    [{ exit: 1, stderr: denied }, { extraSkill: "installed" }],
    "selected plugin contents",
    0,
  ],
  [
    "regular file at the native migration directory",
    [{ exit: 1, stderr: denied }, { invalidMigrationFile: true }],
    "selected plugin contents",
    0,
  ],
  ["backup failure", [{ exit: 1, stderr: backup }, {}], backup, 1],
  ["rollback failure", [{ exit: 1, stderr: rollback }, {}], rollback, 1],
  [
    "rollback failure in the cause chain",
    [
      { exit: 1, stderr: denied + "\nfailed to restore previous cache entry" },
      {},
    ],
    "failed to restore previous cache entry",
    1,
  ],
  [
    "different native error",
    [{ exit: 1, stderr: unrelated }, {}],
    unrelated,
    1,
  ],
  ["stdout alone", [{ exit: 1, stdout: denied }, {}], denied, 1],
  ["different exit status", [{ exit: 2, stderr: denied }, {}], denied, 1],
  ["caller executable", [{ exit: 1, stderr: denied }, {}], denied, 1],
] as const)(
  "%s keeps the previous installation receipt",
  async (scenario, steps, message, remaining) => {
    const f = await fixture([...steps]);
    using resolver = spyOn(
      executable,
      "resolveBundledCodexExecutable",
    ).mockReturnValue(
      scenario === "caller executable" ? f.previous : f.command,
    );
    const error = await f.install().catch((cause: unknown) => cause);
    expect(error).toBeInstanceOf(PluginBootstrapError);
    expect((error as Error).message).toContain(message);
    if (scenario === "persistent denial")
      expect(((error as Error).cause as Error).message).toBe(
        denied + "\nfirst detail",
      );
    expect(await f.remaining()).toHaveLength(remaining);
    expect(await readFile(f.record, "utf8")).toBe(f.previousReceipt);
    expect(await readFile(join(f.previous, "marker.txt"), "utf8")).toBe(
      "previous",
    );
    if (scenario === "caller executable") expect(resolver).toHaveBeenCalled();
  },
);

test.skipIf(process.platform !== "win32")(
  "cancellation during the native activation retry delay preserves the caller's reason",
  async () => {
    const f = await fixture([{ exit: 1, stderr: denied }, {}]);
    const controller = new AbortController();
    const reason = new Error("synthetic canceled recovery");
    using resolver = spyOn(
      executable,
      "resolveBundledCodexExecutable",
    ).mockReturnValue(f.command);
    const originalDelay = timers.setTimeout;
    using delay = spyOn(timers, "setTimeout").mockImplementation((...args) => {
      if (args[2]?.signal === controller.signal) controller.abort(reason);
      return originalDelay(...args);
    });
    await expect(f.install(controller.signal)).rejects.toBe(reason);
    expect(resolver).toHaveBeenCalled();
    expect(delay).toHaveBeenCalled();
    expect(await f.remaining()).toHaveLength(1);
    expect(await readFile(f.record, "utf8")).toBe(f.previousReceipt);
  },
);