Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

14.8 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import Ajv2020 from "ajv/dist/2020.js";
import { main } from "../src/cli.js";
import { CodexSecurityError } from "../src/index.js";
import {
  FakeSignals,
  SYNTHETIC_CREDENTIALS,
  capture,
  dependencies,
  fakePreflight,
  fakeResult,
} from "./cli-fixtures.js";

const commands = [
  ["scan", ["scan", "."]],
  ["rerun", ["scans", "rerun", "saved-scan"]],
  ["resume", ["scans", "resume", "saved-scan"]],
] as const;

function scanDependencies() {
  return dependencies({
    onWorkbench: () => ({
      scanId: "saved-scan",
      scanDir: "/tmp/saved-scan",
      recipe: {
        repository: "/original/repository",
        target: { kind: "repository", paths: [] },
        mode: "standard",
        config: {},
      },
    }),
  });
}

function failingScan(message: string, onRun?: () => void) {
  const deps = scanDependencies();
  deps.createSecurity = () => ({
    run: async () => {
      onRun?.();
      throw new CodexSecurityError(message);
    },
    validate: async () => {
      throw new CodexSecurityError(message);
    },
    preflight: async () => fakePreflight(),
    close: async () => {},
  });
  return deps;
}

function expectFailure(
  text: string,
  fullOutput: boolean,
  code: string,
  message: string,
  data?: unknown,
) {
  const result = JSON.parse(text);
  if (fullOutput) {
    expect(result.ok).toBe(false);
    expect(result.error).toEqual({ code, message });
    if (data === undefined) expect(result).not.toHaveProperty("data");
    else expect(result.data).toEqual(data);
  } else {
    expect(result).toEqual({ status: "failed", code, message });
  }
}

for (const flags of [["--json"], ["--format", "jsonl", "--full-output"]]) {
  describe(`scan failures with ${flags.join(" ")}`, () => {
    const fullOutput = flags.includes("--full-output");

    test.each(commands)(
      "formats %s execution failures consistently",
      async (_name, command) => {
        const stdout = capture();
        const stderr = capture();
        const message = `network failure ECONNRESET ${SYNTHETIC_CREDENTIALS}`;
        expect(
          await main(
            [...command, ...flags],
            stdout.stream,
            stderr.stream,
            failingScan(message),
          ),
        ).toBe(2);
        expectFailure(stdout.text(), fullOutput, "SCAN_FAILED", message);
        expect(stderr.text()).toContain(`${message}\n`);
      },
    );

    test.each([
      ["rerun", "SCAN_REPLAY_UNAVAILABLE"],
      ["resume", "SCAN_RESUME_UNAVAILABLE"],
    ])("preserves %s preparation failure codes", async (command, code) => {
      const stdout = capture();
      const stderr = capture();
      const message = "This scan does not have a saved launch recipe.";
      const deps = scanDependencies();
      deps.runWorkbench = async () => ({
        scanId: "saved-scan",
        scanDir: "/tmp/saved-scan",
      });
      expect(
        await main(
          ["scans", command, "saved-scan", ...flags],
          stdout.stream,
          stderr.stream,
          deps,
        ),
      ).toBe(2);
      expectFailure(stdout.text(), fullOutput, code, message);
      expect(stderr.text()).toBe(`codex-security: ${message}\n`);
    });

    test("reports unavailable latest scans with the replay failure code", async () => {
      const stdout = capture();
      const stderr = capture();
      const deps = scanDependencies();
      deps.runWorkbench = async () => ({ scans: [] });
      expect(
        await main(
          ["scans", "rerun", ...flags],
          stdout.stream,
          stderr.stream,
          deps,
        ),
      ).toBe(2);
      const message = "No completed scans found for the current repository.";
      expectFailure(
        stdout.text(),
        fullOutput,
        "SCAN_REPLAY_UNAVAILABLE",
        message,
      );
      expect(stderr.text()).toBe(`codex-security: ${message}\n`);
    });

    test.each(["csv", "json"])(
      "formats rerun failures independently of the %s import format",
      async (format) => {
        const stdout = capture();
        const stderr = capture();
        const message = "The saved import source is unavailable.";
        const deps = scanDependencies();
        deps.runWorkbench = async () => ({
          recipe: {
            import: { sourcePath: `/tmp/findings.${format}`, format },
          },
        });
        deps.importScan = async () => {
          throw new CodexSecurityError(message);
        };
        expect(
          await main(
            ["scans", "rerun", "saved-scan", ...flags],
            stdout.stream,
            stderr.stream,
            deps,
          ),
        ).toBe(2);
        expectFailure(stdout.text(), fullOutput, "SCAN_IMPORT_FAILED", message);
        expect(stderr.text()).toBe(`codex-security: ${message}\n`);
      },
    );
  });
}

test.each(commands)(
  "preserves successful %s JSON output",
  async (_name, command) => {
    const stdout = capture();
    expect(
      await main(
        [...command, "--json"],
        stdout.stream,
        capture().stream,
        scanDependencies(),
      ),
    ).toBe(0);
    expect(JSON.parse(stdout.text())).toEqual(fakeResult().toJSON());
  },
);

test.each(commands)(
  "keeps non-JSON %s failures on stderr",
  async (_name, command) => {
    for (const flags of [[], ["--full-output"], ["--format", "yaml"]]) {
      const stdout = capture();
      const stderr = capture();
      const message = "The scan could not save its results.";
      expect(
        await main(
          [...command, ...flags],
          stdout.stream,
          stderr.stream,
          failingScan(message),
        ),
      ).toBe(2);
      expect(stdout.text()).toBe("");
      expect(stderr.text()).toContain(`${message}\n`);
    }
  },
);

test.each(commands)(
  "retains %s cancellation exit codes",
  async (_name, command) => {
    for (const [signal, exitCode, message] of [
      ["SIGINT", 130, "Scan canceled by Ctrl-C."],
      ["SIGTERM", 143, "Scan terminated by SIGTERM."],
    ] as const) {
      for (const fullOutput of [false, true]) {
        const stdout = capture();
        const signals = new FakeSignals();
        const result = fakeResult();
        const deps = dependencies({
          result,
          signals,
          onRun: () => signals.emit(signal),
        });
        deps.runWorkbench = scanDependencies().runWorkbench;
        expect(
          await main(
            [...command, "--json", ...(fullOutput ? ["--full-output"] : [])],
            stdout.stream,
            capture().stream,
            deps,
          ),
        ).toBe(exitCode);
        expectFailure(
          stdout.text(),
          fullOutput,
          "SCAN_FAILED",
          message,
          fullOutput ? result.toJSON() : undefined,
        );
      }
    }
  },
);

test("retains diagnostics when the latest-scan working directory is unavailable", async () => {
  for (const flags of [[], ["--json"], ["--json", "--full-output"]]) {
    const stdout = capture();
    const stderr = capture();
    const deps = scanDependencies();
    const message = "The working directory is unavailable.";
    deps.currentDirectory = () => {
      throw new Error(message);
    };
    expect(
      await main(
        ["scans", "rerun", ...flags],
        stdout.stream,
        stderr.stream,
        deps,
      ),
    ).toBe(2);
    expect(stderr.text()).toBe(`codex-security: ${message}\n`);
    if (flags.length === 0) expect(stdout.text()).toBe("");
    else
      expectFailure(
        stdout.text(),
        flags.includes("--full-output"),
        "SCAN_REPLAY_UNAVAILABLE",
        message,
      );
  }
});

test.each([
  [["scan"], ["SCAN_FAILED"]],
  [
    ["scans", "rerun"],
    ["SCAN_FAILED", "SCAN_REPLAY_UNAVAILABLE", "SCAN_IMPORT_FAILED"],
  ],
  [
    ["scans", "resume"],
    ["SCAN_FAILED", "SCAN_RESUME_UNAVAILABLE"],
  ],
])(
  "documents failure codes in the %j output schema",
  async (command, codes) => {
    const stdout = capture();
    expect(
      await main(
        [...command, "--schema", "--json"],
        stdout.stream,
        capture().stream,
        dependencies(),
      ),
    ).toBe(0);
    const schema = JSON.parse(stdout.text());
    const failure = schema.output.anyOf.find(
      (variant: { properties?: { status?: { const?: string } } }) =>
        variant.properties?.status?.const === "failed",
    );
    expect(failure.required).toEqual(["status", "code", "message"]);
    expect(
      failure.properties.code.enum ?? [failure.properties.code.const],
    ).toEqual(codes);
  },
);

test.each(commands.slice(1))(
  "preserves diagnostics and token controls for filtered %s failures",
  async (_name, command) => {
    const message =
      "Synthetic saved scan failure with its complete diagnostic.";
    for (const formatArgs of [["--json"], ["--format", "jsonl"]]) {
      for (const tokens of [
        [],
        ["--token-count"],
        ["--token-limit", "4"],
        ["--token-offset=4", "--token-limit=4"],
      ]) {
        const reference = capture();
        expect(
          await main(
            [...command, ...formatArgs, ...tokens],
            reference.stream,
            capture().stream,
            failingScan(message),
          ),
        ).toBe(2);
        for (const filter of [
          ["--filter-output", "manifest,findings"],
          ["--filter-output=manifest,findings"],
          ["--filter-output", "findings"],
          ["--filter-output=manifest"],
        ]) {
          const stdout = capture();
          const stderr = capture();
          let runs = 0;
          expect(
            await main(
              [...command, ...formatArgs, ...filter, ...tokens],
              stdout.stream,
              stderr.stream,
              failingScan(message, () => runs++),
            ),
          ).toBe(2);
          expect(runs).toBe(1);
          expect(stdout.text()).toBe(reference.text());
          expect(stderr.text()).toContain(message);
        }
      }
    }
  },
);

test.each(["rerun", "resume"])(
  "preserves filtered %s preparation failures",
  async (command) => {
    const stdout = capture();
    const deps = scanDependencies();
    deps.runWorkbench = async () => ({
      scanId: "saved-scan",
      scanDir: "/tmp/saved-scan",
    });
    expect(
      await main(
        [
          "scans",
          command,
          "saved-scan",
          "--json",
          "--filter-output",
          "manifest,findings",
        ],
        stdout.stream,
        capture().stream,
        deps,
      ),
    ).toBe(2);
    expectFailure(
      stdout.text(),
      false,
      command === "rerun"
        ? "SCAN_REPLAY_UNAVAILABLE"
        : "SCAN_RESUME_UNAVAILABLE",
      "This scan does not have a saved launch recipe.",
    );
  },
);

test.each(commands.slice(1))(
  "preserves success filtering and full error envelopes for %s",
  async (_name, command) => {
    const success = capture();
    expect(
      await main(
        [...command, "--json", "--filter-output", "manifest,findings"],
        success.stream,
        capture().stream,
        scanDependencies(),
      ),
    ).toBe(0);
    const original = fakeResult().toJSON();
    expect(JSON.parse(success.text())).toEqual({
      manifest: original["manifest"],
      findings: original["findings"],
    });
    const failure = capture();
    const message = "Synthetic full envelope failure.";
    expect(
      await main(
        [
          ...command,
          "--json",
          "--full-output",
          "--filter-output",
          "manifest,findings",
        ],
        failure.stream,
        capture().stream,
        failingScan(message),
      ),
    ).toBe(2);
    expectFailure(failure.text(), true, "SCAN_FAILED", message);
  },
);

for (const command of ["rerun", "resume"]) {
  test.each([true, false, undefined])(
    `${command} restores owner suggestions from its saved recipe (%p)`,
    async (ownerSuggestions) => {
      let observed: boolean | undefined;
      let called = false;
      const deps = dependencies({
        onWorkbench: () => ({
          scanId: "saved-scan",
          scanDir: "/tmp/saved-scan",
          recipe: {
            repository: "/original/repository",
            target: { kind: "repository", paths: [] },
            mode: "deep",
            config: {},
            ...(ownerSuggestions === undefined ? {} : { ownerSuggestions }),
          },
        }),
        onTurn: (_repository, options) => {
          called = true;
          observed = options.ownerSuggestions;
        },
      });
      const stdout = capture();
      const stderr = capture();
      expect(
        await main(
          ["scans", command, "saved-scan", "--json"],
          stdout.stream,
          stderr.stream,
          deps,
        ),
      ).toBe(0);
      expect(called).toBe(true);
      expect(observed).toBe(ownerSuggestions);
    },
  );
}

test("scan discovery accepts caller inputs and validates the reported result variants", async () => {
  const stdout = capture();
  const stderr = capture();
  expect(
    await main(
      ["scan", "--schema", "--json"],
      stdout.stream,
      stderr.stream,
      dependencies(),
    ),
  ).toBe(0);
  const { options, output } = JSON.parse(stdout.text());
  const ajv = new Ajv2020({ strict: false });
  const inputs = ajv.compile(options);
  expect(inputs({ dryRun: true })).toBe(true);
  expect(inputs({ maxCost: "not a number" })).toBe(false);
  const data = ajv.compile(output);
  expect(data({ dryRun: true, ...fakePreflight() })).toBe(true);
  expect(data(JSON.parse(JSON.stringify(fakeResult(["high"]).toJSON())))).toBe(
    true,
  );
  expect(
    data({ status: "failed", code: "SCAN_FAILED", message: "Example" }),
  ).toBe(true);
  const retained = fakeResult(["medium"]);
  expect(
    data({
      mode: retained.mode,
      manifest: retained.manifest,
      findings: retained.findings,
      review: retained.review,
      scanDir: retained.scanDir,
      reportPath: retained.reportPath,
    }),
  ).toBe(true);
  expect(data({})).toBe(false);
  expect(data({ status: "failed", code: 0, message: 42 })).toBe(false);
});

test.each([
  { args: ["--max-cost"] },
  { args: ["--format"] },
  { args: ["--format", "--json"] },
  { args: ["--token-limit", "--json"] },
  { args: ["--max-cost", "invalid"] },
  { args: ["--path", "src", "--diff", "HEAD"] },
  { args: ["--token-limit", "invalid"] },
])(
  "returns a JSON error envelope for invalid scan arguments %j",
  async ({ args }) => {
    for (const flags of [["--format=jsonl"], ["--json"]]) {
      const stdout = capture();
      const stderr = capture();
      expect(
        await main(
          ["scan", ".", "--dry-run", "--full-output", ...flags, ...args],
          stdout.stream,
          stderr.stream,
          dependencies(),
        ),
      ).toBe(2);
      const result = JSON.parse(stdout.text());
      expect(stderr.text()).toContain(result.error.message.split("\n")[0]);
      expect(result).toMatchObject({
        ok: false,
        error: { code: expect.any(String), message: expect.any(String) },
      });
    }
  },
);