Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

21.1 KB
Raw Blame
import { afterEach, describe, expect, test } from "bun:test";
import { spawnSync } from "node:child_process";
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { setImmediate as nextTurn } from "node:timers/promises";
import { cleanup, render } from "ink-testing-library";
import { act, createElement } from "react";
import type { Finding, SeverityLevel } from "../src/index.js";
import { PatchTui, type PatchSelection } from "../src/patch-tui.js";
import { fakeResult } from "./cli-fixtures.js";
import { scopedTemporaryDirectory } from "./support/temporary-directories.js";

afterEach(() => cleanup());

function findings(severities: readonly SeverityLevel[]): Finding[] {
  const result = fakeResult(severities);
  result.findings.findings.forEach((finding, index) => {
    Object.assign(finding, {
      findingId: `csf_${index + 1}`,
      occurrenceId: `occ_${index + 1}`,
      title: `Finding ${index + 1}`,
      summary: `Attacker-controlled input reaches finding ${index + 1}.`,
      severity: {
        level: severities[index],
        rationale: "The public endpoint is reachable without authentication.",
        changeConditions:
          "Raise severity if tenant identifiers are predictable.",
      },
      confidence: { level: "high", rationale: "The source path was traced." },
      locations: [{ path: `src/finding-${index + 1}.ts`, startLine: 18 }],
      rootCause: {
        summary: "The request bypasses the tenant boundary.",
        evidenceRefs: ["evidence-1"],
      },
      validation: {
        method: "static source trace",
        summary: "A crafted request reproduces the vulnerability.",
        disposition: "confirmed",
        checks: ["The exploit reproduces.", "The permission check fails."],
        environment: { mode: "local", sandboxed: true },
        limitations: ["No live exploit was run."],
        evidenceRefs: ["evidence-1"],
      },
      attackPath: {
        summary: "Public route → database query",
        dataflow: {
          source: "Attacker-controlled request parameter",
          sink: "Tenant-scoped database query",
          outcome: "Another tenant's account records are exposed.",
        },
        reachability: {
          attacker: "Anonymous caller",
          entrypoint: "GET /accounts",
          preconditions: ["A valid tenant identifier."],
        },
        impact: {
          level: "high",
          why: "Reads another tenant's account records.",
        },
        likelihood: {
          level: "medium",
          why: "Tenant identifiers are predictable.",
        },
      },
      codeEvidence: [
        {
          id: "evidence-1",
          label: "Unsafe query",
          path: "src/query.ts",
          startLine: 18,
          code: "if user_input:\n    cursor.execute(user_input)",
          explanation: "Untrusted input reaches the query.",
        },
      ],
      remediation: "Bind the attacker-controlled value as a query parameter.",
      remediationTests: ["Reject cross-tenant requests."],
      preventiveControls: ["Use the shared tenant-scoped query helper."],
      taxonomy: { category: "sql-injection", cwe: ["CWE-89"] },
    });
  });
  return result.findings.findings;
}

function renderSelection(inputFindings: Finding[]) {
  const selected: (PatchSelection | null)[] = [];
  const app = render(
    createElement(PatchTui, {
      repository: "/work/example",
      findings: inputFindings,
      color: false,
      onComplete: (value) => selected.push(value),
    }),
  );
  return { app, selected };
}

async function update(action: () => void): Promise<void> {
  const environment = globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean };
  const previous = environment.IS_REACT_ACT_ENVIRONMENT;
  environment.IS_REACT_ACT_ENVIRONMENT = true;
  try {
    await act(async () => {
      action();
      await nextTurn();
    });
  } finally {
    if (previous === undefined) delete environment.IS_REACT_ACT_ENVIRONMENT;
    else environment.IS_REACT_ACT_ENVIRONMENT = previous;
  }
}

function press(app: ReturnType<typeof render>, input: string): Promise<void> {
  return update(() => app.stdin.write(input));
}

describe("interactive patch finding browser", () => {
  test("orders findings by severity while keeping navigation and selection aligned", async () => {
    const inputFindings = findings([
      "low",
      "high",
      "medium",
      "critical",
      "high",
    ]);
    const originalOrder = inputFindings.map(({ occurrenceId }) => occurrenceId);
    const { app, selected } = renderSelection(inputFindings);

    const displayedFindings = (app.lastFrame() ?? "")
      .split("\n")
      .flatMap(
        (line) =>
          line.match(/(?:CRITICAL|HIGH|MEDIUM|LOW) Finding \d+/gu) ?? [],
      );
    expect(displayedFindings).toEqual([
      "CRITICAL Finding 4",
      "HIGH Finding 2",
      "HIGH Finding 5",
      "MEDIUM Finding 3",
      "LOW Finding 1",
    ]);
    expect(app.lastFrame()).toContain("input reaches finding 4.");

    await press(app, "n");
    await press(app, " ");
    await press(app, "j");
    expect(app.lastFrame()).toContain("input reaches finding 2.");
    await press(app, " ");
    await press(app, "\r");

    expect(selected).toEqual([
      { severity: "high", occurrenceIds: ["occ_4", "occ_2"] },
    ]);
    expect(inputFindings.map(({ occurrenceId }) => occurrenceId)).toEqual(
      originalOrder,
    );
  });

  test("shows complete finding details and scrolls through source evidence", async () => {
    const app = render(
      createElement(PatchTui, {
        repository: "/work/example",
        findings: findings(["high", "medium", "low"]),
        color: false,
        onComplete: () => {},
      }),
    );

    expect(app.lastFrame()).toContain("CODEX SECURITY");
    expect(app.lastFrame()).toContain("FINDINGS");
    expect(app.lastFrame()).toContain("DETAILS");
    expect(app.lastFrame()).toContain("PATCH INSTRUCTIONS");
    expect(app.lastFrame()).toContain("Add instructions for this finding.");
    expect(app.lastFrame()).toContain(
      "[ ] Create draft pull request or merge request after patching",
    );
    expect(app.lastFrame()).toContain("3/3 selected");
    expect(app.lastFrame()).toContain("SUMMARY");
    expect(app.lastFrame()).toContain("Attacker-controlled input");
    expect(app.lastFrame()).toContain("SEVERITY");
    expect(app.lastFrame()).toContain("High");
    expect(app.lastFrame()).not.toContain('"level"');
    expect(app.lastFrame()).not.toContain('"rationale"');

    const frames = [app.lastFrame() ?? ""];
    await press(app, "\t");
    for (let page = 0; page < 12; page += 1) {
      await press(app, "\u001B[6~");
      frames.push(app.lastFrame() ?? "");
    }

    const reviewed = frames.join("\n");
    expect(reviewed).toContain("ROOT CAUSE");
    expect(reviewed).toContain("Evidence:");
    expect(reviewed).toContain("• Unsafe query · src/query.ts:18");
    expect(reviewed).toContain("VALIDATION");
    expect(reviewed).toContain("Method: static source trace");
    expect(reviewed).toContain("Disposition: confirmed");
    expect(reviewed).toContain("• The exploit reproduces.");
    expect(reviewed).toContain("• No live exploit was run.");
    expect(reviewed).toContain("Mode: local");
    expect(reviewed).toContain("Sandboxed: true");
    expect(reviewed).toContain("ATTACK PATH");
    expect(reviewed).toContain("Source: Attacker-controlled request parameter");
    expect(reviewed).toContain("Sink: Tenant-scoped database query");
    expect(reviewed).toContain("Attacker: Anonymous caller");
    expect(reviewed).toContain("Entrypoint: GET /accounts");
    expect(reviewed).toContain("• A valid tenant identifier.");
    expect(reviewed).toContain("Impact:");
    expect(reviewed).toContain("Likelihood:");
    expect(reviewed).toContain("CODE EVIDENCE");
    expect(reviewed).toContain("Unsafe query · src/query.ts:18");
    expect(reviewed).toContain("18 │ if user_input:");
    expect(reviewed).toContain("    cursor.execute(user_input)");
    expect(reviewed).toContain("authentication.");
    expect(reviewed).toContain("src/finding-1.ts:18");
    expect(reviewed).toContain("REMEDIATION");
    expect(reviewed).toContain("• Reject cross-tenant requests.");
    expect(reviewed).toContain("PREVENTIVE CONTROLS");
    expect(reviewed).toContain("Category: sql-injection");
    expect(reviewed).toContain("CWE: CWE-89");
    expect(reviewed).toContain("occ_1");
    expect(reviewed).not.toContain('"summary"');
    expect(reviewed).not.toContain('"path"');
  });

  test("shows surrounding source without reading outside the scanned repository", async () => {
    const directory = await mkdtemp(join(tmpdir(), "codex-security-patch-"));
    const repository = join(directory, "repository");
    try {
      await mkdir(join(repository, "src"), { recursive: true });
      await writeFile(
        join(repository, "src", "finding-1.ts"),
        [
          'import { database } from "./database";',
          "export function loadTenant(request) {",
          "  const tenant = request.params.tenant;",
          "  const input = request.query.id;",
          "  const query = buildQuery(input);",
          "  return database.query(query, { tenant, includeDeleted: false });",
          "  audit.record(tenant);",
          "}",
          'const token = "sk-proj-SYNTHETIC_KEY_123";',
        ].join("\n"),
      );
      await writeFile(join(repository, "eof.ts"), "\r\n".repeat(10));
      const outside = join(directory, "outside.ts");
      await writeFile(outside, "OUTSIDE_PRIVATE_SOURCE");
      await symlink(outside, join(repository, "src", "outside-link.ts"));

      const [finding] = findings(["high"]);
      finding!.locations = [
        {
          path: "src/finding-1.ts",
          startLine: 5,
          endLine: 6,
          role: "sink",
        },
        { path: "eof.ts", startLine: 10 },
        { path: "../outside.ts", startLine: 1 },
        { path: "src/outside-link.ts", startLine: 1 },
      ];
      const app = render(
        createElement(PatchTui, {
          repository,
          findings: [finding!],
          color: false,
          onComplete: () => {},
        }),
      );

      // Resizing rewraps the source excerpt already displayed for this finding.
      await writeFile(join(repository, "src", "finding-1.ts"), "file changed");
      Object.defineProperty(app.stdout, "columns", { value: 110 });
      await update(() => app.stdout.emit("resize"));
      const frames = [app.lastFrame() ?? ""];
      for (let page = 0; page < 12; page += 1) {
        await press(app, "\u001B[6~");
        frames.push(app.lastFrame() ?? "");
      }
      const reviewed = frames.join("\n");

      expect(reviewed).toContain("src/finding-1.ts:5–6 · sink");
      expect(reviewed).toContain("2 │ export function loadTenant(request)");
      expect(reviewed).toContain("› 5 │   const query = buildQuery(input);");
      expect(reviewed).toContain("› 6 │   return database.query(query,");
      expect(reviewed).toContain("›   │");
      expect(reviewed).toContain("includeDeleted: false");
      expect(reviewed).toContain("7 │   audit.record(tenant);");
      expect(reviewed).toContain('const token = "sk-proj-SYNTHETIC_KEY_123";');
      expect(reviewed).toContain("› 10 │");
      expect(reviewed).not.toContain(" 11 │");
      expect(reviewed).toContain("../outside.ts:1");
      expect(reviewed).toContain("src/outside-link.ts:1");
      expect(reviewed).not.toContain("OUTSIDE_PRIVATE_SOURCE");
    } finally {
      await rm(directory, { force: true, recursive: true });
    }
  });

  test("combines severity presets with individual finding selection", async () => {
    const { app, selected } = renderSelection(
      findings(["high", "medium", "low"]),
    );

    await press(app, "2");
    expect(app.lastFrame()).toContain("1/3 selected");
    expect(app.lastFrame()).toContain("high and above");

    await press(app, "\u001B[B ");
    expect(app.lastFrame()).toContain("2/3 selected");
    expect(app.lastFrame()).toContain("custom");

    await press(app, "\r");
    expect(selected).toEqual([
      { severity: "medium", occurrenceIds: ["occ_1", "occ_2"] },
    ]);
  });

  test("edits instructions per finding and only returns selected guidance", async () => {
    const { app, selected } = renderSelection(findings(["high", "medium"]));

    await press(app, "i");
    expect(app.lastFrame()).toContain("Enter save");

    await press(app, "Use the shared 2FA helper, not a new dependency.");
    expect(app.lastFrame()).toContain("Use the shared 2FA helper");
    expect(app.lastFrame()).toContain("2/2 selected");

    await press(app, "\r");
    expect(app.lastFrame()).toContain("PATCH INSTRUCTIONS");
    expect(app.lastFrame()).toContain("Use the shared 2FA helper");
    expect(app.lastFrame()).toContain("✎");
    expect(app.lastFrame()?.match(/PATCH INSTRUCTIONS/gu)).toHaveLength(1);

    await press(app, "\u001B[B");
    await press(app, "i");
    await press(app, "Keep the existing middleware.");
    await press(app, "\r");
    expect(app.lastFrame()).toContain("Keep the existing middleware.");

    await press(app, " ");
    await press(app, "\r");

    expect(selected).toEqual([
      {
        severity: "high",
        occurrenceIds: ["occ_1"],
        instructions: {
          occ_1: "Use the shared 2FA helper, not a new dependency.",
        },
      },
    ]);
  });

  test("optionally creates a draft pull request after selected patches", async () => {
    const { app, selected } = renderSelection(findings(["high"]));

    expect(app.lastFrame()).toContain(
      "[ ] Create draft pull request or merge request after patching",
    );
    await press(app, "r");
    expect(app.lastFrame()).toContain(
      "[✓] Create draft pull request or merge request after patching",
    );
    await press(app, "\r");

    expect(selected).toEqual([
      {
        severity: "high",
        occurrenceIds: ["occ_1"],
        createPullRequest: true,
      },
    ]);
  });

  test("cancels and clears finding instructions without leaving the browser", async () => {
    const { app, selected } = renderSelection(findings(["high"]));

    await press(app, "i");
    await press(app, "Discard this guidance.");
    // A complete Escape sequence avoids Ink's delay for an ambiguous bare Escape.
    await press(app, "\u001B[27u");
    expect(selected).toEqual([]);
    expect(app.lastFrame()).not.toContain("Discard this guidance.");

    await press(app, "i");
    await press(app, "x");
    await press(app, "\u007F");
    await press(app, "\r");
    await press(app, "\r");

    expect(selected).toEqual([{ severity: "high", occurrenceIds: ["occ_1"] }]);
  });

  test("deletes complete characters from finding instructions", async () => {
    const { app, selected } = renderSelection(findings(["high"]));

    await press(app, "i");
    await press(app, "Ask 👩‍💻");
    await press(app, "\u007F");
    await press(app, "the owner 👍🏽");
    await press(app, "\u007F");
    await press(app, "\r");
    await press(app, "\r");

    expect(selected).toEqual([
      {
        severity: "high",
        occurrenceIds: ["occ_1"],
        instructions: { occ_1: "Ask the owner" },
      },
    ]);
  });

  test("shows the final source line of a large excerpt under Node", async () => {
    await using repository = await scopedTemporaryDirectory(
      "patch-large-excerpt-",
    );
    const lineCount = 150_000;
    const path = "large.ts";
    await writeFile(
      join(repository.path, path),
      "x\n".repeat(lineCount - 1) + "last source line\n",
    );
    const finding = {
      ...fakeResult(["high"]).findings.findings[0]!,
      title: "Large excerpt",
      occurrenceId: "occ_large",
      locations: [{ path, startLine: 1, endLine: lineCount }],
    };
    const built = await Bun.build({
      entrypoints: [
        join(import.meta.dirname, "fixtures", "patch-large-excerpt.ts"),
      ],
      target: "node",
      packages: "external",
    });
    expect(built.success).toBe(true);
    const result = spawnSync(
      "node",
      ["--input-type=module", "-", repository.path, JSON.stringify(finding)],
      {
        cwd: join(import.meta.dirname, ".."),
        input: await built.outputs[0]!.text(),
        encoding: "utf8",
      },
    );
    expect(result.status, result.stderr).toBe(0);
  });

  test("uses the current finding and viewport for every buffered scroll key", async () => {
    const items = findings(["high", "medium"]);
    items[1]!.summary = "Review this long finding. ".repeat(80);
    const { app } = renderSelection(items);
    const down = "\u001B[B";
    const pageDown = "\u001B[6~";
    const pageUp = "\u001B[5~";
    const detailRange = () =>
      app.lastFrame()!.match(/DETAILS\s+(\d+)–\d+\/(\d+)/u)!;

    await press(app, down + "\t" + pageDown.repeat(40));
    expect(app.lastFrame()).toContain("occ_2");
    await press(app, pageUp.repeat(40) + down.repeat(3));
    expect(app.lastFrame()).toContain("DETAILS  4–16/");
    await press(app, pageDown.repeat(2));
    expect(app.lastFrame()).toContain("DETAILS  30–42/");

    const wideRows = Number(detailRange()[2]);
    Object.defineProperty(app.stdout, "columns", { value: 50 });
    await update(() => app.stdout.emit("resize"));
    expect(Number(detailRange()[2])).toBeGreaterThan(wideRows);
    await press(app, pageDown.repeat(40));
    expect(app.lastFrame()).toContain("occ_2");
    const lastPageStart = Number(detailRange()[1]);
    Object.defineProperty(app.stdout, "rows", { value: 38 });
    await update(() => app.stdout.emit("resize"));
    expect(app.lastFrame()).toContain(`DETAILS  ${lastPageStart - 10}–`);
    await press(app, pageDown + pageUp);
    const previousPage = app.lastFrame();
    expect(previousPage).not.toContain("occ_2");
    await press(app, pageDown);
    await press(app, pageUp);
    expect(app.lastFrame()).toBe(previousPage);
  });

  test("allows selecting none and canceling without patching", async () => {
    for (const input of ["q", "\r"]) {
      const { app, selected } = renderSelection(findings(["high"]));
      if (input === "\r") {
        await press(app, "n");
        expect(app.lastFrame()).toContain("0/1 selected");
      }
      await press(app, input);
      expect(selected).toEqual([null]);
      app.unmount();
    }
  });

  test("strips terminal escapes and preserves credential-shaped finding details", () => {
    const [finding] = findings(["high"]);
    finding!.title = "\u001B[31mUnsafe title\u001B[0m\nforged line";
    finding!.summary = "sk-proj-SYNTHETIC_KEY_123";
    const app = render(
      createElement(PatchTui, {
        repository: "/work/example",
        findings: [finding!],
        color: false,
        onComplete: () => {},
      }),
    );

    expect(app.lastFrame()).toContain("Unsafe title forged line");
    expect(app.lastFrame()).toContain("sk-proj-SYNTHETIC_KEY_123");
    expect(app.lastFrame()).not.toContain("\u001B[31m");
  });

  test("keeps finding details restrained while honoring NO_COLOR", () => {
    const source = [
      'import {render} from "ink-testing-library";',
      'import {createElement} from "react";',
      `import {PatchTui} from ${JSON.stringify(new URL("../src/patch-tui.tsx", import.meta.url).href)};`,
      `const findings=${JSON.stringify(findings(["critical", "high", "medium", "low"]))};`,
      'const app=render(createElement(PatchTui,{repository:"/work/example",findings,onComplete(){}}));',
      'const frames=[app.lastFrame() ?? ""];app.stdin.write("\\t");',
      'for(let page=0;page<12;page+=1){app.stdin.write("\\u001B[6~");await new Promise(resolve=>setTimeout(resolve,30));frames.push(app.lastFrame()??"");}',
      'process.stdout.write(frames.join("\\n"));app.unmount();',
    ].join("");
    const run = (color: boolean) =>
      spawnSync(process.execPath, ["--eval", source], {
        encoding: "utf8",
        env: {
          ...process.env,
          FORCE_COLOR: color ? "1" : undefined,
          NO_COLOR: color ? undefined : "1",
          TERM: "xterm-256color",
        },
      });
    const colored = run(true);

    expect(colored.status).toBe(0);
    expect(colored.stdout).toContain("\u001B[91m");
    expect(colored.stdout).toContain("\u001B[31m");
    expect(colored.stdout).toContain("\u001B[33m");
    expect(colored.stdout).toContain("\u001B[95m");
    expect(colored.stdout).toContain("\u001B[32m");
    expect(colored.stdout).toContain("\u001B[91m● Critical\u001B[39m");
    expect(colored.stdout).toContain("\u001B[1mSEVERITY\u001B[22m");
    expect(colored.stdout).toContain("\u001B[1mROOT CAUSE\u001B[22m");
    expect(colored.stdout).toContain("\u001B[1mVALIDATION\u001B[22m");
    expect(colored.stdout).toContain(
      "\u001B[90mDisposition:\u001B[39m confirmed",
    );
    expect(colored.stdout).toContain("\u001B[90m• \u001B[39m");
    expect(colored.stdout).toContain("\u001B[36msrc/finding-1.ts:18\u001B[39m");
    expect(colored.stdout).not.toContain("\u001B[92m");
    expect(colored.stdout).not.toContain("\u001B[95mCODE EVIDENCE");
    expect(colored.stdout).not.toContain("\u001B[32mVALIDATION");
    expect(colored.stdout).not.toContain("\u001B[33m→");
    expect(colored.stdout).not.toContain("\u001B[4m");

    const plain = run(false);

    expect(plain.status).toBe(0);
    expect(plain.stdout).not.toMatch(/\u001B\[\d+m/u);
  });
});