Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

14.1 KB
Raw Blame
import assert from "node:assert/strict";
import {
  copyFile,
  mkdir,
  readFile,
  realpath,
  writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { afterEach, test } from "node:test";
import { setTimeout } from "node:timers/promises";
import {
  DIRECT_DEEP_SCAN_CONTROL_USAGE,
  runDeepScan,
} from "../src/deep-scan.js";
import { ScanCostTracker } from "../src/cost.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { writeSession } from "./support/usage-rollout.js";

const { temporaryDirectory, cleanup } = createApiTestFixtures();
afterEach(cleanup);

async function fixture() {
  const root = await temporaryDirectory();
  const pluginRoot = join(root, "plugin");
  await mkdir(join(pluginRoot, "mcp"), { recursive: true });
  await mkdir(join(pluginRoot, "scripts"));
  await mkdir(join(root, "codex-home"), { mode: 0o700 });
  await copyFile(
    join(PLUGIN_ROOT, "scripts", "codex_profile.mjs"),
    join(pluginRoot, "scripts", "codex_profile.mjs"),
  );
  await writeFile(
    join(pluginRoot, "mcp", "permission-profile-preflight.mjs"),
    `
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
let initializing = false;
export async function prepareCliDeepScanSession(options) {
  // Reproduce the native SQLite cold-start conflict while initialization does I/O.
  if (initializing) throw new Error('Concurrent native SQLite initialization');
  initializing = true;
  try {
    await writeFile(join(options.env.CODEX_HOME, 'initialized'), 'ready');
    return { threadId: options.env.SYNTHETIC_THREAD_ID,
    model: 'synthetic-model', reasoningEffort: 'high',
    permissionProfile: { configOverrides: options.configOverrides, commandArgs: options.commandArgs } };
  } finally { initializing = false; }
}
`,
  );
  await writeFile(
    join(pluginRoot, "mcp", "server.mjs"),
    `
import { readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { createInterface } from 'node:readline';
const lines = createInterface({input: process.stdin});
const request = JSON.parse((await lines[Symbol.asyncIterator]().next()).value);
writeFileSync(process.env.SYNTHETIC_RECEIPT, JSON.stringify({
  request, ipc: !!process.send, argv: process.argv.slice(2), key: process.env.CODEX_API_KEY,
  execPath: process.execPath, bun: process.versions.bun,
  awsProfile: process.env.AWS_PROFILE, snapshot: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH,
  skill: process.env.CODEX_SECURITY_PLUGIN_ROOT === undefined ? undefined
    : readFileSync(join(process.env.CODEX_SECURITY_PLUGIN_ROOT, 'skills', 'security-scan', 'SKILL.md'), 'utf8'),
}));
if (request.diagnosticsVersion === 1 && process.env.SYNTHETIC_OLD_ENGINE !== '1') {
  process.send({ event: 'discovery_worker_replaced', scanId: request.scanId, reason: 'tool error' });
}
if (process.env.SYNTHETIC_FAIL === '1') {
  process.stderr.write('first detail\\nlast detail');
  process.exitCode = 1;
  lines.close();
  process.stdin.destroy();
} else if (process.env.SYNTHETIC_WAIT === '1') {
  lines.once('close', () => { writeFileSync(process.env.SYNTHETIC_RECEIPT + '.stopped', 'stopped'); process.exit(0); });
  setInterval(() => {}, 1000);
} else {
  process.stdout.write(JSON.stringify({ scanId: request.scanId,
    manifestPath: join(process.env.CODEX_SECURITY_SCAN_DIR, 'scan-manifest.json') }) + '\\n');
  lines.close();
  process.stdin.destroy();
}
`,
  );
  return { root, pluginRoot };
}

test("isolates direct engine credentials and runtime snapshots across concurrent scans", async () => {
  const { root, pluginRoot } = await fixture();
  const events = await Promise.all(
    ["first", "second"].map(async (name) => {
      const scanDir = join(root, name);
      const receipt = join(root, `${name}.json`);
      await mkdir(scanDir);
      const staged = join(root, `staged plugin ${name}`);
      const skills = join(staged, "skills", "security-scan");
      await mkdir(skills, { recursive: true });
      await writeFile(join(skills, "SKILL.md"), `synthetic skill ${name}`);
      const environment = {
        PATH: process.versions["bun"] ? process.env["PATH"]! : "",
        CODEX_HOME: join(root, "codex-home"),
        SYNTHETIC_RECEIPT: receipt,
        SYNTHETIC_THREAD_ID: `thread-${name}`,
        CODEX_SECURITY_SCAN_DIR: scanDir,
        CODEX_SECURITY_PLUGIN_ROOT: staged,
        CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH: join(
          scanDir,
          "private-snapshot.toml",
        ),
        AWS_PROFILE: `profile-${name}`,
      };
      const before = { ...environment };
      const collected = [];
      for await (const event of runDeepScan({
        preflightCommand: {
          command: "/synthetic/codex",
          args: ["--config", `model_provider="provider_${name}"`],
        },
        pluginRoot,
        repository: root,
        scanDir,
        scanId: `scan-${name}`,
        prompt: "Synthetic audit request.",
        signal: new AbortController().signal,
        codexOptions: {
          codexPathOverride: "/synthetic/codex",
          nativeProfile: `provider_${name}`,
          apiKey: `synthetic-key-${name}`,
          env: environment,
          config: { model: "synthetic-model" },
          configOverrides: ['permissions.scan.filesystem={":root"="read"}'],
        },
      }))
        collected.push(event);
      const saved = JSON.parse(await readFile(receipt, "utf8"));
      assert.partialDeepStrictEqual(saved, {
        key: `synthetic-key-${name}`,
        awsProfile: `profile-${name}`,
        skill: `synthetic skill ${name}`,
        snapshot: environment.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH,
        request: {
          scanId: `scan-${name}`,
          threadId: `thread-${name}`,
        },
      });
      assert.deepEqual(saved.argv, ["--deep-scan-engine"]);
      assert.equal(saved.ipc, false);
      assert.equal(saved.request.diagnosticsVersion, undefined);
      assert.deepEqual(saved.request.permissionProfile, {
        commandArgs: ["--config", `model_provider="provider_${name}"`],
        configOverrides: [
          'model="synthetic-model"',
          'permissions.scan.filesystem={":root"="read"}',
          "features.plugins=false",
        ],
      });
      assert.equal(saved.bun, undefined);
      if (!process.versions["bun"])
        assert.equal(
          await realpath(saved.execPath),
          await realpath(process.execPath),
        );
      assert.deepEqual(environment, before);
      return collected;
    }),
  );
  for (const run of events)
    assert.deepEqual(
      run.map((event) => event.type),
      ["thread.started", "turn.completed"],
    );
  const firstUsage = events[0]![1]!["usage"] as Record<string, number>;
  const secondUsage = events[1]![1]!["usage"] as Record<string, number>;
  firstUsage["input_tokens"] = 100;
  assert.equal(secondUsage["input_tokens"], 0);
  assert.equal(DIRECT_DEEP_SCAN_CONTROL_USAGE.input_tokens, 0);
});

test("keeps direct coordinator events separate from results and child errors across helper versions", async () => {
  const { root, pluginRoot } = await fixture();
  for (const [old, failed] of [
    [false, false],
    [true, false],
    [false, true],
    [true, true],
  ]) {
    const observed: string[] = [];
    const events = runDeepScan({
      preflightCommand: { command: "/synthetic/codex" },
      repository: root,
      pluginRoot,
      scanDir: root,
      scanId: "fixture",
      prompt: "Review fixture",
      signal: new AbortController().signal,
      async onDiagnostic(fields) {
        observed.push(String(fields["event"]));
        throw new Error("unavailable sink");
      },
      codexOptions: {
        env: {
          PATH: process.versions["bun"] ? process.env["PATH"]! : "",
          CODEX_HOME: join(root, "codex-home"),
          CODEX_SECURITY_SCAN_DIR: root,
          SYNTHETIC_RECEIPT: join(root, "receipt"),
          SYNTHETIC_THREAD_ID: "parent",
          SYNTHETIC_OLD_ENGINE: old ? "1" : "0",
          SYNTHETIC_FAIL: failed ? "1" : "0",
        },
      },
    });
    assert.equal((await events.next()).value?.type, "thread.started");
    if (failed)
      await assert.rejects(events.next(), {
        message: "first detail\nlast detail",
      });
    else assert.equal((await events.next()).value?.type, "turn.completed");
    assert.deepEqual(observed, old ? [] : ["discovery_worker_replaced"]);
  }
});

test("preserves complete worker usage and cost after direct engine completion", async () => {
  const { root, pluginRoot } = await fixture();
  const codexHome = join(root, "codex-home");
  const scanDir = join(root, "scan");
  await mkdir(scanDir);
  await writeSession(
    codexHome,
    "discovery-session",
    {
      input_tokens: 1_000,
      cached_input_tokens: 200,
      cache_write_input_tokens: 100,
      output_tokens: 100,
    },
    {
      cwd: join(
        scanDir,
        "artifacts",
        "deep_discovery",
        "workers",
        "1",
        "output",
      ),
      timestamp: "2026-07-26T12:00:01.000Z",
    },
  );
  await writeSession(
    codexHome,
    "reducer-session",
    {
      input_tokens: 500,
      cached_input_tokens: 50,
      cache_write_input_tokens: 40,
      output_tokens: 60,
    },
    {
      cwd: join(scanDir, "artifacts"),
      timestamp: "2026-07-26T12:00:02.000Z",
    },
  );
  await writeFile(
    join(codexHome, "sessions", "owner.jsonl"),
    JSON.stringify({
      type: "session_meta",
      payload: {
        id: "engine-session",
        cwd: scanDir,
        timestamp: "2026-07-26T12:00:00.000Z",
      },
    }) + "\n",
  );
  const events = runDeepScan({
    preflightCommand: { command: "/synthetic/codex" },
    pluginRoot,
    repository: root,
    scanDir,
    scanId: "usage-scan",
    prompt: "Synthetic audit.",
    signal: new AbortController().signal,
    codexOptions: {
      env: {
        PATH: process.versions["bun"] ? process.env["PATH"]! : "",
        CODEX_HOME: codexHome,
        CODEX_SECURITY_SCAN_DIR: scanDir,
        SYNTHETIC_RECEIPT: join(root, "engine.json"),
        SYNTHETIC_THREAD_ID: "engine-session",
      },
    },
  });
  assert.partialDeepStrictEqual((await events.next()).value, {
    type: "thread.started",
    thread_id: "engine-session",
  });
  const tracker = new ScanCostTracker({
    codexHome,
    scanDirectory: scanDir,
    model: "gpt-6.1-sol",
  });
  tracker.recordUsage(DIRECT_DEEP_SCAN_CONTROL_USAGE, "engine-session");
  tracker.start("engine-session");
  const running = await tracker.refresh();
  await writeSession(
    codexHome,
    "waiting-session",
    {},
    {
      parent: "engine-session",
    },
  );
  const partial = await tracker.refresh();
  assert.equal(partial.usage, null);
  assert.equal(partial.cost, null);
  assert.partialDeepStrictEqual(partial.measurement, {
    usage: running.usage,
    cost: running.cost,
    coverage: "partial",
    threadCount: 3,
    missingThreadCount: 1,
  });
  tracker.recordUsage(
    {
      input_tokens: 0,
      cached_input_tokens: 0,
      cache_write_input_tokens: 0,
      output_tokens: 0,
    },
    "waiting-session",
  );
  const completed = (await events.next()).value;
  assert.equal(completed?.type, "turn.completed");
  const final = await tracker.stop(completed?.["usage"]);
  assert.deepEqual(running.usage, {
    input_tokens: 1_500,
    cached_input_tokens: 250,
    cache_write_input_tokens: 140,
    output_tokens: 160,
    reasoning_output_tokens: 0,
    total_tokens: 1_660,
  });
  assert.notEqual(running.cost, null);
  assert.deepEqual(final.usage, running.usage);
  assert.deepEqual(final.cost, running.cost);
  assert.partialDeepStrictEqual(final.measurement, {
    usage: running.usage,
    cost: running.cost,
    coverage: "complete",
    threadCount: 4,
    missingThreadCount: 0,
  });
  assert.equal(final.cost?.cacheWriteInputTokensReported, undefined);
  assert.equal((await events.next()).done, true);
});

test("preserves the engine diagnostic when the child exits before reading its request", async () => {
  const { root, pluginRoot } = await fixture();
  await writeFile(
    join(pluginRoot, "mcp", "server.mjs"),
    'process.stderr.write("synthetic engine startup failure"); process.exitCode = 23;',
  );
  // Exceed the control pipe capacity so the engine closes with a write pending.
  const events = runDeepScan({
    preflightCommand: { command: "/synthetic/codex" },
    pluginRoot,
    repository: root,
    scanDir: root,
    scanId: "startup-failure",
    prompt: "Synthetic audit.",
    signal: new AbortController().signal,
    codexOptions: {
      configOverrides: [`synthetic="${"x".repeat(2 * 1024 * 1024)}"`],
      env: {
        PATH: process.versions["bun"] ? process.env["PATH"]! : "",
        CODEX_HOME: join(root, "codex-home"),
        SYNTHETIC_THREAD_ID: "startup-session",
      },
    },
  });
  await events.next();
  await assert.rejects(events.next(), /synthetic engine startup failure/);
});

test("waits for the direct engine to stop when the scan is canceled", async () => {
  const { root, pluginRoot } = await fixture();
  const receipt = join(root, "engine.json");
  const controller = new AbortController();
  const events = runDeepScan({
    preflightCommand: { command: "/synthetic/codex" },
    pluginRoot,
    repository: root,
    scanDir: root,
    scanId: "canceled-scan",
    onDiagnostic() {
      throw new Error("closed presentation");
    },
    prompt: "Synthetic audit.",
    signal: controller.signal,
    codexOptions: {
      codexPathOverride: "/synthetic/codex",
      env: {
        PATH: process.versions["bun"] ? process.env["PATH"]! : "",
        CODEX_HOME: join(root, "codex-home"),
        SYNTHETIC_RECEIPT: receipt,
        SYNTHETIC_THREAD_ID: "cancel-session",
        SYNTHETIC_WAIT: "1",
      },
    },
  });
  assert.partialDeepStrictEqual((await events.next()).value, {
    type: "thread.started",
  });
  const running = events.next();
  const outcome = running.then(
    () => undefined,
    (error: unknown) => error,
  );
  try {
    while (!(await readFile(receipt).catch(() => undefined)))
      await setTimeout(10);
    controller.abort();
    assert.ok((await outcome) instanceof Error);
    assert.equal(await readFile(receipt + ".stopped", "utf8"), "stopped");
  } finally {
    controller.abort();
    await outcome;
  }
});