Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

45.8 KB
Raw Blame
import { mkdir, readFile, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import { afterEach, expect, test, mock } from "bun:test";
import { ScanCostTracker } from "../src/cost.js";
import type { ScanTokenUsage } from "../src/index.js";
import {
  CodexSecurity,
  type ScanOptions,
  type ValidationOptions,
} from "../src/api.js";
import { ScanResult } from "../src/result.js";
import { main } from "../src/cli.js";
import {
  mergedCodexConfig,
  type CodexSecurityConfig,
  type JsonObject,
} from "../src/config.js";
import type { ProjectConfigInput } from "../src/project-config-schema.js";
import { readProjectConfig } from "../src/project-config.js";
import { git } from "./git-fixture.js";
import { dependencies, fakeResult, fakeSecurity } from "./cli-fixtures.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { copyCompletedScanFixture } from "./plugin-root.js";
import { rejecting, throwing } from "./support/errors.js";
import {
  createCliTest,
  captureCli,
  runCapturedCli,
} from "./support/cli-run.js";

const { temporaryDirectory, cleanup } = createApiTestFixtures(
  "cli-project-config-",
);
afterEach(cleanup);

const cliClientFactory =
  (deps: NonNullable<ConstructorParameters<typeof CodexSecurity>[1]>) =>
  (config: CodexSecurityConfig) =>
    new CodexSecurity(config, { ...deps }, { surface: "cli" });

async function fixture(input: ProjectConfigInput | string) {
  const root = await temporaryDirectory();
  const repository = join(root, "repository");
  const configDirectory = join(root, "settings");
  await mkdir(repository);
  await mkdir(configDirectory);
  await mkdir(join(repository, "src"));
  await mkdir(join(repository, "lib"));
  const config = join(
    configDirectory,
    typeof input === "string" ? "scan.yaml" : "scan.json",
  );
  await writeFile(
    config,
    typeof input === "string" ? input : JSON.stringify(input),
  );
  return { root, repository, configDirectory, config };
}

async function bulkInventory(
  input: Awaited<ReturnType<typeof fixture>>,
  rowModes: readonly string[],
) {
  git(input.repository, "init", "-q");
  git(
    input.repository,
    "commit",
    "--no-gpg-sign",
    "--allow-empty",
    "-qm",
    "fixture",
  );
  const revision = git(input.repository, "rev-parse", "HEAD");
  const csv = join(input.root, "repositories.csv");
  await writeFile(
    csv,
    [
      "id,repository,revision,mode",
      ...rowModes.map(
        (mode, index) =>
          `source-${index},${input.repository},${revision},${mode}`,
      ),
      "",
    ].join("\n"),
  );
  return csv;
}

test.each([
  undefined,
  "starter.json",
  "settings/starter.yaml",
  "settings/starter.json",
])(
  "init writes a valid unpinned starter and refuses overwrites: %s",
  async (file) => {
    const input = await fixture({});
    const args = ["init", ...(file === undefined ? [] : [file]), "--json"];
    const output = captureCli(main, "stdout");
    const deps = dependencies({
      currentDirectory: input.root,
      onConfig: throwing("No runtime for init"),
    });
    expect(await output.run(args, deps)).toBe(0);
    const path = join(input.root, file ?? "codex-security.yaml");
    expect(JSON.parse(output.text())).toEqual({ path });
    const selected = await readProjectConfig(path);
    expect(selected.input).toEqual({});
    const contents = await readFile(path, "utf8");
    expect(await runCapturedCli(main, args, deps)).toBe(2);
    expect(await readFile(path, "utf8")).toBe(contents);
  },
);

test("init shows how to select a custom config and info reports the same preview", async () => {
  const input = await fixture({});
  const file = "config for review.yaml";
  const deps = dependencies({ currentDirectory: input.root });
  const init = captureCli(main, "stdout", true);
  expect(await init.run(["init", file], deps)).toBe(0);
  const preview = "codex-security scan . -c 'config for review.yaml' --dry-run";
  expect(init.text()).toContain(preview);
  const info = captureCli(main, "stdout", true);
  expect(await info.run(["info", "-c", file], deps)).toBe(0);
  expect(info.text()).toContain(preview);
  expect(info.text()).toContain("Scope: repository");
  expect(info.text()).not.toContain("codex.reasoning");
});

test.each(["yaml", "json"])(
  "init preserves URI-sensitive directory names in the %s schema reference",
  async (extension) => {
    const input = await fixture({});
    const workspace = join(
      input.root,
      process.platform === "win32"
        ? "project #draft %25"
        : 'project #draft %25 ?review: "quoted"',
    );
    const schemaDirectory = join(
      workspace,
      "node_modules",
      "@openai",
      "codex-security",
      "schemas",
    );
    await mkdir(schemaDirectory, { recursive: true });
    const schema = join(schemaDirectory, "project-config.schema.json");
    const contents = '{"type":"object"}\n';
    await writeFile(schema, contents);
    const path = join(input.configDirectory, `starter.${extension}`);

    expect(
      await runCapturedCli(
        main,
        ["init", path, "--json"],
        dependencies({ currentDirectory: workspace }),
      ),
    ).toBe(0);

    const selected = await readProjectConfig(path);
    const reference = new URL(selected.input.$schema!, pathToFileURL(path));
    expect(reference.hash).toBe("");
    expect(reference.search).toBe("");
    expect(await readFile(reference, "utf8")).toBe(contents);
    expect(reference.href).toBe(pathToFileURL(schema).href);
  },
);

test("info resolves a config and its sources without a target, prompt reads, or runtime", async () => {
  const input = await fixture({
    scan: {
      mode: "deep",
      instructions_file: "not-read.md",
      deep: { workers: 2 },
    },
    codex: {
      model: "gpt-5.6-terra",
      synthetic_private_setting: "synthetic-private-value",
    },
  });
  const home = join(input.root, "home");
  await mkdir(join(home, "codex-security"), { recursive: true });
  await writeFile(
    join(home, "codex-security", "config.toml"),
    "[deep_scan]\nsubagents = 1\n",
  );
  const stdout = captureCli(main, "stdout");
  expect(
    await stdout.run(
      ["info", "-c", input.config, "--json"],
      dependencies({
        currentDirectory: input.configDirectory,
        environment: { CODEX_HOME: home },
        onConfig: throwing("No runtime for info"),
      }),
    ),
  ).toBe(0);
  expect(stdout.text()).not.toContain("synthetic-private-value");
  expect(JSON.parse(stdout.text())).toMatchObject({
    model: "gpt-5.6-terra",
    nextStep: "codex-security scan . -c scan.json --dry-run",
    configuration: {
      path: input.config,
      settings: {
        mode: "deep",
        workers: 2,
        subagents: 1,
        scanPromptFile: join(input.configDirectory, "not-read.md"),
      },
      sources: {
        "scan.deep.workers": "project",
        "scan.deep.subagents_per_worker": "legacy",
        "scan.deep.max_time_hours": "default",
        "policy.fail_on_severity": "default",
      },
    },
  });
});

test("info without a file reports default sources, including unset settings", async () => {
  const input = await fixture({});
  const stdout = captureCli(main, "stdout");
  expect(
    await stdout.run(
      ["info", "--json", "--filter-output", "configuration"],
      dependencies({ currentDirectory: input.root }),
    ),
  ).toBe(0);
  expect(JSON.parse(stdout.text())).toMatchObject({
    configuration: {
      sources: {
        auth: "default",
        "scan.mode": "default",
        "scan.scope": "default",
        "scan.knowledge_base": "default",
        "scan.instructions_file": "default",
        "scan.validation_file": "default",
        "output.directory": "default",
        "policy.fail_on_severity": "default",
        "limits.max_cost_usd_per_scan": "default",
      },
    },
  });
});

test("an explicit config flag overrides the operator-selected environment file", async () => {
  const input = await fixture({ codex: { model: "gpt-5.6-terra" } });
  const override = join(input.root, "override.json");
  await writeFile(
    override,
    JSON.stringify({ codex: { model: "gpt-5.6-sol" } }),
  );
  for (const [flags, expected] of [
    [[], "gpt-5.6-terra"],
    [["-c", override], "gpt-5.6-sol"],
  ] as const) {
    const onConfig = mock<(config: CodexSecurityConfig) => void>();
    expect(
      await runCapturedCli(
        main,
        ["scan", ...flags, "--json"],
        dependencies({
          currentDirectory: input.repository,
          environment: { CODEX_SECURITY_PROJECT_CONFIG: input.config },
          onConfig,
        }),
      ),
    ).toBe(0);
    const selected = onConfig.mock.lastCall?.[0];
    expect(selected?.codexOverrides?.["model"]).toBe(expected);
  }
});

test.each([undefined, "standard", "daybreak_red"] as const)(
  "scan and follow-up validation use Cyber selection from project or CLI override: %s",
  async (override) => {
    const input = await fixture({
      scan: { cyber_access_program: "daybreak_blue" },
    });
    const onTurn = mock<(_target: string, options: ScanOptions) => void>();
    const onValidate = mock(async (_options: ValidationOptions) => ({
      disposition: "deferred" as const,
      report: "Additional evidence is needed.",
      outputDir: join(input.root, "evidence"),
      threadId: "validation-thread",
    }));
    const result = new ScanResult({
      ...fakeResult(["high"]),
      scanDir: await temporaryDirectory(),
    });
    expect(
      await runCapturedCli(
        main,
        [
          "scan",
          input.repository,
          "-c",
          input.config,
          "--validate",
          "--json",
          ...(override === undefined
            ? []
            : ["--cyber-access-program", override]),
        ],
        dependencies({
          currentDirectory: input.repository,
          onTurn,
          onValidate,
          result,
        }),
      ),
    ).toBe(0);
    const selected = onTurn.mock.lastCall?.[1];
    expect(selected?.cyberAccessProgram).toBe(override ?? "daybreak_blue");
    expect(onValidate).toHaveBeenCalledTimes(1);
    expect(onValidate.mock.lastCall?.[0].cyberAccessProgram).toBe(
      override ?? "daybreak_blue",
    );
  },
);

test("validation rejects knowledge selected through a checkout link", async () => {
  const input = await fixture({});
  git(input.repository, "init", "--quiet");
  const link = join(input.repository, "documents");
  await symlink(input.configDirectory, link, "junction");
  for (const command of [
    ["scan", "--validate"],
    ["validate", "candidate"],
  ]) {
    const stderr = captureCli(main, "stderr");
    expect(
      await stderr.run(
        [...command, "--knowledge-base", join(link, "scan.json")],
        dependencies({ currentDirectory: input.repository }),
      ),
    ).toBe(2);
    expect(stderr.text()).toContain("repository directory links");
  }
});

test("a missing operator-selected environment file fails without falling back", async () => {
  const input = await fixture({});
  const onConfig = mock();
  expect(
    await runCapturedCli(
      main,
      ["scan", "--json"],
      dependencies({
        currentDirectory: input.repository,
        environment: {
          CODEX_SECURITY_PROJECT_CONFIG: join(input.root, "missing.yaml"),
        },
        onConfig,
      }),
    ),
  ).toBe(2);
  expect(onConfig).not.toHaveBeenCalled();
});

test.each([123, "", "   "])(
  "a selected profile's invalid model is reported at the provider flag: %j",
  async (model) => {
    const input = await fixture({
      codex: { profile: "selected", profiles: { selected: { model } } },
    });
    const onConfig = mock();
    const stderr = captureCli(main, "stderr");
    expect(
      await stderr.run(
        ["scan", "-c", input.config, "--provider", "amazon-bedrock", "--json"],
        dependencies({
          currentDirectory: input.repository,
          onConfig,
        }),
      ),
    ).toBe(2);
    expect(stderr.text()).toContain(
      "--model must be a nonempty string when using --provider amazon-bedrock",
    );
    expect(onConfig).not.toHaveBeenCalled();
  },
);

test("rerun rejects a blank replacement when scan instructions are required", async () => {
  const input = await fixture({});
  const prompt = join(input.root, "empty.md");
  await writeFile(prompt, " \n");
  const stderr = captureCli(main, "stderr");
  const onConfig = mock();
  expect(
    await stderr.run(
      ["scans", "rerun", "saved", "--scan-prompt-file", prompt, "--json"],
      dependencies({
        currentDirectory: input.root,
        onWorkbench: async () => ({
          recipe: {
            repository: input.repository,
            target: { kind: "repository", paths: [] },
            mode: "standard",
            config: {},
            requiresScanPrompt: true,
          },
        }),
        onConfig,
      }),
    ),
  ).toBe(2);
  expect(stderr.text()).toContain("--scan-prompt-file must not be empty");
  expect(onConfig).not.toHaveBeenCalled();
});

test.each(["project", "flag"])(
  "rejects patching before starting a scan with a %s budget",
  async (source) => {
    const input = await fixture({ limits: { max_cost_usd_per_scan: 1 } });
    const onRun = mock(throwing("unexpected scan"));
    const onCodex = mock(throwing("unexpected patch"));
    const stderr = captureCli(main, "stderr");
    const budget =
      source === "project" ? ["-c", input.config] : ["--max-cost", "1"];
    expect(
      await stderr.run(
        ["scan", input.repository, "--patch", ...budget, "--json"],
        dependencies({ onRun, onCodex }),
      ),
    ).toBe(2);
    expect(stderr.text()).toContain(
      "--patch cannot be combined with a scan cost limit",
    );
    expect(onRun).not.toHaveBeenCalled();
    expect(onCodex).not.toHaveBeenCalled();
  },
);

test.each(["instructions_file", "validation_file"] as const)(
  "bulk %s retains directory-link protection for local CSV repositories",
  async (file) => {
    const input = await fixture({
      scan: { [file]: "../repository/linked/prompt.md" },
      output: { directory: "../batch-results" },
    });
    git(input.repository, "init", "--quiet");
    const external = join(input.root, "external");
    await mkdir(external);
    await writeFile(join(external, "prompt.md"), "Synthetic external data.");
    await symlink(external, join(input.repository, "linked"), "junction");
    const csv = join(input.root, "repositories.csv");
    await writeFile(
      csv,
      `id,repository,revision\nsource,${input.repository},${"a".repeat(40)}\n`,
    );
    const onConfig = mock(throwing("Runtime must not start"));
    const stderr = captureCli(main, "stderr");
    const exit = await stderr.run(
      ["bulk-scan", csv, "-c", input.config, "--json"],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
      }),
    );
    expect(exit).toBe(2);
    expect(stderr.text()).toContain(
      "Input files must not follow repository directory links",
    );
    expect(onConfig).not.toHaveBeenCalled();
  },
);

test.each([
  ["standard", [""], { max_time_minutes: 6000 }, undefined],
  [
    "deep",
    ["standard"],
    { max_time_minutes: 15, deep: { max_time_hours: 2 } },
    undefined,
  ],
  [
    "standard",
    ["standard", "deep"],
    { max_time_minutes: 15, deep: { max_time_hours: 2 } },
    "Conflicting",
  ],
  ["deep", [""], { max_time_minutes: 6000 }, "5760 minutes"],
] as const)(
  "bulk time settings use effective row modes: default %s, rows %j",
  async (defaultMode, rowModes, time, message) => {
    const input = await fixture({
      scan: { mode: defaultMode, ...time },
      output: { directory: "../batch-results" },
    });
    const csv = await bulkInventory(input, rowModes);
    const selected: ScanOptions[] = [];
    const { runCli, stderr } = createCliTest(main);
    const result = await runCli(
      ["bulk-scan", csv, "-c", input.config, "--json"],
      dependencies({
        currentDirectory: input.root,
        onTurn: (_repository, options) => selected.push(options),
      }),
    );
    expect(result, stderr.text()).toBe(message ? 2 : 0);
    if (message) {
      expect(stderr.text()).toContain(message);
      expect(selected).toEqual([]);
    } else {
      expect(selected).toHaveLength(1);
      expect(selected[0]).toMatchObject({
        mode: "standard",
        maxTimeMinutes: time.max_time_minutes,
      });
      expect(selected[0]?.maxTimeHours).toBeUndefined();
    }
  },
);

test("bulk scans apply config and linked operator prompts, preserve CSV scope overrides, and retain the policy on resume", async () => {
  const config: ProjectConfigInput = {
    auth: "api-key",
    scan: {
      scope: { paths: ["src"] },
      knowledge_base: ["context.md"],
      instructions_file: "linked/instructions.md",
      deep: { workers: 2, subagents_per_worker: 0 },
    },
    codex: { model: "gpt-5.6-terra" },
    limits: { max_cost_usd_per_scan: 5 },
    policy: { fail_on_severity: "high" },
    output: { directory: "../batch-results" },
  };
  const input = await fixture(config);
  const promptDirectory = await temporaryDirectory("bulk-operator-prompts-");
  await symlink(
    promptDirectory,
    join(input.configDirectory, "linked"),
    "junction",
  );
  await writeFile(
    join(input.configDirectory, "context.md"),
    "Synthetic context.",
  );
  await writeFile(
    join(promptDirectory, "instructions.md"),
    "Review synthetic boundaries.",
  );
  await writeFile(
    join(input.repository, "src", "index.ts"),
    "export const value = 1;\n",
  );
  await writeFile(
    join(input.repository, "lib", "index.ts"),
    "export const value = 2;\n",
  );
  git(input.repository, "init", "-q");
  git(input.repository, "config", "--local", "core.autocrlf", "false");
  git(input.repository, "add", ".");
  git(input.repository, "commit", "--no-gpg-sign", "-qm", "fixture");
  const revision = git(input.repository, "rev-parse", "HEAD");
  const csv = join(input.root, "repositories.csv");
  await writeFile(
    csv,
    `id,repository,revision,mode,scope\nstandard,${input.repository},${revision},standard,lib\ndeep,${input.repository},${revision},deep,\n`,
  );
  const selected: ScanOptions[] = [];
  const deps = dependencies({ currentDirectory: input.root });
  deps.createSecurity = (native) => {
    expect(native.codexOverrides?.["model"]).toBe("gpt-5.6-terra");
    return fakeSecurity(async (_repository, options = {}) => {
      selected.push(options);
      const result = fakeResult(["high"]);
      await mkdir(options.outputDir!, { recursive: true, mode: 0o700 });
      await copyCompletedScanFixture(options.outputDir!);
      await writeFile(
        join(options.outputDir!, "report.md"),
        "Synthetic report.",
      );
      return result;
    });
  };
  const args = [
    "bulk-scan",
    csv,
    "-c",
    input.config,
    "--max-cost",
    "3",
    "--json",
  ];
  const output = captureCli(main, "stdout");
  expect(await output.run(args, deps)).toBe(1);
  expect(JSON.parse(output.text())).toMatchObject({
    completed: 2,
    failed: 0,
    policyFailed: true,
  });
  expect(selected.find((options) => options.mode === "standard")).toMatchObject(
    {
      target: [
        join(input.root, "batch-results", "checkouts", "standard", "lib"),
      ],
      auth: "api-key",
      maxCostUsd: 3,
      failureSeverity: "high",
      knowledgeBasePaths: [join(input.configDirectory, "context.md")],
      scanPrompt: "Review synthetic boundaries.",
    },
  );
  expect(
    selected.find((options) => options.mode === "standard")?.workers,
  ).toBeUndefined();
  expect(selected.find((options) => options.mode === "deep")).toMatchObject({
    target: ["src"],
    workers: 2,
    subagents: 0,
  });
  const resumed = captureCli(main, "stdout");
  expect(await resumed.run(args, deps)).toBe(1);
  expect(JSON.parse(resumed.text())).toMatchObject({
    skipped: 2,
    policyFailed: true,
  });
  expect(selected).toHaveLength(2);
  await writeFile(
    input.config,
    JSON.stringify({ ...config, policy: { fail_on_severity: "low" } }),
  );
  const stderr = captureCli(main, "stderr");
  expect(await stderr.run(args, deps)).toBe(2);
  expect(stderr.text()).toContain("manifest does not match");
  expect(selected).toHaveLength(2);
});

test.each(["standard", "deep"] as const)(
  "component scans honor shared %s settings and severity policy",
  async (mode) => {
    const input = await fixture({
      scan: {
        mode,
        scope: { paths: ["src"] },
        instructions_file: "instructions.md",
        ...(mode === "standard" ? { validation_file: "validation.md" } : {}),
        deep: { workers: 2, subagents_per_worker: 0 },
      },
      codex: { model: "gpt-5.6-terra" },
      limits: { max_cost_usd_per_scan: 7 },
      policy: { fail_on_severity: "high" },
      output: { directory: "../component-results" },
    });
    await writeFile(
      join(input.repository, "lib", "index.ts"),
      "export const value = 1;\n",
    );
    await writeFile(
      join(input.configDirectory, "instructions.md"),
      "Review synthetic boundaries.",
    );
    await writeFile(
      join(input.configDirectory, "validation.md"),
      "Validate synthetic boundaries.",
    );
    const onTurn = mock<(_target: string, options: ScanOptions) => void>();
    const { stdout, stderr, runCli } = createCliTest(main);

    const result = fakeResult(["high"], "complete", {
      input_tokens: 1250,
      cached_input_tokens: 200,
      output_tokens: 30,
    });
    expect(
      await runCli(
        [
          "scan-components",
          input.repository,
          "-c",
          input.config,
          "--component",
          "lib",
          "--headless",
          "--json",
        ],
        dependencies({
          currentDirectory: input.root,
          result,
          costUpdates: [result.cost!],
          onTurn,
        }),
      ),
    ).toBe(1);
    const selected = onTurn.mock.lastCall?.[1];
    expect(selected).toMatchObject({
      mode,
      target: [join(input.repository, "lib")],
      scanPrompt: "Review synthetic boundaries.",
      failureSeverity: "high",
      maxCostUsd: 7,
    });
    expect(stderr.text()).toContain("Cost: $0.00488");
    if (mode === "deep")
      expect(selected).toMatchObject({ workers: 2, subagents: 0 });
    else
      expect(selected?.validationPrompt).toBe("Validate synthetic boundaries.");
    expect(JSON.parse(stdout.text())).toMatchObject({
      completed: 1,
      failed: 0,
      policyFailed: true,
    });
  },
);

test("actual CLI parsing preserves file values when flags are absent", async () => {
  const input = await fixture({
    auth: "api-key",
    scan: {
      mode: "deep",
      scope: { paths: ["src"] },
      deep: {
        workers: 8,
        subagents_per_worker: 0,
        stop_after_consecutive_errors: 2,
      },
    },
    limits: { max_cost_usd_per_scan: 7 },
    policy: { fail_on_severity: "high" },
    codex: { model: "gpt-5.6-terra", model_reasoning_effort: "high" },
  });
  const onTurn = mock<(_target: string, options: ScanOptions) => void>();
  const onConfig = mock<(config: CodexSecurityConfig) => void>();
  const { stdout, stderr, runCli } = createCliTest(main);

  expect(
    await runCli(
      ["scan", "-c", input.config, "--json"],
      dependencies({
        currentDirectory: input.repository,
        environment: { OPENAI_API_KEY: "synthetic-test-key" },
        onConfig,
        onTurn,
        result: fakeResult(["high"]),
      }),
    ),
  ).toBe(1);
  const repository = onTurn.mock.lastCall?.[0];
  const selected = onTurn.mock.lastCall?.[1];
  const native = onConfig.mock.lastCall?.[0];
  expect(repository).toBe(input.repository);
  expect(selected).toMatchObject({
    auth: "api-key",
    mode: "deep",
    target: ["src"],
    workers: 8,
    subagents: 0,
    stopAfterConsecutiveErrors: 2,
    maxCostUsd: 7,
    failureSeverity: "high",
  });
  expect(native?.codexOverrides).toMatchObject({
    model: "gpt-5.6-terra",
    model_reasoning_effort: "high",
  });
  expect(JSON.parse(stdout.text())).toMatchObject({
    manifest: { scan: { status: "completed" } },
  });
  expect(stderr.text()).toContain("COST");
});

test("CLI values override matching file values, including native objects and lists", async () => {
  const input = await fixture({
    auth: "chatgpt",
    scan: {
      mode: "deep",
      scope: { paths: ["src"] },
      knowledge_base: ["file-context.md"],
      deep: { workers: 8, subagents_per_worker: 3 },
    },
    limits: { max_cost_usd_per_scan: 7 },
    policy: { fail_on_severity: "high" },
    codex: {
      model: "gpt-5.6-sol",
      synthetic_setting: { enabled: true, names: ["first"] },
    },
  });
  const onTurn = mock<(_target: string, options: ScanOptions) => void>();
  const onConfig = mock<(config: CodexSecurityConfig) => void>();
  expect(
    await runCapturedCli(
      main,
      [
        "scan",
        "--config",
        input.config,
        "--auth",
        "auto",
        "--path",
        "lib",
        "--knowledge-base",
        "cli-context.md",
        "--subagents",
        "0",
        "--workers",
        "2",
        "--max-cost",
        "3",
        "--fail-on-severity",
        "low",
        "--model",
        "gpt-5.6-terra",
        "--codex",
        "synthetic_setting.enabled=false",
        "--codex",
        "synthetic_setting.names=[]",
        "--json",
      ],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
        onTurn,
      }),
    ),
  ).toBe(0);
  const selected = onTurn.mock.lastCall?.[1];
  const native = onConfig.mock.lastCall?.[0];
  expect(selected).toMatchObject({
    auth: "auto",
    mode: "deep",
    target: ["lib"],
    knowledgeBasePaths: [join(input.repository, "cli-context.md")],
    workers: 2,
    subagents: 0,
    maxCostUsd: 3,
    failureSeverity: "low",
  });
  expect(native?.codexOverrides).toMatchObject({
    model: "gpt-5.6-terra",
    synthetic_setting: { enabled: false, names: [] },
  });
});

test("rerun accepts replacement scan and validation files relative to the invocation directory", async () => {
  const input = await fixture({});
  await writeFile(
    join(input.configDirectory, "instructions.md"),
    "Review the synthetic boundary.",
  );
  await writeFile(
    join(input.configDirectory, "validation.md"),
    "Validate the synthetic boundary.",
  );
  const recipe: JsonObject = {
    repository: input.repository,
    target: { kind: "repository", paths: [] },
    mode: "standard",
    config: {},
    requiresScanPrompt: true,
    validationMode: "custom",
    cyberAccessProgram: "daybreak_blue",
  };
  const onTurn = mock<(_target: string, options: ScanOptions) => void>();
  const stderr = captureCli(main, "stderr");
  expect(
    await stderr.run(
      [
        "scans",
        "rerun",
        "saved",
        "--scan-prompt-file",
        "instructions.md",
        "--validation-prompt-file",
        "validation.md",
        "--json",
      ],
      dependencies({
        currentDirectory: input.configDirectory,
        onWorkbench: async () => ({ scanId: "saved", recipe }),
        onTurn,
      }),
    ),
  ).toBe(0);
  const selected = onTurn.mock.lastCall?.[1];
  expect(selected).toMatchObject({
    scanPrompt: "Review the synthetic boundary.",
    validationPrompt: "Validate the synthetic boundary.",
    parentScanId: "saved",
    cyberAccessProgram: "daybreak_blue",
  });
});

test.each([
  [
    { paths: ["src"] },
    ["--diff", "HEAD"],
    { kind: "refs", base: "HEAD", head: "HEAD" },
  ],
  [{ diff: { base: "HEAD", head: "HEAD~1" } }, ["--path", "lib"], ["lib"]],
  [
    { diff: { base: "HEAD", head: "HEAD~1" } },
    ["--head", "HEAD"],
    { kind: "refs", base: "HEAD", head: "HEAD" },
  ],
  [
    { working_tree: {} },
    ["--base", "HEAD~1"],
    { kind: "working_tree", base: "HEAD~1" },
  ],
  [{ working_tree: {} }, ["--no-working-tree"], "repository"],
] as const)(
  "resolves scope %j with overrides %j",
  async (scope, flags, target) => {
    const input = await fixture({
      scan: { scope: structuredClone(scope) },
    } as ProjectConfigInput);
    const onTurn = mock<(_target: string, options: ScanOptions) => void>();
    const stderr = captureCli(main, "stderr");
    expect(
      await stderr.run(
        ["scan", "-c", input.config, ...flags, "--json"],
        dependencies({
          currentDirectory: input.repository,
          onTurn,
        }),
      ),
    ).toBe(0);
    const selected = onTurn.mock.lastCall?.[1];
    if (typeof target === "string" || Array.isArray(target))
      expect(selected?.target).toEqual(target);
    else expect(selected?.target).toMatchObject(target);
  },
);

test.each([
  [["--path", "src", "--diff", "HEAD"], "mutually exclusive"],
  [["--head", "HEAD"], "--head requires --diff"],
  [["--base", "HEAD"], "--base requires --working-tree"],
  [["--workers", "2", "--mode", "standard"], "require deep mode"],
  [
    ["--model", "gpt-5.6-terra", "--codex", 'model="gpt-5.6-sol"'],
    "--model conflicts",
  ],
] as const)(
  "rejects incompatible explicit overrides: %j",
  async (flags, message) => {
    const input = await fixture({});
    const onConfig = mock();
    const stderr = captureCli(main, "stderr");
    expect(
      await stderr.run(
        ["scan", "-c", input.config, ...flags, "--json"],
        dependencies({
          currentDirectory: input.repository,
          onConfig,
        }),
      ),
    ).toBe(2);
    expect(stderr.text()).toContain(message);
    expect(onConfig).not.toHaveBeenCalled();
  },
);

test("selecting standard mode leaves inactive file deep settings out of the active scan", async () => {
  const input = await fixture({
    scan: {
      mode: "deep",
      deep: { workers: 8, stop_after_consecutive_errors: 2 },
    },
  });
  const onTurn = mock<(_target: string, options: ScanOptions) => void>();
  expect(
    await runCapturedCli(
      main,
      ["scan", "-c", input.config, "--mode", "standard", "--json"],
      dependencies({
        currentDirectory: input.repository,
        onTurn,
      }),
    ),
  ).toBe(0);
  const selected = onTurn.mock.lastCall?.[1];
  expect(selected?.mode).toBe("standard");
  expect(selected?.workers).toBeUndefined();
  expect(selected?.stopAfterConsecutiveErrors).toBeUndefined();
});

test("file prompts use the config directory and CLI prompt overrides use the invocation directory", async () => {
  const input = await fixture({
    scan: { instructions_file: "scan.md", validation_file: "validate.md" },
  });
  await writeFile(
    join(input.configDirectory, "scan.md"),
    "Synthetic file instructions.",
  );
  await writeFile(
    join(input.configDirectory, "validate.md"),
    "Synthetic file validation.",
  );
  await writeFile(
    join(input.repository, "validate.md"),
    "Synthetic CLI validation.",
  );
  const onTurn = mock<(_target: string, options: ScanOptions) => void>();
  expect(
    await runCapturedCli(
      main,
      [
        "scan",
        "-c",
        input.config,
        "--validation-prompt-file",
        "validate.md",
        "--json",
      ],
      dependencies({
        currentDirectory: input.repository,
        onTurn,
      }),
    ),
  ).toBe(0);
  const selected = onTurn.mock.lastCall?.[1];
  expect(selected).toMatchObject({
    scanPrompt: "Synthetic file instructions.",
    validationPrompt: "Synthetic CLI validation.",
  });
});

test("an explicit file can supply the model required by a provider override", async () => {
  const input = await fixture({
    codex: { model: "synthetic-model" },
  });
  const onConfig = mock<(config: CodexSecurityConfig) => void>();
  expect(
    await runCapturedCli(
      main,
      ["scan", "-c", input.config, "--provider", "amazon-bedrock", "--json"],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
      }),
    ),
  ).toBe(0);
  const native = onConfig.mock.lastCall?.[0];
  expect(native?.codexOverrides).toMatchObject({
    model: "synthetic-model",
    model_provider: "amazon-bedrock",
  });
});

test.each(["file", "CLI"])(
  "a profile selected by the %s supplies the provider model",
  async (selection) => {
    const overrideProfile = selection === "CLI";
    const input = await fixture({
      codex: {
        profile: overrideProfile ? "other" : "review",
        profiles: {
          review: { model: "synthetic-profile-model" },
          other: { model: "synthetic-other-model" },
        },
      },
    });
    const onConfig = mock<(config: CodexSecurityConfig) => void>();
    expect(
      await runCapturedCli(
        main,
        [
          "scan",
          "-c",
          input.config,
          "--provider",
          "amazon-bedrock",
          ...(overrideProfile ? ["--codex", 'profile="review"'] : []),
          "--json",
        ],
        dependencies({
          currentDirectory: input.repository,
          onConfig,
        }),
      ),
    ).toBe(0);
    const native = onConfig.mock.lastCall?.[0];
    expect(native?.codexOverrides).toMatchObject({
      profile: "review",
      profiles: { review: { model: "synthetic-profile-model" } },
      model_provider: "amazon-bedrock",
    });
  },
);

test("an unselected file profile does not satisfy the provider model requirement", async () => {
  const input = await fixture({
    codex: {
      profiles: { review: { model: "synthetic-profile-model" } },
    },
  });
  const stderr = captureCli(main, "stderr");
  const onConfig = mock();
  expect(
    await stderr.run(
      ["scan", "-c", input.config, "--provider", "amazon-bedrock", "--json"],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
      }),
    ),
  ).toBe(2);
  expect(onConfig).not.toHaveBeenCalled();
  expect(stderr.text()).toContain(
    "--model is required when using --provider amazon-bedrock",
  );
});

test.each([
  { argv: ["scan", "--help"] },
  { argv: ["scan", "--schema", "--json"] },
  { argv: ["info", "--json"] },
])("malformed unselected files do not affect %j", async ({ argv }) => {
  const input = await fixture("scan: [");
  await writeFile(join(input.repository, "codex-security.yaml"), "scan: [");
  const onConfig = mock();
  expect(
    await runCapturedCli(
      main,
      [...argv],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
      }),
    ),
  ).toBe(0);
  expect(onConfig).not.toHaveBeenCalled();
});

test.each(["--help", "--schema"])(
  "%s does not load even an explicitly selected invalid file",
  async (flag) => {
    const input = await fixture("scan: [");
    const stdout = captureCli(main, "stdout");
    expect(
      await stdout.run(
        ["scan", "-c", input.config, flag, "--json"],
        dependencies({ currentDirectory: input.repository }),
      ),
    ).toBe(0);
    if (flag === "--schema")
      expect(JSON.parse(stdout.text())).toMatchObject({
        options: { properties: { config: { type: "string" } } },
      });
  },
);

test("a malformed selected file fails before constructing a client", async () => {
  const input = await fixture("scan: [");
  const onConfig = mock();
  const { stdout, stderr, runCli } = createCliTest(main);

  expect(
    await runCli(
      ["scan", "-c", input.config, "--dry-run", "--json"],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
      }),
    ),
  ).toBe(2);
  expect(onConfig).not.toHaveBeenCalled();
  expect(stderr.text()).toContain(
    `Cannot parse project configuration at ${input.config}: `,
  );
  expect(stderr.text()).toContain("end with a ]");
  expect(JSON.parse(stdout.text())).toEqual({
    status: "failed",
    code: "SCAN_FAILED",
    message: expect.stringContaining("end with a ]"),
  });
});

test("dry-run reports native defaults restored by a CLI table", async () => {
  const input = await fixture({
    codex: { features: { multi_agent_v2: null } },
  });
  const onConfig = mock<(config: CodexSecurityConfig) => void>();
  const { stdout, runCli } = createCliTest(main);
  expect(
    await runCli(
      [
        "scan",
        input.repository,
        "-c",
        input.config,
        "--codex",
        "features.multi_agent_v2.max_concurrent_threads_per_session=2",
        "--dry-run",
        "--json",
      ],
      dependencies({
        currentDirectory: input.repository,
        onConfig,
        onRun: throwing("Dry-run must not start a scan"),
      }),
    ),
  ).toBe(0);
  expect(JSON.parse(stdout.text())).toMatchObject({
    dryRun: true,
    projectConfig: {
      sources: {
        "codex.features.multi_agent_v2.enabled": "default",
        "codex.features.multi_agent_v2.max_concurrent_threads_per_session":
          "cli",
      },
    },
  });
  expect(
    (await mergedCodexConfig(onConfig.mock.lastCall![0]))["features"],
  ).toMatchObject({
    multi_agent_v2: { enabled: true, max_concurrent_threads_per_session: 2 },
  });
});

test("dry-run uses the real SDK without initializing its runtime and reports provenance", async () => {
  const input = await fixture({
    scan: {
      mode: "deep",
      scope: { paths: ["src"] },
      deep: { workers: 8, stop_after_consecutive_errors: 2 },
    },
    codex: {
      profile: "review",
      model: "gpt-5.6-sol",
      profiles: {
        review: { model: "gpt-5.6-terra", model_reasoning_effort: "high" },
      },
    },
    policy: { fail_on_severity: "high" },
  });
  const ambient = join(input.root, "ambient");
  await mkdir(join(ambient, "codex-security"), { recursive: true });
  await writeFile(
    join(ambient, "codex-security", "config.toml"),
    "[deep_scan]\nworkers = 6\nsubagents = 1\nstop_after_no_new = 5\n",
  );
  const environment = {
    codex_home: ambient,
    CODEX_SECURITY_STATE_DIR: join(input.root, "state"),
  };
  const { stdout, runCli } = createCliTest(main);

  const createCodex = mock(throwing("No inference in dry-run"));
  const prepareRuntime = mock(rejecting("No runtime in dry-run"));
  const deps = dependencies({
    currentDirectory: input.repository,
    environment,
  });
  deps.createSecurity = cliClientFactory({
    environment,
    createCodex,
    prepareRuntime,
  });
  expect(
    await runCli(
      [
        "scan",
        "-c",
        input.config,
        "--subagents",
        "0",
        "--model",
        "gpt-5.6-sol",
        "--dry-run",
        "--json",
      ],
      deps,
    ),
  ).toBe(0);
  expect(createCodex).not.toHaveBeenCalled();
  expect(prepareRuntime).not.toHaveBeenCalled();
  expect(JSON.parse(stdout.text())).toMatchObject({
    dryRun: true,
    repository: input.repository,
    mode: "deep",
    model: "gpt-5.6-terra",
    reasoningEffort: "high",
    target: { kind: "paths", paths: ["src"] },
    workers: 8,
    subagents: 0,
    stopAfterNoNew: 5,
    stopAfterConsecutiveErrors: 2,
    maxDiscoveryRuns: 40,
    maxTimeHours: 96,
    projectConfig: {
      path: input.config,
      sources: {
        "scan.deep.workers": "project",
        "scan.deep.subagents_per_worker": "cli",
        "scan.deep.stop_after_no_new": "legacy",
        "scan.deep.max_time_hours": "default",
        "codex.model": "cli",
        "codex.profiles.review.model": "project",
      },
    },
    failOnSeverity: "high",
  });
  const nativeProject = JSON.parse(
    await readFile(input.config, "utf8"),
  ) as Record<string, JsonObject>;
  delete nativeProject["codex"]!["profiles"];
  delete nativeProject["codex"]!["model"];
  await writeFile(input.config, JSON.stringify(nativeProject));
  await writeFile(
    join(ambient, "review.config.toml"),
    'model = "synthetic-native-model"\nmodel_provider = "openrouter"\nmodel_reasoning_effort = "high"\n',
  );
  Object.assign(environment, {
    OPENROUTER_API_KEY: "synthetic-cli-provider-key",
  });
  const nativeCli = createCliTest(main);
  expect(
    await nativeCli.runCli(
      ["scan", "-c", input.config, "--dry-run", "--json"],
      deps,
    ),
    nativeCli.stderr.text(),
  ).toBe(0);
  expect(JSON.parse(nativeCli.stdout.text())).toMatchObject({
    model: "synthetic-native-model",
    modelProvider: "openrouter",
    reasoningEffort: "high",
  });
  expect(createCodex).not.toHaveBeenCalled();
  expect(prepareRuntime).not.toHaveBeenCalled();
});

test.each([
  [{ output: { directory: "../repository/artifacts" } }, "outside"],
  [{ codex: { plugins: {} } }, "plugin"],
  [{ scan: { mode: "deep", validation_file: "validate.md" } }, "Deep"],
] as const)(
  "project files retain active scan checks: %j",
  async (config, message) => {
    const input = await fixture(config as ProjectConfigInput);
    await writeFile(
      join(input.configDirectory, "validate.md"),
      "Synthetic validation instructions.",
    );
    const environment = {
      CODEX_HOME: join(input.root, "ambient"),
      CODEX_SECURITY_STATE_DIR: join(input.root, "state"),
    };
    const stderr = captureCli(main, "stderr");
    const createCodex = mock(throwing("No inference"));
    const prepareRuntime = mock(rejecting("No runtime"));
    const deps = dependencies({
      currentDirectory: input.repository,
      environment,
    });
    deps.createSecurity = cliClientFactory({
      environment,
      createCodex,
      prepareRuntime,
    });
    expect(
      await stderr.run(
        ["scan", "-c", input.config, "--dry-run", "--json"],
        deps,
      ),
    ).toBe(2);
    expect(stderr.text().toLowerCase()).toContain(message.toLowerCase());
    expect(createCodex).not.toHaveBeenCalled();
    expect(prepareRuntime).not.toHaveBeenCalled();
  },
);

test("rerun restores all saved deep settings and authentication without loading project files", async () => {
  const input = await fixture("scan: [");
  const deep = {
    workers: 7,
    subagents: 0,
    stopAfterNoNew: 5,
    stopAfterConsecutiveErrors: 2,
    maxDiscoveryRuns: 12,
    maxTimeHours: 3,
  };
  const recipe: JsonObject = {
    repository: input.repository,
    target: { kind: "repository", paths: [] },
    mode: "deep",
    config: {},
    auth: "api-key",
    deepScan: deep,
    deepScanResolved: true,
  };
  const onTurn = mock<(_target: string, options: ScanOptions) => void>();
  expect(
    await runCapturedCli(
      main,
      ["scans", "rerun", "saved", "--json"],
      dependencies({
        currentDirectory: input.repository,
        environment: {
          OPENAI_API_KEY: "synthetic-test-key",
          CODEX_SECURITY_PROJECT_CONFIG: input.config,
        },
        onWorkbench: async () => ({ scanId: "saved", recipe }),
        onTurn,
      }),
    ),
  ).toBe(0);
  const selected = onTurn.mock.lastCall?.[1];
  expect(selected).toMatchObject({
    ...deep,
    auth: "api-key",
    parentScanId: "saved",
  });
});

test.each([
  [{ requiresScanPrompt: true }, "additional instructions"],
  [{ maxTimeMinutes: 0 }, "invalid investigation time limit"],
  [{ maxTimeMinutes: "15" }, "invalid investigation time limit"],
  [{ maxTimeMinutes: 1e12 }, "deadline within the supported date range"],
  [
    { mode: "deep", deepScan: { workers: 7 }, deepScanResolved: true },
    "missing resolved deep scan settings",
  ],
] as const)(
  "rerun rejects an incomplete saved input: %j",
  async (extra, message) => {
    const input = await fixture({});
    const recipe: JsonObject = {
      repository: input.repository,
      target: { kind: "repository", paths: [] },
      mode: "standard",
      config: {},
      ...extra,
    };
    const { stdout, stderr, runCli } = createCliTest(main);
    const onRun = mock();
    expect(
      await runCli(
        ["scans", "rerun", "saved", "--json", "--full-output"],
        dependencies({
          currentDirectory: input.repository,
          onWorkbench: async () => ({ recipe, scanId: "saved" }),
          onRun,
        }),
      ),
    ).toBe(2);
    expect(stderr.text()).toContain(message);
    expect(JSON.parse(stdout.text())).toMatchObject({
      ok: false,
      error: { message: expect.stringContaining(message) },
    });
    expect(onRun).not.toHaveBeenCalled();
  },
);

test.each([
  { model: "gpt-5.6-sol", showCost: false },
  { model: "gpt-5.6-sol", showCost: true },
  { model: "synthetic-unpriced-model", showCost: false },
  { model: "synthetic-unpriced-model", showCost: true },
])(
  "headless component usage is printed once for %p",
  async ({ model, showCost }) => {
    const input = await fixture({
      output: { directory: "../component-results" },
    });
    await writeFile(
      join(input.repository, "lib", "index.ts"),
      "export const value = 1;\n",
    );
    const usage: ScanTokenUsage = {
      input_tokens: 1250,
      cached_input_tokens: 200,
      cache_write_input_tokens: 0,
      output_tokens: 30,
      reasoning_output_tokens: 0,
      total_tokens: 1280,
    };
    const { stderr, runCli } = createCliTest(main);
    const deps = dependencies({ currentDirectory: input.root });
    deps.createSecurity = () =>
      fakeSecurity(async (_repository, options) => {
        const tracker = new ScanCostTracker({
          codexHome: input.root,
          model,
          onUsage: options?.onUsage,
          onCost: (cost, complete) => {
            if (complete) options?.onCost?.(cost);
          },
        });
        tracker.start("synthetic-scan-thread");
        await tracker.stop(usage);
        return fakeResult([], "complete", null);
      });
    expect(
      await runCli(
        [
          "scan-components",
          input.repository,
          "-c",
          input.config,
          "--component",
          "lib",
          "--headless",
          "--json",
          ...(showCost ? ["--show-cost"] : []),
        ],
        deps,
      ),
    ).toBe(0);
    expect(stderr.text().match(/Tokens:/g)).toHaveLength(1);
    expect(stderr.text()).toContain("Tokens: 1,050 uncached input");
    expect(stderr.text()).toContain("30 output");
    expect(stderr.text().includes("Cost:")).toBe(
      showCost && model === "gpt-5.6-sol",
    );
  },
);

test.each(["yaml", "toon", "json"])(
  "a long framework error keeps its whole message with --format %s",
  async (format) => {
    const root = await temporaryDirectory();
    const missing = join(
      root,
      "a-directory-name-long-enough-to-make-yaml-fold-the-line",
      "nested\\scan.yaml",
    );
    const { stderr, runCli } = createCliTest(main);
    expect(
      await runCli(
        ["info", "-c", missing, "--format", format],
        dependencies({ currentDirectory: root }),
      ),
    ).toBe(2);
    const message =
      format === "json"
        ? JSON.parse(stderr.text().replace(/^codex-security: /u, "")).message
        : stderr.text();
    expect(message).toContain(missing);
  },
);