OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import { describe, expect, test } from "bun:test";
import { readdir } from "node:fs/promises";
import { scopedTemporaryDirectory } from "./support/temporary-directories.js";
import { main } from "../src/cli.js";
import { DEFAULT_CODEX_CONFIG, scanModelConfiguration } from "../src/config.js";
import { DEFAULT_DEEP_SCAN_SETTINGS } from "../src/deep-scan-defaults.js";
import { capture, dependencies } from "./cli-fixtures.js";
import { createCliTest } from "./support/cli-run.js";
async function help(args: readonly string[], columns?: number) {
const stdout = capture(true);
const stderr = capture(true);
const calls: string[] = [];
const unexpected = (name: string): never => {
calls.push(name);
throw new Error(`Help must not call ${name}.`);
};
const deps = dependencies({
onConfig: () => unexpected("createSecurity"),
onCodex: () => unexpected("runCodex"),
onWorkbench: () => unexpected("runWorkbench"),
onRepositoryCommand: () => unexpected("runRepositoryCommand"),
onUpdateCheck: async () => unexpected("checkForUpdate"),
});
deps.importScan = async () => unexpected("importScan");
const code = await main(
args,
{ ...stdout.stream, ...(columns === undefined ? {} : { columns }) },
stderr.stream,
deps,
);
expect(code).toBe(0);
expect(stderr.text()).toBe("");
expect(calls).toEqual([]);
return stdout.text();
}
function option(helpText: string, flag: string): string {
const lines = helpText.split("\n");
const start = lines.findIndex((line) => line.startsWith(` ${flag} `));
if (start === -1) return "";
let end = start + 1;
while (
end < lines.length &&
lines[end]!.trim() !== "" &&
!/^ --[a-z]/u.test(lines[end]!) &&
/^\s/u.test(lines[end]!)
) {
end += 1;
}
return lines.slice(start, end).join(" ").replace(/\s+/gu, " ");
}
describe("CLI help", () => {
test.each(["scan", "patch", "verify-fix"])(
"%s advertises hosted mode without entering the protocol",
async (command) => {
const text = await help([command, "--host", "--help"]);
expect(option(text, "--host")).toContain("JSON-RPC stdin/stdout");
expect(option(text, "--host")).toContain("use alone");
if (command !== "scan") {
expect(option(text, "--verification-mode")).toContain("static");
expect(option(text, "--verification-mode")).toContain("standard");
const schema = JSON.parse(await help([command, "--schema", "--json"]));
expect(schema.options.properties.verificationMode.enum).toEqual([
"static",
"standard",
]);
}
},
);
test("groups the common workflow ahead of setup and integrations", async () => {
const text = await help(["--help"]);
const sections = text.split(/\n(?=[A-Z][^\n]+:\n)/u);
const sectionFor = (command: string) =>
sections.findIndex((section) =>
new RegExp(`^ ${command} {2,}`, "mu").test(section),
);
expect(sectionFor("scan")).toBeGreaterThanOrEqual(0);
expect(sectionFor("patch")).toBe(sectionFor("scan"));
expect(sectionFor("scans")).toBeGreaterThan(sectionFor("scan"));
expect(sectionFor("findings")).toBe(sectionFor("scans"));
expect(sectionFor("login")).toBeGreaterThan(sectionFor("scan"));
expect(sectionFor("init")).toBe(sectionFor("login"));
expect(sectionFor("update")).toBe(sectionFor("login"));
expect(text).not.toContain("--update");
expect(sectionFor("completions")).toBeGreaterThan(sectionFor("scan"));
expect(text).toContain("Aliases: cs");
for (const command of [
"scan . --dry-run",
"login",
"scan .",
"scans show",
]) {
expect(text).toContain(` codex-security ${command}`);
}
expect(text.indexOf("codex-security scan . --dry-run")).toBeLessThan(
text.indexOf("codex-security login"),
);
expect(text).toContain("<command> --help");
});
test.each(["bulk-scan", "scan-components"])(
"labels %s cost limits in USD",
async (command) => {
const text = await help([command, "--help"]);
expect(option(text, "--max-total-cost")).toContain("<usd>");
},
);
test("shows semantic values, repeatable inputs, choices, and scan defaults", async () => {
const text = await help(["scan", "--help"]);
expect(text).toContain("Usage: codex-security scan [repository]");
expect(text).not.toMatch(/<(?:array|string|number)>/u);
expect(option(text, "--path")).toContain("<path>");
expect(option(text, "--path")).toMatch(/repeat/iu);
expect(option(text, "--codex")).toContain("<key=value>");
expect(option(text, "--codex")).toMatch(/repeat/iu);
expect(option(text, "--max-cost")).toContain("<usd>");
expect(option(text, "--owner-suggestions")).toContain(
"owner-suggestions.json",
);
expect(option(text, "--mode")).toContain("<mode>");
expect(option(text, "--mode")).toContain("standard");
expect(option(text, "--mode")).toContain("deep");
expect(option(text, "--provider")).toContain("amazon-bedrock");
for (const label of [
"--model <model>",
"--provider <provider>",
"--effort <effort>",
"--max-time-hours <hours>",
"--max-time <minutes>",
])
expect(text).toContain(label);
const { model, reasoningEffort } =
scanModelConfiguration(DEFAULT_CODEX_CONFIG);
expect(text).toContain(model);
expect(text).toContain(reasoningEffort);
expect(text).toContain('model_reasoning_effort="high"');
expect(text).toContain("CODEX_SECURITY_STATE_DIR");
expect(text).not.toContain("openai:gpt");
expect(text).toContain("--path src --path tests");
expect(text).not.toContain("codex-security scan . --path src,tests");
expect(text).toMatch(
/^ codex-security scan \. --working-tree(?:\s+#.*)?$/mu,
);
for (const [flag, value] of [
["--workers", DEFAULT_DEEP_SCAN_SETTINGS.workers],
["--subagents", DEFAULT_DEEP_SCAN_SETTINGS.subagents],
["--stop-after-no-new", DEFAULT_DEEP_SCAN_SETTINGS.stopAfterNoNew],
["--max-discovery-runs", DEFAULT_DEEP_SCAN_SETTINGS.maxDiscoveryRuns],
] as const) {
expect(option(text, flag)).toContain("<count>");
expect(option(text, flag)).toMatch(
new RegExp(`default: ${value}\\b`, "iu"),
);
}
});
test("uses bare booleans in copyable publish examples", async () => {
const publish = await help(["publish", "scan", "--help"]);
expect(publish).not.toMatch(/--dry-run true\b/u);
expect(publish).toMatch(
/^ codex-security publish scan .*--dry-run(?:\s+#.*)?$/mu,
);
});
test.each([
{ command: ["scan"] },
{ command: ["scan", "import"] },
{ command: ["scans", "rerun"] },
{ command: ["scans", "resume"] },
])("advertises supported formats for $command", async ({ command }) => {
const text = await help([...command, "--help"]);
const format = option(text, "--format");
expect(format).toContain("json");
expect(format).not.toMatch(/\bmd\b/u);
if (command[0] === "scan") {
expect(option(text, "--filter-output")).toBe("");
}
});
test.each(["validate", "login", "logout"])(
"%s does not promise unsupported JSON command output",
async (command) => {
const text = await help([command, "--help"]);
expect(option(text, "--format")).not.toMatch(/\bjsonl?\b/u);
expect(option(text, "--json")).toBe("");
const schema = JSON.parse(await help([command, "--schema", "--json"]));
expect(schema).toEqual(expect.any(Object));
},
);
test("documents optional validation context and structured output", async () => {
const text = await help(["validate", "--help"]);
expect(option(text, "--context")).toContain("<text>");
expect(option(text, "--context-file")).toContain("<file>");
expect(option(text, "--knowledge-base")).toContain("<path>");
expect(option(text, "--knowledge-base")).toMatch(/repeat/iu);
expect(option(text, "--output-schema")).toContain("<file>");
});
test("documents raw export formats without promising output transforms", async () => {
const text = await help(["export", "--help"]);
expect(text).toContain("Usage: codex-security export [scan-dir]");
expect(text).toContain("--artifact <artifact>");
expect(text).toContain("--scan <scan-id>");
expect(text).toContain("--source-root <path>");
expect(text).toContain("threatmodel.md");
expect(option(text, "--export-format")).toContain("<format>");
expect(option(text, "--export-format")).toContain("sarif");
expect(text).toContain("--avit-template <file>");
for (const flag of [
"--format",
"--json",
"--filter-output",
"--full-output",
"--token-count",
"--token-limit",
"--token-offset",
]) {
expect(option(text, flag)).toBe("");
}
const schema = await help(["export", "--schema", "--json"]);
expect(JSON.parse(schema)).toMatchObject({
options: {
properties: {
artifact: { enum: ["findings", "threat-model"], default: "findings" },
exportFormat: {
enum: ["csv", "json", "sarif", "md", "servicenow-avit"],
},
},
},
});
});
test("distinguishes JSON file import from JSON output", async () => {
const text = await help(["scan", "import", "--help"]);
expect(option(text, "--json")).toContain("--format json");
expect(option(text, "--input-format")).toContain("Input format");
expect(text.replace(/\s+/gu, " ")).toContain("--format json");
expect(option(text, "--format")).toContain("json");
});
test.each([
{ args: ["-h"], usage: "codex-security <command>" },
{ args: ["--help", "scan"], usage: "codex-security scan [repository]" },
{
args: ["scan", "--mode", "bogus", "--help"],
usage: "codex-security scan [repository]",
},
{
args: ["scan", "--config", "missing.yaml", "--help"],
usage: "codex-security scan [repository]",
},
{
args: ["scan", "import", "--json", "--help"],
usage: "codex-security scan import",
},
{
args: ["--format", "json", "scan", "import", "--help"],
usage: "codex-security scan import",
},
{ args: ["scans", "--help"], usage: "codex-security scans [command]" },
{
args: ["findings", "--help"],
usage: "codex-security findings [command]",
},
{ args: ["login", "status", "--help"], usage: "codex-security login" },
])("preserves help routing for $args", async ({ args, usage }) => {
const text = await help(args);
expect(text).toContain(`Usage: ${usage}`);
});
test.each([
{
args: ["sacn", "--help"],
missing: "sacn",
recovery: "codex-security --help",
},
{
args: ["scans", "foo", "--help"],
missing: "scans foo",
recovery: "codex-security scans --help",
},
])(
"identifies an unresolved help path $args",
async ({ args, missing, recovery }) => {
const out = capture();
const err = capture();
expect(await main(args, out.stream, err.stream, dependencies())).toBe(2);
expect(out.text()).toBe("");
expect(err.text()).toContain(missing);
expect(err.text()).toContain(recovery);
},
);
test.each([
["imports", "--json"],
["--json", "./import"],
])(
"inspects repository scans with shared flags in either position (%s, %s)",
async (first, second) => {
const args = ["scan", first, second];
expect(await help([...args, "--help"])).toContain(
"Usage: codex-security scan [repository]",
);
const schema = JSON.parse(await help([...args, "--schema"]));
expect(schema.args.properties.repository.type).toBe("string");
},
);
test.each([
["scans", "show"],
["findings", "false-positive"],
["scan", "import"],
["login", "status"],
["publish", "findings"],
])("resolves %s %s around global options", async (group, verb) => {
const flags = ["--format", "json", "--schema"];
const expected = JSON.parse(await help([group, verb, ...flags]));
expect(JSON.parse(await help([group, ...flags, verb]))).toEqual(expected);
});
test.each(
[
["bogus", "--llms-full"],
["scans", "--json", "bogus", "--llms"],
["mcp", "bogus", "--llms-full"],
["info", "extra", "--help"],
["info", "extra", "--schema"],
["info", "extra", "--llms"],
["completions", "zsh", "extra"],
["completions", "zsh", "--", "--version"],
["skills", "list", "--", "--no-global"],
].map((args) => ({ args })),
)(
"rejects unknown discovery paths or surplus words (%j)",
async ({ args }) => {
const stdout = capture();
const stderr = capture();
expect(
await main(args, stdout.stream, stderr.stream, dependencies()),
).toBe(2);
expect(stdout.text()).toBe("");
expect(stderr.text()).toMatch(
/Unknown command|not a command|Unexpected positional/,
);
},
);
test.each([
["scans", "shwo", "show"],
["mcp", "doctr", "doctor"],
["skills", "lsit", "list"],
])("preserves suggested verbs for %s %s", async (group, typo, verb) => {
const stderr = capture();
expect(
await main(
[group, typo],
capture().stream,
stderr.stream,
dependencies(),
),
).toBe(2);
expect(stderr.text()).toContain(`Did you mean '${verb}'?`);
});
test.each([
{ command: [], expected: "Usage: codex-security" },
...["skills", "mcp", "import", "publish", "completions"].map((name) => ({
command: [name],
expected: `Usage: codex-security ${name}`,
})),
{ command: ["completions", "zsh"], expected: "#compdef" },
])(
"formats the text from $command before applying JSON token controls",
async ({ command, expected }) => {
async function run(...flags: string[]) {
const { runCli, stdout, stderr } = createCliTest(main);
expect(await runCli([...command, ...flags], dependencies())).toBe(0);
expect(stderr.text()).toBe("");
return stdout.text();
}
const plain = await run();
expect(plain).toContain(expected);
expect(JSON.parse(await run("--json"))).toBe(plain);
const flags = ["--format=jsonl", "--full-output"];
expect(JSON.parse(await run(...flags))).toMatchObject({
ok: true,
data: plain,
});
expect(JSON.parse(await run(...flags, "--token-count"))).toBeGreaterThan(
0,
);
expect(JSON.parse(await run(...flags, "--token-limit=1"))).toMatchObject({
ok: true,
meta: { nextOffset: 1 },
});
},
);
test("uses the completion shell schema without generating a script", async () => {
const schema = JSON.parse(
await help(["completions", "--json", "--schema"]),
);
expect(schema.args.properties.shell.enum).toContain("zsh");
});
test.each(
[
["scan", ".", "--dry-run", "--update"],
["scans", "list", "--mcp"],
["policy", "--full-output", "--mcp"],
["--mcp", "--update"],
["scan", ".", "--dry-run", "--incur-update-check"],
].map((args) => ({ args })),
)("rejects mixed or private actions (%j)", async ({ args }) => {
const stdout = capture();
const stderr = capture();
expect(await main(args, stdout.stream, stderr.stream, dependencies())).toBe(
2,
);
expect(stdout.text()).toBe("");
expect(stderr.text()).toMatch(/must be used alone|Unknown flag/);
});
test.each(["--filter-output", "--format", "--token-limit", "--token-offset"])(
"%s cannot turn a following switch into a value and dispatch a write",
async (flag) => {
await using temp = await scopedTemporaryDirectory("cli-missing-global-");
const deps = dependencies({ currentDirectory: temp.path });
for (const next of [
"-h",
"--help",
"--version",
"--schema",
"--llms",
"--llms-full",
"--mcp",
"--update",
"--incur-update-check",
"--json",
"--full-output",
"-c",
]) {
const stderr = capture();
expect(
await main(
[flag, next, "init", "new.yaml"],
capture().stream,
stderr.stream,
deps,
),
).toBe(2);
expect(stderr.text()).toMatch(
/Missing value|must be used alone|Unknown flag/,
);
}
expect(await readdir(temp.path)).toEqual([]);
},
);
test("the equals form keeps a dash-leading filter value literal", async () => {
await using temp = await scopedTemporaryDirectory("cli-explicit-filter-");
expect(
await main(
["--filter-output=-h", "init", "new.yaml"],
capture().stream,
capture().stream,
dependencies({ currentDirectory: temp.path }),
),
).toBe(0);
expect(await readdir(temp.path)).toEqual(["new.yaml"]);
});
test("passes option-shaped filenames after -- through positional validation", async () => {
await using temp = await scopedTemporaryDirectory("cli-literal-args-");
const deps = dependencies({ currentDirectory: temp.path });
const stdout = capture();
const stderr = capture();
expect(
await main(
["init", "--", "--version.yaml"],
stdout.stream,
stderr.stream,
deps,
),
).toBe(0);
expect(
await main(
["--", "init", "--mcp.json"],
stdout.stream,
stderr.stream,
deps,
),
).toBe(0);
expect((await readdir(temp.path)).sort()).toEqual([
"--mcp.json",
"--version.yaml",
]);
expect(
await main(
["init", "--", "--version"],
stdout.stream,
stderr.stream,
deps,
),
).toBe(2);
expect(stderr.text()).toContain(".yaml");
});
test("links both file import routes and keeps the vendor preview copyable", async () => {
const menu = (await help(["import", "--help"])).replace(/\s+/gu, " ");
expect(menu).toContain("publish findings --help");
expect(menu).toContain("scan import --help");
expect(menu).toContain("publish-findings skill");
expect(menu).toContain("publish a stored import");
const vendor = await help(["publish", "findings", "--help"], 60);
const preview = vendor
.split("\n")
.find((line) => line.startsWith(" codex-security publish findings "));
for (const flag of [
"--provider",
"--source-key",
"--repository",
"--to",
"--dry-run",
]) {
expect(preview).toContain(flag);
}
});
test("exposes the scan and its import child in both manifests", async () => {
for (const flag of ["--llms", "--llms-full"]) {
const manifest = JSON.parse(await help([flag, "--format", "json"]));
const names = manifest.commands.map(({ name }: { name: string }) => name);
expect(names.filter((name: string) => name === "scan")).toHaveLength(1);
expect(
names.filter((name: string) => name === "scan import"),
).toHaveLength(1);
}
});
test("publishes runnable examples with the same flags in help and the full manifest", async () => {
const manifest = JSON.parse(
await help(["--llms-full", "--format", "json"]),
) as {
commands: { name: string; examples?: { command: string }[] }[];
};
for (const [name, flags] of [
["scan", ["--path src --path tests"]],
["scan-components", ["--auto --plan-only", "--output-dir"]],
["publish findings", ["--source-key", "--dry-run"]],
] as const) {
const example = manifest.commands
.find((command) => command.name === name)
?.examples?.find(({ command }) =>
flags.every((flag) => command.includes(flag)),
)?.command;
expect(example).toBeDefined();
expect(example).not.toMatch(
/--[a-z-]+[A-Z]|--(?:auto|plan-only|dry-run) true/u,
);
expect(await help([...name.split(" "), "--help"], 60)).toContain(
` codex-security ${example}`,
);
}
});
test("completes scan import and its input format option from the command registry", async () => {
const previousShell = process.env["COMPLETE"];
const previousIndex = process.env["_COMPLETE_INDEX"];
try {
process.env["COMPLETE"] = "bash";
process.env["_COMPLETE_INDEX"] = "2";
expect((await help(["--", "cs", "scan", "im"])).trim()).toBe("import");
process.env["_COMPLETE_INDEX"] = "3";
expect(
(await help(["--", "cs", "scan", "import", "--input-f"])).trim(),
).toBe("--input-format");
} finally {
if (previousShell === undefined) delete process.env["COMPLETE"];
else process.env["COMPLETE"] = previousShell;
if (previousIndex === undefined) delete process.env["_COMPLETE_INDEX"];
else process.env["_COMPLETE_INDEX"] = previousIndex;
}
});
test.each([
{ command: ["mcp"] },
{ command: ["mcp", "add"] },
{ command: ["skills"] },
{ command: ["skills", "list"] },
{ command: ["completions"] },
])("retains framework help for $command", async ({ command }) => {
const text = await help([...command, "--help"]);
expect(text).toContain(`Usage: codex-security ${command.join(" ")}`);
if (command[0] === "skills") {
expect(text).toMatch(
command.length === 1 ? /^Aliases: skill$/mu : /^Aliases: ls$/mu,
);
}
if (command[0] === "completions") {
for (const shell of ["bash", "fish", "nushell", "zsh"]) {
expect(text).toContain(shell);
}
}
});
test.each([60, 80])(
"wraps descriptions at %i columns without breaking examples",
async (columns) => {
for (const [command, examples] of [
[[], []],
[["scan"], [/^ codex-security scan \.(?:\s+#.*)?$/mu]],
[
["login"],
[
/^ ssh -L 1455:localhost:1455 user@remote$/mu,
/^ codex-security login$/mu,
],
],
[["publish", "scan"], []],
[["scan", "import"], []],
] as const) {
const text = await help([...command, "--help"], columns);
expect(text).not.toMatch(/\u001b\[/u);
for (const line of text.split("\n")) {
if (line.startsWith(" codex-security ")) continue;
const indent = line.length - line.trimStart().length;
const longestToken = Math.max(
...line
.trim()
.split(/\s+/u)
.map((word) => word.length),
);
expect(line.length, line).toBeLessThanOrEqual(
Math.max(columns, indent + longestToken),
);
}
for (const example of examples) expect(text).toMatch(example);
}
},
);
});