OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import type { JsonObject, ScanOptions } from "../src/index.js";
import { BUNDLED_PLUGIN_VERSION, VERSION } from "../src/version.js";
import {
mkdir,
mkdtemp,
readFile,
readdir,
rm,
realpath,
symlink,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
test,
mock,
} from "bun:test";
import { main } from "../src/cli.js";
import { DiffTarget } from "../src/targets.js";
import { resolvePluginPython, runWorkbench } from "../src/runtime.js";
import { loadContract } from "../src/contract.js";
import { ScanInterruptedError } from "../src/errors.js";
import { TestClient } from "./support/api-client.js";
import { capture, dependencies, fakeResult } from "./cli-fixtures.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { runCommand } from "./support/shell.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { rejecting, throwing } from "./support/errors.js";
const { temporaryDirectories: roots, cleanup } = createApiTestFixtures();
afterEach(cleanup);
async function fixture(workbench: typeof runWorkbench = runWorkbench) {
const root = await mkdtemp(join(await realpath(tmpdir()), "mock-scan-test-"));
roots.track(root);
const repository = join(root, "repository");
await mkdir(repository);
await writeFile(
join(repository, "example.ts"),
"export const example = 1;\n",
);
const environment = {
PATH: process.env["PATH"],
SystemRoot: process.env["SystemRoot"],
CODEX_HOME: join(root, "codex-home"),
CODEX_SECURITY_STATE_DIR: join(root, "state"),
};
const python = await resolvePluginPython({
environment,
protectedRoot: repository,
});
const client = new TestClient(
{ pythonPath: python },
{
environment,
runWorkbench: workbench,
resolvePluginPython: async () => python,
prepareRuntime: rejecting("Mock scan initialized Codex"),
matchFindings: rejecting("Mock scan called model matching"),
},
);
return { root, repository, client, environment, python };
}
test("mock scans seal real artifacts and index shared and unique findings without Codex", async () => {
const { repository, client, environment, python } = await fixture();
const callbacks: string[] = [];
const registrations: Array<
Parameters<NonNullable<ScanOptions["onScanRegistered"]>>[0]
> = [];
const onLifecycle = mock<NonNullable<ScanOptions["onLifecycle"]>>();
onLifecycle.mockImplementationOnce(rejecting("observer unavailable"));
const onObserverError = mock<NonNullable<ScanOptions["onObserverError"]>>();
try {
const first = await client.run(repository, {
mock: true,
auth: "api-key",
maxCostUsd: 0.01,
failureSeverity: "informational",
onAuthentication: () => callbacks.push("authentication"),
onLifecycle,
onObserverError,
onScanRegistered: (scan) => {
callbacks.push("registered");
registrations.push(scan);
},
onScanStarted: () => callbacks.push("started"),
});
const second = await client.run(repository, { mock: true });
expect(callbacks).toEqual(["registered", "started"]);
const launch = {
scanId: first.manifest.scan.id,
mode: "standard",
startedAt: first.manifest.scan.startedAt,
status: "running",
finishedAt: null,
hasReportableFindings: null,
surface: "sdk",
packageVersion: VERSION,
pluginVersion: BUNDLED_PLUGIN_VERSION,
mock: true,
} as const;
expect(onLifecycle.mock.calls).toEqual([
[launch],
[
{
...launch,
status: "completed",
finishedAt: first.manifest.scan.completedAt,
hasReportableFindings: true,
},
],
]);
expect(onObserverError).toHaveBeenCalledWith(
"onLifecycle",
expect.any(Error),
);
expect(registrations).toEqual([
{
scanId: first.manifest.scan.id,
scanDir: first.scanDir,
},
]);
expect(first.findings.findings).toHaveLength(12);
expect(first.review).not.toHaveProperty("status");
expect(first.manifest.scan.status).toBe("completed");
expect(first.manifest.scan.sealedAt).toBeTruthy();
expect(first.turnResult["mock"]).toBe(true);
expect(first.threadId).toBe("");
expect(first.cost?.estimatedUsd).toBe(0);
expect(first.cost?.inputTokens).toBe(0);
expect(first.cost?.outputTokens).toBe(0);
expect(await readFile(first.reportPath, "utf8")).toContain(
"Synthetic mock scan",
);
expect(
await loadContract(first.scanDir, { pluginRoot: PLUGIN_ROOT }),
).toEqual({
manifest: first.manifest,
findings: first.findings,
review: first.review,
mode: first.mode,
});
const previousIds = new Set(
first.findings.findings.map((finding) => finding.findingId),
);
const recurring = second.findings.findings.filter((finding) =>
previousIds.has(finding.findingId),
);
expect(recurring).toHaveLength(8);
expect(
new Set([
...previousIds,
...second.findings.findings.map((finding) => finding.findingId),
]).size,
).toBe(16);
expect(
new Set(first.findings.findings.map((finding) => finding.severity.level))
.size,
).toBe(5);
for (const id of ["command-injection", "path-traversal"]) {
const pair = first.findings.findings.filter(
(finding) => finding.extensions?.["mockGroup"] === id,
);
expect(pair).toHaveLength(2);
expect(pair[0]!.findingId).not.toBe(pair[1]!.findingId);
expect(pair[0]!.locations).toEqual(pair[1]!.locations);
}
expect(second.repositoryFindings).toHaveLength(16);
expect(
second.repositoryFindings?.filter(
(finding) => finding.knownScanIds?.length === 2,
),
).toHaveLength(8);
expect(await readdir(repository)).toEqual(["example.ts"]);
expect(await readFile(join(repository, "example.ts"), "utf8")).toBe(
"export const example = 1;\n",
);
const history = await runWorkbench(
{ python, pluginRoot: PLUGIN_ROOT, environment },
["list-scans", "--repository", repository],
);
expect(history["scans"]).toHaveLength(2);
const saved = await runWorkbench(
{ python, pluginRoot: PLUGIN_ROOT, environment },
["get-scan", "--scan-id", first.manifest.scan.id],
);
const savedScan = saved["scan"] as JsonObject;
expect(typeof savedScan["updatedAt"]).toBe("string");
const showOutput = capture(true);
const showDiagnostics = capture();
expect(
await main(
["scans", "show", first.manifest.scan.id],
showOutput.stream,
showDiagnostics.stream,
dependencies({ onWorkbench: async () => saved }),
),
).toBe(0);
expect(showDiagnostics.text()).toContain(
`updated ${savedScan["updatedAt"]}`,
);
expect(showOutput.text()).toContain("UPDATED");
expect(showOutput.text()).toContain(savedScan["updatedAt"] as string);
const recipe = await runWorkbench(
{ python, pluginRoot: PLUGIN_ROOT, environment },
["get-scan-recipe", "--scan-id", first.manifest.scan.id],
);
const onTurn = mock((_repository: string, { mock }: ScanOptions) => mock);
expect(
await main(
["scans", "rerun", first.manifest.scan.id],
capture().stream,
capture().stream,
dependencies({
currentDirectory: repository,
onWorkbench: async () => recipe,
onTurn,
}),
),
).toBe(0);
expect(onTurn.mock.results.at(-1)?.value).toBe(true);
expect(onTurn.mock.lastCall?.[1].failureSeverity).toBe("informational");
} finally {
await client.close();
}
});
test("mock scans preserve output protection and archive existing completed results", async () => {
const { root, repository, client } = await fixture();
try {
await expect(
client.run(repository, {
mock: true,
outputDir: join(repository, "results"),
}),
).rejects.toThrow();
const outputDir = join(root, "results");
const firstOptions = {
mock: true,
outputDir,
target: ["example.ts"],
workflowId: "synthetic-archived-workflow",
};
const first = await client.run(repository, firstOptions);
const original = await readFile(first.manifestPath, "utf8");
await expect(
client.run(repository, { mock: true, outputDir }),
).rejects.toThrow();
expect(await readFile(first.manifestPath, "utf8")).toBe(original);
const onOutputArchived = mock((_path: string) => {});
const second = await client.run(repository, {
mock: true,
outputDir,
archiveExisting: true,
onOutputArchived,
});
const archive = onOutputArchived.mock.lastCall?.[0] ?? "";
expect(await readFile(join(archive, "scan-manifest.json"), "utf8")).toBe(
original,
);
expect(second.manifest.scan.id).not.toBe(first.manifest.scan.id);
expect(first.manifest.scan.mode).toBe("scoped_path");
expect(first.manifest.scan.scope.includePaths).toEqual(["example.ts"]);
const resumed = await client.run(repository, firstOptions);
expect(resumed.manifest).toEqual(first.manifest);
expect(resumed.scanDir).toBe(archive);
expect(resumed.sarifPath).toBe(join(archive, "exports", "results.sarif"));
} finally {
await client.close();
}
});
test.skipIf(process.platform === "win32")(
"durable mock scans resume only from the same physical output directory",
async () => {
const { root, repository, client } = await fixture();
const linkedOutput = (id: string) => `${join(root, `${id}-link`)}/../scan`;
const run = async (
workflowId: string,
outputDir = linkedOutput(workflowId),
) => {
const { manifest, scanDir } = await client.run(repository, {
mock: true,
workflowId,
outputDir,
});
return { scanId: manifest.scan.id, scanDir };
};
try {
for (const id of ["first", "second"]) {
const child = join(root, id, "child");
await mkdir(child, { recursive: true, mode: 0o700 });
await symlink(child, join(root, `${id}-link`));
}
const first = await run("first");
const second = await run("second");
expect(first.scanDir).toBe(join(root, "first", "scan"));
expect(second.scanDir).toBe(join(root, "second", "scan"));
await expect(run("first", linkedOutput("second"))).rejects.toThrow(
"already bound to a different",
);
expect(await run("first", first.scanDir)).toEqual(first);
expect(await run("second", second.scanDir)).toEqual(second);
} finally {
await client.close();
}
},
);
test("mock scan CLI forwards the flag and never offers authentication or patching", async () => {
const stdout = capture();
const stderr = capture(true);
const onTurn = mock((_repository: string, { mock }: ScanOptions) => mock);
const code = await main(
["scan", ".", "--mock", "--fail-on-severity", "high", "--auth", "api-key"],
stdout.stream,
stderr.stream,
{
...dependencies({
result: fakeResult(["high"]),
onTurn,
}),
confirmPatchReview: rejecting("Unexpected patch prompt"),
},
);
expect(code).toBe(1);
expect(onTurn.mock.results.at(-1)?.value).toBe(true);
expect(stderr.text()).toContain("Mock scan");
});
test("mock scans bind clean Git, committed diff, and working-tree snapshots", async () => {
const { root, repository, client } = await fixture();
const git = async (...args: string[]) => {
const result = await runCommand("git", [
"-C",
repository,
"-c",
`core.hooksPath=${join(root, "hooks")}`,
"-c",
"commit.gpgsign=false",
"-c",
"user.name=Example",
"-c",
"[email protected]",
...args,
]);
expect(result.status).toBe(0);
};
try {
await git("init");
await git("add", ".");
await git("commit", "-m", "Initial synthetic fixture");
const clean = await client.run(repository, { mock: true });
expect(clean.manifest.scan.target.kind).toBe("git_revision");
await writeFile(
join(repository, "example.ts"),
"export const example = 2;\n",
);
await git("commit", "-am", "Update synthetic fixture");
const onLifecycle = mock<NonNullable<ScanOptions["onLifecycle"]>>();
const diff = await client.run(repository, {
mock: true,
target: DiffTarget.refs({ base: "HEAD~1" }),
onLifecycle,
});
expect(
onLifecycle.mock.calls.map(([{ mode, status }]) => [mode, status]),
).toEqual([
["diff", "running"],
["diff", "completed"],
]);
expect(diff.manifest.scan.target.kind).toBe("git_diff");
expect(diff.manifest.scan.mode).toBe("branch_diff");
await writeFile(
join(repository, "example.ts"),
"export const example = 3;\n",
);
const workingTree = await client.run(repository, {
mock: true,
target: DiffTarget.workingTree({}),
});
expect(workingTree.manifest.scan.target.kind).toBe("git_diff");
expect(workingTree.manifest.scan.mode).toBe("working_tree");
} finally {
await client.close();
}
});
test.each(["abort", "close", "timeout"] as const)(
"stopping mock generation via %s records its terminal outcome",
async (stop) => {
const { client, repository, environment, python } = await fixture();
const controller = new AbortController();
const onLifecycle = mock<NonNullable<ScanOptions["onLifecycle"]>>();
try {
await expect(
client.run(repository, {
mock: true,
signal: controller.signal,
onLifecycle,
onScanStarted: () => {
if (stop === "close") void client.close();
else if (stop === "timeout")
controller.abort(new DOMException("deadline", "TimeoutError"));
else controller.abort();
},
}),
).rejects.toThrow(
stop === "close" ? "CodexSecurity is closed" : ScanInterruptedError,
);
const terminal = stop === "timeout" ? "failed" : "canceled";
const history = await runWorkbench(
{ python, pluginRoot: PLUGIN_ROOT, environment },
["list-scans", "--repository", repository],
);
expect(history["scans"]).toMatchObject([
{ progress: { status: terminal } },
]);
const [saved] = history["scans"] as JsonObject[];
expect(
onLifecycle.mock.calls.map(([snapshot]) => snapshot.status),
).toEqual(["running", terminal]);
expect(saved?.["lifecycle"]).toEqual(onLifecycle.mock.lastCall?.[0]);
} finally {
await client.close();
}
},
);
test("reports committed completion after the workbench response is lost", async () => {
const { client, repository } = await fixture(async (options, args, input) => {
if (args[0] === "get-scan")
throw new Error("Full scan details unavailable");
const saved = await runWorkbench(options, args, input);
if (args[0] === "complete-scan") throw new Error("completion reply lost");
return saved;
});
const onLifecycle = mock<NonNullable<ScanOptions["onLifecycle"]>>();
try {
await expect(
client.run(repository, { mock: true, onLifecycle }),
).rejects.toThrow("completion reply lost");
expect(onLifecycle.mock.calls.map(([snapshot]) => snapshot.status)).toEqual(
["running", "completed"],
);
} finally {
await client.close();
}
});
test.each(["--dry-run", "--patch"])(
"mock scan CLI rejects %s",
async (flag) => {
const stderr = capture();
const code = await main(
["scan", "--mock", flag],
capture().stream,
stderr.stream,
dependencies({
onRun: throwing("Unexpected scan"),
}),
);
expect(code).toBe(2);
expect(stderr.text()).toContain("--mock cannot be combined");
},
);
test.each([
{ mode: "deep" as const },
{ validationPrompt: "Validate" },
{ postScanPrompt: "Follow up" },
])("mock scans reject model-dependent workflows: %j", async (options) => {
const { client, repository } = await fixture();
try {
await expect(
client.run(repository, { mock: true, ...options }),
).rejects.toThrow("require model calls");
} finally {
await client.close();
}
});
test("history archive compatibility restores completed mock output after a checkout is replaced", async () => {
const { root, repository, client, environment, python } = await fixture();
const git = async (...args: string[]) => {
const result = await runCommand(
"git",
[
"-C",
repository,
"-c",
"core.hooksPath=/dev/null",
"-c",
"commit.gpgsign=false",
"-c",
"user.name=Example",
"-c",
"[email protected]",
...args,
],
{ timeout: 10000 },
);
expect(result.status).toBe(0);
};
const output = join(root, "completed-output");
try {
await git("init");
await git("add", ".");
await git("commit", "-m", "Initial synthetic fixture");
const first = await client.run(repository, {
mock: true,
outputDir: output,
});
const manifest = await readFile(join(output, "scan-manifest.json"));
await rm(join(repository, ".git"), { recursive: true, force: true });
await git("init");
await git("add", ".");
await git("commit", "-m", "Replacement synthetic fixture");
await expect(
client.run(repository, {
mock: true,
outputDir: output,
archiveExisting: true,
}),
).rejects.toThrow();
expect(await readFile(join(output, "scan-manifest.json"))).toEqual(
manifest,
);
const history = await runWorkbench(
{ python, pluginRoot: PLUGIN_ROOT, environment },
["list-scans", "--scan-root", output],
);
expect(
(history["scans"] as { scanId: string; scanDir: string }[]).find(
({ scanId }) => scanId === first.manifest.scan.id,
)?.scanDir,
).toBe(output);
expect(
(await readdir(root)).filter((name) =>
name.startsWith("completed-output.previous-"),
),
).toEqual([]);
} finally {
await client.close();
}
});
describe("interrupted workbench registration", () => {
let registeredId: string | undefined;
let saved: { id: string; status: string } | undefined;
afterAll(cleanup);
beforeAll(async () => {
const controller = new AbortController();
const { client, repository, environment, python } = await fixture(
async (options, args, input) => {
const reply = await runWorkbench(options, args, input);
if (args[0] === "register-cli-scan") {
if (typeof reply["scanId"] !== "string")
throw new Error("Missing registered scan ID");
registeredId = reply["scanId"];
controller.abort();
throw new DOMException(
"Synthetic registration reply lost",
"AbortError",
);
}
return reply;
},
);
const workbench = { environment, python, pluginRoot: PLUGIN_ROOT };
try {
await expect(
client.run(repository, { mock: true, signal: controller.signal }),
).rejects.toThrow(ScanInterruptedError);
if (registeredId === undefined)
throw new Error("Registration did not commit");
const { scan } = await runWorkbench(workbench, [
"get-scan",
"--scan-id",
registeredId,
]);
const row = scan as { scanId: string; lifecycle: { status: string } };
saved = { id: row.scanId, status: row.lifecycle.status };
} finally {
await client.close();
}
}, 60_000);
test("finds the exact saved scan after interruption", () => {
expect(registeredId).toEqual(expect.any(String));
expect(saved?.id).toBe(registeredId);
expect(saved?.status).toEqual(expect.any(String));
});
test.todo("marks that saved scan canceled when its reply was lost", () => {
expect(saved?.status).toBe("canceled");
});
});