OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import {
chmod,
cp,
mkdir,
readFile,
readdir,
rename,
rm,
stat,
symlink,
writeFile,
} from "node:fs/promises";
import * as fsPromises from "node:fs/promises";
import { execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import { dirname, join, relative } from "node:path";
import { describe, expect, spyOn, test } from "bun:test";
import { parse as parseToml } from "smol-toml";
import { resolveCodexProfile, type JsonObject } from "../src/config.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { shellEnvironmentReference } from "./support/api-client.js";
import { readKnowledgeBaseSnapshot } from "../src/knowledge-base.js";
import { createValidationFixtures } from "./support/validation.js";
describe("CodexSecurity finding validation", () => {
const {
actualLstat,
assessment,
validationEvents,
validationClient,
persistedValidationClient,
temporaryDirectory,
} = createValidationFixtures();
for (const sourceOwned of [false, true]) {
test(`validates knowledge directory links against the traversed repository (source-owned=${sourceOwned})`, async () => {
const fixture = await validationClient();
await using client = fixture.client;
const spelling = join(fixture.root, "spelling");
const child = join(fixture.root, "child");
const documents = join(fixture.root, "documents");
await Promise.all([mkdir(spelling), mkdir(child), mkdir(documents)]);
await symlink(child, join(spelling, "jump"), "junction");
const lexical = join(spelling, "repository");
if (sourceOwned || process.platform === "win32") {
await mkdir(lexical);
execFileSync("git", ["init", "--quiet", lexical]);
}
const repository =
process.platform === "win32" ? lexical : fixture.options.repositoryPath;
const alias = join(sourceOwned ? repository : spelling, "knowledge");
await symlink(documents, alias, "junction");
await writeFile(
join(documents, "policy.md"),
"Synthetic operator guidance.",
);
const result = client.validate({
...fixture.options,
repositoryPath: `${spelling}/jump/../repository`,
knowledgeBasePaths: [join(alias, "policy.md")],
});
if (sourceOwned) {
await expect(result).rejects.toThrow(
"repository directory links outside the selected repository",
);
expect(fixture.captured.thread).toBeUndefined();
} else {
await expect(result).resolves.toMatchObject(assessment);
expect(fixture.captured.codex?.env?.["CODEX_SECURITY_REPOSITORY"]).toBe(
await fsPromises.realpath(repository),
);
}
});
}
test("invalidates cached validation when inline instructions change", async () => {
let modelCalls = 0;
let instructions = "Original standalone validation instructions.";
const instructionPath = join(
PLUGIN_ROOT,
"references",
"standalone-validation.md",
);
const actualReadFile = fsPromises.readFile;
const readInstructions = spyOn(fsPromises, "readFile").mockImplementation(
(async (...args: Parameters<typeof actualReadFile>) =>
args[0] === instructionPath
? instructions
: await actualReadFile(...args)) as typeof actualReadFile,
);
try {
async function* events() {
modelCalls += 1;
yield* validationEvents();
}
const fixture = await persistedValidationClient(events);
await using client = fixture.client;
const workflowId = await fixture.bindWorkflow("validation-instructions");
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
};
const first = await client.validate(request);
expect(await client.validate(request)).toEqual(first);
expect(modelCalls).toBe(1);
instructions = "Updated standalone validation instructions.";
await client.validate(request);
expect(modelCalls).toBe(2);
expect(fixture.captured.prompt).toContain(instructions);
} finally {
readInstructions.mockRestore();
}
});
test.each([true, false])(
"reuses a validation only when bundled references are used (custom: %j)",
async (custom) => {
const pluginRoot = custom
? join(await temporaryDirectory(), "selected-plugin")
: PLUGIN_ROOT;
const reference = join(
await temporaryDirectory(),
"additional-guidance.md",
);
await writeFile(reference, "Original synthetic guidance.");
if (custom) {
await cp(PLUGIN_ROOT, pluginRoot, { recursive: true });
await writeFile(
join(pluginRoot, "references", "standalone-validation.md"),
`Read the validation guidance at ${JSON.stringify(reference)}.\n`,
);
}
let modelCalls = 0;
async function* events() {
modelCalls += 1;
yield* validationEvents(
JSON.stringify({
...assessment,
report: await readFile(reference, "utf8"),
}),
);
}
const fixture = await persistedValidationClient(events, pluginRoot);
await using client = fixture.client;
const workflowId = await fixture.bindWorkflow(
"additional-validation-reference",
);
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
};
const first = await client.validate(request);
expect((await client.validate(request)).report).toBe(first.report);
expect(modelCalls).toBe(custom ? 2 : 1);
if (custom) {
await writeFile(reference, "Updated synthetic guidance.");
expect((await client.validate(request)).report).toBe(
"Updated synthetic guidance.",
);
expect(modelCalls).toBe(3);
}
expect(fixture.captured.prompt).toContain(
await readFile(
join(pluginRoot, "references", "standalone-validation.md"),
"utf8",
),
);
},
);
test("reads unchanged and updated system-configured validation instructions", async () => {
const root = await temporaryDirectory("native-system-validation-");
const instructions = join(root, "instructions.md");
const nativeConfig = join(root, "config.toml");
await writeFile(instructions, "Original synthetic system guidance.");
await writeFile(
nativeConfig,
`model_instructions_file = ${JSON.stringify(instructions)}\n`,
);
let modelCalls = 0;
async function* events() {
modelCalls += 1;
const configuration = parseToml(await readFile(nativeConfig, "utf8"));
yield* validationEvents(
JSON.stringify({
...assessment,
report: await readFile(
configuration["model_instructions_file"] as string,
"utf8",
),
}),
);
}
const fixture = await persistedValidationClient(
events,
PLUGIN_ROOT,
undefined,
{ ProgramData: root },
);
await using client = fixture.client;
const workflowId = await fixture.bindWorkflow(
"system-validation-instructions",
);
const nativePath =
process.platform === "win32"
? join(root, "OpenAI", "Codex", "config.toml")
: "/etc/codex/config.toml";
const originalLstat = actualLstat;
const nativeLookup = spyOn(fsPromises, "lstat").mockImplementation(((
...args: Parameters<typeof originalLstat>
) =>
originalLstat(
args[0] === nativePath ? nativeConfig : args[0],
args[1],
)) as typeof originalLstat);
try {
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
};
const first = await client.validate(request);
expect((await client.validate(request)).report).toBe(first.report);
expect(modelCalls).toBe(2);
await writeFile(instructions, "Updated synthetic system guidance.");
expect((await client.validate(request)).report).toBe(
"Updated synthetic system guidance.",
);
expect(modelCalls).toBe(3);
} finally {
nativeLookup.mockRestore();
}
});
test.each([
{ setting: "top-level", changed: true },
{ setting: "top-level", changed: false },
{ setting: "selected profile", changed: true },
{ setting: "selected profile", changed: false },
{ setting: "none", changed: false },
] as const)(
"runs file-backed validation instructions through Codex, %p",
async ({ setting, changed }) => {
const instructions = join(await temporaryDirectory(), "instructions.md");
await writeFile(instructions, "Original synthetic instructions.");
let modelCalls = 0;
async function* events() {
modelCalls += 1;
yield* validationEvents(
JSON.stringify({
...assessment,
report: await readFile(instructions, "utf8"),
}),
);
}
const fixture = await persistedValidationClient(events);
await using client = fixture.client;
if (setting === "top-level") {
client.config.codexOverrides!["model_instructions_file"] = instructions;
} else if (setting === "selected profile") {
client.config.codexOverrides!["profile"] = "selected";
client.config.codexOverrides!["profiles"] = {
selected: { model_instructions_file: instructions },
};
}
const workflowId = await fixture.bindWorkflow("file-backed-validation");
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
};
const first = await client.validate(request);
expect(modelCalls).toBe(1);
if (changed)
await writeFile(instructions, "Updated synthetic instructions.");
const second = await client.validate(request);
expect(modelCalls).toBe(setting === "none" ? 1 : 2);
expect(second.report).toBe(
changed ? "Updated synthetic instructions." : first.report,
);
expect(
resolveCodexProfile(fixture.captured.codex!.config as JsonObject)[
"model_instructions_file"
],
).toBe(setting === "none" ? undefined : instructions);
},
);
test.each([
{ modelFails: false, cleanupFails: true },
{ modelFails: true, cleanupFails: true },
{ modelFails: false, cleanupFails: false },
{ modelFails: true, cleanupFails: false },
])(
"preserves the validation outcome when temporary knowledge cleanup fails: %p",
async ({ modelFails, cleanupFails }) => {
const modelError = new Error("Synthetic validation failure.");
const cleanupError = new Error(
"Synthetic temporary knowledge cleanup failure.",
);
async function* events() {
if (modelFails) throw modelError;
yield* validationEvents();
}
const fixture = await validationClient(events);
await using client = fixture.client;
const document = join(fixture.root, "knowledge.md");
await writeFile(document, "Synthetic primary project guidance.");
const actualRm = fsPromises.rm;
let cleanupPath: string | undefined;
const cleanupSpy = spyOn(fsPromises, "rm").mockImplementation(
async (path, options) => {
if (
path ===
fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"]
) {
cleanupPath = String(path);
if (cleanupFails) throw cleanupError;
}
return await actualRm(path, options);
},
);
try {
const result = client.validate({
...fixture.options,
knowledgeBasePaths: [document],
});
if (modelFails) await expect(result).rejects.toBe(modelError);
else await expect(result).resolves.toMatchObject(assessment);
expect(cleanupPath).toBeDefined();
expect(await readFile(document, "utf8")).toBe(
"Synthetic primary project guidance.",
);
} finally {
cleanupSpy.mockRestore();
if (cleanupPath !== undefined)
await actualRm(cleanupPath, { recursive: true, force: true });
}
},
);
test.each([
["unchanged", false],
["modified", false],
["modified", true],
["renamed", true],
["deleted", true],
] as const)(
"retains supplied knowledge context for workflow validation; change=%s, snapshot=%p",
async (change, snapshot) => {
let modelCalls = 0;
const normalizedPaths: string[] = [];
let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
async function* events() {
modelCalls += 1;
const path =
fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"];
expect(path).toBeDefined();
normalizedPaths.push(path!);
const files = await readdir(path!);
const context = await readFile(join(path!, files[0]!), "utf8");
yield* validationEvents(
JSON.stringify({ ...assessment, report: context }),
);
}
fixture = await persistedValidationClient(events);
await using client = fixture.client;
const document = join(fixture.root, "policy.md");
await writeFile(document, "Original synthetic policy.");
const workflowId = await fixture.bindWorkflow("knowledge-validation");
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
knowledgeBasePaths: [document],
...(snapshot
? {
knowledgeBaseSnapshot: await readKnowledgeBaseSnapshot([
document,
]),
}
: {}),
};
const first = await client.validate(request);
expect(first.report).toBe("Original synthetic policy.");
expect(fixture.captured.prompt).toContain(
shellEnvironmentReference("CODEX_SECURITY_KNOWLEDGE_BASE"),
);
expect(fixture.captured.prompt).not.toContain(
"Original synthetic policy.",
);
const movedDocument = join(fixture.root, "moved-policy.md");
if (change === "modified")
await writeFile(document, "Updated synthetic policy.");
else if (change === "renamed") await rename(document, movedDocument);
else if (change === "deleted") await rm(document);
const second = await client.validate(request);
expect(modelCalls).toBe(change === "modified" && !snapshot ? 2 : 1);
expect(second.report).toBe(
change === "modified" && !snapshot
? "Updated synthetic policy."
: first.report,
);
if (change === "renamed" || change === "deleted") {
const fresh = await client.validate({
...request,
finding: "Another synthetic candidate.",
});
expect(fresh.report).toBe(first.report);
expect(modelCalls).toBe(2);
}
expect(normalizedPaths.every((path) => !existsSync(path))).toBe(true);
if (change === "deleted") expect(existsSync(document)).toBe(false);
else
expect(
await readFile(
change === "renamed" ? movedDocument : document,
"utf8",
),
).toBe(
change === "modified" ? "Updated synthetic policy." : first.report,
);
},
);
test
.skipIf(process.platform === "win32")
.each(["workflow", "file-backed workflow", "no workflow"] as const)(
"validates an ignored FIFO without requiring a cache snapshot: %s",
async (mode) => {
let modelCalls = 0;
async function* events() {
modelCalls += 1;
yield* validationEvents();
}
const fixture = await persistedValidationClient(events);
await using client = fixture.client;
const repository = fixture.options.repositoryPath;
execFileSync("git", ["init", "-q", repository]);
await writeFile(join(repository, ".gitignore"), "ignored-pipe\n");
execFileSync(fixture.python, [
"-c",
"import os,sys; os.mkfifo(sys.argv[1])",
join(repository, "ignored-pipe"),
]);
const workflowId = await fixture.bindWorkflow(
"unsupported-validation-snapshot",
);
if (mode === "file-backed workflow") {
const instructions = join(fixture.root, "instructions.md");
await writeFile(instructions, "Synthetic validation instructions.");
client.config.codexOverrides!["model_instructions_file"] = instructions;
}
const request = {
...fixture.options,
outputDir: undefined,
...(mode === "no workflow" ? {} : { workflowId }),
};
await client.validate(request);
await client.validate(request);
expect(modelCalls).toBe(2);
expect(fixture.captured.prompt).toContain("Candidate finding");
},
);
test
.skipIf(process.platform === "win32" || process.geteuid?.() === 0)
.each(["file", "directory"] as const)(
"validates unreadable ignored cache entries: %s",
async (kind) => {
let modelCalls = 0;
async function* events() {
modelCalls += 1;
yield* validationEvents();
}
const fixture = await persistedValidationClient(events);
await using client = fixture.client;
const repository = fixture.options.repositoryPath;
execFileSync("git", ["init", "-q", repository]);
await writeFile(join(repository, ".gitignore"), "ignored-cache\n");
const entry = join(repository, "ignored-cache");
if (kind === "directory") {
await mkdir(entry);
await writeFile(join(entry, "cache.txt"), "Synthetic cache data.");
} else await writeFile(entry, "Synthetic cache data.");
const mode = (await stat(entry)).mode;
const workflowId = await fixture.bindWorkflow(
"unreadable-validation-cache",
);
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
};
await chmod(entry, 0);
try {
expect(await client.validate(request)).toMatchObject(assessment);
expect(await client.validate(request)).toMatchObject(assessment);
expect(modelCalls).toBe(2);
} finally {
await chmod(entry, mode);
}
expect(await client.validate(request)).toMatchObject(assessment);
expect(await client.validate(request)).toMatchObject(assessment);
expect(modelCalls).toBe(3);
},
);
test.each([
"home",
"output worktree",
"output project configuration",
"none",
] as const)("uses current native validation guidance: %s", async (scope) => {
const stateRoot = await temporaryDirectory("native-validation-guidance-");
if (scope === "output worktree" || scope === "output project configuration")
execFileSync("git", ["init", "-q", stateRoot]);
let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
let guide: string | undefined;
let modelCalls = 0;
async function* events() {
modelCalls += 1;
expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(
fixture.codexHome,
);
if (
scope === "output worktree" ||
scope === "output project configuration"
)
expect(
relative(
stateRoot,
fixture.captured.thread!.workingDirectory!,
).startsWith(".."),
).toBe(false);
yield* validationEvents(
JSON.stringify({
...assessment,
report:
guide === undefined
? "Synthetic assessment."
: scope === "output project configuration"
? parseToml(await readFile(guide, "utf8"))[
"developer_instructions"
]
: await readFile(guide, "utf8"),
}),
);
}
fixture = await persistedValidationClient(events, PLUGIN_ROOT, undefined, {
CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),
});
await using client = fixture.client;
guide =
scope === "home"
? join(fixture.codexHome, "AGENTS.md")
: scope === "output worktree"
? join(stateRoot, "AGENTS.md")
: scope === "output project configuration"
? join(stateRoot, ".codex", "config.toml")
: undefined;
if (guide !== undefined) {
await mkdir(dirname(guide), { recursive: true });
await writeFile(
guide,
scope === "output project configuration"
? 'developer_instructions = "Original synthetic guidance."\n'
: "Original synthetic guidance.",
);
}
const workflowId = await fixture.bindWorkflow("native-validation-guidance");
const request = {
...fixture.options,
outputDir: undefined,
workflowId,
};
const first = await client.validate(request);
expect((await client.validate(request)).report).toBe(first.report);
expect(modelCalls).toBe(guide === undefined ? 1 : 2);
if (guide !== undefined) {
await writeFile(
guide,
scope === "output project configuration"
? 'developer_instructions = "Updated synthetic guidance."\n'
: "Updated synthetic guidance.",
);
expect((await client.validate(request)).report).toBe(
"Updated synthetic guidance.",
);
expect(modelCalls).toBe(3);
}
});
});