OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import { execFileSync } from "node:child_process";
import { resolveCodexProfile } from "../src/config.js";
import { createHash, randomUUID } from "node:crypto";
import { mkdirSync, renameSync, writeFileSync } from "node:fs";
import {
appendFile,
chmod,
cp,
mkdir,
readFile,
rm,
stat,
symlink,
writeFile,
} from "node:fs/promises";
import { join, resolve } from "node:path";
import type { ScanReadFile } from "../../../plugins/codex-security/native/scan-read-binding.mjs";
import type { ThreadEvent } from "@openai/codex-sdk";
import type { ScanActivity } from "../src/scan-activity.js";
import type { ScanSessionEvent } from "../src/cost.js";
import { progressMarker, writeSession } from "./support/usage-rollout.js";
import Ajv2020 from "ajv/dist/2020.js";
import { afterEach, describe, expect, mock, spyOn, test } from "bun:test";
import {
runCustomValidation,
type CustomValidationResult,
} from "../src/custom-validation.js";
import {
customDiscoveryPrompt,
customValidationConfig,
} from "../src/custom-validation-prompt.js";
import {
ContractValidationError,
DiffTarget,
type CoverageDocument,
type FindingsDocument,
type ScanManifest,
type ScanProgress,
} from "../src/index.js";
import { createMarketplace, resolveCodexCommand } from "../src/runtime.js";
import { copyCompletedScanFixture, PLUGIN_ROOT } from "./plugin-root.js";
import { runWorkbench } from "../src/runtime.js";
import { TestClient } from "./support/api-client.js";
import { completedEvents, preparedRuntime } from "./support/api-events.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { readJson as json, jsonLines } from "./support/json.js";
import { rejecting } from "./support/errors.js";
import { runTestInSubprocess } from "./support/test-subprocess.js";
import { scanReadBinding } from "./support/scan-read-binding.js";
import { readReviewState } from "./support/review-state.js";
const { cleanup, temporaryDirectory } = createApiTestFixtures();
const resultName = "artifacts/custom-validation/results.json";
afterEach(cleanup);
async function save(path: string, value: unknown) {
await writeFile(path, JSON.stringify(value));
}
async function draft(scanDir: string, scanId: string, count = 1, diff = false) {
await copyCompletedScanFixture(scanDir);
const manifest = await json<ScanManifest>(
join(scanDir, "scan-manifest.json"),
);
const { sealedAt: _sealed, artifacts: _artifacts, ...scan } = manifest.scan;
scan.id = scanId;
scan.target.kind = diff ? "git_diff" : "directory_snapshot";
scan.scope.validationMode = "custom_pending";
await save(join(scanDir, "scan-manifest.json"), { ...manifest, scan });
const findings = await json<FindingsDocument>(join(scanDir, "findings.json"));
const original = findings.findings[0]!;
findings.scanId = scanId;
findings.findings = Array.from({ length: count }, (_, index) => ({
...structuredClone(original),
identity: { anchor: `fixture-${index}` },
title: `Fixture ${index}`,
extensions: { customValidationSurfaceIds: [`surface-${index}`] },
}));
await save(join(scanDir, "findings.json"), findings);
const coverage = await json<CoverageDocument>(join(scanDir, "coverage.json"));
coverage.scanId = scanId;
coverage.mode = diff ? "diff" : "repository";
coverage.inventoryStrategy = diff ? "diff" : "directory";
coverage.surfaces = findings.findings.map((finding, index) => ({
id: `surface-${index}`,
label: finding.title,
disposition: "reported",
receiptRefs: [],
}));
if (count === 0)
coverage.surfaces.push({
id: "reviewed",
label: "Reviewed source",
disposition: "no_issue_found",
receiptRefs: [],
});
await save(join(scanDir, "coverage.json"), coverage);
await rm(join(scanDir, "report.md"), { force: true });
return findings;
}
async function publishDraft(
scanDir: string,
scanId: string,
workbench: (args: readonly string[]) => Promise<Record<string, unknown>>,
complete?: boolean,
) {
const manifest = await json<ScanManifest>(
join(scanDir, "scan-manifest.json"),
);
const findings = await json<FindingsDocument>(join(scanDir, "findings.json"));
const coverage = await json<CoverageDocument>(join(scanDir, "coverage.json"));
const staged = {
manifest: {
scan: {
target: manifest.scan.target,
scope: manifest.scan.scope,
...(complete === undefined ? {} : { complete }),
},
},
findings: {
findings: findings.findings.map(
({
findingId: _id,
occurrenceId: _occurrence,
fingerprints: _fingerprints,
...finding
}) => finding,
),
},
coverage: {
inventoryStrategy: coverage.inventoryStrategy,
surfaces: coverage.surfaces,
explicitExclusions: coverage.explicitExclusions,
deferred: coverage.deferred,
},
};
await mkdir(join(scanDir, "drafts"), { recursive: true });
const draftPath = join(scanDir, "drafts", `${scanId}.json`);
await save(draftPath, staged);
expect(
await workbench([
"write-scan-draft",
"--scan-id",
scanId,
"--draft-path",
draftPath,
]),
).toMatchObject({ scanId, status: "draft_written" });
}
function result(
...dispositions: CustomValidationResult["validations"][number]["validation"]["disposition"][]
): CustomValidationResult {
return {
status: "complete",
reason: null,
validations: dispositions.map((disposition, index) => ({
candidateId: `candidate-${index + 1}`,
validation: {
disposition,
method: "integration test",
confidence: "medium",
confidence_rationale: "The test exercised the selected path.",
rubric: "Check the protected operation.",
evidence: ["The test returned the observed result."],
counterevidence_or_proof_gap:
disposition === "deferred"
? "The required service was unavailable."
: "",
remaining_uncertainty: "",
artifact_paths: [],
},
severity: null,
impact: null,
})),
};
}
async function fixture(count = 1) {
const root = await temporaryDirectory();
const scanDir = join(root, "scan");
await mkdir(scanDir, { mode: 0o700 });
const scanId = randomUUID();
const findings = await draft(scanDir, scanId, count);
return {
root,
repository: root,
target: { kind: "repository" as const, paths: [] },
scanDir,
scanId,
findings,
pluginRoot: PLUGIN_ROOT,
prompt: "Run the selected fixture workflow.",
signal: new AbortController().signal,
};
}
async function* responseEvents(
value: unknown,
activity?: string,
): AsyncGenerator<ThreadEvent> {
for await (const event of completedEvents("validation-thread")) {
if (
event.type === "item.completed" &&
event.item.type === "agent_message"
) {
if (activity !== undefined)
yield { ...event, item: { ...event.item, text: activity } };
yield { ...event, item: { ...event.item, text: JSON.stringify(value) } };
} else yield event;
}
}
describe("custom validation", () => {
test.each(["scan-manifest.json", "findings.json", "coverage.json"])(
"rejects %s replaced by a directory before opening the provisional document",
async (name) => {
const title = `rejects ${name} replaced by a directory before opening the provisional document`;
if (runTestInSubprocess(import.meta.path, title)) return;
const f = await fixture();
const path = join(f.scanDir, name);
const native = scanReadBinding();
const original = native.openScanReadRoot;
let replaced = false;
const opening = spyOn(native, "openScanReadRoot").mockImplementation(
(...args) => {
const root = original(...args);
const open = root.openFile.bind(root);
spyOn(root, "openFile").mockImplementation((relativePath) => {
if (relativePath === name && !replaced) {
renameSync(path, `${path}.original`);
mkdirSync(path);
replaced = true;
}
return open(relativePath);
});
return root;
},
);
const run = mock(async () => JSON.stringify(result("reportable")));
try {
const pending = runCustomValidation({ ...f, run });
await expect(pending).rejects.toBeInstanceOf(ContractValidationError);
await expect(pending).rejects.toThrow(
name === "coverage.json" ? "legacy review state" : name,
);
await expect(pending).rejects.toThrow(/regular.*file/i);
expect(replaced).toBe(true);
expect(run).not.toHaveBeenCalled();
} finally {
opening.mockRestore();
}
},
);
test.each(["scan-manifest.json", "findings.json", "coverage.json"])(
"reads and closes held %s after a pathname replacement attempt",
async (name) => {
const title = `reads and closes held ${name} after a pathname replacement attempt`;
if (runTestInSubprocess(import.meta.path, title)) return;
const f = await fixture();
const path = join(f.scanDir, name);
const native = scanReadBinding();
const original = native.openScanReadRoot;
let checked: ScanReadFile | undefined;
let attempted = false;
const originalBytes = await readFile(path);
const chunks: Buffer[] = [];
const opening = spyOn(native, "openScanReadRoot").mockImplementation(
(...args) => {
const root = original(...args);
const open = root.openFile.bind(root);
spyOn(root, "openFile").mockImplementation((relativePath) => {
const file = open(relativePath);
if (relativePath === name && file !== null) {
checked = file;
const read = file.read.bind(file);
spyOn(file, "read").mockImplementation((size) => {
if (!attempted) {
attempted = true;
if (process.platform === "win32") {
expect(() =>
renameSync(path, `${path}.original`),
).toThrow();
} else {
renameSync(path, `${path}.original`);
writeFileSync(path, "not JSON");
}
}
const chunk = read(size);
chunks.push(chunk);
return chunk;
});
}
return file;
});
return root;
},
);
const run = mock(async () => JSON.stringify(result("reportable")));
try {
await runCustomValidation({ ...f, run });
expect(run).toHaveBeenCalledTimes(1);
expect(attempted).toBe(true);
expect(Buffer.concat(chunks)).toEqual(originalBytes);
expect(checked).toBeDefined();
expect(() => checked!.read(1)).toThrow("Scan file is closed");
} finally {
opening.mockRestore();
}
},
);
test.each(["malformed", "directory"])(
"rejects a %s provisional document before invoking validation",
async (kind) => {
const f = await fixture();
const path = join(f.scanDir, "findings.json");
if (kind === "malformed") await writeFile(path, "not JSON");
else {
await rm(path);
await mkdir(path);
}
const run = mock(async () => JSON.stringify(result("reportable")));
const pending = runCustomValidation({ ...f, run });
if (kind === "malformed") {
await expect(pending).rejects.toThrow(/JSON/);
} else {
await expect(pending).rejects.toBeInstanceOf(ContractValidationError);
await expect(pending).rejects.toThrow("findings.json");
await expect(pending).rejects.toThrow(/regular.*file/i);
}
expect(run).not.toHaveBeenCalled();
},
);
test.skipIf(process.platform === "win32")(
"rejects provisional documents from a scan owned by another user",
async () => {
const title =
"rejects provisional documents from a scan owned by another user";
if (runTestInSubprocess(import.meta.path, title)) return;
const f = await fixture();
const files = await import("node:fs/promises");
const original = files.lstat;
const inspecting = spyOn(files, "lstat").mockImplementation((async (
...args: Parameters<typeof original>
) => {
const metadata = await original(...args);
if (
args[0] === f.scanDir ||
(Buffer.isBuffer(args[0]) && args[0].equals(Buffer.from(f.scanDir)))
)
Object.defineProperty(metadata, "uid", {
value: process.geteuid!() + 1,
});
return metadata;
}) as typeof original);
const run = mock(async () => JSON.stringify(result("reportable")));
try {
await expect(runCustomValidation({ ...f, run })).rejects.toThrow(
"must be owned by the current user",
);
expect(run).not.toHaveBeenCalled();
} finally {
inspecting.mockRestore();
}
},
);
test("applies dispositions and assessments without changing source identity or output permissions", async () => {
const f = await fixture(4);
if (process.platform !== "win32") await chmod(f.scanDir, 0o755);
const output = result(
"reportable",
"suppressed",
"not_applicable",
"deferred",
);
const counterEvidence =
"The protected caller checks a separate precondition.";
const limitations = "The alternate configuration was not exercised.";
output.validations[0]!.validation.counterevidence_or_proof_gap =
counterEvidence;
output.validations[0]!.validation.remaining_uncertainty = limitations;
output.validations[0]!.severity = {
level: "medium",
rationale: "Requires an uncommon configuration.",
};
output.validations[0]!.impact = {
level: "high",
rationale: "Can modify protected files.",
};
output.validations[0]!.validation.artifact_paths = [
"artifacts/custom-validation/proof.txt",
];
await runCustomValidation({
...f,
run: async (prompt, schema) => {
expect(prompt).toContain(JSON.stringify(f.target));
await writeFile(
join(f.scanDir, "artifacts/custom-validation/proof.txt"),
"Synthetic proof.\n",
);
expect(prompt).toContain(f.prompt);
// Structured outputs cannot resolve plugin URIs or use regex lookaround.
expect(JSON.stringify(schema)).not.toContain("codex-security://");
expect(JSON.stringify(schema)).not.toMatch(/\(\?[=!<]/);
const common = await json<object>(
join(PLUGIN_ROOT, "schemas/definitions/artifact-common.schema.json"),
);
const existing = await json<object>(
join(PLUGIN_ROOT, "schemas/tools/candidate-validations.schema.json"),
);
const ajv = new Ajv2020({ strict: false, validateFormats: false });
ajv.addSchema(common);
expect(
ajv.validate(existing, {
scanId: f.scanId,
validations: output.validations.map(
({ candidateId, validation }) => ({ candidateId, validation }),
),
}),
).toBe(true);
return JSON.stringify(output);
},
});
if (process.platform !== "win32")
expect((await stat(f.scanDir)).mode & 0o777).toBe(0o755);
const findings = await json<FindingsDocument>(
join(f.scanDir, "findings.json"),
);
expect(findings.findings).toHaveLength(1);
expect(findings.findings[0]).toMatchObject({
identity: f.findings.findings[0]!.identity,
locations: f.findings.findings[0]!.locations,
severity: output.validations[0]!.severity,
confidence: { level: "medium" },
attackPath: { impact: output.validations[0]!.impact },
validation: {
disposition: "reportable",
counterEvidence: [counterEvidence],
limitations: [limitations],
},
});
const coverage = await json<CoverageDocument>(
join(f.scanDir, "coverage.json"),
);
expect(coverage.surfaces.map((surface) => surface.disposition)).toEqual([
"reported",
"rejected",
"not_applicable",
"needs_follow_up",
]);
expect(coverage).not.toHaveProperty("completeness");
expect(coverage.surfaces[0]!.receiptRefs).toContain(
"artifacts/custom-validation/proof.txt",
);
expect(coverage.deferred).toHaveLength(1);
expect(await json(join(f.scanDir, resultName))).toMatchObject({
scanId: f.scanId,
...output,
});
expect(
await json(
join(f.scanDir, "artifacts/custom-validation/candidates.json"),
),
).toMatchObject({
target: f.target,
candidates: f.findings.findings.map((finding, index) => ({
candidateId: `candidate-${index + 1}`,
finding,
})),
});
});
test.each([{ artifacts: [] }, { artifacts: null }])(
"validates an unfinished draft with %j",
async ({ artifacts }) => {
const f = await fixture();
const manifest = await json<{ scan: Record<string, unknown> }>(
join(f.scanDir, "scan-manifest.json"),
);
manifest.scan["artifacts"] = artifacts;
manifest.scan["sealedAt"] = null;
await save(join(f.scanDir, "scan-manifest.json"), manifest);
const run = mock(async () => JSON.stringify(result("reportable")));
await runCustomValidation({ ...f, run });
expect(run).toHaveBeenCalledTimes(1);
const findings = await json<FindingsDocument>(
join(f.scanDir, "findings.json"),
);
expect(findings.findings).toHaveLength(1);
expect(findings.findings[0]).toMatchObject({
identity: f.findings.findings[0]!.identity,
validation: { disposition: "reportable" },
});
},
);
test.each([
"sealedAt",
"artifacts",
"manifest ID",
"findings ID",
"coverage ID",
])("rejects an invalid discovery handoff: %s", async (kind) => {
const f = await fixture();
const manifest = await json<ScanManifest>(
join(f.scanDir, "scan-manifest.json"),
);
if (kind === "sealedAt") manifest.scan.sealedAt = "2026-01-01T00:00:00Z";
if (kind === "artifacts")
manifest.scan.artifacts = [
{
path: "findings.json",
sha256: "0".repeat(64),
mediaType: "application/json",
},
];
if (kind === "manifest ID") manifest.scan.id = randomUUID();
if (kind === "findings ID" || kind === "coverage ID") {
const name = kind === "findings ID" ? "findings.json" : "coverage.json";
const document = await json<{ scanId: string }>(join(f.scanDir, name));
document.scanId = randomUUID();
await save(join(f.scanDir, name), document);
}
await save(join(f.scanDir, "scan-manifest.json"), manifest);
await expect(
runCustomValidation({
...f,
run: unexpectedValidation,
}),
).rejects.toThrow("unsealed custom-validation draft");
});
test.each([
["empty title", { title: "" }],
["empty optional detail", { attackPath: { dataflow: { sink: "" } } }],
["legacy field", { root_cause: "Synthetic legacy detail." }],
])(
"rejects %s before validation without changing the draft",
async (_name, fields) => {
const f = await fixture(2);
Object.assign(f.findings.findings[1]!, fields);
await save(join(f.scanDir, "findings.json"), f.findings);
await expect(
runCustomValidation({
...f,
run: unexpectedValidation,
}),
).rejects.toThrow(/valid provisional finding set.*findings\[1\]/);
expect(
await json<FindingsDocument>(join(f.scanDir, "findings.json")),
).toEqual(f.findings);
},
);
test.each([
["bundled", ""],
["unmarked oneOf", "coverage/deferred/0/paths/1"],
["additional pattern", "coverage/deferred/0/paths/1"],
["referenced array", "coverage/checkedPaths"],
["conditional", "must have required property 'reviewReceipt'"],
["referenced predicate", "must have required property 'reviewReceipt'"],
])(
"honors %s deferred path rules before custom validation",
async (rules, expectedError) => {
const f = await fixture();
const path = join(f.scanDir, "coverage.json");
const coverage = await json<CoverageDocument>(path);
coverage.deferred = [
{
id: "pending",
reason: "Review pending.",
paths: ["src/valid", "../outside"],
},
];
await save(path, coverage);
const run = mock(async () => JSON.stringify(result("reportable")));
if (rules === "bundled") {
await runCustomValidation({ ...f, run });
expect(run).toHaveBeenCalledTimes(1);
expect((await json<CoverageDocument>(path)).deferred).toEqual(
coverage.deferred,
);
return;
}
const pluginRoot = join(f.root, "plugin");
await cp(join(f.pluginRoot, "schemas"), join(pluginRoot, "schemas"), {
recursive: true,
});
const schemaPath = join(pluginRoot, "schemas", "coverage.schema.json");
const schema = JSON.parse(await readFile(schemaPath, "utf8"));
const deferred = schema.properties.deferred.items;
const paths = deferred.properties.paths;
if (rules === "unmarked oneOf") delete paths.items.oneOf[1].$comment;
if (rules === "additional pattern") paths.items.pattern = "^src/";
if (rules === "referenced array") {
schema.$id = "relative/coverage.json";
paths.$id = "deferred-paths.json";
schema.$defs = { ...schema.$defs, sharedPaths: paths };
schema.properties.deferred.items.properties.paths = {
$ref: "#/$defs/sharedPaths",
};
schema.properties.checkedPaths = {
anyOf: [{ $ref: "#/$defs/sharedPaths" }, { type: "null" }],
};
await save(path, {
...coverage,
checkedPaths: ["src/other", "../outside"],
});
}
if (rules === "conditional") {
schema.if = {
properties: { deferred: { $ref: "#/properties/deferred" } },
};
schema.then = { required: ["reviewReceipt"] };
}
if (rules === "referenced predicate") {
deferred.properties.reviewReceipt = { type: "boolean" };
deferred.if = {
properties: {
paths: {
contains: {
$ref: "#/properties/deferred/items/properties/paths/items/oneOf/1",
},
},
},
};
deferred.then = { required: ["reviewReceipt"] };
}
await save(schemaPath, schema);
await expect(
runCustomValidation({ ...f, pluginRoot, run }),
).rejects.toThrow(expectedError);
expect(run).not.toHaveBeenCalled();
if (rules === "referenced array" || rules === "referenced predicate") {
if (rules === "referenced array")
Object.assign(coverage, { checkedPaths: ["src/other"] });
else Object.assign(coverage.deferred[0]!, { reviewReceipt: true });
await save(path, coverage);
await runCustomValidation({ ...f, pluginRoot, run });
expect(run).toHaveBeenCalledTimes(1);
expect((await json<CoverageDocument>(path)).deferred).toEqual(
coverage.deferred,
);
}
},
);
test("identifies invalid provisional coverage before validation", async () => {
const f = await fixture();
const path = join(f.scanDir, "coverage.json");
const coverage = await json<CoverageDocument>(path);
coverage.surfaces[0]!.label = "";
await save(path, coverage);
await expect(
runCustomValidation({
...f,
run: unexpectedValidation,
}),
).rejects.toThrow("coverage/surfaces/0/label");
expect(await json<CoverageDocument>(path)).toEqual(coverage);
});
test.each([
"missing",
"duplicate",
"unknown",
"malformed",
"incomplete",
"unsafe artifact",
])("rejects %s results without losing the draft", async (kind) => {
const f = await fixture();
const original = await readFile(join(f.scanDir, "findings.json"), "utf8");
const output = result("reportable");
if (kind === "missing") output.validations = [];
if (kind === "duplicate") output.validations.push(output.validations[0]!);
if (kind === "unknown") output.validations[0]!.candidateId = "unknown";
if (kind === "unsafe artifact")
output.validations[0]!.validation.artifact_paths = ["../outside.txt"];
if (kind === "incomplete") {
output.status = "incomplete";
output.reason = "Setup failed.";
}
await expect(
runCustomValidation({
...f,
run: async () => {
await writeFile(join(f.scanDir, "findings.json"), "{}");
return kind === "malformed" ? "not JSON" : JSON.stringify(output);
},
}),
).rejects.toThrow("Custom validation is incomplete");
expect(await json(join(f.scanDir, "findings.json"))).toEqual(
JSON.parse(original),
);
});
test("rejects output directories linked outside the scan", async () => {
const f = await fixture();
const outside = join(f.root, "outside");
await mkdir(outside);
await mkdir(join(f.scanDir, "artifacts"), { recursive: true });
await symlink(
outside,
join(f.scanDir, "artifacts/custom-validation"),
process.platform === "win32" ? "junction" : "dir",
);
await expect(
runCustomValidation({
...f,
run: async () => JSON.stringify(result("reportable")),
}),
).rejects.toThrow("inside the scan directory");
await expect(readFile(join(outside, "candidates.json"))).rejects.toThrow();
});
const completionScenarios = [
"standard",
"progress",
"diff",
"empty",
"incomplete",
"dismissed",
"late-progress",
"live-progress",
"late-progress-observer-error",
"unfinished-source",
];
test.each(completionScenarios)(
"SDK owns real workbench completion: %s",
async (scenario) => {
const diff = scenario === "diff";
const maxTimeMinutes = scenario === "standard" || diff ? 5 : undefined;
const count = scenario === "empty" ? 0 : 1;
const root = await temporaryDirectory();
const repository = join(root, "repository");
const scanDir = join(root, "scan");
const codexHome = join(root, "codex-home");
const stateDir = join(root, "state");
const python = Bun.which("python3") ?? Bun.which("python");
expect(python).not.toBeNull();
await mkdir(join(repository, "src"), { recursive: true });
await writeFile(join(repository, "src/extract.py"), "# source fixture\n");
await mkdir(scanDir, { mode: 0o700 });
await mkdir(codexHome);
if (diff) {
const git = (...args: string[]) =>
execFileSync("git", [
"-C",
repository,
"-c",
"user.name=Fixture",
"-c",
"[email protected]",
...args,
]);
git("init", "-q");
git("config", "--local", "core.autocrlf", "false");
git("add", ".");
git("commit", "-qm", "base");
await writeFile(
join(repository, "src/extract.py"),
"# changed fixture\n",
);
}
const workflow = "Run the synthetic validation script, then clean up.";
const markdown = "# Custom severity policy\nGrade by verified impact.";
const severityRubric = {
markdown,
revision: 4,
sha256: createHash("sha256").update(markdown).digest("hex"),
};
const workflowFile = join(root, "validation.md");
if (scenario === "standard") await writeFile(workflowFile, workflow);
const falsePositive = {
reason: "The fixture is not included in the deployed application.",
};
let scanId = "";
let turns = 0;
const workingDirectories: Array<string | undefined> = [];
const commands: string[] = [];
const activities: ScanActivity[] = [];
const progressScenario =
scenario !== "progress" && scenario.includes("progress");
const progressUpdates: Array<
ScanProgress & { validationStarted: boolean }
> = [];
const observerErrors: string[] = [];
let validationStarted = false;
let parentRollout = "";
const parentSeen = Promise.withResolvers<void>();
const parentMessage = (
id: string,
phase: ScanProgress["phase"],
filesCompleted: number,
) => ({
type: "response_item",
payload: {
type: "message",
id,
role: "assistant",
content: [
{
type: "output_text",
text: progressMarker(phase, filesCompleted, 1),
},
],
},
});
const progress: ScanProgress[] = [];
const validationActivity = "Synthetic validation activity.";
const profileDisabledTools =
scenario === "standard"
? []
: diff
? ["profile_disabled_tool"]
: undefined;
const workbench = (args: readonly string[], input?: string) =>
runWorkbench(
{
python: python!,
pluginRoot: PLUGIN_ROOT,
environment: {
PATH: process.env["PATH"],
CODEX_SECURITY_STATE_DIR: stateDir,
},
},
args,
input,
);
const client = new TestClient(
profileDisabledTools === undefined
? {}
: {
codexOverrides: {
profile: "synthetic.validation",
profiles: {
"synthetic.validation": {
mcp_servers: {
"codex-security": {
disabled_tools: profileDisabledTools,
},
},
},
},
},
},
{
environment: { CODEX_SECURITY_STATE_DIR: stateDir },
now: () => Date.UTC(2026, 0, 1),
prepareRuntime: async () => {
const runtime = preparedRuntime(codexHome);
runtime.plugin.version = (
await json<{ version: string }>(
join(PLUGIN_ROOT, ".codex-plugin/plugin.json"),
)
).version;
return runtime;
},
resolvePluginPython: async () => python!,
prepareOutputDir: async () => scanDir,
createCodex: (options) => {
expect(options.config?.["mcp_servers"]).toMatchObject({
"codex-security": {
disabled_tools: expect.arrayContaining([
"start_codex_security_standard_scan",
"start_codex_security_prompt_only_scan",
"start_codex_security_deep_scan",
"complete_codex_security_scan",
"record_codex_security_candidate_validations",
"record_candidate_attack_paths",
...(profileDisabledTools ?? []),
]),
},
});
return {
startThread: (threadOptions) => {
expect(threadOptions.threadSource).toBe("security_scan");
workingDirectories.push(threadOptions.workingDirectory);
return {
id:
workingDirectories.length === 1
? "thread-1"
: "validation-thread",
async runStreamed(prompt, turnOptions) {
expect(turnOptions.cyberAccessProgram).toBe(
"daybreak_blue",
);
turns += 1;
expect(prompt).toContain(
JSON.stringify({ severityRubric }),
);
if (turns === 1) {
expect(prompt.includes("2026-01-01T00:05:00.000Z")).toBe(
maxTimeMinutes !== undefined,
);
expect(prompt).not.toContain(workflow);
expect(prompt).toContain("SDK-owned discovery workflow");
expect(prompt).not.toContain(
"Independently validate each unique finding",
);
expect(prompt).not.toContain("run `$validation` once");
expect(turnOptions.outputSchema).toBeUndefined();
await draft(scanDir, scanId, 0, diff);
await publishDraft(scanDir, scanId, workbench, false);
await draft(scanDir, scanId, count, diff);
if (scenario === "unfinished-source") {
const coverage = await json<CoverageDocument>(
join(scanDir, "coverage.json"),
);
coverage.deferred = [
{
id: "unfinished-source",
reason:
"Requested source review remains unfinished.",
},
];
await save(join(scanDir, "coverage.json"), coverage);
}
// Follow the adapter's terminal draft instruction at the real workbench boundary.
await publishDraft(
scanDir,
scanId,
workbench,
scenario !== "unfinished-source" &&
/\bcomplete:\s*true\b/u.test(prompt),
);
expect(commands).not.toContain("prepare-scan-completion");
expect(commands).not.toContain("complete-scan");
if (progressScenario) {
parentRollout = await writeSession(
codexHome,
"thread-1",
{ input_tokens: 10, output_tokens: 3 },
);
const messages = [
parentMessage("inventory", "discovery", 0),
parentMessage("review", "discovery", 1),
];
if (scenario === "live-progress")
messages.push(
parentMessage("parent-final", "reporting", 1),
);
await appendFile(
parentRollout,
jsonLines(messages) + "\n",
);
async function* progressEvents(): AsyncGenerator<ThreadEvent> {
for (const message of messages)
yield {
type: "item.completed",
item: {
id: message.payload.id,
type: "agent_message",
text: message.payload.content[0]!.text,
},
};
}
return {
events: completedEvents("thread-1", progressEvents()),
};
}
if (scenario === "progress") {
return {
events:
(async function* (): AsyncGenerator<ThreadEvent> {
yield {
type: "item.completed",
item: {
id: "primary-reporting",
type: "agent_message",
text: 'CODEX_SECURITY_SCAN_PROGRESS {"phase":"reporting","filesCompleted":1,"filesTotal":1}',
},
};
yield* completedEvents();
})(),
};
}
return { events: completedEvents() };
}
if (scenario === "progress") {
await Promise.resolve();
expect(
progress.some(({ phase }) => phase === "reporting"),
).toBe(true);
expect(progress.at(-1)?.phase).toBe("validation");
}
expect(prompt).toContain(workflow);
expect(prompt).not.toContain("2026-01-01T00:05:00.000Z");
const pendingManifest = await json<ScanManifest>(
join(scanDir, "scan-manifest.json"),
);
expect(pendingManifest.scan.id).toBe(scanId);
expect(pendingManifest.scan.scope.validationMode).toBe(
"custom_pending",
);
expect(pendingManifest.scan).not.toHaveProperty("sealedAt");
expect(pendingManifest.scan).not.toHaveProperty(
"artifacts",
);
expect(turnOptions.outputSchema).toBeDefined();
if (scenario === "dismissed") {
expect(prompt).toContain("untrusted reviewer feedback");
expect(prompt).toContain("reason still applies");
expect(
await json(
join(
scanDir,
"artifacts/custom-validation/candidates.json",
),
),
).toMatchObject({ falsePositives: [falsePositive] });
}
const output = result(
scenario === "dismissed" ? "suppressed" : "reportable",
);
output.validations[0]!.validation.counterevidence_or_proof_gap =
"Synthetic counterevidence from validation.";
output.validations[0]!.validation.remaining_uncertainty =
"Synthetic limitation from validation.";
if (scenario === "incomplete") {
output.status = "incomplete";
output.reason =
"The validation environment did not start.";
}
if (scenario === "standard") {
await mkdir(join(codexHome, "sessions"), {
recursive: true,
});
for (const [id, cwd] of [
["thread-1", scanDir],
["validation-thread", join(scanDir, "artifacts")],
]) {
const records = [
{
type: "session_meta",
payload: {
id,
cwd,
timestamp: "2026-08-21T00:00:00Z",
},
},
{
type: "event_msg",
payload: {
type: "token_count",
info: {
total_token_usage: {
input_tokens: 10,
output_tokens: 3,
},
},
},
},
...(id === "validation-thread"
? [
{
type: "event_msg",
payload: {
type: "agent_message",
message: validationActivity,
},
},
]
: []),
];
await writeFile(
join(codexHome, "sessions", `rollout-${id}.jsonl`),
jsonLines(records) + "\n",
);
}
}
if (progressScenario) {
validationStarted = true;
if (scenario !== "live-progress") {
await appendFile(
parentRollout,
JSON.stringify(
parentMessage("parent-final", "reporting", 1),
) + "\n",
);
// The observer fires during the same synchronous batch that
// processes progress, so this awaits the actual saved receipt.
await parentSeen.promise;
}
expect(progressUpdates.at(-1)?.phase).toBe("validation");
expect(
progressUpdates
.filter((update) => update.validationStarted)
.every((update) => update.phase === "validation"),
).toBe(true);
}
return {
events: responseEvents(
output,
scenario === "standard"
? validationActivity
: undefined,
),
};
},
};
},
};
},
runWorkbench: async (_options, args, input) => {
commands.push(args[0]!);
const value = await workbench(args, input);
if (args[0] === "register-cli-scan")
scanId = String(value["scanId"]);
if (args[0] === "get-scan-feedback" && scenario === "dismissed")
value["falsePositives"] = [falsePositive];
return value;
},
},
);
// Keep the synthetic validation turn alive while it awaits an unref'ed poll.
const keepAlive = progressScenario
? setInterval(() => {}, 1_000)
: undefined;
try {
const pending = client.run(repository, {
severityRubric,
cyberAccessProgram: "daybreak_blue",
maxTimeMinutes,
...(scenario === "standard"
? { validationPromptFile: workflowFile }
: { validationPrompt: workflow }),
onActivity: (activity) => activities.push(activity),
...(progressScenario
? {
onProgress: (progress: ScanProgress) => {
progressUpdates.push({ ...progress, validationStarted });
if (
scenario === "late-progress-observer-error" &&
progress.phase === "validation"
)
throw new Error("Synthetic optional progress failure.");
},
onSessionEvent: ({ event }: ScanSessionEvent) => {
if (
(event["payload"] as { id?: string } | undefined)?.id ===
"parent-final"
)
parentSeen.resolve();
},
onObserverError: (observer: string) =>
observerErrors.push(observer),
}
: {}),
...(scenario === "progress"
? { onProgress: (update: ScanProgress) => progress.push(update) }
: {}),
...(diff ? { target: DiffTarget.workingTree({}) } : {}),
});
if (scenario === "unfinished-source") {
await expect(pending).rejects.toThrow(
"The latest saved scan draft is incomplete",
);
expect(turns).toBe(2);
expect(commands).not.toContain("complete-scan");
expect(commands).toContain("fail-scan");
expect(await json(join(scanDir, resultName))).toMatchObject({
status: "complete",
});
expect(await readReviewState(scanDir)).toMatchObject({
deferred: expect.arrayContaining([
expect.objectContaining({ id: "unfinished-source" }),
]),
});
return;
}
if (scenario === "incomplete") {
await expect(pending).rejects.toThrow(
"The validation environment did not start",
);
expect(turns).toBe(2);
expect(commands).not.toContain("prepare-scan-completion");
expect(commands).not.toContain("complete-scan");
expect(commands).toContain("fail-scan");
expect(await json(join(scanDir, resultName))).toMatchObject({
status: "incomplete",
});
expect(
(await json<FindingsDocument>(join(scanDir, "findings.json")))
.findings,
).toHaveLength(1);
expect(
(
await json<{ scan: { status: string } }>(
join(scanDir, "scan-manifest.json"),
)
).scan.status,
).toBe("failed");
expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain(
"Fixture 0",
);
return;
}
const completed = await pending;
if (progressScenario) {
expect(observerErrors).toEqual(
scenario === "late-progress-observer-error" ? ["onProgress"] : [],
);
if (scenario === "live-progress")
expect(
progressUpdates.some(
(update) =>
update.phase === "reporting" && !update.validationStarted,
),
).toBe(true);
}
if (scenario === "standard") {
const report = await readFile(join(scanDir, "report.md"), "utf8");
expect(report).toContain(
"Synthetic counterevidence from validation.",
);
expect(report).toContain("Synthetic limitation from validation.");
expect(
activities.filter(
({ description }) => description === validationActivity,
),
).toHaveLength(1);
expect(completed.cost?.inputTokens).toBe(20);
}
expect(turns).toBe(count === 0 ? 1 : 2);
expect(workingDirectories).toEqual(
count === 0 ? [scanDir] : [scanDir, join(scanDir, "artifacts")],
);
expect(commands.indexOf("prepare-scan-completion")).toBeLessThan(
commands.indexOf("complete-scan"),
);
expect(completed.findings.findings).toHaveLength(
scenario === "dismissed" ? 0 : count,
);
const receipt = await json<CustomValidationResult>(
join(scanDir, resultName),
);
expect(receipt.status).toBe("complete");
expect(receipt.validations).toHaveLength(count);
if (count > 0 && scenario !== "dismissed")
expect(completed.findings.findings[0]?.validation?.disposition).toBe(
"reportable",
);
expect(completed.manifest.scan.scope.validationMode).toBe("custom");
expect(
completed.manifest.scan.artifacts.map((artifact) => artifact.path),
).toContain(resultName);
expect(
completed.manifest.scan.artifacts.map((artifact) => artifact.path),
).toContain("artifacts/custom-validation/candidates.json");
expect(completed.turnResult.usage).toMatchObject({
input_tokens: count === 0 ? 10 : 20,
output_tokens: count === 0 ? 3 : 6,
});
expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain(
count === 0 || scenario === "dismissed" ? "No findings" : "Fixture 0",
);
} finally {
clearInterval(keepAlive);
await client.close();
}
},
);
test("rejects Deep and empty prompts before starting Codex", async () => {
const root = await temporaryDirectory();
const client = TestClient.withDependencies({});
await expect(
client.run(root, { mode: "deep", validationPrompt: "Validate." }),
).rejects.toThrow("not supported for Deep");
await expect(client.run(root, { validationPrompt: " \n" })).rejects.toThrow(
"must not be empty",
);
await client.close();
});
test("renders only the discovery portion of the shipped workflows", async () => {
const standard = await customDiscoveryPrompt(
PLUGIN_ROOT,
"security-scan",
true,
);
const diff = await customDiscoveryPrompt(
PLUGIN_ROOT,
"security-diff-scan",
false,
);
expect(standard).toContain("<plugin_dir>/references/core-scan.md");
expect(standard).not.toContain("validate-scan-findings.md");
expect(standard).not.toContain("security-scan/SKILL.md");
expect(diff).toContain("<plugin_dir>/references/scan-runtime.md");
expect(standard).not.toContain("undefined");
expect(standard).not.toContain(
"Independently validate each unique finding",
);
expect(diff).toContain("Run `$finding-discovery`");
expect(diff).toContain(
"Immediately save a `complete: false` semantic draft",
);
expect(diff).toContain('"format": "markdown", "content": "<model text>"');
expect(diff).not.toContain("run `$validation` once");
expect(diff).not.toContain("Call `complete_codex_security_scan` once");
for (const prompt of [standard, diff]) {
expect(prompt).toContain("custom_pending");
expect(prompt).toContain("extensions.customValidationSurfaceIds");
expect(prompt).not.toContain("finalize_scan_contract.py");
}
const changed = await temporaryDirectory();
await mkdir(join(changed, "skills/security-diff-scan"), {
recursive: true,
});
await writeFile(
join(changed, "skills/security-diff-scan/SKILL.md"),
"Changed workflow\n",
);
await expect(
customDiscoveryPrompt(changed, "security-diff-scan", false),
).rejects.toThrow("incompatible");
});
for (const profiled of [false, true]) {
test(`the bundled Codex honors the invocation-only completion restriction with inline profile: ${profiled}`, async () => {
const home = await temporaryDirectory();
const marketplace = await createMarketplace(home, PLUGIN_ROOT);
const command = resolveCodexCommand({}).command;
const run = (args: string[]) =>
execFileSync(command, args, {
env: {
PATH: process.env["PATH"],
SystemRoot: process.env["SystemRoot"],
TEMP: process.env["TEMP"],
TMP: process.env["TMP"],
CODEX_HOME: home,
},
encoding: "utf8",
windowsHide: true,
});
run(["plugin", "marketplace", "add", marketplace]);
run(["plugin", "add", "--json", "codex-security@codex-security-sdk"]);
const settings = {
mcp_servers: {
"codex-security": { disabled_tools: ["user_disabled_tool"] },
},
};
const config = resolveCodexProfile(
await customValidationConfig(
profiled
? { profile: "review", profiles: { review: settings } }
: settings,
PLUGIN_ROOT,
),
);
const server = (config["mcp_servers"] as Record<string, unknown>)[
"codex-security"
] as Record<string, unknown>;
const overrides = Object.entries(server).flatMap(([key, value]) => [
"-c",
`mcp_servers.codex-security.${key}=${JSON.stringify(value)}`,
]);
const effective = JSON.parse(
run([...overrides, "mcp", "get", "codex-security", "--json"]),
);
expect(effective.disabled_tools).toContain(
"complete_codex_security_scan",
);
expect(effective.disabled_tools).toContain("user_disabled_tool");
expect(config).not.toHaveProperty("profile");
expect(config).not.toHaveProperty("profiles");
expect(effective.transport.cwd).toBe(resolve(PLUGIN_ROOT));
const ordinary = JSON.parse(
run(["mcp", "get", "codex-security", "--json"]),
);
expect(ordinary.disabled_tools).toBeNull();
});
}
});
const unexpectedValidation = rejecting("unexpected validation");