Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(sdk): streamline CLI and validation coverage (#1971)

Michael D'Angelo0036b153d0fb
diff --git a/sdk/typescript/scripts/test-timings.json b/sdk/typescript/scripts/test-timings.json
index aea906dbe..33bdcd98c 100644
--- a/sdk/typescript/scripts/test-timings.json
+++ b/sdk/typescript/scripts/test-timings.json
@@ -109,10 +109,25 @@
"darwin": 1.1,
"win32": 2
},
+ "api-validation-evidence.test.ts": {
+ "linux": 74.9,
+ "darwin": 128.8,
+ "win32": 139
+ },
+ "api-validation-guidance.test.ts": {
+ "linux": 42.9,
+ "darwin": 66.5,
+ "win32": 66.1
+ },
+ "api-validation-persistence.test.ts": {
+ "linux": 77.3,
+ "darwin": 121.6,
+ "win32": 136.7
+ },
"api-validation.test.ts": {
- "linux": 190.4,
- "darwin": 197.1,
- "win32": 319.2
+ "linux": 0.3,
+ "darwin": 0.8,
+ "win32": 3.6
},
"api-workflow.test.ts": {
"linux": 27.1,
diff --git a/sdk/typescript/tests-ts/api-events.test.ts b/sdk/typescript/tests-ts/api-events.test.ts
index f9103d7b3..368da5c46 100644
--- a/sdk/typescript/tests-ts/api-events.test.ts
+++ b/sdk/typescript/tests-ts/api-events.test.ts
@@ -763,83 +763,39 @@ describe("one-shot scan events", () => {
});
});
- test("fails the scan for every turn.failed error payload shape", async () => {
- const usage = {
- input_tokens: 10,
- cached_input_tokens: 2,
- cache_write_input_tokens: 0,
- output_tokens: 3,
- reasoning_output_tokens: 1,
- };
- const payloads: Array<[string, unknown]> = [
- ["object message", { message: "model refused the turn" }],
- ["null", null],
- ["undefined", undefined],
- ["string", "model refused the turn"],
- ["non-string message", { message: { text: "model refused the turn" } }],
- ["array", ["model refused the turn"]],
- ["blank message", { message: " " }],
+ test("fails the scan for every turn.failed payload, reporting only readable nested messages", async () => {
+ const fallback =
+ "The Codex Security scan turn failed without a readable error message.";
+ const scanDir = await copyCompletedScan(await temporaryDirectory());
+ const payloads: Array<[string, unknown, string]> = [
+ [
+ "object message",
+ { message: "model refused the turn" },
+ "model refused the turn",
+ ],
+ ["null", null, fallback],
+ ["undefined", undefined, fallback],
+ ["raw string", "token sk-proj-EXAMPLE1234567890 rejected", fallback],
+ [
+ "non-string message",
+ { message: { text: "model refused the turn" } },
+ fallback,
+ ],
+ ["array", ["model refused the turn"], fallback],
+ ["blank message", { message: " " }, fallback],
+ ["missing message", { code: 500 }, fallback],
];
- for (const [label, error] of payloads) {
- // A complete, valid artifact bundle so the failed turn is the only variable.
- const scanDir = await copyCompletedScan(await temporaryDirectory());
+ for (const [label, error, message] of payloads) {
async function* failedEvents(): AsyncGenerator<ThreadEvent> {
yield { type: "thread.started", thread_id: "thread-1" };
- yield { type: "turn.completed", usage };
+ yield completedTurn();
yield { type: "turn.failed", error } as unknown as ThreadEvent;
}
await expect(
runEvents(scanDir, failedEvents()),
- `turn.failed carrying ${label} must fail the scan`,
- ).rejects.toMatchObject({ name: CodexSecurityError.name });
- }
- });
-
- test("reuses only a nested turn.failed message and falls back otherwise", async () => {
- const usage = {
- input_tokens: 10,
- cached_input_tokens: 2,
- cache_write_input_tokens: 0,
- output_tokens: 3,
- reasoning_output_tokens: 1,
- };
- async function* failedWith(error: unknown): AsyncGenerator<ThreadEvent> {
- yield { type: "thread.started", thread_id: "thread-1" };
- yield { type: "turn.completed", usage };
- yield { type: "turn.failed", error } as unknown as ThreadEvent;
+ `turn.failed carrying ${label}`,
+ ).rejects.toMatchObject({ name: CodexSecurityError.name, message });
}
-
- await expect(
- runEvents(
- await copyCompletedScan(await temporaryDirectory()),
- failedWith({ message: "retry budget exhausted" }),
- ),
- ).rejects.toMatchObject({
- name: CodexSecurityError.name,
- message: "retry budget exhausted",
- });
-
- await expect(
- runEvents(
- await copyCompletedScan(await temporaryDirectory()),
- failedWith("token sk-proj-EXAMPLE1234567890 rejected"),
- ),
- ).rejects.toMatchObject({
- name: CodexSecurityError.name,
- message:
- "The Codex Security scan turn failed without a readable error message.",
- });
-
- await expect(
- runEvents(
- await copyCompletedScan(await temporaryDirectory()),
- failedWith({ code: 500 }),
- ),
- ).rejects.toMatchObject({
- name: CodexSecurityError.name,
- message:
- "The Codex Security scan turn failed without a readable error message.",
- });
});
test("extracts bounded rate-limit context from reconnect notifications", async () => {
diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.ts
index 6a2473c0e..bdcd4183c 100644
--- a/sdk/typescript/tests-ts/api-policy.test.ts
+++ b/sdk/typescript/tests-ts/api-policy.test.ts
@@ -771,13 +771,15 @@ describe("CodexSecurity policy API", () => {
const gitDirectory = policyGitDirectory(repository);
for (const metadata of new Set([common, gitDirectory])) {
const outputDir = join(metadata, "policy-artifacts");
- for (const operation of [
- () => f.security.preflightPolicy(repository, { outputDir }),
- () => f.security.generatePolicy(repository, { outputDir }),
- ])
- await expect(operation()).rejects.toThrow(
- "outside the protected scan root",
- );
+ await expect(
+ f.security.preflightPolicy(repository, { outputDir }),
+ ).rejects.toThrow("outside the protected scan root");
+ // Generation uses the same checks as preflight; exercise each destination type once.
+ if (kind === "separate") {
+ await expect(
+ f.security.generatePolicy(repository, { outputDir }),
+ ).rejects.toThrow("outside the protected scan root");
+ }
const state = new InternalSecurity(
{},
{ environment: { CODEX_SECURITY_STATE_DIR: outputDir } },
@@ -785,9 +787,11 @@ describe("CodexSecurity policy API", () => {
await expect(state.preflightPolicy(repository)).rejects.toThrow(
"outside the protected scan root",
);
- await expect(state.generatePolicy(repository)).rejects.toThrow(
- "outside the protected scan root",
- );
+ if (kind === "linked" && metadata === gitDirectory) {
+ await expect(state.generatePolicy(repository)).rejects.toThrow(
+ "outside the protected scan root",
+ );
+ }
await state.close();
await expect(readdir(outputDir)).rejects.toMatchObject({
code: "ENOENT",
diff --git a/sdk/typescript/tests-ts/api-provider-isolation.test.ts b/sdk/typescript/tests-ts/api-provider-isolation.test.ts
index b2f7f6386..fb6b940c9 100644
--- a/sdk/typescript/tests-ts/api-provider-isolation.test.ts
+++ b/sdk/typescript/tests-ts/api-provider-isolation.test.ts
@@ -612,11 +612,12 @@ for (const resume of [false, true]) {
await mkdir(repository);
await mkdir(scan, { mode: 0o700 });
await mkdir(home, { mode: 0o700 });
- await cp(PLUGIN_ROOT, plugin, { recursive: true });
- const manifestPath = join(plugin, ".codex-plugin", "plugin.json");
- const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
- manifest.codexSecurity = capabilities;
- await writeFile(manifestPath, JSON.stringify(manifest));
+ const manifestDirectory = join(plugin, ".codex-plugin");
+ await mkdir(manifestDirectory, { recursive: true });
+ await writeFile(
+ join(manifestDirectory, "plugin.json"),
+ JSON.stringify({ codexSecurity: capabilities }),
+ );
const runtime = preparedRuntime(home);
const createCodex = mock(() => {
throw new Error("Model execution must not begin.");
diff --git a/sdk/typescript/tests-ts/api-validation-evidence.test.ts b/sdk/typescript/tests-ts/api-validation-evidence.test.ts
new file mode 100644
index 000000000..500b6f2ab
--- /dev/null
+++ b/sdk/typescript/tests-ts/api-validation-evidence.test.ts
@@ -0,0 +1,404 @@
+import {
+ mkdir,
+ readFile,
+ realpath,
+ rename,
+ rm,
+ symlink,
+ writeFile,
+} from "node:fs/promises";
+import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { describe, expect, test } from "bun:test";
+import { runWorkbench } from "../src/runtime.js";
+
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { gitText } from "./support/shell.js";
+import { createValidationFixtures } from "./support/validation.js";
+
+describe("CodexSecurity finding validation", () => {
+ const {
+ assessment,
+ validationEvents,
+ persistedValidationClient,
+ temporaryDirectory,
+ } = createValidationFixtures();
+
+ async function registerValidationScan(
+ fixture: Awaited<ReturnType<typeof persistedValidationClient>>,
+ repository: string,
+ environment: NodeJS.ProcessEnv,
+ ): Promise<string> {
+ const scanDir = join(fixture.root, "scan");
+ await mkdir(scanDir, { mode: 0o700 });
+ const registration = await runWorkbench(
+ { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },
+ [
+ "register-cli-scan",
+ "--repository",
+ repository,
+ "--scan-dir",
+ scanDir,
+ "--recipe-json",
+ JSON.stringify({
+ repository,
+ mode: "standard",
+ target: { kind: "repository", paths: [] },
+ config: {},
+ }),
+ ],
+ );
+ return registration["scanId"] as string;
+ }
+
+ test.each([
+ "AGENTS.md",
+ "REVIEW.md",
+ ".codex/config.toml",
+ "unrelated.txt",
+ "unchanged",
+ ] as const)(
+ "rechecks native guidance created while cached evidence is fingerprinted: %s",
+ async (scope) => {
+ const stateRoot = await temporaryDirectory("validation-guidance-race-");
+ execFileSync("git", ["init", "-q", stateRoot]);
+ const guide = join(stateRoot, scope);
+ let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls++;
+ await writeFile(
+ join(fixture.captured.thread!.workingDirectory!, "proof.bin"),
+ Buffer.alloc(256 * 1024, 7),
+ );
+ yield* validationEvents(
+ JSON.stringify({
+ ...assessment,
+ report: existsSync(guide)
+ ? await readFile(guide, "utf8")
+ : "Original assessment.",
+ }),
+ );
+ }
+ fixture = await persistedValidationClient(
+ events,
+ PLUGIN_ROOT,
+ undefined,
+ {
+ CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),
+ },
+ );
+ await using client = fixture.client;
+ client.config.codexOverrides!["project_doc_fallback_filenames"] = [
+ "REVIEW.md",
+ ];
+ let changeDuringEvidence = false;
+ let changed = false;
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ const payload =
+ args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;
+ const pending = runWorkbench(options, args, input);
+ if (
+ changeDuringEvidence &&
+ !changed &&
+ payload?.action === "source" &&
+ payload.evidence === true
+ ) {
+ changed = true;
+ if (scope !== "unchanged") {
+ await mkdir(dirname(guide), { recursive: true });
+ await writeFile(
+ guide,
+ scope === ".codex/config.toml"
+ ? 'developer_instructions = "New native guidance."\n'
+ : "New native guidance.\n",
+ );
+ }
+ }
+ return await pending;
+ });
+ const workflowId = await fixture.bindWorkflow(
+ "native-guidance-during-evidence",
+ );
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect(await client.validate(request)).toEqual(first);
+ expect(modelCalls).toBe(1);
+ changeDuringEvidence = true;
+ const second = await client.validate(request);
+ const nativeGuidance = !["unrelated.txt", "unchanged"].includes(scope);
+ expect(changed).toBe(true);
+ expect(modelCalls).toBe(nativeGuidance ? 2 : 1);
+ expect(second.report).toBe(
+ nativeGuidance ? (await readFile(guide, "utf8")).trim() : first.report,
+ );
+ const sameGuidance = await client.validate(request);
+ expect(sameGuidance.report).toBe(second.report);
+ expect(modelCalls).toBe(nativeGuidance ? 3 : 1);
+ },
+ );
+
+ test.each([
+ "unchanged",
+ "source contents",
+ "recorded contents",
+ "recorded checkout",
+ ] as const)(
+ "rechecks %s after reading a saved validation assessment",
+ async (change) => {
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const repository = fixture.options.repositoryPath;
+ const source = join(repository, "source.ts");
+ await writeFile(source, "export const value = 'original';\n");
+ const environment = {
+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
+ };
+ const scanId =
+ change === "source contents"
+ ? undefined
+ : await registerValidationScan(fixture, repository, environment);
+ const workflowId = "cached-validation-recheck";
+ await fixture.bindWorkflow(workflowId, environment);
+ let readSavedAssessment = false;
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ const response = await runWorkbench(options, args, input);
+ if (
+ args[0] === "finding-workflow" &&
+ JSON.parse(input!)["action"] === "get-review" &&
+ response["review"] !== null
+ ) {
+ readSavedAssessment = true;
+ if (change === "recorded checkout") {
+ await rename(repository, join(fixture.root, "original-repository"));
+ await mkdir(repository);
+ await writeFile(source, "export const value = 'original';\n");
+ } else if (change !== "unchanged") {
+ await writeFile(source, "export const value = 'changed';\n");
+ }
+ }
+ return response;
+ });
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ ...(scanId === undefined ? {} : { scanId }),
+ };
+ const first = await client.validate(request);
+ if (change === "unchanged") {
+ expect(await client.validate(request)).toEqual(first);
+ } else {
+ await expect(client.validate(request)).rejects.toThrow(
+ change === "source contents"
+ ? "Repository changed during validation"
+ : change === "recorded contents"
+ ? "Scan target contents changed"
+ : "checkout path was replaced",
+ );
+ }
+ expect(readSavedAssessment).toBe(true);
+ expect(modelCalls).toBe(1);
+ },
+ );
+
+ test.each([
+ ["cached", "source contents", "unchanged"],
+ ["cached", "recorded contents", "unchanged"],
+ ["cached", "source contents", "changed"],
+ ["cached", "recorded contents", "missing"],
+ ["fresh", "source contents", "unchanged"],
+ ["fresh", "recorded contents", "missing"],
+ ] as const)(
+ "rechecks %s validation %s after reading %s evidence",
+ async (phase, change, evidence) => {
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls++;
+ await writeFile(
+ join(fixture.captured.thread!.workingDirectory!, "proof.txt"),
+ "Synthetic evidence.\n",
+ );
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const repository = fixture.options.repositoryPath;
+ const source = join(repository, "source.ts");
+ await writeFile(source, "export const value = 'original';\n");
+ const environment = {
+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
+ };
+ const scanId =
+ change === "source contents"
+ ? undefined
+ : await registerValidationScan(fixture, repository, environment);
+ const workflowId = "evidence-validation-recheck";
+ await fixture.bindWorkflow(workflowId, environment);
+ let changeDuringEvidence = false;
+ let changed = false;
+ let savedReviews = 0;
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ const payload =
+ args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;
+ const changeNow =
+ changeDuringEvidence &&
+ !changed &&
+ payload?.action === "source" &&
+ payload.evidence === true;
+ if (changeNow && evidence === "changed")
+ await writeFile(
+ join(payload.repository, "proof.txt"),
+ "Changed evidence.\n",
+ );
+ if (changeNow && evidence === "missing")
+ await rm(payload.repository, { recursive: true });
+ const response = await runWorkbench(options, args, input);
+ if (changeNow) {
+ await writeFile(source, "export const value = 'changed';\n");
+ changed = true;
+ }
+ if (payload?.action === "save-review") savedReviews++;
+ return response;
+ });
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ ...(scanId === undefined ? {} : { scanId }),
+ };
+ const first = await client.validate(request);
+ expect(await client.validate(request)).toEqual(first);
+ expect(modelCalls).toBe(1);
+ changeDuringEvidence = true;
+ await expect(
+ client.validate({
+ ...request,
+ ...(phase === "fresh" ? { finding: "Another candidate" } : {}),
+ }),
+ ).rejects.toThrow(
+ change === "source contents"
+ ? "Repository changed during validation"
+ : "Scan target contents changed",
+ );
+ expect(changed).toBe(true);
+ expect(modelCalls).toBe(phase === "fresh" ? 2 : 1);
+ expect(savedReviews).toBe(1);
+ },
+ );
+
+ test.each(["canonical", "repository alias"])(
+ "validates an unchanged recorded Git target with %s tool paths",
+ async (kind) => {
+ const root = await temporaryDirectory();
+ const repository = join(root, "repository");
+ await mkdir(repository);
+ const hostGit = Bun.which("git");
+ expect(hostGit).not.toBeNull();
+ const tools = join(root, "external-tools");
+ await mkdir(tools);
+ const selectedGit = join(
+ tools,
+ process.platform === "win32" ? "git.exe" : "git",
+ );
+ await symlink(await realpath(hostGit!), selectedGit);
+ const alias = join(repository, "tools");
+ await symlink(
+ tools,
+ alias,
+ process.platform === "win32" ? "junction" : "dir",
+ );
+ await writeFile(join(repository, ".gitignore"), "tools\n");
+ await writeFile(
+ join(repository, "source.ts"),
+ "export const value = 'original';\n",
+ );
+ gitText(["init", "--initial-branch=main"], { cwd: repository });
+ gitText(
+ [
+ "-c",
+ "user.name=Synthetic User",
+ "-c",
+ "add",
+ ".",
+ ],
+ { cwd: repository },
+ );
+ gitText(
+ [
+ "-c",
+ "user.name=Synthetic User",
+ "-c",
+ "commit",
+ "-m",
+ "Synthetic validation baseline",
+ ],
+ { cwd: repository },
+ );
+ const fixture = await persistedValidationClient(
+ undefined,
+ PLUGIN_ROOT,
+ root,
+ {
+ PATH: kind === "canonical" ? tools : alias,
+ },
+ );
+ await using client = fixture.client;
+ const environment = {
+ PATH: tools,
+ CODEX_SECURITY_GIT: selectedGit,
+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
+ };
+ const scanDir = join(root, "scan");
+ await mkdir(scanDir, { mode: 0o700 });
+ const registration = await runWorkbench(
+ { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },
+ [
+ "register-cli-scan",
+ "--repository",
+ repository,
+ "--scan-dir",
+ scanDir,
+ "--recipe-json",
+ JSON.stringify({
+ repository,
+ mode: "standard",
+ target: { kind: "repository", paths: [] },
+ config: {},
+ }),
+ ],
+ );
+ const workflowId = "git-tool-validation";
+ await fixture.bindWorkflow(workflowId, environment);
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ scanId: registration["scanId"] as string,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect(await client.validate(request)).toEqual(first);
+ expect(first.disposition).toBe(assessment.disposition);
+ for (const [options] of fixture.workbench.mock.calls) {
+ expect(options.environment?.["CODEX_SECURITY_GIT"]).toBe(selectedGit);
+ expect(options.environment?.["PATH"]).toBe(tools);
+ }
+ expect(fixture.captured.codex?.env?.["CODEX_SECURITY_GIT"]).toBe(
+ selectedGit,
+ );
+ expect(fixture.captured.codex?.env?.["PATH"]).toContain(tools);
+ },
+ );
+});
diff --git a/sdk/typescript/tests-ts/api-validation-guidance.test.ts b/sdk/typescript/tests-ts/api-validation-guidance.test.ts
new file mode 100644
index 000000000..be4c8df4e
--- /dev/null
+++ b/sdk/typescript/tests-ts/api-validation-guidance.test.ts
@@ -0,0 +1,554 @@
+import {
+ chmod,
+ cp,
+ mkdir,
+ readFile,
+ readdir,
+ rename,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import * as fsPromises from "node:fs/promises";
+import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
+import { describe, expect, spyOn, test } from "bun:test";
+import { parse as parseToml } from "smol-toml";
+import { resolveCodexProfile, type JsonObject } from "../src/config.js";
+
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { shellEnvironmentReference } from "./support/api-client.js";
+import { readKnowledgeBaseSnapshot } from "../src/knowledge-base.js";
+import { createValidationFixtures } from "./support/validation.js";
+
+describe("CodexSecurity finding validation", () => {
+ const {
+ actualLstat,
+ assessment,
+ validationEvents,
+ validationClient,
+ persistedValidationClient,
+ temporaryDirectory,
+ } = createValidationFixtures();
+
+ test("invalidates cached validation when inline instructions change", async () => {
+ let modelCalls = 0;
+ let instructions = "Original standalone validation instructions.";
+ const instructionPath = join(
+ PLUGIN_ROOT,
+ "references",
+ "standalone-validation.md",
+ );
+ const actualReadFile = fsPromises.readFile;
+ const readInstructions = spyOn(fsPromises, "readFile").mockImplementation(
+ (async (...args: Parameters<typeof actualReadFile>) =>
+ args[0] === instructionPath
+ ? instructions
+ : await actualReadFile(...args)) as typeof actualReadFile,
+ );
+ try {
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const workflowId = await fixture.bindWorkflow("validation-instructions");
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect(await client.validate(request)).toEqual(first);
+ expect(modelCalls).toBe(1);
+ instructions = "Updated standalone validation instructions.";
+ await client.validate(request);
+ expect(modelCalls).toBe(2);
+ expect(fixture.captured.prompt).toContain(instructions);
+ } finally {
+ readInstructions.mockRestore();
+ }
+ });
+
+ test.each([true, false])(
+ "reuses a validation only when bundled references are used (custom: %j)",
+ async (custom) => {
+ const pluginRoot = custom
+ ? join(await temporaryDirectory(), "selected-plugin")
+ : PLUGIN_ROOT;
+ const reference = join(
+ await temporaryDirectory(),
+ "additional-guidance.md",
+ );
+ await writeFile(reference, "Original synthetic guidance.");
+ if (custom) {
+ await cp(PLUGIN_ROOT, pluginRoot, { recursive: true });
+ await writeFile(
+ join(pluginRoot, "references", "standalone-validation.md"),
+ `Read the validation guidance at ${JSON.stringify(reference)}.\n`,
+ );
+ }
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents(
+ JSON.stringify({
+ ...assessment,
+ report: await readFile(reference, "utf8"),
+ }),
+ );
+ }
+ const fixture = await persistedValidationClient(events, pluginRoot);
+ await using client = fixture.client;
+ const workflowId = await fixture.bindWorkflow(
+ "additional-validation-reference",
+ );
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect((await client.validate(request)).report).toBe(first.report);
+ expect(modelCalls).toBe(custom ? 2 : 1);
+ if (custom) {
+ await writeFile(reference, "Updated synthetic guidance.");
+ expect((await client.validate(request)).report).toBe(
+ "Updated synthetic guidance.",
+ );
+ expect(modelCalls).toBe(3);
+ }
+ expect(fixture.captured.prompt).toContain(
+ await readFile(
+ join(pluginRoot, "references", "standalone-validation.md"),
+ "utf8",
+ ),
+ );
+ },
+ );
+
+ test("reads unchanged and updated system-configured validation instructions", async () => {
+ const root = await temporaryDirectory("native-system-validation-");
+ const instructions = join(root, "instructions.md");
+ const nativeConfig = join(root, "config.toml");
+ await writeFile(instructions, "Original synthetic system guidance.");
+ await writeFile(
+ nativeConfig,
+ `model_instructions_file = ${JSON.stringify(instructions)}\n`,
+ );
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ const configuration = parseToml(await readFile(nativeConfig, "utf8"));
+ yield* validationEvents(
+ JSON.stringify({
+ ...assessment,
+ report: await readFile(
+ configuration["model_instructions_file"] as string,
+ "utf8",
+ ),
+ }),
+ );
+ }
+ const fixture = await persistedValidationClient(
+ events,
+ PLUGIN_ROOT,
+ undefined,
+ { ProgramData: root },
+ );
+ await using client = fixture.client;
+ const workflowId = await fixture.bindWorkflow(
+ "system-validation-instructions",
+ );
+ const nativePath =
+ process.platform === "win32"
+ ? join(root, "OpenAI", "Codex", "config.toml")
+ : "/etc/codex/config.toml";
+ const originalLstat = actualLstat;
+ const nativeLookup = spyOn(fsPromises, "lstat").mockImplementation(((
+ ...args: Parameters<typeof originalLstat>
+ ) =>
+ originalLstat(
+ args[0] === nativePath ? nativeConfig : args[0],
+ args[1],
+ )) as typeof originalLstat);
+ try {
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect((await client.validate(request)).report).toBe(first.report);
+ expect(modelCalls).toBe(2);
+ await writeFile(instructions, "Updated synthetic system guidance.");
+ expect((await client.validate(request)).report).toBe(
+ "Updated synthetic system guidance.",
+ );
+ expect(modelCalls).toBe(3);
+ } finally {
+ nativeLookup.mockRestore();
+ }
+ });
+
+ test.each([
+ { setting: "top-level", changed: true },
+ { setting: "top-level", changed: false },
+ { setting: "selected profile", changed: true },
+ { setting: "selected profile", changed: false },
+ { setting: "none", changed: false },
+ ] as const)(
+ "runs file-backed validation instructions through Codex, %p",
+ async ({ setting, changed }) => {
+ const instructions = join(await temporaryDirectory(), "instructions.md");
+ await writeFile(instructions, "Original synthetic instructions.");
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents(
+ JSON.stringify({
+ ...assessment,
+ report: await readFile(instructions, "utf8"),
+ }),
+ );
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ if (setting === "top-level") {
+ client.config.codexOverrides!["model_instructions_file"] = instructions;
+ } else if (setting === "selected profile") {
+ client.config.codexOverrides!["profile"] = "selected";
+ client.config.codexOverrides!["profiles"] = {
+ selected: { model_instructions_file: instructions },
+ };
+ }
+ const workflowId = await fixture.bindWorkflow("file-backed-validation");
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect(modelCalls).toBe(1);
+ if (changed)
+ await writeFile(instructions, "Updated synthetic instructions.");
+ const second = await client.validate(request);
+ expect(modelCalls).toBe(setting === "none" ? 1 : 2);
+ expect(second.report).toBe(
+ changed ? "Updated synthetic instructions." : first.report,
+ );
+ expect(
+ resolveCodexProfile(fixture.captured.codex!.config as JsonObject)[
+ "model_instructions_file"
+ ],
+ ).toBe(setting === "none" ? undefined : instructions);
+ },
+ );
+
+ test.each([
+ { modelFails: false, cleanupFails: true },
+ { modelFails: true, cleanupFails: true },
+ { modelFails: false, cleanupFails: false },
+ { modelFails: true, cleanupFails: false },
+ ])(
+ "preserves the validation outcome when temporary knowledge cleanup fails: %p",
+ async ({ modelFails, cleanupFails }) => {
+ const modelError = new Error("Synthetic validation failure.");
+ const cleanupError = new Error(
+ "Synthetic temporary knowledge cleanup failure.",
+ );
+ async function* events() {
+ if (modelFails) throw modelError;
+ yield* validationEvents();
+ }
+ const fixture = await validationClient(events);
+ await using client = fixture.client;
+ const document = join(fixture.root, "knowledge.md");
+ await writeFile(document, "Synthetic primary project guidance.");
+ const actualRm = fsPromises.rm;
+ let cleanupPath: string | undefined;
+ const cleanupSpy = spyOn(fsPromises, "rm").mockImplementation(
+ async (path, options) => {
+ if (
+ path ===
+ fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"]
+ ) {
+ cleanupPath = String(path);
+ if (cleanupFails) throw cleanupError;
+ }
+ return await actualRm(path, options);
+ },
+ );
+ try {
+ const result = client.validate({
+ ...fixture.options,
+ knowledgeBasePaths: [document],
+ });
+ if (modelFails) await expect(result).rejects.toBe(modelError);
+ else await expect(result).resolves.toMatchObject(assessment);
+ expect(cleanupPath).toBeDefined();
+ expect(await readFile(document, "utf8")).toBe(
+ "Synthetic primary project guidance.",
+ );
+ } finally {
+ cleanupSpy.mockRestore();
+ if (cleanupPath !== undefined)
+ await actualRm(cleanupPath, { recursive: true, force: true });
+ }
+ },
+ );
+
+ test.each([
+ ["unchanged", false],
+ ["modified", false],
+ ["modified", true],
+ ["renamed", true],
+ ["deleted", true],
+ ] as const)(
+ "retains supplied knowledge context for workflow validation; change=%s, snapshot=%p",
+ async (change, snapshot) => {
+ let modelCalls = 0;
+ const normalizedPaths: string[] = [];
+ let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
+ async function* events() {
+ modelCalls += 1;
+ const path =
+ fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"];
+ expect(path).toBeDefined();
+ normalizedPaths.push(path!);
+ const files = await readdir(path!);
+ const context = await readFile(join(path!, files[0]!), "utf8");
+ yield* validationEvents(
+ JSON.stringify({ ...assessment, report: context }),
+ );
+ }
+ fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const document = join(fixture.root, "policy.md");
+ await writeFile(document, "Original synthetic policy.");
+ const workflowId = await fixture.bindWorkflow("knowledge-validation");
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ knowledgeBasePaths: [document],
+ ...(snapshot
+ ? {
+ knowledgeBaseSnapshot: await readKnowledgeBaseSnapshot([
+ document,
+ ]),
+ }
+ : {}),
+ };
+ const first = await client.validate(request);
+ expect(first.report).toBe("Original synthetic policy.");
+ expect(fixture.captured.prompt).toContain(
+ shellEnvironmentReference("CODEX_SECURITY_KNOWLEDGE_BASE"),
+ );
+ expect(fixture.captured.prompt).not.toContain(
+ "Original synthetic policy.",
+ );
+ const movedDocument = join(fixture.root, "moved-policy.md");
+ if (change === "modified")
+ await writeFile(document, "Updated synthetic policy.");
+ else if (change === "renamed") await rename(document, movedDocument);
+ else if (change === "deleted") await rm(document);
+ const second = await client.validate(request);
+ expect(modelCalls).toBe(change === "modified" && !snapshot ? 2 : 1);
+ expect(second.report).toBe(
+ change === "modified" && !snapshot
+ ? "Updated synthetic policy."
+ : first.report,
+ );
+ if (change === "renamed" || change === "deleted") {
+ const fresh = await client.validate({
+ ...request,
+ finding: "Another synthetic candidate.",
+ });
+ expect(fresh.report).toBe(first.report);
+ expect(modelCalls).toBe(2);
+ }
+ expect(normalizedPaths.every((path) => !existsSync(path))).toBe(true);
+ if (change === "deleted") expect(existsSync(document)).toBe(false);
+ else
+ expect(
+ await readFile(
+ change === "renamed" ? movedDocument : document,
+ "utf8",
+ ),
+ ).toBe(
+ change === "modified" ? "Updated synthetic policy." : first.report,
+ );
+ },
+ );
+
+ test
+ .skipIf(process.platform === "win32")
+ .each(["workflow", "file-backed workflow", "no workflow"] as const)(
+ "validates an ignored FIFO without requiring a cache snapshot: %s",
+ async (mode) => {
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const repository = fixture.options.repositoryPath;
+ execFileSync("git", ["init", "-q", repository]);
+ await writeFile(join(repository, ".gitignore"), "ignored-pipe\n");
+ execFileSync(fixture.python, [
+ "-c",
+ "import os,sys; os.mkfifo(sys.argv[1])",
+ join(repository, "ignored-pipe"),
+ ]);
+ const workflowId = await fixture.bindWorkflow(
+ "unsupported-validation-snapshot",
+ );
+ if (mode === "file-backed workflow") {
+ const instructions = join(fixture.root, "instructions.md");
+ await writeFile(instructions, "Synthetic validation instructions.");
+ client.config.codexOverrides!["model_instructions_file"] = instructions;
+ }
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ ...(mode === "no workflow" ? {} : { workflowId }),
+ };
+ await client.validate(request);
+ await client.validate(request);
+ expect(modelCalls).toBe(2);
+ expect(fixture.captured.prompt).toContain("Candidate finding");
+ },
+ );
+
+ test
+ .skipIf(process.platform === "win32" || process.geteuid?.() === 0)
+ .each(["file", "directory"] as const)(
+ "validates unreadable ignored cache entries: %s",
+ async (kind) => {
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const repository = fixture.options.repositoryPath;
+ execFileSync("git", ["init", "-q", repository]);
+ await writeFile(join(repository, ".gitignore"), "ignored-cache\n");
+ const entry = join(repository, "ignored-cache");
+ if (kind === "directory") {
+ await mkdir(entry);
+ await writeFile(join(entry, "cache.txt"), "Synthetic cache data.");
+ } else await writeFile(entry, "Synthetic cache data.");
+ const mode = (await stat(entry)).mode;
+ const workflowId = await fixture.bindWorkflow(
+ "unreadable-validation-cache",
+ );
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ await chmod(entry, 0);
+ try {
+ expect(await client.validate(request)).toMatchObject(assessment);
+ expect(await client.validate(request)).toMatchObject(assessment);
+ expect(modelCalls).toBe(2);
+ } finally {
+ await chmod(entry, mode);
+ }
+ expect(await client.validate(request)).toMatchObject(assessment);
+ expect(await client.validate(request)).toMatchObject(assessment);
+ expect(modelCalls).toBe(3);
+ },
+ );
+
+ test.each([
+ "home",
+ "output worktree",
+ "output project configuration",
+ "none",
+ ] as const)("uses current native validation guidance: %s", async (scope) => {
+ const stateRoot = await temporaryDirectory("native-validation-guidance-");
+ if (scope === "output worktree" || scope === "output project configuration")
+ execFileSync("git", ["init", "-q", stateRoot]);
+ let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
+ let guide: string | undefined;
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(
+ fixture.codexHome,
+ );
+ if (
+ scope === "output worktree" ||
+ scope === "output project configuration"
+ )
+ expect(
+ relative(
+ stateRoot,
+ fixture.captured.thread!.workingDirectory!,
+ ).startsWith(".."),
+ ).toBe(false);
+ yield* validationEvents(
+ JSON.stringify({
+ ...assessment,
+ report:
+ guide === undefined
+ ? "Synthetic assessment."
+ : scope === "output project configuration"
+ ? parseToml(await readFile(guide, "utf8"))[
+ "developer_instructions"
+ ]
+ : await readFile(guide, "utf8"),
+ }),
+ );
+ }
+ fixture = await persistedValidationClient(events, PLUGIN_ROOT, undefined, {
+ CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),
+ });
+ await using client = fixture.client;
+ guide =
+ scope === "home"
+ ? join(fixture.codexHome, "AGENTS.md")
+ : scope === "output worktree"
+ ? join(stateRoot, "AGENTS.md")
+ : scope === "output project configuration"
+ ? join(stateRoot, ".codex", "config.toml")
+ : undefined;
+ if (guide !== undefined) {
+ await mkdir(dirname(guide), { recursive: true });
+ await writeFile(
+ guide,
+ scope === "output project configuration"
+ ? 'developer_instructions = "Original synthetic guidance."\n'
+ : "Original synthetic guidance.",
+ );
+ }
+ const workflowId = await fixture.bindWorkflow("native-validation-guidance");
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect((await client.validate(request)).report).toBe(first.report);
+ expect(modelCalls).toBe(guide === undefined ? 1 : 2);
+ if (guide !== undefined) {
+ await writeFile(
+ guide,
+ scope === "output project configuration"
+ ? 'developer_instructions = "Updated synthetic guidance."\n'
+ : "Updated synthetic guidance.",
+ );
+ expect((await client.validate(request)).report).toBe(
+ "Updated synthetic guidance.",
+ );
+ expect(modelCalls).toBe(3);
+ }
+ });
+});
diff --git a/sdk/typescript/tests-ts/api-validation-persistence.test.ts b/sdk/typescript/tests-ts/api-validation-persistence.test.ts
new file mode 100644
index 000000000..4ab4fb12f
--- /dev/null
+++ b/sdk/typescript/tests-ts/api-validation-persistence.test.ts
@@ -0,0 +1,531 @@
+import { Database } from "bun:sqlite";
+import {
+ chmod,
+ mkdir,
+ readFile,
+ realpath,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { join, relative } from "node:path";
+import { type ThreadEvent } from "@openai/codex-sdk";
+import { describe, expect, test } from "bun:test";
+import { OutputDirectoryError, ScanInterruptedError } from "../src/index.js";
+import { resolveCodexProfile, type JsonObject } from "../src/config.js";
+import { runWorkbench } from "../src/runtime.js";
+
+import { reviewSqliteHome } from "../src/deduplication/codex-review.js";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { gitText } from "./support/shell.js";
+import { createValidationFixtures } from "./support/validation.js";
+
+describe("CodexSecurity finding validation", () => {
+ const {
+ validationEvents,
+ validationClient,
+ persistedValidationClient,
+ temporaryDirectory,
+ } = createValidationFixtures();
+
+ test.each([
+ "environment",
+ "root configuration",
+ "selected profile",
+ "repository root",
+ "repository ancestor",
+ ] as const)(
+ "keeps native SQLite state out of validation cache snapshots: %s",
+ async (setting) => {
+ const root = await temporaryDirectory();
+ const repository = join(root, "repository");
+ const coversRepository =
+ setting === "repository root" || setting === "repository ancestor";
+ const sqliteHome =
+ setting === "repository root"
+ ? repository
+ : setting === "repository ancestor"
+ ? root
+ : join(repository, ".native-state", "nested");
+ let modelCalls = 0;
+ let changeSource = false;
+ const writeNativeState = async () => {
+ await mkdir(sqliteHome, { recursive: true, mode: 0o700 });
+ const database = new Database(join(sqliteHome, "state.sqlite"));
+ try {
+ database.run("CREATE TABLE IF NOT EXISTS events (message TEXT)");
+ database
+ .query("INSERT INTO events VALUES (?)")
+ .run("Synthetic validation thread");
+ } finally {
+ database.close();
+ }
+ };
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ modelCalls++;
+ expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(
+ fixture.codexHome,
+ );
+ expect(
+ reviewSqliteHome(
+ fixture.captured.codex!.env!,
+ resolveCodexProfile(fixture.captured.codex!.config as JsonObject),
+ ),
+ ).toBe(sqliteHome);
+ await writeNativeState();
+ if (changeSource)
+ await writeFile(join(repository, "source.txt"), "Changed source.\n");
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(
+ events,
+ PLUGIN_ROOT,
+ root,
+ {
+ CODEX_HOME: join(root, "ambient-home"),
+ CODEX_SQLITE_HOME:
+ setting === "root configuration" || setting === "selected profile"
+ ? join(root, "unused-environment-storage")
+ : sqliteHome,
+ },
+ );
+ await using client = fixture.client;
+ if (setting === "root configuration")
+ client.config.codexOverrides!["sqlite_home"] = relative(
+ fixture.codexHome,
+ sqliteHome,
+ );
+ if (setting === "selected profile") {
+ client.config.codexOverrides!["sqlite_home"] = join(
+ root,
+ "unused-root-storage",
+ );
+ client.config.codexOverrides!["profile"] = "selected";
+ client.config.codexOverrides!["profiles"] = {
+ selected: { sqlite_home: relative(fixture.codexHome, sqliteHome) },
+ };
+ }
+ await writeFile(join(repository, "source.txt"), "Original source.\n");
+ await writeFile(join(repository, ".gitignore"), ".native-state/\n");
+ gitText(["-C", repository, "init", "-q"]);
+ gitText(["-C", repository, "add", "."]);
+ gitText([
+ "-C",
+ repository,
+ "-c",
+ "user.name=Fixture",
+ "-c",
+ "commit",
+ "-qm",
+ "initial",
+ ]);
+ const snapshots: JsonObject[] = [];
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ if (args[0] === "finding-workflow" && input !== undefined) {
+ const request = JSON.parse(input) as JsonObject;
+ if (
+ request["action"] === "source" &&
+ request["repository"] === repository
+ )
+ snapshots.push(request);
+ }
+ return await runWorkbench(options, args, input);
+ });
+ const workflowId = await fixture.bindWorkflow("native-sqlite-validation");
+ const request = { ...fixture.options, workflowId };
+ const first = await client.validate(request);
+ expect(first.disposition).toBe("reportable");
+ expect(modelCalls).toBe(1);
+ await writeNativeState();
+ const second = await client.validate(request);
+ if (coversRepository) {
+ expect(second.disposition).toBe("reportable");
+ expect(modelCalls).toBe(2);
+ expect(snapshots).toEqual([]);
+ } else {
+ expect(second).toEqual(first);
+ expect(modelCalls).toBe(1);
+ expect(
+ snapshots.map((snapshot) => snapshot["privateStatePaths"]),
+ ).toEqual(Array(4).fill([sqliteHome]));
+ changeSource = true;
+ await expect(
+ client.validate({ ...request, finding: "Another candidate" }),
+ ).rejects.toThrow("Repository changed during validation.");
+ expect(modelCalls).toBe(2);
+ }
+ },
+ );
+
+ test.each([
+ "unchanged",
+ "deleted",
+ "modified",
+ "git metadata",
+ "unbound",
+ ...(process.platform === "win32" ? [] : ["insecure permissions"]),
+ ])(
+ "checks saved validation evidence before cache reuse: %s",
+ async (change) => {
+ let modelCalls = 0;
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ modelCalls++;
+ const directory = fixture.captured.thread!.workingDirectory!;
+ await writeFile(
+ join(directory, "proof.txt"),
+ `Synthetic evidence ${modelCalls}.\n`,
+ );
+ await mkdir(join(directory, ".git"));
+ await writeFile(
+ join(directory, ".git", "HEAD"),
+ "Synthetic Git metadata.\n",
+ );
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ let unbound = change === "unbound";
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ if (args[0] === "finding-workflow" && input !== undefined) {
+ const payload = JSON.parse(input);
+ if (unbound && payload.action === "save-review") {
+ payload.result = payload.result.assessment ?? payload.result;
+ input = JSON.stringify(payload);
+ unbound = false;
+ }
+ }
+ return await runWorkbench(options, args, input);
+ });
+ const workflowId = await fixture.bindWorkflow("validation-evidence");
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ };
+ const first = await client.validate(request);
+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
+ "Synthetic evidence 1.\n",
+ );
+ if (change === "deleted") await rm(first.outputDir, { recursive: true });
+ if (change === "modified")
+ await writeFile(
+ join(first.outputDir, "proof.txt"),
+ "Changed evidence.\n",
+ );
+ if (change === "git metadata")
+ await writeFile(
+ join(first.outputDir, ".git", "HEAD"),
+ "Changed metadata.\n",
+ );
+ if (change === "insecure permissions")
+ await chmod(first.outputDir, 0o777);
+ const second = await client.validate(request);
+ expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);
+ expect(await readFile(join(second.outputDir, "proof.txt"), "utf8")).toBe(
+ `Synthetic evidence ${modelCalls}.\n`,
+ );
+ expect(await client.validate(request)).toEqual(second);
+ expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);
+ if (change === "insecure permissions") {
+ expect(second.outputDir).not.toBe(first.outputDir);
+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
+ "Synthetic evidence 1.\n",
+ );
+ await chmod(first.outputDir, 0o700);
+ }
+ },
+ );
+
+ test("reuses explicit validation output without overwriting changed evidence", async () => {
+ let modelCalls = 0;
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ modelCalls++;
+ await writeFile(
+ join(fixture.captured.thread!.workingDirectory!, "proof.txt"),
+ "Original evidence.\n",
+ );
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const workflowId = await fixture.bindWorkflow(
+ "explicit-validation-evidence",
+ );
+ const request = { ...fixture.options, workflowId };
+ const first = await client.validate(request);
+ expect(await client.validate(request)).toEqual(first);
+ expect(modelCalls).toBe(1);
+ if (process.platform !== "win32") {
+ await chmod(first.outputDir, 0o770);
+ await expect(client.validate(request)).rejects.toThrow(
+ "must not be accessible to other users",
+ );
+ await chmod(first.outputDir, 0o700);
+ }
+ await writeFile(join(first.outputDir, "proof.txt"), "Changed evidence.\n");
+ await expect(client.validate(request)).rejects.toBeInstanceOf(
+ OutputDirectoryError,
+ );
+ expect(modelCalls).toBe(1);
+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
+ "Changed evidence.\n",
+ );
+ });
+
+ test.each(["empty", "nonempty"])(
+ "respects a different explicit validation output directory that is %s",
+ async (contents) => {
+ let modelCalls = 0;
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ modelCalls++;
+ await writeFile(
+ join(fixture.captured.thread!.workingDirectory!, "proof.txt"),
+ `Synthetic evidence ${modelCalls}.\n`,
+ );
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ const workflowId = await fixture.bindWorkflow(
+ "validation-output-destination",
+ );
+ const request = { ...fixture.options, workflowId };
+ const first = await client.validate(request);
+ const outputDir = join(fixture.root, "other-validation");
+ await mkdir(outputDir, { mode: 0o700 });
+ if (contents === "nonempty") {
+ await writeFile(join(outputDir, "proof.txt"), "Existing evidence.\n");
+ await expect(
+ client.validate({ ...request, outputDir }),
+ ).rejects.toBeInstanceOf(OutputDirectoryError);
+ expect(modelCalls).toBe(1);
+ expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(
+ "Existing evidence.\n",
+ );
+ } else {
+ const second = await client.validate({ ...request, outputDir });
+ expect(second.outputDir).toBe(await realpath(outputDir));
+ expect(modelCalls).toBe(2);
+ expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(
+ "Synthetic evidence 2.\n",
+ );
+ expect(await client.validate({ ...request, outputDir })).toEqual(
+ second,
+ );
+ expect(modelCalls).toBe(2);
+ }
+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
+ "Synthetic evidence 1.\n",
+ );
+ },
+ );
+
+ test.each(["unavailable", "aborted"] as const)(
+ "handles %s evidence fingerprints without caching an invalid result",
+ async (mode) => {
+ const controller = new AbortController();
+ let modelCalls = 0;
+ let saved = 0;
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ modelCalls++;
+ yield* validationEvents();
+ }
+ const fixture = await persistedValidationClient(events);
+ await using client = fixture.client;
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ if (args[0] === "finding-workflow" && input !== undefined) {
+ const payload = JSON.parse(input);
+ if (payload.action === "save-review") saved++;
+ if (payload.action === "source" && payload.evidence === true) {
+ if (mode === "unavailable") return { source: null };
+ controller.abort();
+ }
+ }
+ return await runWorkbench(options, args, input);
+ });
+ const workflowId = await fixture.bindWorkflow(
+ "optional-validation-evidence",
+ );
+ const request = {
+ ...fixture.options,
+ outputDir: undefined,
+ workflowId,
+ signal: controller.signal,
+ };
+ if (mode === "aborted") {
+ await expect(client.validate(request)).rejects.toBeInstanceOf(
+ ScanInterruptedError,
+ );
+ expect(modelCalls).toBe(1);
+ } else {
+ expect((await client.validate(request)).disposition).toBe("reportable");
+ expect((await client.validate(request)).disposition).toBe("reportable");
+ expect(modelCalls).toBe(2);
+ }
+ expect(saved).toBe(0);
+ },
+ );
+
+ function usePersistedValidationWorkbench(
+ fixture: Awaited<ReturnType<typeof validationClient>>,
+ targetChanged: () => boolean = () => false,
+ ) {
+ fixture.workbench.mockImplementation(async (options, args, input) => {
+ if (args[0] === "get-scan") {
+ if (targetChanged())
+ throw new Error("The recorded scan target changed.");
+ return { scan: { targetPath: fixture.options.repositoryPath } };
+ }
+ return await runWorkbench(options, args, input);
+ });
+ }
+
+ test("resumes workflow validation from persisted assessments across clients", async () => {
+ let modelCalls = 0;
+ let targetChanged = false;
+ async function* events() {
+ modelCalls += 1;
+ if (modelCalls === 2) throw new Error("Second assessment failed.");
+ yield* validationEvents();
+ }
+ const original = await persistedValidationClient(events);
+ usePersistedValidationWorkbench(original, () => targetChanged);
+ await using originalClient = original.client;
+ await original.bindWorkflow("validation-retry");
+ const request = {
+ ...original.options,
+ outputDir: undefined,
+ scanId: "recorded-scan",
+ workflowId: "validation-retry",
+ safetyIdentifier: "synthetic-validation-first",
+ };
+ const first = await originalClient.validate(request);
+ await expect(
+ originalClient.validate({ ...request, finding: "Second candidate" }),
+ ).rejects.toThrow("Second assessment failed.");
+ await originalClient.close();
+
+ const resumed = await persistedValidationClient(
+ events,
+ PLUGIN_ROOT,
+ original.root,
+ );
+ usePersistedValidationWorkbench(resumed, () => targetChanged);
+ await using client = resumed.client;
+ expect(await client.validate(request)).toEqual(first);
+ expect(modelCalls).toBe(2);
+ await client.validate({ ...request, finding: "Second candidate" });
+ expect(modelCalls).toBe(3);
+ targetChanged = true;
+ await expect(client.validate(request)).rejects.toThrow(
+ "scan target changed",
+ );
+ expect(modelCalls).toBe(3);
+ });
+
+ test("separates persisted validation by model and access program across clients", async () => {
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents();
+ }
+ const original = await persistedValidationClient(events);
+ usePersistedValidationWorkbench(original);
+ await using originalClient = original.client;
+ await original.bindWorkflow("validation-settings");
+ const request = {
+ ...original.options,
+ outputDir: undefined,
+ scanId: "recorded-scan",
+ workflowId: "validation-settings",
+ safetyIdentifier: "synthetic-validation-first",
+ };
+ await originalClient.validate(request);
+ await originalClient.close();
+
+ const resumed = await persistedValidationClient(
+ events,
+ PLUGIN_ROOT,
+ original.root,
+ );
+ usePersistedValidationWorkbench(resumed);
+ await using client = resumed.client;
+ await client.validate(request);
+ expect(modelCalls).toBe(1);
+ client.config.codexOverrides!["model"] = "changed-validation-model";
+ const changedModel = await client.validate(request);
+ expect(modelCalls).toBe(2);
+ const blue = await client.validate({
+ ...request,
+ cyberAccessProgram: "daybreak_blue",
+ });
+ expect(modelCalls).toBe(3);
+ const red = await client.validate({
+ ...request,
+ cyberAccessProgram: "daybreak_red",
+ });
+ expect(modelCalls).toBe(4);
+ expect(
+ await client.validate({
+ ...request,
+ cyberAccessProgram: "daybreak_blue",
+ }),
+ ).toEqual(blue);
+ expect(
+ await client.validate({ ...request, cyberAccessProgram: "daybreak_red" }),
+ ).toEqual(red);
+ expect(modelCalls).toBe(4);
+ expect(await client.validate(request)).toEqual(changedModel);
+ expect(modelCalls).toBe(4);
+ });
+
+ test("separates persisted validation by safety identifier across clients", async () => {
+ let modelCalls = 0;
+ async function* events() {
+ modelCalls += 1;
+ yield* validationEvents();
+ }
+ const original = await persistedValidationClient(events);
+ original.client.config.codexOverrides!["model"] =
+ "changed-validation-model";
+ usePersistedValidationWorkbench(original);
+ await using originalClient = original.client;
+ await original.bindWorkflow("validation-safety-identifier");
+ const request = {
+ ...original.options,
+ outputDir: undefined,
+ scanId: "recorded-scan",
+ workflowId: "validation-safety-identifier",
+ safetyIdentifier: "synthetic-validation-first",
+ };
+ const first = await originalClient.validate(request);
+ await originalClient.close();
+
+ const resumed = await persistedValidationClient(
+ events,
+ PLUGIN_ROOT,
+ original.root,
+ );
+ usePersistedValidationWorkbench(resumed);
+ await using client = resumed.client;
+ client.config.codexOverrides!["model"] = "changed-validation-model";
+ await client.validate({ ...request, finding: "Second candidate" });
+ expect(modelCalls).toBe(2);
+ expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(
+ request.safetyIdentifier,
+ );
+ for (const safetyIdentifier of ["synthetic-validation-second", undefined]) {
+ const previousCalls = modelCalls;
+ const changedRequest = { ...request, safetyIdentifier };
+ const assessment = await client.validate(changedRequest);
+ expect(modelCalls).toBe(previousCalls + 1);
+ expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(
+ safetyIdentifier,
+ );
+ expect(await client.validate(changedRequest)).toEqual(assessment);
+ expect(modelCalls).toBe(previousCalls + 1);
+ }
+ expect(await client.validate(request)).toEqual(first);
+ expect(modelCalls).toBe(4);
+ });
+});
diff --git a/sdk/typescript/tests-ts/api-validation.test.ts b/sdk/typescript/tests-ts/api-validation.test.ts
index 5ad28ece7..e3af96221 100644
--- a/sdk/typescript/tests-ts/api-validation.test.ts
+++ b/sdk/typescript/tests-ts/api-validation.test.ts
@@ -1,267 +1,37 @@
import { requestMetadata } from "./support/request-metadata.js";
import { once } from "node:events";
-import { Database } from "bun:sqlite";
-import {
- chmod,
- cp,
- mkdir,
- readFile,
- readdir,
- realpath,
- rename,
- rm,
- stat,
- symlink,
- writeFile,
-} from "node:fs/promises";
-import * as fsPromises from "node:fs/promises";
-import { execFileSync } from "node:child_process";
+import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import { existsSync } from "node:fs";
-import { dirname, join, relative } from "node:path";
+import { join } from "node:path";
import { pathToFileURL } from "node:url";
-import {
- Codex,
- type CodexOptions,
- type ThreadEvent,
- type ThreadOptions,
- type TurnOptions,
-} from "@openai/codex-sdk";
-import {
- afterEach,
- beforeEach,
- describe,
- expect,
- mock,
- spyOn,
- test,
-} from "bun:test";
+import { Codex } from "@openai/codex-sdk";
+import { describe, expect, mock, test } from "bun:test";
import { parse as parseToml } from "smol-toml";
import {
- OutputDirectoryError,
OutputInsideProtectedRootError,
ScanInterruptedError,
} from "../src/index.js";
-import { resolveCodexProfile, type JsonObject } from "../src/config.js";
-import { resolvePluginPython, runWorkbench } from "../src/runtime.js";
+import { type JsonObject } from "../src/config.js";
-import { reviewSqliteHome } from "../src/deduplication/codex-review.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
-import { shellEnvironmentReference, TestClient } from "./support/api-client.js";
+import { TestClient } from "./support/api-client.js";
import { preparedRuntime } from "./support/api-events.js";
-import { createApiTestFixtures } from "./support/temporary-directories.js";
-import { FindingWorkflow } from "../src/finding-workflow.js";
import { readKnowledgeBaseSnapshot } from "../src/knowledge-base.js";
import { VERSION } from "../src/version.js";
-import { nodeCommand, gitText } from "./support/shell.js";
+import { nodeCommand } from "./support/shell.js";
import { fail, throwing } from "./support/errors.js";
-
-const { cleanup, temporaryDirectory } = createApiTestFixtures();
-afterEach(cleanup);
-
-async function repositoryDirectories() {
- const root = await temporaryDirectory();
- const repository = join(root, "repository");
- await mkdir(repository);
- return { root, repository };
-}
-
-async function runtimeDirectories() {
- const { root, repository } = await repositoryDirectories();
- const codexHome = join(root, "codex-home");
- await mkdir(codexHome);
- return { root, repository, codexHome };
-}
-
-function runtimePreparer(
- codexHome: string,
- overrides: () => Partial<ReturnType<typeof preparedRuntime>>,
-) {
- return async () => ({
- ...preparedRuntime(codexHome),
- ...overrides(),
- });
-}
+import { createValidationFixtures } from "./support/validation.js";
describe("CodexSecurity finding validation", () => {
- const actualLstat = fsPromises.lstat;
- let systemConfigurationLookup: ReturnType<
- typeof spyOn<typeof fsPromises, "lstat">
- >;
- beforeEach(() => {
- const nativePath =
- process.platform === "win32"
- ? join("C:\\ProgramData", "OpenAI", "Codex", "config.toml")
- : "/etc/codex/config.toml";
- systemConfigurationLookup = spyOn(fsPromises, "lstat").mockImplementation(
- (async (...args: Parameters<typeof actualLstat>) => {
- if (args[0] === nativePath)
- throw Object.assign(
- new Error("Synthetic absent native system configuration"),
- { code: "ENOENT" },
- );
- return await actualLstat(...args);
- }) as typeof actualLstat,
- );
- });
- afterEach(() => systemConfigurationLookup.mockRestore());
-
- const assessment = {
- disposition: "reportable",
- report: "Static trace reaches the SQL sink; runtime proof is still needed.",
- } as const;
-
- async function* validationEvents(
- response = JSON.stringify(assessment),
- complete = true,
- ): AsyncGenerator<ThreadEvent> {
- yield { type: "thread.started", thread_id: "validation-thread" };
- yield {
- type: "item.completed",
- item: { id: "result", type: "agent_message", text: response },
- };
- if (complete) {
- yield {
- type: "turn.completed",
- usage: {
- input_tokens: 10,
- cached_input_tokens: 0,
- cache_write_input_tokens: 0,
- output_tokens: 3,
- reasoning_output_tokens: 0,
- },
- };
- }
- }
-
- let fixturePython: Promise<string> | undefined;
-
- async function validationClient(
- events: (signal: AbortSignal) => AsyncGenerator<ThreadEvent> = () =>
- validationEvents(),
- pluginRoot = PLUGIN_ROOT,
- pythonPath = "/managed/python",
- fixtureRoot?: string,
- environmentOverrides: Record<string, string> = {},
- ) {
- const root = fixtureRoot ?? (await temporaryDirectory());
- const repository = join(root, "repository");
- const codexHome = await temporaryDirectory();
- const stateDirectory =
- environmentOverrides["CODEX_SECURITY_STATE_DIR"] ?? join(root, "state");
- await mkdir(repository, { recursive: true });
- const captured: {
- codex?: CodexOptions;
- thread?: ThreadOptions;
- prompt?: string;
- turn?: TurnOptions;
- } = {};
- const workbench = mock(
- async (
- _options: Parameters<typeof runWorkbench>[0],
- _args: readonly string[],
- _input?: string,
- ): Promise<JsonObject> => ({
- scan: { targetPath: repository },
- }),
- );
- const environment = {
- ...Object.fromEntries(
- ["PATH", "SystemRoot", "TEMP", "TMP"].flatMap((name) =>
- process.env[name] === undefined ? [] : [[name, process.env[name]!]],
- ),
- ),
- CODEX_SECURITY_STATE_DIR: stateDirectory,
- OPENAI_API_KEY: "synthetic-validation-key",
- ...environmentOverrides,
- };
- const client = new TestClient(
- {
- pluginPath: pluginRoot === PLUGIN_ROOT ? undefined : pluginRoot,
- pythonPath,
- codexOverrides: {
- model: "test-model",
- model_reasoning_effort: "high",
- approval_policy: "never",
- analytics: { enabled: false },
- },
- },
- {
- environment,
- prepareRuntime: runtimePreparer(codexHome, () => ({
- plugin: { ...preparedRuntime(codexHome).plugin, pluginRoot },
- environment,
- })),
- resolvePluginPython: async ({ configuredPath } = {}) => configuredPath!,
- runWorkbench: workbench,
- createCodex: (options) => {
- captured.codex = options;
- return {
- startThread: (options) => {
- captured.thread = options;
- return {
- id: null,
- async runStreamed(prompt, options) {
- captured.prompt = prompt;
- captured.turn = options;
- return { events: events(options.signal!) };
- },
- };
- },
- };
- },
- },
- );
- const options = {
- repositoryPath: repository,
- finding: "Candidate finding",
- outputDir: join(root, "validation"),
- };
- return {
- client,
- options,
- stateDirectory,
- captured,
- workbench,
- root,
- codexHome,
- };
- }
-
- async function persistedValidationClient(
- events?: Parameters<typeof validationClient>[0],
- pluginRoot = PLUGIN_ROOT,
- fixtureRoot?: string,
- environmentOverrides?: Record<string, string>,
- ) {
- const python = await (fixturePython ??= resolvePluginPython());
- const fixture = await validationClient(
- events,
- pluginRoot,
- python,
- fixtureRoot,
- environmentOverrides,
- );
- fixture.workbench.mockImplementation(runWorkbench);
- async function bindWorkflow(
- id: string,
- environment: NodeJS.ProcessEnv = {
- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
- },
- ) {
- const workflow = new FindingWorkflow(
- id,
- environment,
- runWorkbench,
- python,
- {
- workbenchOptions: { python, pluginRoot: PLUGIN_ROOT, environment },
- },
- );
- await workflow.bind({ repositoryPath: fixture.options.repositoryPath });
- return id;
- }
- return { ...fixture, python, bindWorkflow };
- }
+ const {
+ assessment,
+ validationEvents,
+ validationClient,
+ temporaryDirectory,
+ repositoryDirectories,
+ runtimeDirectories,
+ runtimePreparer,
+ } = createValidationFixtures();
test.each(
["output", "state", "runtime"].flatMap((destination) =>
@@ -641,1431 +411,6 @@ describe("CodexSecurity finding validation", () => {
expect(await readFile(evidence, "utf8")).toBe("synthetic evidence");
});
- test.each([
- "environment",
- "root configuration",
- "selected profile",
- "repository root",
- "repository ancestor",
- ] as const)(
- "keeps native SQLite state out of validation cache snapshots: %s",
- async (setting) => {
- const root = await temporaryDirectory();
- const repository = join(root, "repository");
- const coversRepository =
- setting === "repository root" || setting === "repository ancestor";
- const sqliteHome =
- setting === "repository root"
- ? repository
- : setting === "repository ancestor"
- ? root
- : join(repository, ".native-state", "nested");
- let modelCalls = 0;
- let changeSource = false;
- const writeNativeState = async () => {
- await mkdir(sqliteHome, { recursive: true, mode: 0o700 });
- const database = new Database(join(sqliteHome, "state.sqlite"));
- try {
- database.run("CREATE TABLE IF NOT EXISTS events (message TEXT)");
- database
- .query("INSERT INTO events VALUES (?)")
- .run("Synthetic validation thread");
- } finally {
- database.close();
- }
- };
- async function* events(): AsyncGenerator<ThreadEvent> {
- modelCalls++;
- expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(
- fixture.codexHome,
- );
- expect(
- reviewSqliteHome(
- fixture.captured.codex!.env!,
- resolveCodexProfile(fixture.captured.codex!.config as JsonObject),
- ),
- ).toBe(sqliteHome);
- await writeNativeState();
- if (changeSource)
- await writeFile(join(repository, "source.txt"), "Changed source.\n");
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- root,
- {
- CODEX_HOME: join(root, "ambient-home"),
- CODEX_SQLITE_HOME:
- setting === "root configuration" || setting === "selected profile"
- ? join(root, "unused-environment-storage")
- : sqliteHome,
- },
- );
- await using client = fixture.client;
- if (setting === "root configuration")
- client.config.codexOverrides!["sqlite_home"] = relative(
- fixture.codexHome,
- sqliteHome,
- );
- if (setting === "selected profile") {
- client.config.codexOverrides!["sqlite_home"] = join(
- root,
- "unused-root-storage",
- );
- client.config.codexOverrides!["profile"] = "selected";
- client.config.codexOverrides!["profiles"] = {
- selected: { sqlite_home: relative(fixture.codexHome, sqliteHome) },
- };
- }
- await writeFile(join(repository, "source.txt"), "Original source.\n");
- await writeFile(join(repository, ".gitignore"), ".native-state/\n");
- gitText(["-C", repository, "init", "-q"]);
- gitText(["-C", repository, "add", "."]);
- gitText([
- "-C",
- repository,
- "-c",
- "user.name=Fixture",
- "-c",
- "commit",
- "-qm",
- "initial",
- ]);
- const snapshots: JsonObject[] = [];
- fixture.workbench.mockImplementation(async (options, args, input) => {
- if (args[0] === "finding-workflow" && input !== undefined) {
- const request = JSON.parse(input) as JsonObject;
- if (
- request["action"] === "source" &&
- request["repository"] === repository
- )
- snapshots.push(request);
- }
- return await runWorkbench(options, args, input);
- });
- const workflowId = await fixture.bindWorkflow("native-sqlite-validation");
- const request = { ...fixture.options, workflowId };
- const first = await client.validate(request);
- expect(first.disposition).toBe("reportable");
- expect(modelCalls).toBe(1);
- await writeNativeState();
- const second = await client.validate(request);
- if (coversRepository) {
- expect(second.disposition).toBe("reportable");
- expect(modelCalls).toBe(2);
- expect(snapshots).toEqual([]);
- } else {
- expect(second).toEqual(first);
- expect(modelCalls).toBe(1);
- expect(
- snapshots.map((snapshot) => snapshot["privateStatePaths"]),
- ).toEqual(Array(4).fill([sqliteHome]));
- changeSource = true;
- await expect(
- client.validate({ ...request, finding: "Another candidate" }),
- ).rejects.toThrow("Repository changed during validation.");
- expect(modelCalls).toBe(2);
- }
- },
- );
-
- test.each([
- "unchanged",
- "deleted",
- "modified",
- "git metadata",
- "unbound",
- ...(process.platform === "win32" ? [] : ["insecure permissions"]),
- ])(
- "checks saved validation evidence before cache reuse: %s",
- async (change) => {
- let modelCalls = 0;
- async function* events(): AsyncGenerator<ThreadEvent> {
- modelCalls++;
- const directory = fixture.captured.thread!.workingDirectory!;
- await writeFile(
- join(directory, "proof.txt"),
- `Synthetic evidence ${modelCalls}.\n`,
- );
- await mkdir(join(directory, ".git"));
- await writeFile(
- join(directory, ".git", "HEAD"),
- "Synthetic Git metadata.\n",
- );
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- let unbound = change === "unbound";
- fixture.workbench.mockImplementation(async (options, args, input) => {
- if (args[0] === "finding-workflow" && input !== undefined) {
- const payload = JSON.parse(input);
- if (unbound && payload.action === "save-review") {
- payload.result = payload.result.assessment ?? payload.result;
- input = JSON.stringify(payload);
- unbound = false;
- }
- }
- return await runWorkbench(options, args, input);
- });
- const workflowId = await fixture.bindWorkflow("validation-evidence");
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
- "Synthetic evidence 1.\n",
- );
- if (change === "deleted") await rm(first.outputDir, { recursive: true });
- if (change === "modified")
- await writeFile(
- join(first.outputDir, "proof.txt"),
- "Changed evidence.\n",
- );
- if (change === "git metadata")
- await writeFile(
- join(first.outputDir, ".git", "HEAD"),
- "Changed metadata.\n",
- );
- if (change === "insecure permissions")
- await chmod(first.outputDir, 0o777);
- const second = await client.validate(request);
- expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);
- expect(await readFile(join(second.outputDir, "proof.txt"), "utf8")).toBe(
- `Synthetic evidence ${modelCalls}.\n`,
- );
- expect(await client.validate(request)).toEqual(second);
- expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);
- if (change === "insecure permissions") {
- expect(second.outputDir).not.toBe(first.outputDir);
- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
- "Synthetic evidence 1.\n",
- );
- await chmod(first.outputDir, 0o700);
- }
- },
- );
-
- test("reuses explicit validation output without overwriting changed evidence", async () => {
- let modelCalls = 0;
- async function* events(): AsyncGenerator<ThreadEvent> {
- modelCalls++;
- await writeFile(
- join(fixture.captured.thread!.workingDirectory!, "proof.txt"),
- "Original evidence.\n",
- );
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const workflowId = await fixture.bindWorkflow(
- "explicit-validation-evidence",
- );
- const request = { ...fixture.options, workflowId };
- const first = await client.validate(request);
- expect(await client.validate(request)).toEqual(first);
- expect(modelCalls).toBe(1);
- if (process.platform !== "win32") {
- await chmod(first.outputDir, 0o770);
- await expect(client.validate(request)).rejects.toThrow(
- "must not be accessible to other users",
- );
- await chmod(first.outputDir, 0o700);
- }
- await writeFile(join(first.outputDir, "proof.txt"), "Changed evidence.\n");
- await expect(client.validate(request)).rejects.toBeInstanceOf(
- OutputDirectoryError,
- );
- expect(modelCalls).toBe(1);
- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
- "Changed evidence.\n",
- );
- });
-
- test.each(["empty", "nonempty"])(
- "respects a different explicit validation output directory that is %s",
- async (contents) => {
- let modelCalls = 0;
- async function* events(): AsyncGenerator<ThreadEvent> {
- modelCalls++;
- await writeFile(
- join(fixture.captured.thread!.workingDirectory!, "proof.txt"),
- `Synthetic evidence ${modelCalls}.\n`,
- );
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const workflowId = await fixture.bindWorkflow(
- "validation-output-destination",
- );
- const request = { ...fixture.options, workflowId };
- const first = await client.validate(request);
- const outputDir = join(fixture.root, "other-validation");
- await mkdir(outputDir, { mode: 0o700 });
- if (contents === "nonempty") {
- await writeFile(join(outputDir, "proof.txt"), "Existing evidence.\n");
- await expect(
- client.validate({ ...request, outputDir }),
- ).rejects.toBeInstanceOf(OutputDirectoryError);
- expect(modelCalls).toBe(1);
- expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(
- "Existing evidence.\n",
- );
- } else {
- const second = await client.validate({ ...request, outputDir });
- expect(second.outputDir).toBe(await realpath(outputDir));
- expect(modelCalls).toBe(2);
- expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(
- "Synthetic evidence 2.\n",
- );
- expect(await client.validate({ ...request, outputDir })).toEqual(
- second,
- );
- expect(modelCalls).toBe(2);
- }
- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(
- "Synthetic evidence 1.\n",
- );
- },
- );
-
- test.each(["unavailable", "aborted"] as const)(
- "handles %s evidence fingerprints without caching an invalid result",
- async (mode) => {
- const controller = new AbortController();
- let modelCalls = 0;
- let saved = 0;
- async function* events(): AsyncGenerator<ThreadEvent> {
- modelCalls++;
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- fixture.workbench.mockImplementation(async (options, args, input) => {
- if (args[0] === "finding-workflow" && input !== undefined) {
- const payload = JSON.parse(input);
- if (payload.action === "save-review") saved++;
- if (payload.action === "source" && payload.evidence === true) {
- if (mode === "unavailable") return { source: null };
- controller.abort();
- }
- }
- return await runWorkbench(options, args, input);
- });
- const workflowId = await fixture.bindWorkflow(
- "optional-validation-evidence",
- );
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- signal: controller.signal,
- };
- if (mode === "aborted") {
- await expect(client.validate(request)).rejects.toBeInstanceOf(
- ScanInterruptedError,
- );
- expect(modelCalls).toBe(1);
- } else {
- expect((await client.validate(request)).disposition).toBe("reportable");
- expect((await client.validate(request)).disposition).toBe("reportable");
- expect(modelCalls).toBe(2);
- }
- expect(saved).toBe(0);
- },
- );
-
- function usePersistedValidationWorkbench(
- fixture: Awaited<ReturnType<typeof validationClient>>,
- targetChanged: () => boolean = () => false,
- ) {
- fixture.workbench.mockImplementation(async (options, args, input) => {
- if (args[0] === "get-scan") {
- if (targetChanged())
- throw new Error("The recorded scan target changed.");
- return { scan: { targetPath: fixture.options.repositoryPath } };
- }
- return await runWorkbench(options, args, input);
- });
- }
-
- test("resumes workflow validation from persisted assessments across clients", async () => {
- let modelCalls = 0;
- let targetChanged = false;
- async function* events() {
- modelCalls += 1;
- if (modelCalls === 2) throw new Error("Second assessment failed.");
- yield* validationEvents();
- }
- const original = await persistedValidationClient(events);
- usePersistedValidationWorkbench(original, () => targetChanged);
- await using originalClient = original.client;
- await original.bindWorkflow("validation-retry");
- const request = {
- ...original.options,
- outputDir: undefined,
- scanId: "recorded-scan",
- workflowId: "validation-retry",
- safetyIdentifier: "synthetic-validation-first",
- };
- const first = await originalClient.validate(request);
- await expect(
- originalClient.validate({ ...request, finding: "Second candidate" }),
- ).rejects.toThrow("Second assessment failed.");
- await originalClient.close();
-
- const resumed = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- original.root,
- );
- usePersistedValidationWorkbench(resumed, () => targetChanged);
- await using client = resumed.client;
- expect(await client.validate(request)).toEqual(first);
- expect(modelCalls).toBe(2);
- await client.validate({ ...request, finding: "Second candidate" });
- expect(modelCalls).toBe(3);
- targetChanged = true;
- await expect(client.validate(request)).rejects.toThrow(
- "scan target changed",
- );
- expect(modelCalls).toBe(3);
- });
-
- test("separates persisted validation by model and access program across clients", async () => {
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents();
- }
- const original = await persistedValidationClient(events);
- usePersistedValidationWorkbench(original);
- await using originalClient = original.client;
- await original.bindWorkflow("validation-settings");
- const request = {
- ...original.options,
- outputDir: undefined,
- scanId: "recorded-scan",
- workflowId: "validation-settings",
- safetyIdentifier: "synthetic-validation-first",
- };
- await originalClient.validate(request);
- await originalClient.close();
-
- const resumed = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- original.root,
- );
- usePersistedValidationWorkbench(resumed);
- await using client = resumed.client;
- await client.validate(request);
- expect(modelCalls).toBe(1);
- client.config.codexOverrides!["model"] = "changed-validation-model";
- const changedModel = await client.validate(request);
- expect(modelCalls).toBe(2);
- const blue = await client.validate({
- ...request,
- cyberAccessProgram: "daybreak_blue",
- });
- expect(modelCalls).toBe(3);
- const red = await client.validate({
- ...request,
- cyberAccessProgram: "daybreak_red",
- });
- expect(modelCalls).toBe(4);
- expect(
- await client.validate({
- ...request,
- cyberAccessProgram: "daybreak_blue",
- }),
- ).toEqual(blue);
- expect(
- await client.validate({ ...request, cyberAccessProgram: "daybreak_red" }),
- ).toEqual(red);
- expect(modelCalls).toBe(4);
- expect(await client.validate(request)).toEqual(changedModel);
- expect(modelCalls).toBe(4);
- });
-
- test("separates persisted validation by safety identifier across clients", async () => {
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents();
- }
- const original = await persistedValidationClient(events);
- original.client.config.codexOverrides!["model"] =
- "changed-validation-model";
- usePersistedValidationWorkbench(original);
- await using originalClient = original.client;
- await original.bindWorkflow("validation-safety-identifier");
- const request = {
- ...original.options,
- outputDir: undefined,
- scanId: "recorded-scan",
- workflowId: "validation-safety-identifier",
- safetyIdentifier: "synthetic-validation-first",
- };
- const first = await originalClient.validate(request);
- await originalClient.close();
-
- const resumed = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- original.root,
- );
- usePersistedValidationWorkbench(resumed);
- await using client = resumed.client;
- client.config.codexOverrides!["model"] = "changed-validation-model";
- await client.validate({ ...request, finding: "Second candidate" });
- expect(modelCalls).toBe(2);
- expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(
- request.safetyIdentifier,
- );
- for (const safetyIdentifier of ["synthetic-validation-second", undefined]) {
- const previousCalls = modelCalls;
- const changedRequest = { ...request, safetyIdentifier };
- const assessment = await client.validate(changedRequest);
- expect(modelCalls).toBe(previousCalls + 1);
- expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(
- safetyIdentifier,
- );
- expect(await client.validate(changedRequest)).toEqual(assessment);
- expect(modelCalls).toBe(previousCalls + 1);
- }
- expect(await client.validate(request)).toEqual(first);
- expect(modelCalls).toBe(4);
- });
-
- test("invalidates cached validation when inline instructions change", async () => {
- let modelCalls = 0;
- let instructions = "Original standalone validation instructions.";
- const instructionPath = join(
- PLUGIN_ROOT,
- "references",
- "standalone-validation.md",
- );
- const actualReadFile = fsPromises.readFile;
- const readInstructions = spyOn(fsPromises, "readFile").mockImplementation(
- (async (...args: Parameters<typeof actualReadFile>) =>
- args[0] === instructionPath
- ? instructions
- : await actualReadFile(...args)) as typeof actualReadFile,
- );
- try {
- async function* events() {
- modelCalls += 1;
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const workflowId = await fixture.bindWorkflow("validation-instructions");
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- expect(await client.validate(request)).toEqual(first);
- expect(modelCalls).toBe(1);
- instructions = "Updated standalone validation instructions.";
- await client.validate(request);
- expect(modelCalls).toBe(2);
- expect(fixture.captured.prompt).toContain(instructions);
- } finally {
- readInstructions.mockRestore();
- }
- });
-
- test.each([
- { custom: true, changed: true },
- { custom: true, changed: false },
- { custom: false, changed: false },
- ] as const)(
- "uses current custom validation references: %p",
- async ({ custom, changed }) => {
- const pluginRoot = custom
- ? join(await temporaryDirectory(), "selected-plugin")
- : PLUGIN_ROOT;
- const reference = join(
- await temporaryDirectory(),
- "additional-guidance.md",
- );
- await writeFile(reference, "Original synthetic guidance.");
- if (custom) {
- await cp(PLUGIN_ROOT, pluginRoot, { recursive: true });
- await writeFile(
- join(pluginRoot, "references", "standalone-validation.md"),
- `Read the validation guidance at ${JSON.stringify(reference)}.\n`,
- );
- }
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents(
- JSON.stringify({
- ...assessment,
- report: await readFile(reference, "utf8"),
- }),
- );
- }
- const fixture = await persistedValidationClient(events, pluginRoot);
- await using client = fixture.client;
- const workflowId = await fixture.bindWorkflow(
- "additional-validation-reference",
- );
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- if (changed) await writeFile(reference, "Updated synthetic guidance.");
- const second = await client.validate(request);
- expect(second.report).toBe(
- changed ? "Updated synthetic guidance." : first.report,
- );
- if (!custom) expect(modelCalls).toBe(1);
- expect(fixture.captured.prompt).toContain(
- await readFile(
- join(pluginRoot, "references", "standalone-validation.md"),
- "utf8",
- ),
- );
- },
- );
-
- for (const changed of [true, false]) {
- test(`preserves system-configured validation instructions: changed=${changed}`, async () => {
- const root = await temporaryDirectory("native-system-validation-");
- const instructions = join(root, "instructions.md");
- const nativeConfig = join(root, "config.toml");
- await writeFile(instructions, "Original synthetic system guidance.");
- await writeFile(
- nativeConfig,
- `model_instructions_file = ${JSON.stringify(instructions)}\n`,
- );
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- const configuration = parseToml(await readFile(nativeConfig, "utf8"));
- yield* validationEvents(
- JSON.stringify({
- ...assessment,
- report: await readFile(
- configuration["model_instructions_file"] as string,
- "utf8",
- ),
- }),
- );
- }
- const fixture = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- undefined,
- { ProgramData: root },
- );
- await using client = fixture.client;
- const workflowId = await fixture.bindWorkflow(
- "system-validation-instructions",
- );
- const nativePath =
- process.platform === "win32"
- ? join(root, "OpenAI", "Codex", "config.toml")
- : "/etc/codex/config.toml";
- const originalLstat = actualLstat;
- const nativeLookup = spyOn(fsPromises, "lstat").mockImplementation(((
- ...args: Parameters<typeof originalLstat>
- ) =>
- originalLstat(
- args[0] === nativePath ? nativeConfig : args[0],
- args[1],
- )) as typeof originalLstat);
- try {
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- if (changed)
- await writeFile(instructions, "Updated synthetic system guidance.");
- const second = await client.validate(request);
- expect(second.report).toBe(
- changed ? "Updated synthetic system guidance." : first.report,
- );
- } finally {
- nativeLookup.mockRestore();
- }
- });
- }
-
- test.each([
- { setting: "top-level", changed: true },
- { setting: "top-level", changed: false },
- { setting: "selected profile", changed: true },
- { setting: "selected profile", changed: false },
- { setting: "none", changed: false },
- ] as const)(
- "runs file-backed validation instructions through Codex, %p",
- async ({ setting, changed }) => {
- const instructions = join(await temporaryDirectory(), "instructions.md");
- await writeFile(instructions, "Original synthetic instructions.");
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents(
- JSON.stringify({
- ...assessment,
- report: await readFile(instructions, "utf8"),
- }),
- );
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- if (setting === "top-level") {
- client.config.codexOverrides!["model_instructions_file"] = instructions;
- } else if (setting === "selected profile") {
- client.config.codexOverrides!["profile"] = "selected";
- client.config.codexOverrides!["profiles"] = {
- selected: { model_instructions_file: instructions },
- };
- }
- const workflowId = await fixture.bindWorkflow("file-backed-validation");
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- expect(modelCalls).toBe(1);
- if (changed)
- await writeFile(instructions, "Updated synthetic instructions.");
- const second = await client.validate(request);
- expect(modelCalls).toBe(setting === "none" ? 1 : 2);
- expect(second.report).toBe(
- changed ? "Updated synthetic instructions." : first.report,
- );
- expect(
- resolveCodexProfile(fixture.captured.codex!.config as JsonObject)[
- "model_instructions_file"
- ],
- ).toBe(setting === "none" ? undefined : instructions);
- },
- );
-
- test.each([
- { modelFails: false, cleanupFails: true },
- { modelFails: true, cleanupFails: true },
- { modelFails: false, cleanupFails: false },
- { modelFails: true, cleanupFails: false },
- ])(
- "preserves the validation outcome when temporary knowledge cleanup fails: %p",
- async ({ modelFails, cleanupFails }) => {
- const modelError = new Error("Synthetic validation failure.");
- const cleanupError = new Error(
- "Synthetic temporary knowledge cleanup failure.",
- );
- async function* events() {
- if (modelFails) throw modelError;
- yield* validationEvents();
- }
- const fixture = await validationClient(events);
- await using client = fixture.client;
- const document = join(fixture.root, "knowledge.md");
- await writeFile(document, "Synthetic primary project guidance.");
- const actualRm = fsPromises.rm;
- let cleanupPath: string | undefined;
- const cleanupSpy = spyOn(fsPromises, "rm").mockImplementation(
- async (path, options) => {
- if (
- path ===
- fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"]
- ) {
- cleanupPath = String(path);
- if (cleanupFails) throw cleanupError;
- }
- return await actualRm(path, options);
- },
- );
- try {
- const result = client.validate({
- ...fixture.options,
- knowledgeBasePaths: [document],
- });
- if (modelFails) await expect(result).rejects.toBe(modelError);
- else await expect(result).resolves.toMatchObject(assessment);
- expect(cleanupPath).toBeDefined();
- expect(await readFile(document, "utf8")).toBe(
- "Synthetic primary project guidance.",
- );
- } finally {
- cleanupSpy.mockRestore();
- if (cleanupPath !== undefined)
- await actualRm(cleanupPath, { recursive: true, force: true });
- }
- },
- );
-
- test.each([
- ["unchanged", false],
- ["modified", false],
- ["modified", true],
- ["renamed", true],
- ["deleted", true],
- ] as const)(
- "retains supplied knowledge context for workflow validation; change=%s, snapshot=%p",
- async (change, snapshot) => {
- let modelCalls = 0;
- const normalizedPaths: string[] = [];
- let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
- async function* events() {
- modelCalls += 1;
- const path =
- fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"];
- expect(path).toBeDefined();
- normalizedPaths.push(path!);
- const files = await readdir(path!);
- const context = await readFile(join(path!, files[0]!), "utf8");
- yield* validationEvents(
- JSON.stringify({ ...assessment, report: context }),
- );
- }
- fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const document = join(fixture.root, "policy.md");
- await writeFile(document, "Original synthetic policy.");
- const workflowId = await fixture.bindWorkflow("knowledge-validation");
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- knowledgeBasePaths: [document],
- ...(snapshot
- ? {
- knowledgeBaseSnapshot: await readKnowledgeBaseSnapshot([
- document,
- ]),
- }
- : {}),
- };
- const first = await client.validate(request);
- expect(first.report).toBe("Original synthetic policy.");
- expect(fixture.captured.prompt).toContain(
- shellEnvironmentReference("CODEX_SECURITY_KNOWLEDGE_BASE"),
- );
- expect(fixture.captured.prompt).not.toContain(
- "Original synthetic policy.",
- );
- const movedDocument = join(fixture.root, "moved-policy.md");
- if (change === "modified")
- await writeFile(document, "Updated synthetic policy.");
- else if (change === "renamed") await rename(document, movedDocument);
- else if (change === "deleted") await rm(document);
- const second = await client.validate(request);
- expect(modelCalls).toBe(change === "modified" && !snapshot ? 2 : 1);
- expect(second.report).toBe(
- change === "modified" && !snapshot
- ? "Updated synthetic policy."
- : first.report,
- );
- if (change === "renamed" || change === "deleted") {
- const fresh = await client.validate({
- ...request,
- finding: "Another synthetic candidate.",
- });
- expect(fresh.report).toBe(first.report);
- expect(modelCalls).toBe(2);
- }
- expect(normalizedPaths.every((path) => !existsSync(path))).toBe(true);
- if (change === "deleted") expect(existsSync(document)).toBe(false);
- else
- expect(
- await readFile(
- change === "renamed" ? movedDocument : document,
- "utf8",
- ),
- ).toBe(
- change === "modified" ? "Updated synthetic policy." : first.report,
- );
- },
- );
-
- test
- .skipIf(process.platform === "win32")
- .each(["workflow", "file-backed workflow", "no workflow"] as const)(
- "validates an ignored FIFO without requiring a cache snapshot: %s",
- async (mode) => {
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const repository = fixture.options.repositoryPath;
- execFileSync("git", ["init", "-q", repository]);
- await writeFile(join(repository, ".gitignore"), "ignored-pipe\n");
- execFileSync(fixture.python, [
- "-c",
- "import os,sys; os.mkfifo(sys.argv[1])",
- join(repository, "ignored-pipe"),
- ]);
- const workflowId = await fixture.bindWorkflow(
- "unsupported-validation-snapshot",
- );
- if (mode === "file-backed workflow") {
- const instructions = join(fixture.root, "instructions.md");
- await writeFile(instructions, "Synthetic validation instructions.");
- client.config.codexOverrides!["model_instructions_file"] = instructions;
- }
- const request = {
- ...fixture.options,
- outputDir: undefined,
- ...(mode === "no workflow" ? {} : { workflowId }),
- };
- await client.validate(request);
- await client.validate(request);
- expect(modelCalls).toBe(2);
- expect(fixture.captured.prompt).toContain("Candidate finding");
- },
- );
-
- test
- .skipIf(process.platform === "win32" || process.geteuid?.() === 0)
- .each(["file", "directory"] as const)(
- "validates unreadable ignored cache entries: %s",
- async (kind) => {
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const repository = fixture.options.repositoryPath;
- execFileSync("git", ["init", "-q", repository]);
- await writeFile(join(repository, ".gitignore"), "ignored-cache\n");
- const entry = join(repository, "ignored-cache");
- if (kind === "directory") {
- await mkdir(entry);
- await writeFile(join(entry, "cache.txt"), "Synthetic cache data.");
- } else await writeFile(entry, "Synthetic cache data.");
- const mode = (await stat(entry)).mode;
- const workflowId = await fixture.bindWorkflow(
- "unreadable-validation-cache",
- );
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- await chmod(entry, 0);
- try {
- expect(await client.validate(request)).toMatchObject(assessment);
- expect(await client.validate(request)).toMatchObject(assessment);
- expect(modelCalls).toBe(2);
- } finally {
- await chmod(entry, mode);
- }
- expect(await client.validate(request)).toMatchObject(assessment);
- expect(await client.validate(request)).toMatchObject(assessment);
- expect(modelCalls).toBe(3);
- },
- );
-
- test.each([
- "AGENTS.md",
- "REVIEW.md",
- ".codex/config.toml",
- "unrelated.txt",
- "unchanged",
- ] as const)(
- "rechecks native guidance created while cached evidence is fingerprinted: %s",
- async (scope) => {
- const stateRoot = await temporaryDirectory("validation-guidance-race-");
- execFileSync("git", ["init", "-q", stateRoot]);
- const guide = join(stateRoot, scope);
- let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
- let modelCalls = 0;
- async function* events() {
- modelCalls++;
- await writeFile(
- join(fixture.captured.thread!.workingDirectory!, "proof.bin"),
- Buffer.alloc(256 * 1024, 7),
- );
- yield* validationEvents(
- JSON.stringify({
- ...assessment,
- report: existsSync(guide)
- ? await readFile(guide, "utf8")
- : "Original assessment.",
- }),
- );
- }
- fixture = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- undefined,
- {
- CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),
- },
- );
- await using client = fixture.client;
- client.config.codexOverrides!["project_doc_fallback_filenames"] = [
- "REVIEW.md",
- ];
- let changeDuringEvidence = false;
- let changed = false;
- fixture.workbench.mockImplementation(async (options, args, input) => {
- const payload =
- args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;
- const pending = runWorkbench(options, args, input);
- if (
- changeDuringEvidence &&
- !changed &&
- payload?.action === "source" &&
- payload.evidence === true
- ) {
- changed = true;
- if (scope !== "unchanged") {
- await mkdir(dirname(guide), { recursive: true });
- await writeFile(
- guide,
- scope === ".codex/config.toml"
- ? 'developer_instructions = "New native guidance."\n'
- : "New native guidance.\n",
- );
- }
- }
- return await pending;
- });
- const workflowId = await fixture.bindWorkflow(
- "native-guidance-during-evidence",
- );
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- expect(await client.validate(request)).toEqual(first);
- expect(modelCalls).toBe(1);
- changeDuringEvidence = true;
- const second = await client.validate(request);
- const nativeGuidance = !["unrelated.txt", "unchanged"].includes(scope);
- expect(changed).toBe(true);
- expect(modelCalls).toBe(nativeGuidance ? 2 : 1);
- expect(second.report).toBe(
- nativeGuidance ? (await readFile(guide, "utf8")).trim() : first.report,
- );
- const sameGuidance = await client.validate(request);
- expect(sameGuidance.report).toBe(second.report);
- expect(modelCalls).toBe(nativeGuidance ? 3 : 1);
- },
- );
-
- test.each([
- { scope: "home", changed: true },
- { scope: "home", changed: false },
- { scope: "output worktree", changed: true },
- { scope: "output worktree", changed: false },
- { scope: "output project configuration", changed: true },
- { scope: "output project configuration", changed: false },
- { scope: "none", changed: false },
- ] as const)(
- "preserves native validation guidance: %p",
- async ({ scope, changed }) => {
- const stateRoot = await temporaryDirectory("native-validation-guidance-");
- if (
- scope === "output worktree" ||
- scope === "output project configuration"
- )
- execFileSync("git", ["init", "-q", stateRoot]);
- let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;
- let guide: string | undefined;
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(
- fixture.codexHome,
- );
- if (
- scope === "output worktree" ||
- scope === "output project configuration"
- )
- expect(
- relative(
- stateRoot,
- fixture.captured.thread!.workingDirectory!,
- ).startsWith(".."),
- ).toBe(false);
- yield* validationEvents(
- JSON.stringify({
- ...assessment,
- report:
- guide === undefined
- ? "Synthetic assessment."
- : scope === "output project configuration"
- ? parseToml(await readFile(guide, "utf8"))[
- "developer_instructions"
- ]
- : await readFile(guide, "utf8"),
- }),
- );
- }
- fixture = await persistedValidationClient(
- events,
- PLUGIN_ROOT,
- undefined,
- {
- CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),
- },
- );
- await using client = fixture.client;
- guide =
- scope === "home"
- ? join(fixture.codexHome, "AGENTS.md")
- : scope === "output worktree"
- ? join(stateRoot, "AGENTS.md")
- : scope === "output project configuration"
- ? join(stateRoot, ".codex", "config.toml")
- : undefined;
- if (guide !== undefined) {
- await mkdir(dirname(guide), { recursive: true });
- await writeFile(
- guide,
- scope === "output project configuration"
- ? 'developer_instructions = "Original synthetic guidance."\n'
- : "Original synthetic guidance.",
- );
- }
- const workflowId = await fixture.bindWorkflow(
- "native-validation-guidance",
- );
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- };
- const first = await client.validate(request);
- if (changed)
- await writeFile(
- guide!,
- scope === "output project configuration"
- ? 'developer_instructions = "Updated synthetic guidance."\n'
- : "Updated synthetic guidance.",
- );
- const second = await client.validate(request);
- expect(second.report).toBe(
- changed ? "Updated synthetic guidance." : first.report,
- );
- if (changed) expect(modelCalls).toBe(2);
- if (scope === "none") expect(modelCalls).toBe(1);
- },
- );
-
- test.each([
- "unchanged",
- "source contents",
- "recorded contents",
- "recorded checkout",
- ] as const)(
- "rechecks %s after reading a saved validation assessment",
- async (change) => {
- let modelCalls = 0;
- async function* events() {
- modelCalls += 1;
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const repository = fixture.options.repositoryPath;
- const source = join(repository, "source.ts");
- await writeFile(source, "export const value = 'original';\n");
- const environment = {
- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
- };
- const workbenchOptions = {
- python: fixture.python,
- pluginRoot: PLUGIN_ROOT,
- environment,
- };
- const scanDir = join(fixture.root, "scan");
- await mkdir(scanDir, { mode: 0o700 });
- const registration = await runWorkbench(workbenchOptions, [
- "register-cli-scan",
- "--repository",
- repository,
- "--scan-dir",
- scanDir,
- "--recipe-json",
- JSON.stringify({
- repository,
- mode: "standard",
- target: { kind: "repository", paths: [] },
- config: {},
- }),
- ]);
- const workflowId = "cached-validation-recheck";
- await fixture.bindWorkflow(workflowId, environment);
- let readSavedAssessment = false;
- fixture.workbench.mockImplementation(async (options, args, input) => {
- const response = await runWorkbench(options, args, input);
- if (
- args[0] === "finding-workflow" &&
- JSON.parse(input!)["action"] === "get-review" &&
- response["review"] !== null
- ) {
- readSavedAssessment = true;
- if (change === "recorded checkout") {
- await rename(repository, join(fixture.root, "original-repository"));
- await mkdir(repository);
- await writeFile(source, "export const value = 'original';\n");
- } else if (change !== "unchanged") {
- await writeFile(source, "export const value = 'changed';\n");
- }
- }
- return response;
- });
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- ...(change === "source contents"
- ? {}
- : { scanId: registration["scanId"] as string }),
- };
- const first = await client.validate(request);
- if (change === "unchanged") {
- expect(await client.validate(request)).toEqual(first);
- } else {
- await expect(client.validate(request)).rejects.toThrow(
- change === "source contents"
- ? "Repository changed during validation"
- : change === "recorded contents"
- ? "Scan target contents changed"
- : "checkout path was replaced",
- );
- }
- expect(readSavedAssessment).toBe(true);
- expect(modelCalls).toBe(1);
- },
- );
-
- test.each([
- ["cached", "source contents", "unchanged"],
- ["cached", "recorded contents", "unchanged"],
- ["cached", "source contents", "changed"],
- ["cached", "recorded contents", "missing"],
- ["fresh", "source contents", "unchanged"],
- ["fresh", "recorded contents", "missing"],
- ] as const)(
- "rechecks %s validation %s after reading %s evidence",
- async (phase, change, evidence) => {
- let modelCalls = 0;
- async function* events() {
- modelCalls++;
- await writeFile(
- join(fixture.captured.thread!.workingDirectory!, "proof.txt"),
- "Synthetic evidence.\n",
- );
- yield* validationEvents();
- }
- const fixture = await persistedValidationClient(events);
- await using client = fixture.client;
- const repository = fixture.options.repositoryPath;
- const source = join(repository, "source.ts");
- await writeFile(source, "export const value = 'original';\n");
- const environment = {
- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
- };
- const scanDir = join(fixture.root, "scan");
- await mkdir(scanDir, { mode: 0o700 });
- const registration = await runWorkbench(
- { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },
- [
- "register-cli-scan",
- "--repository",
- repository,
- "--scan-dir",
- scanDir,
- "--recipe-json",
- JSON.stringify({
- repository,
- mode: "standard",
- target: { kind: "repository", paths: [] },
- config: {},
- }),
- ],
- );
- const workflowId = "evidence-validation-recheck";
- await fixture.bindWorkflow(workflowId, environment);
- let changeDuringEvidence = false;
- let changed = false;
- let savedReviews = 0;
- fixture.workbench.mockImplementation(async (options, args, input) => {
- const payload =
- args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;
- const changeNow =
- changeDuringEvidence &&
- !changed &&
- payload?.action === "source" &&
- payload.evidence === true;
- if (changeNow && evidence === "changed")
- await writeFile(
- join(payload.repository, "proof.txt"),
- "Changed evidence.\n",
- );
- if (changeNow && evidence === "missing")
- await rm(payload.repository, { recursive: true });
- const response = await runWorkbench(options, args, input);
- if (changeNow) {
- await writeFile(source, "export const value = 'changed';\n");
- changed = true;
- }
- if (payload?.action === "save-review") savedReviews++;
- return response;
- });
- const request = {
- ...fixture.options,
- outputDir: undefined,
- workflowId,
- ...(change === "recorded contents"
- ? { scanId: registration["scanId"] as string }
- : {}),
- };
- const first = await client.validate(request);
- expect(await client.validate(request)).toEqual(first);
- expect(modelCalls).toBe(1);
- changeDuringEvidence = true;
- await expect(
- client.validate({
- ...request,
- ...(phase === "fresh" ? { finding: "Another candidate" } : {}),
- }),
- ).rejects.toThrow(
- change === "source contents"
- ? "Repository changed during validation"
- : "Scan target contents changed",
- );
- expect(changed).toBe(true);
- expect(modelCalls).toBe(phase === "fresh" ? 2 : 1);
- expect(savedReviews).toBe(1);
- },
- );
-
- test.each(["canonical", "repository alias"])(
- "validates an unchanged recorded Git target with %s tool paths",
- async (kind) => {
- const root = await temporaryDirectory();
- const repository = join(root, "repository");
- await mkdir(repository);
- const hostGit = Bun.which("git");
- expect(hostGit).not.toBeNull();
- const tools = join(root, "external-tools");
- await mkdir(tools);
- const selectedGit = join(
- tools,
- process.platform === "win32" ? "git.exe" : "git",
- );
- await symlink(await realpath(hostGit!), selectedGit);
- const alias = join(repository, "tools");
- await symlink(
- tools,
- alias,
- process.platform === "win32" ? "junction" : "dir",
- );
- await writeFile(join(repository, ".gitignore"), "tools\n");
- await writeFile(
- join(repository, "source.ts"),
- "export const value = 'original';\n",
- );
- gitText(["init", "--initial-branch=main"], { cwd: repository });
- gitText(
- [
- "-c",
- "user.name=Synthetic User",
- "-c",
- "add",
- ".",
- ],
- { cwd: repository },
- );
- gitText(
- [
- "-c",
- "user.name=Synthetic User",
- "-c",
- "commit",
- "-m",
- "Synthetic validation baseline",
- ],
- { cwd: repository },
- );
- const fixture = await persistedValidationClient(
- undefined,
- PLUGIN_ROOT,
- root,
- {
- PATH: kind === "canonical" ? tools : alias,
- },
- );
- await using client = fixture.client;
- const environment = {
- PATH: tools,
- CODEX_SECURITY_GIT: selectedGit,
- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
- };
- const scanDir = join(root, "scan");
- await mkdir(scanDir, { mode: 0o700 });
- const registration = await runWorkbench(
- { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },
- [
- "register-cli-scan",
- "--repository",
- repository,
- "--scan-dir",
- scanDir,
- "--recipe-json",
- JSON.stringify({
- repository,
- mode: "standard",
- target: { kind: "repository", paths: [] },
- config: {},
- }),
- ],
- );
- const workflowId = "git-tool-validation";
- await fixture.bindWorkflow(workflowId, environment);
- const request = {
- ...fixture.options,
- outputDir: undefined,
- scanId: registration["scanId"] as string,
- workflowId,
- };
- const first = await client.validate(request);
- expect(await client.validate(request)).toEqual(first);
- expect(first.disposition).toBe(assessment.disposition);
- for (const [options] of fixture.workbench.mock.calls) {
- expect(options.environment?.["CODEX_SECURITY_GIT"]).toBe(selectedGit);
- expect(options.environment?.["PATH"]).toBe(tools);
- }
- expect(fixture.captured.codex?.env?.["CODEX_SECURITY_GIT"]).toBe(
- selectedGit,
- );
- expect(fixture.captured.codex?.env?.["PATH"]).toContain(tools);
- },
- );
-
test("rejects invalid inputs, unsafe output, and cancellation before preparing credentials", async () => {
const repositoryPath = await temporaryDirectory();
const prepareRuntime = mock(async () => fail("runtime must not start"));
diff --git a/sdk/typescript/tests-ts/api-workflow.test.ts b/sdk/typescript/tests-ts/api-workflow.test.ts
index df0d8ae6a..2ea1a50ff 100644
--- a/sdk/typescript/tests-ts/api-workflow.test.ts
+++ b/sdk/typescript/tests-ts/api-workflow.test.ts
@@ -194,9 +194,11 @@ test.each([
(await new FindingWorkflow(workflowId, environment).get())?.stages.scan
.status,
).toBe("completed");
- await expect(
- resumed.run(repository, { workflowId, mode: "deep" }),
- ).rejects.toThrow("already bound to a different");
+ if (scenario === "completed") {
+ await expect(
+ resumed.run(repository, { workflowId, mode: "deep" }),
+ ).rejects.toThrow("already bound to a different");
+ }
if (knowledge) {
if (scenario === "knowledge-snapshot")
await writeFile(promptFile, "Changed synthetic project guidance.");
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index c1b77c0c3..00de9e092 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -68,11 +68,7 @@ import {
matchScanFindingsInternal,
} from "../src/scan-comparison.js";
import { normalizeTarget } from "../src/targets.js";
-import {
- copyCompletedScan,
- INTEGRATION_TARGET,
- PLUGIN_ROOT,
-} from "./plugin-root.js";
+import { copyCompletedScan, PLUGIN_ROOT } from "./plugin-root.js";
import {
mockScanRegistration,
mockWorkbench,
@@ -102,7 +98,6 @@ import { pythonExecutable, nodeCommand, gitText } from "./support/shell.js";
import { fail, rejecting, throwing } from "./support/errors.js";
import { mockFs, restoreFs } from "./support/module-mocks.js";
-const REPOSITORY_ROOT = fileURLToPath(new URL("../../..", import.meta.url));
const EXAMPLE = join(PLUGIN_ROOT, "examples", "completed-scan");
const { cleanup, temporaryDirectory } = createApiTestFixtures();
afterEach(cleanup);
@@ -539,12 +534,6 @@ describe("CodexSecurity orchestration", () => {
},
);
- test("selects a real-scan target in the active repository layout", async () => {
- await expect(
- stat(join(REPOSITORY_ROOT, INTEGRATION_TARGET)),
- ).resolves.toBeDefined();
- });
-
test("validates local inputs before runtime or plugin Python discovery", async () => {
const client = new CodexSecurity({
pythonPath: "/definitely/missing/python",
@@ -4836,14 +4825,6 @@ describe("CodexSecurity orchestration", () => {
}
});
- const pricedModels = [
- "gpt-5.5",
- "gpt-6-astra",
- "gpt-5.6-terra",
- "gpt-5.6-cyber",
- "gpt-daybreak-blue-latest",
- "gpt-daybreak-red-latest",
- ];
test("reports unpriced model usage without letting its observer stop the scan", async () => {
const { root, repository, codexHome, scanDir } = await scanDirectories();
const usages: unknown[] = [];
@@ -4874,20 +4855,14 @@ describe("CodexSecurity orchestration", () => {
await client.close();
}
});
- test.each(pricedModels)("tracks live and saved %s costs", async (model) => {
+ const prices = [
+ { model: "gpt-6-astra", min: 0.000232, max: 0.000389 },
+ { model: "gpt-5.6-cyber", min: 0.0003275, max: null },
+ ];
+ test.each(prices)("tracks live and saved $model costs", async (price) => {
+ const { model, min, max } = price;
const { root, repository, codexHome, scanDir } = await scanDirectories();
- const usage = {
- input_tokens: 10,
- cached_input_tokens: 2,
- cache_write_input_tokens: 0,
- output_tokens: 3,
- reasoning_output_tokens: 1,
- };
- const expectedCost = estimateScanCost(model, usage);
- expect(expectedCost).not.toBeNull();
- if (expectedCost === null) throw new Error("Missing selected-model price");
-
const costs: ScanCost[] = [];
const commands: Array<readonly string[]> = [];
const client = new TestClient(
@@ -4917,15 +4892,24 @@ describe("CodexSecurity orchestration", () => {
});
expect(result.turnResult.model).toBe(model);
- expect(result.cost).toEqual(expectedCost);
- expect(costs).toEqual([expectedCost]);
+ expect(result.cost).toMatchObject({
+ model,
+ inputTokens: 10,
+ cachedInputTokens: 2,
+ cacheWriteInputTokens: 0,
+ outputTokens: 3,
+ estimatedUsd: min,
+ estimatedUsdRange: { min, max, context: "unknown" },
+ pricing: { serviceTier: "standard", context: "short" },
+ });
+ expect([result.cost]).toEqual(costs);
const completion = commands.find((args) => args[0] === "complete-scan");
expect(completion).toBeDefined();
const costIndex = completion?.indexOf("--cost-json") ?? -1;
expect(costIndex).toBeGreaterThan(0);
expect(JSON.parse(completion?.[costIndex + 1] ?? "null")).toEqual(
- expectedCost,
+ result.cost,
);
await client.close();
@@ -7629,8 +7613,10 @@ process.exit(2);
},
prepareRuntime: unauthenticatedRuntime(codexHome, () => ({
CODEX_HOME: codexHome,
- OpenAi_Api_Key: "forwarded-openai-key",
- codex_api_key: "forwarded-codex-key",
+ OPENAI_API_KEY: "forwarded-openai-key",
+ OpenAi_Api_Key: "mixed-case-openai-key",
+ CODEX_API_KEY: "forwarded-codex-key",
+ codex_api_key: "lowercase-codex-key",
...fakeCommand.environment,
})),
resolveCodexCommand: () => fakeCommand.command,
@@ -8546,42 +8532,6 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s
await expect(login.wait()).resolves.toMatchObject({ success: false });
});
- test("keeps ambient credentials available to scans", async () => {
- const root = await temporaryDirectory();
- const repository = join(root, "repository");
- const codexHome = join(root, "codex-home");
- const scanDir = join(root, "scan");
- await mkdir(repository);
- await mkdir(codexHome, { mode: 0o700 });
- await mkdir(scanDir, { mode: 0o700 });
- const createCodex = mock(completedCodex(root));
- const client = TestClient.withDependencies({
- environment: {
- CODEX_SECURITY_STATE_DIR: join(root, "state"),
- OPENAI_API_KEY: "ambient-key",
- },
- ...scanRuntimeDependencies(codexHome, scanDir),
- prepareRuntime: unauthenticatedRuntime(codexHome, () => ({
- ...process.env,
- CODEX_HOME: codexHome,
- OPENAI_API_KEY: "ambient-key",
- CODEX_API_KEY: "secondary-ambient-key",
- })),
- createCodex,
- });
- try {
- await client.run(repository);
- expect(createCodex.mock.lastCall?.[0]?.apiKey).toBe("ambient-key");
- expect(
- Object.keys(createCodex.mock.lastCall?.[0]?.env ?? {}).some((name) =>
- ["OPENAI_API_KEY", "CODEX_API_KEY"].includes(name.toUpperCase()),
- ),
- ).toBe(false);
- } finally {
- await client.close();
- }
- });
-
test("aborts and waits for an in-flight API-key login during close", async () => {
const root = await temporaryDirectory();
const codexHome = join(root, "codex-home");
diff --git a/sdk/typescript/tests-ts/cli-authentication.test.ts b/sdk/typescript/tests-ts/cli-authentication.test.ts
index 103fcb8ef..0314343a4 100644
--- a/sdk/typescript/tests-ts/cli-authentication.test.ts
+++ b/sdk/typescript/tests-ts/cli-authentication.test.ts
@@ -26,7 +26,6 @@ import {
import {
codexSecurityCredentialAllowsAmbientImport,
prepareCodexSecurityCredentialHome,
- resolveCodexCommand,
setCodexSecurityCredentialLogout,
} from "../src/runtime.js";
import {
@@ -929,25 +928,14 @@ describe("CLI authentication", () => {
expect(run(["login", "--with-api-key"], "synthetic-key\n")).toBe(0);
expect(await stat(join(credentialHome, "auth.json"))).toBeDefined();
await expect(stat(join(repository, "auth.json"))).rejects.toThrow();
- expect(run(["login", "status"])).toBe(0);
- expect(run(["logout"])).toBe(0);
+ if (configuredHome === mountedHome) {
+ expect(run(["login", "status"])).toBe(0);
+ expect(run(["logout"])).toBe(0);
+ } else {
+ // Empty and blank settings share the default. Each login must create its own file.
+ await rm(join(credentialHome, "auth.json"));
+ }
}
- expect(
- spawnSync(
- process.execPath,
- [join(import.meta.dir, "../src/cli.ts"), "login", "--help"],
- {
- cwd: repository,
- env: {
- PATH: process.env["PATH"],
- HOME: root,
- USERPROFILE: root,
- Codex_Home: " ",
- },
- encoding: "utf8",
- },
- ).status,
- ).toBe(0);
} finally {
await rm(root, { recursive: true, force: true });
}
@@ -2246,27 +2234,40 @@ describe("skill authentication", () => {
},
);
- test.each(["validate", "patch", "verify-fix"] as const)(
- "%s stops before starting a model command without a stored login",
- async (command) => {
- for (const auth of ["auto", "chatgpt"] as const) {
- await expect(
- runCodexSkillCommand(
- ["invalid-synthetic-command"],
- {
- command,
- auth,
- stdout: capture().stream,
- stderr: capture().stream,
- },
- resolveCodexCommand({}),
- {
- CODEX_HOME: join(stateDirectory, "ambient"),
- CODEX_SECURITY_STATE_DIR: stateDirectory,
- },
- ),
- ).rejects.toThrow("No credentials were found");
- }
+ test.each(["auto", "chatgpt"] as const)(
+ "%s auth stops after the native status check when no login is stored",
+ async (auth) => {
+ const directory = join(stateDirectory, "checkout");
+ await mkdir(directory);
+ const script = join(stateDirectory, "check-login.mjs");
+ const calls = join(stateDirectory, "calls.jsonl");
+ await writeFile(
+ script,
+ [
+ 'import { appendFileSync } from "node:fs";',
+ 'appendFileSync(process.argv[2], JSON.stringify(process.argv.slice(3)) + "\\n");',
+ 'console.error("Not logged in");',
+ "process.exitCode = 1;",
+ ].join("\n"),
+ );
+ await expect(
+ runCodexSkillCommand(
+ [script, calls, "model"],
+ {
+ command: "validate",
+ directory,
+ auth,
+ stdout: capture().stream,
+ stderr: capture().stream,
+ },
+ { command: process.execPath, args: [script, calls] },
+ {
+ CODEX_HOME: join(stateDirectory, "ambient"),
+ CODEX_SECURITY_STATE_DIR: stateDirectory,
+ },
+ ),
+ ).rejects.toThrow("No credentials were found");
+ expect(await readJsonLines(calls)).toEqual([["login", "status"]]);
},
);
diff --git a/sdk/typescript/tests-ts/cli-patch-results.test.ts b/sdk/typescript/tests-ts/cli-patch-results.test.ts
index 4e85b18d1..97614afcb 100644
--- a/sdk/typescript/tests-ts/cli-patch-results.test.ts
+++ b/sdk/typescript/tests-ts/cli-patch-results.test.ts
@@ -27,13 +27,21 @@ async function repositoryFixture({
} = {}) {
const directory = await mkdtemp(join(tmpdir(), "patch-results-"));
const git = (...args: string[]) =>
- runRepositoryCommand("git", args, directory);
+ runRepositoryCommand(
+ "git",
+ [
+ "-c",
+ "user.name=Synthetic User",
+ "-c",
+ ...args,
+ ],
+ directory,
+ );
try {
await writeFile(join(directory, "app.ts"), "original\n");
if (initializeGit) {
await git("init", "--initial-branch=main");
- await git("config", "user.name", "Synthetic User");
- await git("config", "user.email", "[email protected]");
await git("add", ".");
if (initialCommit) await git("commit", "-m", "Synthetic fixture");
}
diff --git a/sdk/typescript/tests-ts/cli-scan-failures.test.ts b/sdk/typescript/tests-ts/cli-scan-failures.test.ts
index 568386935..ba66831e8 100644
--- a/sdk/typescript/tests-ts/cli-scan-failures.test.ts
+++ b/sdk/typescript/tests-ts/cli-scan-failures.test.ts
@@ -55,8 +55,10 @@ function expectFailure(
) {
const result = JSON.parse(text);
if (fullOutput) {
- expect(result).toMatchObject({ ok: false, error: { code, message } });
- expect(result.data).toEqual(data);
+ expect(result.ok).toBe(false);
+ expect(result.error).toEqual({ code, message });
+ if (data === undefined) expect(result).not.toHaveProperty("data");
+ else expect(result.data).toEqual(data);
} else {
expect(result).toEqual({ status: "failed", code, message });
}
@@ -76,7 +78,8 @@ for (const formatArgs of [
async (_name, command) => {
const stdout = capture();
const stderr = capture();
- const message = "The scan could not save its results.";
+ const message =
+ "network failure ECONNRESET Bearer SYNTHETIC_TOKEN_123";
expect(
await main(
[...command, ...flags],
@@ -169,26 +172,6 @@ for (const formatArgs of [
expect(stderr.text()).toBe(`codex-security: ${message}\n`);
},
);
-
- test.each(commands)(
- "preserves diagnostic text for %s failures",
- async (_name, command) => {
- const stdout = capture();
- const stderr = capture();
- const message =
- "network failure ECONNRESET Bearer SYNTHETIC_TOKEN_123";
- expect(
- await main(
- [...command, ...flags],
- stdout.stream,
- stderr.stream,
- failingScan(message),
- ),
- ).toBe(2);
- expectFailure(stdout.text(), fullOutput, "SCAN_FAILED", message);
- expect(stderr.text()).toContain(message);
- },
- );
});
}
}
diff --git a/sdk/typescript/tests-ts/cli-scan-validation.test.ts b/sdk/typescript/tests-ts/cli-scan-validation.test.ts
index 064fdcbd9..9bea0bef5 100644
--- a/sdk/typescript/tests-ts/cli-scan-validation.test.ts
+++ b/sdk/typescript/tests-ts/cli-scan-validation.test.ts
@@ -228,175 +228,168 @@ test.each([false, true])(
},
);
-for (const changed of ["unchanged", "content", "head"] as const)
- test(`cached empty scan validation checks current target: ${changed}`, async () => {
- const root = await temporaryDirectory("empty-cached-scan-", true);
- const repository = join(root, "repository"),
- scanDir = join(root, "scan"),
- home = join(root, "codex");
- await mkdir(repository);
- await mkdir(home);
- await mkdir(scanDir, { mode: 0o700 });
- git(repository, "init", "-q", "-b", "main");
- await writeFile(join(repository, "source.ts"), "export const value = 1;\n");
- git(repository, "add", ".");
- git(repository, "commit", "-qm", "initial");
- const environment = {
- ...process.env,
- CODEX_HOME: home,
- CODEX_SECURITY_STATE_DIR: join(root, "state"),
- OPENAI_API_KEY: "synthetic-cached-scan-key",
- };
- const python = await resolvePluginPython({
- environment,
- protectedRoot: repository,
- });
- let scanId = "",
- modelCalls = 0;
- let registration:
- | {
- contract: { target: { allowedKinds: string[] } };
- targetId: string;
- targetRevision?: string;
- }
- | undefined;
- let selectedWorkbench: Parameters<typeof runWorkbench>[0] | undefined;
- const workbench = async (
- options: Parameters<typeof runWorkbench>[0],
- args: readonly string[],
- input?: string,
- ) => {
- const answer = await runWorkbench(
- {
- ...options,
- python,
- environment: { ...options.environment, ...environment },
- },
- args,
- input,
- );
- if (args[0] === "register-cli-scan") {
- scanId = String(answer["scanId"]);
- registration = answer as unknown as NonNullable<typeof registration>;
- selectedWorkbench = {
- ...options,
- python,
- environment: { ...options.environment, ...environment },
- };
+test("cached empty scan validation rechecks an unchanged, edited, and committed target", async () => {
+ const root = await temporaryDirectory("empty-cached-scan-", true);
+ const repository = join(root, "repository"),
+ scanDir = join(root, "scan"),
+ home = join(root, "codex");
+ await mkdir(repository);
+ await mkdir(home);
+ await mkdir(scanDir, { mode: 0o700 });
+ git(repository, "init", "-q", "-b", "main");
+ await writeFile(join(repository, "source.ts"), "export const value = 1;\n");
+ git(repository, "add", ".");
+ git(repository, "commit", "-qm", "initial");
+ const environment = {
+ ...process.env,
+ CODEX_HOME: home,
+ CODEX_SECURITY_STATE_DIR: join(root, "state"),
+ OPENAI_API_KEY: "synthetic-cached-scan-key",
+ };
+ const python = await resolvePluginPython({
+ environment,
+ protectedRoot: repository,
+ });
+ let scanId = "",
+ modelCalls = 0;
+ let registration:
+ | {
+ contract: { target: { allowedKinds: string[] } };
+ targetId: string;
+ targetRevision?: string;
}
- return answer;
- };
- const createSecurity = (
- config: ConstructorParameters<typeof TestClient>[0],
- ) =>
- new TestClient(
- { ...config, pythonPath: python, pluginPath: PLUGIN_ROOT },
- {
- environment,
- prepareRuntime: async () => {
- const r = preparedRuntime(home);
- r.environment = environment;
- r.plugin.version = JSON.parse(
- await readFile(
- join(PLUGIN_ROOT, ".codex-plugin", "plugin.json"),
- "utf8",
- ),
- ).version;
- return r;
- },
- prepareOutputDir: async () => scanDir,
- resolvePluginPython: async () => python,
- runWorkbench: workbench,
- createCodex: codexFactory(async () => {
- modelCalls++;
- await copyCompletedScan(root);
- const manifest = JSON.parse(
- await readFile(join(scanDir, "scan-manifest.json"), "utf8"),
- );
- manifest.scan.id = scanId;
- manifest.scan.target.kind =
- registration!.contract.target.allowedKinds[0];
- manifest.scan.target.targetId = registration!.targetId;
- manifest.scan.target.revision = registration!.targetRevision;
- manifest.scan.target.displayName = "repository";
- delete manifest.scan.target.snapshotDigest;
- delete manifest.scan.sealedAt;
- delete manifest.scan.artifacts;
- await writeFile(
- join(scanDir, "scan-manifest.json"),
- JSON.stringify(manifest),
- );
- const findings = JSON.parse(
- await readFile(join(scanDir, "findings.json"), "utf8"),
- );
- findings.scanId = scanId;
- findings.findings = [];
- await writeFile(
- join(scanDir, "findings.json"),
- JSON.stringify(findings),
- );
- const coverage = JSON.parse(
- await readFile(join(scanDir, "coverage.json"), "utf8"),
- );
- coverage.scanId = scanId;
- coverage.surfaces = [];
- await writeFile(
- join(scanDir, "coverage.json"),
- JSON.stringify(coverage),
- );
- return { events: completedEvents() };
- }, "empty-cached-conversation"),
- },
- );
- const cliDependencies = {
- ...dependencies({ currentDirectory: root, environment }),
- createSecurity,
- runWorkbench: async (
- args: readonly string[],
- input?: string,
- signal?: AbortSignal,
- configuredPython?: string,
- protectedRoot?: string,
- ) => {
- expect(configuredPython).toBe(python);
- expect(protectedRoot).toBe(repository);
- expect(selectedWorkbench).toBeDefined();
- return runWorkbench({ ...selectedWorkbench!, signal }, args, input);
+ | undefined;
+ let selectedWorkbench: Parameters<typeof runWorkbench>[0] | undefined;
+ const workbench = async (
+ options: Parameters<typeof runWorkbench>[0],
+ args: readonly string[],
+ input?: string,
+ ) => {
+ const answer = await runWorkbench(
+ {
+ ...options,
+ python,
+ environment: { ...options.environment, ...environment },
},
- };
- try {
- const first = createCliTest(main);
- const args = [
- "scan",
- repository,
- "--python",
+ args,
+ input,
+ );
+ if (args[0] === "register-cli-scan") {
+ scanId = String(answer["scanId"]);
+ registration = answer as unknown as NonNullable<typeof registration>;
+ selectedWorkbench = {
+ ...options,
python,
- "--plugin-path",
- PLUGIN_ROOT,
- "--workflow-id",
- "empty-cached-workflow",
- "--json",
- ];
- const initial = await first.runCli(args, cliDependencies);
- expect(initial, first.stderr.text()).toBe(0);
- expect(JSON.parse(first.stdout.text()).findings.findings).toEqual([]);
- expect(modelCalls).toBe(1);
- if (changed !== "unchanged")
+ environment: { ...options.environment, ...environment },
+ };
+ }
+ return answer;
+ };
+ const createSecurity = (
+ config: ConstructorParameters<typeof TestClient>[0],
+ ) =>
+ new TestClient(
+ { ...config, pythonPath: python, pluginPath: PLUGIN_ROOT },
+ {
+ environment,
+ prepareRuntime: async () => {
+ const r = preparedRuntime(home);
+ r.environment = environment;
+ r.plugin.version = JSON.parse(
+ await readFile(
+ join(PLUGIN_ROOT, ".codex-plugin", "plugin.json"),
+ "utf8",
+ ),
+ ).version;
+ return r;
+ },
+ prepareOutputDir: async () => scanDir,
+ resolvePluginPython: async () => python,
+ runWorkbench: workbench,
+ createCodex: codexFactory(async () => {
+ modelCalls++;
+ await copyCompletedScan(root);
+ const manifest = JSON.parse(
+ await readFile(join(scanDir, "scan-manifest.json"), "utf8"),
+ );
+ manifest.scan.id = scanId;
+ manifest.scan.target.kind =
+ registration!.contract.target.allowedKinds[0];
+ manifest.scan.target.targetId = registration!.targetId;
+ manifest.scan.target.revision = registration!.targetRevision;
+ manifest.scan.target.displayName = "repository";
+ delete manifest.scan.target.snapshotDigest;
+ delete manifest.scan.sealedAt;
+ delete manifest.scan.artifacts;
+ await writeFile(
+ join(scanDir, "scan-manifest.json"),
+ JSON.stringify(manifest),
+ );
+ const findings = JSON.parse(
+ await readFile(join(scanDir, "findings.json"), "utf8"),
+ );
+ findings.scanId = scanId;
+ findings.findings = [];
+ await writeFile(
+ join(scanDir, "findings.json"),
+ JSON.stringify(findings),
+ );
+ const coverage = JSON.parse(
+ await readFile(join(scanDir, "coverage.json"), "utf8"),
+ );
+ coverage.scanId = scanId;
+ coverage.surfaces = [];
+ await writeFile(
+ join(scanDir, "coverage.json"),
+ JSON.stringify(coverage),
+ );
+ return { events: completedEvents() };
+ }, "empty-cached-conversation"),
+ },
+ );
+ const targetChecks: string[] = [];
+ const cliDependencies = {
+ ...dependencies({ currentDirectory: root, environment }),
+ createSecurity,
+ runWorkbench: async (
+ args: readonly string[],
+ input?: string,
+ signal?: AbortSignal,
+ configuredPython?: string,
+ protectedRoot?: string,
+ ) => {
+ expect(configuredPython).toBe(python);
+ expect(protectedRoot).toBe(repository);
+ expect(selectedWorkbench).toBeDefined();
+ expect(args).toEqual(["get-scan", "--scan-id", scanId, "--check-target"]);
+ targetChecks.push(scanId);
+ return runWorkbench({ ...selectedWorkbench!, signal }, args, input);
+ },
+ };
+ try {
+ const first = createCliTest(main);
+ const args = [
+ "scan",
+ repository,
+ "--python",
+ python,
+ "--plugin-path",
+ PLUGIN_ROOT,
+ "--workflow-id",
+ "empty-cached-workflow",
+ "--json",
+ ];
+ const initial = await first.runCli(args, cliDependencies);
+ expect(initial, first.stderr.text()).toBe(0);
+ expect(JSON.parse(first.stdout.text()).findings.findings).toEqual([]);
+ expect(modelCalls).toBe(1);
+ for (const changed of ["unchanged", "content", "head"] as const) {
+ if (changed === "content")
await writeFile(
join(repository, "source.ts"),
"export const value = 2;\n",
);
if (changed === "head") git(repository, "commit", "-qam", "changed");
- const recorded = await runWorkbench(selectedWorkbench!, [
- "get-scan",
- "--scan-id",
- scanId,
- "--check-target",
- ]).then(
- () => "unchanged",
- (e) => (e as Error).message,
- );
- expect(recorded === "unchanged").toBe(changed === "unchanged");
const second = createCliTest(main);
const result = await second.runCli(
[...args, "--validate"],
@@ -417,10 +410,12 @@ for (const changed of ["unchanged", "content", "head"] as const)
findings: 0,
message: expect.stringContaining("changed"),
});
- } finally {
- await rm(root, { recursive: true, force: true });
}
- });
+ expect(targetChecks).toEqual([scanId, scanId, scanId]);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
test("empty scan validation preserves configured Python and Git ignore rules", async () => {
if (
diff --git a/sdk/typescript/tests-ts/cli-skill-native-config.test.ts b/sdk/typescript/tests-ts/cli-skill-native-config.test.ts
index 59a63a8cf..84854ee1c 100644
--- a/sdk/typescript/tests-ts/cli-skill-native-config.test.ts
+++ b/sdk/typescript/tests-ts/cli-skill-native-config.test.ts
@@ -361,11 +361,11 @@ async function withStoredApiKey(
}
}
-test.each(
- (["patch", "verify-fix"] as const).flatMap((command) =>
- [true, false].map((daybreak) => [command, daybreak] as const),
- ),
-)(
+test.each([
+ ["patch", true],
+ ["patch", false],
+ ["verify-fix", true],
+] as const)(
"%s preserves ambient Daybreak %p with a stored API key",
async (command, daybreak) => {
await withStoredApiKey(
@@ -384,95 +384,80 @@ test.each(
},
);
-test.each(["patch", "verify-fix"] as const)(
- "%s preserves the Daybreak opt-out with an explicit toggle and a stored API key",
- async (command) => {
- await withStoredApiKey(command, async ({ ambientHome, run }) => {
- await writeCodexConfig(join(ambientHome, "config.toml"), {
- daybreak: false,
- features: { cli_daybreak: false },
- });
- expect(await run({ daybreak: true })).toMatchObject({
- daybreak: true,
- daybreakEnabled: true,
- daybreakFeatures: { cli_daybreak: false },
- });
+test("patch preserves the Daybreak opt-out with an explicit toggle and a stored API key", async () => {
+ await withStoredApiKey("patch", async ({ ambientHome, run }) => {
+ await writeCodexConfig(join(ambientHome, "config.toml"), {
+ daybreak: false,
+ features: { cli_daybreak: false },
});
- },
-);
+ expect(await run({ daybreak: true })).toMatchObject({
+ daybreak: true,
+ daybreakEnabled: true,
+ daybreakFeatures: { cli_daybreak: false },
+ });
+ });
+});
-test.each(["patch", "verify-fix"] as const)(
- "%s forwards wrapper-resolved ambient Daybreak profiles with a stored API key",
- async (command) => {
- await withStoredApiKey(command, async ({ ambientHome, run }) => {
- await writeCodexConfig(join(ambientHome, "config.toml"), {
- daybreak: false,
- features: { cli_daybreak: true },
- profile: "selected",
- profiles: {
- selected: { daybreak: true, features: disabledDaybreakFeatures },
- },
- });
- expect(await run()).toMatchObject({
- daybreak: true,
- daybreakEnabled: true,
- daybreakFeatures: disabledDaybreakFeatures,
- });
+test("patch forwards wrapper-resolved ambient Daybreak profiles with a stored API key", async () => {
+ await withStoredApiKey("patch", async ({ ambientHome, run }) => {
+ await writeCodexConfig(join(ambientHome, "config.toml"), {
+ daybreak: false,
+ features: { cli_daybreak: true },
+ profile: "selected",
+ profiles: {
+ selected: { daybreak: true, features: disabledDaybreakFeatures },
+ },
});
- },
-);
+ expect(await run()).toMatchObject({
+ daybreak: true,
+ daybreakEnabled: true,
+ daybreakFeatures: disabledDaybreakFeatures,
+ });
+ });
+});
-test.each(["patch", "verify-fix"] as const)(
- "%s clears removed ambient Daybreak settings on the next stored API-key launch",
- async (command) => {
- await withStoredApiKey(command, async ({ ambientHome, run }) => {
- const configPath = join(ambientHome, "config.toml");
- await writeCodexConfig(configPath, {});
- const baseline = await run();
- await writeCodexConfig(configPath, {
- daybreak: true,
- features: disabledDaybreakFeatures,
- });
- expect(await run()).toMatchObject({
- daybreak: true,
- daybreakEnabled: true,
- daybreakFeatures: disabledDaybreakFeatures,
- });
- await writeCodexConfig(configPath, {});
- expect(await run()).toEqual(baseline);
+test("patch clears removed ambient Daybreak settings on the next stored API-key launch", async () => {
+ await withStoredApiKey("patch", async ({ ambientHome, run }) => {
+ const configPath = join(ambientHome, "config.toml");
+ await writeCodexConfig(configPath, {});
+ const baseline = await run();
+ await writeCodexConfig(configPath, {
+ daybreak: true,
+ features: disabledDaybreakFeatures,
});
- },
-);
+ expect(await run()).toMatchObject({
+ daybreak: true,
+ daybreakEnabled: true,
+ daybreakFeatures: disabledDaybreakFeatures,
+ });
+ await writeCodexConfig(configPath, {});
+ expect(await run()).toEqual(baseline);
+ });
+});
-test.each(["patch", "verify-fix"] as const)(
- "%s preserves native project and explicit Daybreak precedence with a stored API key",
- async (command) => {
- await withStoredApiKey(
- command,
- async ({ ambientHome, repository, run }) => {
- await writeCodexConfig(join(ambientHome, "config.toml"), {
- daybreak: true,
- features: { cli_daybreak: true },
- projects: { [repository]: { trust_level: "trusted" } },
- });
- await writeCodexConfig(join(repository, ".codex", "config.toml"), {
- daybreak: false,
- features: { cli_daybreak: false },
- });
- expect(await run()).toMatchObject({
- daybreak: false,
- daybreakEnabled: null,
- daybreakFeatures: { cli_daybreak: false },
- });
- expect(await run({ daybreak: true })).toMatchObject({
- daybreak: true,
- daybreakEnabled: true,
- daybreakFeatures: { cli_daybreak: false },
- });
- },
- );
- },
-);
+test("patch preserves native project and explicit Daybreak precedence with a stored API key", async () => {
+ await withStoredApiKey("patch", async ({ ambientHome, repository, run }) => {
+ await writeCodexConfig(join(ambientHome, "config.toml"), {
+ daybreak: true,
+ features: { cli_daybreak: true },
+ projects: { [repository]: { trust_level: "trusted" } },
+ });
+ await writeCodexConfig(join(repository, ".codex", "config.toml"), {
+ daybreak: false,
+ features: { cli_daybreak: false },
+ });
+ expect(await run()).toMatchObject({
+ daybreak: false,
+ daybreakEnabled: null,
+ daybreakFeatures: { cli_daybreak: false },
+ });
+ expect(await run({ daybreak: true })).toMatchObject({
+ daybreak: true,
+ daybreakEnabled: true,
+ daybreakFeatures: { cli_daybreak: false },
+ });
+ });
+});
test.each(["omitted", "omitted with changed gates", "explicit"] as const)(
"stored API-key patches preserve %s settings for an active scan's later workers",
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index 9acd9fbee..0daca1efd 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -3778,16 +3778,6 @@ describe("CLI", () => {
}
});
- test("honors Incur help before command validation", async () => {
- const { stdout, stderr, runCli } = createCliTest(main);
-
- expect(
- await runCli(["scan", "--mode", "bogus", "--help"], dependencies()),
- ).toBe(0);
- expect(stdout.text()).toContain("Usage: codex-security scan [repository]");
- expect(stderr.text()).toBe("");
- });
-
test("maps configuration and emits JSON only on stdout", async () => {
const { stdout, stderr, runCli } = createCliTest(main);
@@ -4629,49 +4619,18 @@ describe("CLI", () => {
}
});
- test("emits structured failures for machine-readable scan output", async () => {
+ test("preserves a model content refusal in structured scan failures", async () => {
const message = "This content was flagged for possible cybersecurity risk.";
- for (const formatArgs of [
- ["--json"],
- ["--format", "json"],
- ["--format", "jsonl"],
- ] as const) {
- const { stdout, stderr, runCli } = createCliTest(main);
-
- const deps = dependencies();
- deps.createSecurity = () => failingSecurity(message);
-
- expect(await runCli(["scan", ".", ...formatArgs], deps)).toBe(2);
- expect(JSON.parse(stdout.text().trim())).toEqual({
- status: "failed",
- code: "SCAN_FAILED",
- message,
- });
- expect(stderr.text()).toContain(`${message}\n`);
- }
- });
-
- test("reports full-output scan failures in an error envelope", async () => {
- const message = "This content was flagged for possible cybersecurity risk.";
- for (const formatArgs of [
- ["--json"],
- ["--format", "json"],
- ["--format", "jsonl"],
- ] as const) {
- const { stdout, stderr, runCli } = createCliTest(main);
-
- const deps = dependencies();
- deps.createSecurity = () => failingSecurity(message);
-
- expect(
- await runCli(["scan", ".", ...formatArgs, "--full-output"], deps),
- ).toBe(2);
- const envelope = JSON.parse(stdout.text().trim());
- expect(envelope.ok).toBe(false);
- expect(envelope.error).toEqual({ code: "SCAN_FAILED", message });
- expect(envelope).not.toHaveProperty("data");
- expect(stderr.text()).toContain(`${message}\n`);
- }
+ const { stdout, stderr, runCli } = createCliTest(main);
+ const deps = dependencies();
+ deps.createSecurity = () => failingSecurity(message);
+ expect(await runCli(["scan", ".", "--json"], deps)).toBe(2);
+ expect(JSON.parse(stdout.text())).toEqual({
+ status: "failed",
+ code: "SCAN_FAILED",
+ message,
+ });
+ expect(stderr.text()).toContain(message);
});
test("preserves diagnostic text in full-output scan failures", async () => {
diff --git a/sdk/typescript/tests-ts/support/validation.ts b/sdk/typescript/tests-ts/support/validation.ts
new file mode 100644
index 000000000..9dd1a1c88
--- /dev/null
+++ b/sdk/typescript/tests-ts/support/validation.ts
@@ -0,0 +1,240 @@
+import { mkdir } from "node:fs/promises";
+import * as fsPromises from "node:fs/promises";
+import { join } from "node:path";
+import type {
+ CodexOptions,
+ ThreadEvent,
+ ThreadOptions,
+ TurnOptions,
+} from "@openai/codex-sdk";
+import { afterEach, beforeEach, mock, spyOn } from "bun:test";
+import type { JsonObject } from "../../src/config.js";
+import { resolvePluginPython, runWorkbench } from "../../src/runtime.js";
+
+import { PLUGIN_ROOT } from "../plugin-root.js";
+import { TestClient } from "./api-client.js";
+import { preparedRuntime } from "./api-events.js";
+import { createApiTestFixtures } from "./temporary-directories.js";
+import { FindingWorkflow } from "../../src/finding-workflow.js";
+
+export function createValidationFixtures() {
+ const { cleanup, temporaryDirectory } = createApiTestFixtures();
+
+ async function repositoryDirectories() {
+ const root = await temporaryDirectory();
+ const repository = join(root, "repository");
+ await mkdir(repository);
+ return { root, repository };
+ }
+
+ async function runtimeDirectories() {
+ const { root, repository } = await repositoryDirectories();
+ const codexHome = join(root, "codex-home");
+ await mkdir(codexHome);
+ return { root, repository, codexHome };
+ }
+
+ function runtimePreparer(
+ codexHome: string,
+ overrides: () => Partial<ReturnType<typeof preparedRuntime>>,
+ ) {
+ return async () => ({
+ ...preparedRuntime(codexHome),
+ ...overrides(),
+ });
+ }
+ const actualLstat = fsPromises.lstat;
+ let systemConfigurationLookup: ReturnType<
+ typeof spyOn<typeof fsPromises, "lstat">
+ >;
+ beforeEach(() => {
+ const nativePath =
+ process.platform === "win32"
+ ? join("C:\\ProgramData", "OpenAI", "Codex", "config.toml")
+ : "/etc/codex/config.toml";
+ systemConfigurationLookup = spyOn(fsPromises, "lstat").mockImplementation(
+ (async (...args: Parameters<typeof actualLstat>) => {
+ if (args[0] === nativePath)
+ throw Object.assign(
+ new Error("Synthetic absent native system configuration"),
+ { code: "ENOENT" },
+ );
+ return await actualLstat(...args);
+ }) as typeof actualLstat,
+ );
+ });
+ afterEach(async () => {
+ systemConfigurationLookup.mockRestore();
+ await cleanup();
+ });
+
+ const assessment = {
+ disposition: "reportable",
+ report: "Static trace reaches the SQL sink; runtime proof is still needed.",
+ } as const;
+
+ async function* validationEvents(
+ response = JSON.stringify(assessment),
+ complete = true,
+ ): AsyncGenerator<ThreadEvent> {
+ yield { type: "thread.started", thread_id: "validation-thread" };
+ yield {
+ type: "item.completed",
+ item: { id: "result", type: "agent_message", text: response },
+ };
+ if (complete) {
+ yield {
+ type: "turn.completed",
+ usage: {
+ input_tokens: 10,
+ cached_input_tokens: 0,
+ cache_write_input_tokens: 0,
+ output_tokens: 3,
+ reasoning_output_tokens: 0,
+ },
+ };
+ }
+ }
+
+ let fixturePython: Promise<string> | undefined;
+
+ async function validationClient(
+ events: (signal: AbortSignal) => AsyncGenerator<ThreadEvent> = () =>
+ validationEvents(),
+ pluginRoot = PLUGIN_ROOT,
+ pythonPath = "/managed/python",
+ fixtureRoot?: string,
+ environmentOverrides: Record<string, string> = {},
+ ) {
+ const root = fixtureRoot ?? (await temporaryDirectory());
+ const repository = join(root, "repository");
+ const codexHome = await temporaryDirectory();
+ const stateDirectory =
+ environmentOverrides["CODEX_SECURITY_STATE_DIR"] ?? join(root, "state");
+ await mkdir(repository, { recursive: true });
+ const captured: {
+ codex?: CodexOptions;
+ thread?: ThreadOptions;
+ prompt?: string;
+ turn?: TurnOptions;
+ } = {};
+ const workbench = mock(
+ async (
+ _options: Parameters<typeof runWorkbench>[0],
+ _args: readonly string[],
+ _input?: string,
+ ): Promise<JsonObject> => ({
+ scan: { targetPath: repository },
+ }),
+ );
+ const environment = {
+ ...Object.fromEntries(
+ ["PATH", "SystemRoot", "TEMP", "TMP"].flatMap((name) =>
+ process.env[name] === undefined ? [] : [[name, process.env[name]!]],
+ ),
+ ),
+ CODEX_SECURITY_STATE_DIR: stateDirectory,
+ OPENAI_API_KEY: "synthetic-validation-key",
+ ...environmentOverrides,
+ };
+ const client = new TestClient(
+ {
+ pluginPath: pluginRoot === PLUGIN_ROOT ? undefined : pluginRoot,
+ pythonPath,
+ codexOverrides: {
+ model: "test-model",
+ model_reasoning_effort: "high",
+ approval_policy: "never",
+ analytics: { enabled: false },
+ },
+ },
+ {
+ environment,
+ prepareRuntime: runtimePreparer(codexHome, () => ({
+ plugin: { ...preparedRuntime(codexHome).plugin, pluginRoot },
+ environment,
+ })),
+ resolvePluginPython: async ({ configuredPath } = {}) => configuredPath!,
+ runWorkbench: workbench,
+ createCodex: (options) => {
+ captured.codex = options;
+ return {
+ startThread: (options) => {
+ captured.thread = options;
+ return {
+ id: null,
+ async runStreamed(prompt, options) {
+ captured.prompt = prompt;
+ captured.turn = options;
+ return { events: events(options.signal!) };
+ },
+ };
+ },
+ };
+ },
+ },
+ );
+ const options = {
+ repositoryPath: repository,
+ finding: "Candidate finding",
+ outputDir: join(root, "validation"),
+ };
+ return {
+ client,
+ options,
+ stateDirectory,
+ captured,
+ workbench,
+ root,
+ codexHome,
+ };
+ }
+
+ async function persistedValidationClient(
+ events?: Parameters<typeof validationClient>[0],
+ pluginRoot = PLUGIN_ROOT,
+ fixtureRoot?: string,
+ environmentOverrides?: Record<string, string>,
+ ) {
+ const python = await (fixturePython ??= resolvePluginPython());
+ const fixture = await validationClient(
+ events,
+ pluginRoot,
+ python,
+ fixtureRoot,
+ environmentOverrides,
+ );
+ fixture.workbench.mockImplementation(runWorkbench);
+ async function bindWorkflow(
+ id: string,
+ environment: NodeJS.ProcessEnv = {
+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,
+ },
+ ) {
+ const workflow = new FindingWorkflow(
+ id,
+ environment,
+ runWorkbench,
+ python,
+ {
+ workbenchOptions: { python, pluginRoot: PLUGIN_ROOT, environment },
+ },
+ );
+ await workflow.bind({ repositoryPath: fixture.options.repositoryPath });
+ return id;
+ }
+ return { ...fixture, python, bindWorkflow };
+ }
+
+ return {
+ actualLstat,
+ assessment,
+ validationEvents,
+ validationClient,
+ persistedValidationClient,
+ temporaryDirectory,
+ repositoryDirectories,
+ runtimeDirectories,
+ runtimePreparer,
+ };
+}