OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(sdk): streamline CLI and validation coverage (#1971)
Michael D'Angelo0036b153d0fb
diff --git a/sdk/typescript/scripts/test-timings.json b/sdk/typescript/scripts/test-timings.jsonindex aea906dbe..33bdcd98c 100644--- a/sdk/typescript/scripts/test-timings.json+++ b/sdk/typescript/scripts/test-timings.json@@ -109,10 +109,25 @@"darwin": 1.1,"win32": 2},+ "api-validation-evidence.test.ts": {+ "linux": 74.9,+ "darwin": 128.8,+ "win32": 139+ },+ "api-validation-guidance.test.ts": {+ "linux": 42.9,+ "darwin": 66.5,+ "win32": 66.1+ },+ "api-validation-persistence.test.ts": {+ "linux": 77.3,+ "darwin": 121.6,+ "win32": 136.7+ },"api-validation.test.ts": {- "linux": 190.4,- "darwin": 197.1,- "win32": 319.2+ "linux": 0.3,+ "darwin": 0.8,+ "win32": 3.6},"api-workflow.test.ts": {"linux": 27.1,diff --git a/sdk/typescript/tests-ts/api-events.test.ts b/sdk/typescript/tests-ts/api-events.test.tsindex f9103d7b3..368da5c46 100644--- a/sdk/typescript/tests-ts/api-events.test.ts+++ b/sdk/typescript/tests-ts/api-events.test.ts@@ -763,83 +763,39 @@ describe("one-shot scan events", () => {});});- test("fails the scan for every turn.failed error payload shape", async () => {- const usage = {- input_tokens: 10,- cached_input_tokens: 2,- cache_write_input_tokens: 0,- output_tokens: 3,- reasoning_output_tokens: 1,- };- const payloads: Array<[string, unknown]> = [- ["object message", { message: "model refused the turn" }],- ["null", null],- ["undefined", undefined],- ["string", "model refused the turn"],- ["non-string message", { message: { text: "model refused the turn" } }],- ["array", ["model refused the turn"]],- ["blank message", { message: " " }],+ test("fails the scan for every turn.failed payload, reporting only readable nested messages", async () => {+ const fallback =+ "The Codex Security scan turn failed without a readable error message.";+ const scanDir = await copyCompletedScan(await temporaryDirectory());+ const payloads: Array<[string, unknown, string]> = [+ [+ "object message",+ { message: "model refused the turn" },+ "model refused the turn",+ ],+ ["null", null, fallback],+ ["undefined", undefined, fallback],+ ["raw string", "token sk-proj-EXAMPLE1234567890 rejected", fallback],+ [+ "non-string message",+ { message: { text: "model refused the turn" } },+ fallback,+ ],+ ["array", ["model refused the turn"], fallback],+ ["blank message", { message: " " }, fallback],+ ["missing message", { code: 500 }, fallback],];- for (const [label, error] of payloads) {- // A complete, valid artifact bundle so the failed turn is the only variable.- const scanDir = await copyCompletedScan(await temporaryDirectory());+ for (const [label, error, message] of payloads) {async function* failedEvents(): AsyncGenerator<ThreadEvent> {yield { type: "thread.started", thread_id: "thread-1" };- yield { type: "turn.completed", usage };+ yield completedTurn();yield { type: "turn.failed", error } as unknown as ThreadEvent;}await expect(runEvents(scanDir, failedEvents()),- `turn.failed carrying ${label} must fail the scan`,- ).rejects.toMatchObject({ name: CodexSecurityError.name });- }- });-- test("reuses only a nested turn.failed message and falls back otherwise", async () => {- const usage = {- input_tokens: 10,- cached_input_tokens: 2,- cache_write_input_tokens: 0,- output_tokens: 3,- reasoning_output_tokens: 1,- };- async function* failedWith(error: unknown): AsyncGenerator<ThreadEvent> {- yield { type: "thread.started", thread_id: "thread-1" };- yield { type: "turn.completed", usage };- yield { type: "turn.failed", error } as unknown as ThreadEvent;+ `turn.failed carrying ${label}`,+ ).rejects.toMatchObject({ name: CodexSecurityError.name, message });}-- await expect(- runEvents(- await copyCompletedScan(await temporaryDirectory()),- failedWith({ message: "retry budget exhausted" }),- ),- ).rejects.toMatchObject({- name: CodexSecurityError.name,- message: "retry budget exhausted",- });-- await expect(- runEvents(- await copyCompletedScan(await temporaryDirectory()),- failedWith("token sk-proj-EXAMPLE1234567890 rejected"),- ),- ).rejects.toMatchObject({- name: CodexSecurityError.name,- message:- "The Codex Security scan turn failed without a readable error message.",- });-- await expect(- runEvents(- await copyCompletedScan(await temporaryDirectory()),- failedWith({ code: 500 }),- ),- ).rejects.toMatchObject({- name: CodexSecurityError.name,- message:- "The Codex Security scan turn failed without a readable error message.",- });});test("extracts bounded rate-limit context from reconnect notifications", async () => {diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.tsindex 6a2473c0e..bdcd4183c 100644--- a/sdk/typescript/tests-ts/api-policy.test.ts+++ b/sdk/typescript/tests-ts/api-policy.test.ts@@ -771,13 +771,15 @@ describe("CodexSecurity policy API", () => {const gitDirectory = policyGitDirectory(repository);for (const metadata of new Set([common, gitDirectory])) {const outputDir = join(metadata, "policy-artifacts");- for (const operation of [- () => f.security.preflightPolicy(repository, { outputDir }),- () => f.security.generatePolicy(repository, { outputDir }),- ])- await expect(operation()).rejects.toThrow(- "outside the protected scan root",- );+ await expect(+ f.security.preflightPolicy(repository, { outputDir }),+ ).rejects.toThrow("outside the protected scan root");+ // Generation uses the same checks as preflight; exercise each destination type once.+ if (kind === "separate") {+ await expect(+ f.security.generatePolicy(repository, { outputDir }),+ ).rejects.toThrow("outside the protected scan root");+ }const state = new InternalSecurity({},{ environment: { CODEX_SECURITY_STATE_DIR: outputDir } },@@ -785,9 +787,11 @@ describe("CodexSecurity policy API", () => {await expect(state.preflightPolicy(repository)).rejects.toThrow("outside the protected scan root",);- await expect(state.generatePolicy(repository)).rejects.toThrow(- "outside the protected scan root",- );+ if (kind === "linked" && metadata === gitDirectory) {+ await expect(state.generatePolicy(repository)).rejects.toThrow(+ "outside the protected scan root",+ );+ }await state.close();await expect(readdir(outputDir)).rejects.toMatchObject({code: "ENOENT",diff --git a/sdk/typescript/tests-ts/api-provider-isolation.test.ts b/sdk/typescript/tests-ts/api-provider-isolation.test.tsindex b2f7f6386..fb6b940c9 100644--- a/sdk/typescript/tests-ts/api-provider-isolation.test.ts+++ b/sdk/typescript/tests-ts/api-provider-isolation.test.ts@@ -612,11 +612,12 @@ for (const resume of [false, true]) {await mkdir(repository);await mkdir(scan, { mode: 0o700 });await mkdir(home, { mode: 0o700 });- await cp(PLUGIN_ROOT, plugin, { recursive: true });- const manifestPath = join(plugin, ".codex-plugin", "plugin.json");- const manifest = JSON.parse(await readFile(manifestPath, "utf8"));- manifest.codexSecurity = capabilities;- await writeFile(manifestPath, JSON.stringify(manifest));+ const manifestDirectory = join(plugin, ".codex-plugin");+ await mkdir(manifestDirectory, { recursive: true });+ await writeFile(+ join(manifestDirectory, "plugin.json"),+ JSON.stringify({ codexSecurity: capabilities }),+ );const runtime = preparedRuntime(home);const createCodex = mock(() => {throw new Error("Model execution must not begin.");diff --git a/sdk/typescript/tests-ts/api-validation-evidence.test.ts b/sdk/typescript/tests-ts/api-validation-evidence.test.tsnew file mode 100644index 000000000..500b6f2ab--- /dev/null+++ b/sdk/typescript/tests-ts/api-validation-evidence.test.ts@@ -0,0 +1,404 @@+import {+ mkdir,+ readFile,+ realpath,+ rename,+ rm,+ symlink,+ writeFile,+} from "node:fs/promises";+import { execFileSync } from "node:child_process";+import { existsSync } from "node:fs";+import { dirname, join } from "node:path";+import { describe, expect, test } from "bun:test";+import { runWorkbench } from "../src/runtime.js";++import { PLUGIN_ROOT } from "./plugin-root.js";+import { gitText } from "./support/shell.js";+import { createValidationFixtures } from "./support/validation.js";++describe("CodexSecurity finding validation", () => {+ const {+ assessment,+ validationEvents,+ persistedValidationClient,+ temporaryDirectory,+ } = createValidationFixtures();++ async function registerValidationScan(+ fixture: Awaited<ReturnType<typeof persistedValidationClient>>,+ repository: string,+ environment: NodeJS.ProcessEnv,+ ): Promise<string> {+ const scanDir = join(fixture.root, "scan");+ await mkdir(scanDir, { mode: 0o700 });+ const registration = await runWorkbench(+ { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },+ [+ "register-cli-scan",+ "--repository",+ repository,+ "--scan-dir",+ scanDir,+ "--recipe-json",+ JSON.stringify({+ repository,+ mode: "standard",+ target: { kind: "repository", paths: [] },+ config: {},+ }),+ ],+ );+ return registration["scanId"] as string;+ }++ test.each([+ "AGENTS.md",+ "REVIEW.md",+ ".codex/config.toml",+ "unrelated.txt",+ "unchanged",+ ] as const)(+ "rechecks native guidance created while cached evidence is fingerprinted: %s",+ async (scope) => {+ const stateRoot = await temporaryDirectory("validation-guidance-race-");+ execFileSync("git", ["init", "-q", stateRoot]);+ const guide = join(stateRoot, scope);+ let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;+ let modelCalls = 0;+ async function* events() {+ modelCalls++;+ await writeFile(+ join(fixture.captured.thread!.workingDirectory!, "proof.bin"),+ Buffer.alloc(256 * 1024, 7),+ );+ yield* validationEvents(+ JSON.stringify({+ ...assessment,+ report: existsSync(guide)+ ? await readFile(guide, "utf8")+ : "Original assessment.",+ }),+ );+ }+ fixture = await persistedValidationClient(+ events,+ PLUGIN_ROOT,+ undefined,+ {+ CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),+ },+ );+ await using client = fixture.client;+ client.config.codexOverrides!["project_doc_fallback_filenames"] = [+ "REVIEW.md",+ ];+ let changeDuringEvidence = false;+ let changed = false;+ fixture.workbench.mockImplementation(async (options, args, input) => {+ const payload =+ args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;+ const pending = runWorkbench(options, args, input);+ if (+ changeDuringEvidence &&+ !changed &&+ payload?.action === "source" &&+ payload.evidence === true+ ) {+ changed = true;+ if (scope !== "unchanged") {+ await mkdir(dirname(guide), { recursive: true });+ await writeFile(+ guide,+ scope === ".codex/config.toml"+ ? 'developer_instructions = "New native guidance."\n'+ : "New native guidance.\n",+ );+ }+ }+ return await pending;+ });+ const workflowId = await fixture.bindWorkflow(+ "native-guidance-during-evidence",+ );+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect(await client.validate(request)).toEqual(first);+ expect(modelCalls).toBe(1);+ changeDuringEvidence = true;+ const second = await client.validate(request);+ const nativeGuidance = !["unrelated.txt", "unchanged"].includes(scope);+ expect(changed).toBe(true);+ expect(modelCalls).toBe(nativeGuidance ? 2 : 1);+ expect(second.report).toBe(+ nativeGuidance ? (await readFile(guide, "utf8")).trim() : first.report,+ );+ const sameGuidance = await client.validate(request);+ expect(sameGuidance.report).toBe(second.report);+ expect(modelCalls).toBe(nativeGuidance ? 3 : 1);+ },+ );++ test.each([+ "unchanged",+ "source contents",+ "recorded contents",+ "recorded checkout",+ ] as const)(+ "rechecks %s after reading a saved validation assessment",+ async (change) => {+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const repository = fixture.options.repositoryPath;+ const source = join(repository, "source.ts");+ await writeFile(source, "export const value = 'original';\n");+ const environment = {+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,+ };+ const scanId =+ change === "source contents"+ ? undefined+ : await registerValidationScan(fixture, repository, environment);+ const workflowId = "cached-validation-recheck";+ await fixture.bindWorkflow(workflowId, environment);+ let readSavedAssessment = false;+ fixture.workbench.mockImplementation(async (options, args, input) => {+ const response = await runWorkbench(options, args, input);+ if (+ args[0] === "finding-workflow" &&+ JSON.parse(input!)["action"] === "get-review" &&+ response["review"] !== null+ ) {+ readSavedAssessment = true;+ if (change === "recorded checkout") {+ await rename(repository, join(fixture.root, "original-repository"));+ await mkdir(repository);+ await writeFile(source, "export const value = 'original';\n");+ } else if (change !== "unchanged") {+ await writeFile(source, "export const value = 'changed';\n");+ }+ }+ return response;+ });+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ ...(scanId === undefined ? {} : { scanId }),+ };+ const first = await client.validate(request);+ if (change === "unchanged") {+ expect(await client.validate(request)).toEqual(first);+ } else {+ await expect(client.validate(request)).rejects.toThrow(+ change === "source contents"+ ? "Repository changed during validation"+ : change === "recorded contents"+ ? "Scan target contents changed"+ : "checkout path was replaced",+ );+ }+ expect(readSavedAssessment).toBe(true);+ expect(modelCalls).toBe(1);+ },+ );++ test.each([+ ["cached", "source contents", "unchanged"],+ ["cached", "recorded contents", "unchanged"],+ ["cached", "source contents", "changed"],+ ["cached", "recorded contents", "missing"],+ ["fresh", "source contents", "unchanged"],+ ["fresh", "recorded contents", "missing"],+ ] as const)(+ "rechecks %s validation %s after reading %s evidence",+ async (phase, change, evidence) => {+ let modelCalls = 0;+ async function* events() {+ modelCalls++;+ await writeFile(+ join(fixture.captured.thread!.workingDirectory!, "proof.txt"),+ "Synthetic evidence.\n",+ );+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const repository = fixture.options.repositoryPath;+ const source = join(repository, "source.ts");+ await writeFile(source, "export const value = 'original';\n");+ const environment = {+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,+ };+ const scanId =+ change === "source contents"+ ? undefined+ : await registerValidationScan(fixture, repository, environment);+ const workflowId = "evidence-validation-recheck";+ await fixture.bindWorkflow(workflowId, environment);+ let changeDuringEvidence = false;+ let changed = false;+ let savedReviews = 0;+ fixture.workbench.mockImplementation(async (options, args, input) => {+ const payload =+ args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;+ const changeNow =+ changeDuringEvidence &&+ !changed &&+ payload?.action === "source" &&+ payload.evidence === true;+ if (changeNow && evidence === "changed")+ await writeFile(+ join(payload.repository, "proof.txt"),+ "Changed evidence.\n",+ );+ if (changeNow && evidence === "missing")+ await rm(payload.repository, { recursive: true });+ const response = await runWorkbench(options, args, input);+ if (changeNow) {+ await writeFile(source, "export const value = 'changed';\n");+ changed = true;+ }+ if (payload?.action === "save-review") savedReviews++;+ return response;+ });+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ ...(scanId === undefined ? {} : { scanId }),+ };+ const first = await client.validate(request);+ expect(await client.validate(request)).toEqual(first);+ expect(modelCalls).toBe(1);+ changeDuringEvidence = true;+ await expect(+ client.validate({+ ...request,+ ...(phase === "fresh" ? { finding: "Another candidate" } : {}),+ }),+ ).rejects.toThrow(+ change === "source contents"+ ? "Repository changed during validation"+ : "Scan target contents changed",+ );+ expect(changed).toBe(true);+ expect(modelCalls).toBe(phase === "fresh" ? 2 : 1);+ expect(savedReviews).toBe(1);+ },+ );++ test.each(["canonical", "repository alias"])(+ "validates an unchanged recorded Git target with %s tool paths",+ async (kind) => {+ const root = await temporaryDirectory();+ const repository = join(root, "repository");+ await mkdir(repository);+ const hostGit = Bun.which("git");+ expect(hostGit).not.toBeNull();+ const tools = join(root, "external-tools");+ await mkdir(tools);+ const selectedGit = join(+ tools,+ process.platform === "win32" ? "git.exe" : "git",+ );+ await symlink(await realpath(hostGit!), selectedGit);+ const alias = join(repository, "tools");+ await symlink(+ tools,+ alias,+ process.platform === "win32" ? "junction" : "dir",+ );+ await writeFile(join(repository, ".gitignore"), "tools\n");+ await writeFile(+ join(repository, "source.ts"),+ "export const value = 'original';\n",+ );+ gitText(["init", "--initial-branch=main"], { cwd: repository });+ gitText(+ [+ "-c",+ "user.name=Synthetic User",+ "-c",+ "[email protected]",+ "add",+ ".",+ ],+ { cwd: repository },+ );+ gitText(+ [+ "-c",+ "user.name=Synthetic User",+ "-c",+ "[email protected]",+ "commit",+ "-m",+ "Synthetic validation baseline",+ ],+ { cwd: repository },+ );+ const fixture = await persistedValidationClient(+ undefined,+ PLUGIN_ROOT,+ root,+ {+ PATH: kind === "canonical" ? tools : alias,+ },+ );+ await using client = fixture.client;+ const environment = {+ PATH: tools,+ CODEX_SECURITY_GIT: selectedGit,+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,+ };+ const scanDir = join(root, "scan");+ await mkdir(scanDir, { mode: 0o700 });+ const registration = await runWorkbench(+ { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },+ [+ "register-cli-scan",+ "--repository",+ repository,+ "--scan-dir",+ scanDir,+ "--recipe-json",+ JSON.stringify({+ repository,+ mode: "standard",+ target: { kind: "repository", paths: [] },+ config: {},+ }),+ ],+ );+ const workflowId = "git-tool-validation";+ await fixture.bindWorkflow(workflowId, environment);+ const request = {+ ...fixture.options,+ outputDir: undefined,+ scanId: registration["scanId"] as string,+ workflowId,+ };+ const first = await client.validate(request);+ expect(await client.validate(request)).toEqual(first);+ expect(first.disposition).toBe(assessment.disposition);+ for (const [options] of fixture.workbench.mock.calls) {+ expect(options.environment?.["CODEX_SECURITY_GIT"]).toBe(selectedGit);+ expect(options.environment?.["PATH"]).toBe(tools);+ }+ expect(fixture.captured.codex?.env?.["CODEX_SECURITY_GIT"]).toBe(+ selectedGit,+ );+ expect(fixture.captured.codex?.env?.["PATH"]).toContain(tools);+ },+ );+});diff --git a/sdk/typescript/tests-ts/api-validation-guidance.test.ts b/sdk/typescript/tests-ts/api-validation-guidance.test.tsnew file mode 100644index 000000000..be4c8df4e--- /dev/null+++ b/sdk/typescript/tests-ts/api-validation-guidance.test.ts@@ -0,0 +1,554 @@+import {+ chmod,+ cp,+ mkdir,+ readFile,+ readdir,+ rename,+ rm,+ stat,+ writeFile,+} from "node:fs/promises";+import * as fsPromises from "node:fs/promises";+import { execFileSync } from "node:child_process";+import { existsSync } from "node:fs";+import { dirname, join, relative } from "node:path";+import { describe, expect, spyOn, test } from "bun:test";+import { parse as parseToml } from "smol-toml";+import { resolveCodexProfile, type JsonObject } from "../src/config.js";++import { PLUGIN_ROOT } from "./plugin-root.js";+import { shellEnvironmentReference } from "./support/api-client.js";+import { readKnowledgeBaseSnapshot } from "../src/knowledge-base.js";+import { createValidationFixtures } from "./support/validation.js";++describe("CodexSecurity finding validation", () => {+ const {+ actualLstat,+ assessment,+ validationEvents,+ validationClient,+ persistedValidationClient,+ temporaryDirectory,+ } = createValidationFixtures();++ test("invalidates cached validation when inline instructions change", async () => {+ let modelCalls = 0;+ let instructions = "Original standalone validation instructions.";+ const instructionPath = join(+ PLUGIN_ROOT,+ "references",+ "standalone-validation.md",+ );+ const actualReadFile = fsPromises.readFile;+ const readInstructions = spyOn(fsPromises, "readFile").mockImplementation(+ (async (...args: Parameters<typeof actualReadFile>) =>+ args[0] === instructionPath+ ? instructions+ : await actualReadFile(...args)) as typeof actualReadFile,+ );+ try {+ async function* events() {+ modelCalls += 1;+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const workflowId = await fixture.bindWorkflow("validation-instructions");+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect(await client.validate(request)).toEqual(first);+ expect(modelCalls).toBe(1);+ instructions = "Updated standalone validation instructions.";+ await client.validate(request);+ expect(modelCalls).toBe(2);+ expect(fixture.captured.prompt).toContain(instructions);+ } finally {+ readInstructions.mockRestore();+ }+ });++ test.each([true, false])(+ "reuses a validation only when bundled references are used (custom: %j)",+ async (custom) => {+ const pluginRoot = custom+ ? join(await temporaryDirectory(), "selected-plugin")+ : PLUGIN_ROOT;+ const reference = join(+ await temporaryDirectory(),+ "additional-guidance.md",+ );+ await writeFile(reference, "Original synthetic guidance.");+ if (custom) {+ await cp(PLUGIN_ROOT, pluginRoot, { recursive: true });+ await writeFile(+ join(pluginRoot, "references", "standalone-validation.md"),+ `Read the validation guidance at ${JSON.stringify(reference)}.\n`,+ );+ }+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents(+ JSON.stringify({+ ...assessment,+ report: await readFile(reference, "utf8"),+ }),+ );+ }+ const fixture = await persistedValidationClient(events, pluginRoot);+ await using client = fixture.client;+ const workflowId = await fixture.bindWorkflow(+ "additional-validation-reference",+ );+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect((await client.validate(request)).report).toBe(first.report);+ expect(modelCalls).toBe(custom ? 2 : 1);+ if (custom) {+ await writeFile(reference, "Updated synthetic guidance.");+ expect((await client.validate(request)).report).toBe(+ "Updated synthetic guidance.",+ );+ expect(modelCalls).toBe(3);+ }+ expect(fixture.captured.prompt).toContain(+ await readFile(+ join(pluginRoot, "references", "standalone-validation.md"),+ "utf8",+ ),+ );+ },+ );++ test("reads unchanged and updated system-configured validation instructions", async () => {+ const root = await temporaryDirectory("native-system-validation-");+ const instructions = join(root, "instructions.md");+ const nativeConfig = join(root, "config.toml");+ await writeFile(instructions, "Original synthetic system guidance.");+ await writeFile(+ nativeConfig,+ `model_instructions_file = ${JSON.stringify(instructions)}\n`,+ );+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ const configuration = parseToml(await readFile(nativeConfig, "utf8"));+ yield* validationEvents(+ JSON.stringify({+ ...assessment,+ report: await readFile(+ configuration["model_instructions_file"] as string,+ "utf8",+ ),+ }),+ );+ }+ const fixture = await persistedValidationClient(+ events,+ PLUGIN_ROOT,+ undefined,+ { ProgramData: root },+ );+ await using client = fixture.client;+ const workflowId = await fixture.bindWorkflow(+ "system-validation-instructions",+ );+ const nativePath =+ process.platform === "win32"+ ? join(root, "OpenAI", "Codex", "config.toml")+ : "/etc/codex/config.toml";+ const originalLstat = actualLstat;+ const nativeLookup = spyOn(fsPromises, "lstat").mockImplementation(((+ ...args: Parameters<typeof originalLstat>+ ) =>+ originalLstat(+ args[0] === nativePath ? nativeConfig : args[0],+ args[1],+ )) as typeof originalLstat);+ try {+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect((await client.validate(request)).report).toBe(first.report);+ expect(modelCalls).toBe(2);+ await writeFile(instructions, "Updated synthetic system guidance.");+ expect((await client.validate(request)).report).toBe(+ "Updated synthetic system guidance.",+ );+ expect(modelCalls).toBe(3);+ } finally {+ nativeLookup.mockRestore();+ }+ });++ test.each([+ { setting: "top-level", changed: true },+ { setting: "top-level", changed: false },+ { setting: "selected profile", changed: true },+ { setting: "selected profile", changed: false },+ { setting: "none", changed: false },+ ] as const)(+ "runs file-backed validation instructions through Codex, %p",+ async ({ setting, changed }) => {+ const instructions = join(await temporaryDirectory(), "instructions.md");+ await writeFile(instructions, "Original synthetic instructions.");+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents(+ JSON.stringify({+ ...assessment,+ report: await readFile(instructions, "utf8"),+ }),+ );+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ if (setting === "top-level") {+ client.config.codexOverrides!["model_instructions_file"] = instructions;+ } else if (setting === "selected profile") {+ client.config.codexOverrides!["profile"] = "selected";+ client.config.codexOverrides!["profiles"] = {+ selected: { model_instructions_file: instructions },+ };+ }+ const workflowId = await fixture.bindWorkflow("file-backed-validation");+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect(modelCalls).toBe(1);+ if (changed)+ await writeFile(instructions, "Updated synthetic instructions.");+ const second = await client.validate(request);+ expect(modelCalls).toBe(setting === "none" ? 1 : 2);+ expect(second.report).toBe(+ changed ? "Updated synthetic instructions." : first.report,+ );+ expect(+ resolveCodexProfile(fixture.captured.codex!.config as JsonObject)[+ "model_instructions_file"+ ],+ ).toBe(setting === "none" ? undefined : instructions);+ },+ );++ test.each([+ { modelFails: false, cleanupFails: true },+ { modelFails: true, cleanupFails: true },+ { modelFails: false, cleanupFails: false },+ { modelFails: true, cleanupFails: false },+ ])(+ "preserves the validation outcome when temporary knowledge cleanup fails: %p",+ async ({ modelFails, cleanupFails }) => {+ const modelError = new Error("Synthetic validation failure.");+ const cleanupError = new Error(+ "Synthetic temporary knowledge cleanup failure.",+ );+ async function* events() {+ if (modelFails) throw modelError;+ yield* validationEvents();+ }+ const fixture = await validationClient(events);+ await using client = fixture.client;+ const document = join(fixture.root, "knowledge.md");+ await writeFile(document, "Synthetic primary project guidance.");+ const actualRm = fsPromises.rm;+ let cleanupPath: string | undefined;+ const cleanupSpy = spyOn(fsPromises, "rm").mockImplementation(+ async (path, options) => {+ if (+ path ===+ fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"]+ ) {+ cleanupPath = String(path);+ if (cleanupFails) throw cleanupError;+ }+ return await actualRm(path, options);+ },+ );+ try {+ const result = client.validate({+ ...fixture.options,+ knowledgeBasePaths: [document],+ });+ if (modelFails) await expect(result).rejects.toBe(modelError);+ else await expect(result).resolves.toMatchObject(assessment);+ expect(cleanupPath).toBeDefined();+ expect(await readFile(document, "utf8")).toBe(+ "Synthetic primary project guidance.",+ );+ } finally {+ cleanupSpy.mockRestore();+ if (cleanupPath !== undefined)+ await actualRm(cleanupPath, { recursive: true, force: true });+ }+ },+ );++ test.each([+ ["unchanged", false],+ ["modified", false],+ ["modified", true],+ ["renamed", true],+ ["deleted", true],+ ] as const)(+ "retains supplied knowledge context for workflow validation; change=%s, snapshot=%p",+ async (change, snapshot) => {+ let modelCalls = 0;+ const normalizedPaths: string[] = [];+ let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;+ async function* events() {+ modelCalls += 1;+ const path =+ fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"];+ expect(path).toBeDefined();+ normalizedPaths.push(path!);+ const files = await readdir(path!);+ const context = await readFile(join(path!, files[0]!), "utf8");+ yield* validationEvents(+ JSON.stringify({ ...assessment, report: context }),+ );+ }+ fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const document = join(fixture.root, "policy.md");+ await writeFile(document, "Original synthetic policy.");+ const workflowId = await fixture.bindWorkflow("knowledge-validation");+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ knowledgeBasePaths: [document],+ ...(snapshot+ ? {+ knowledgeBaseSnapshot: await readKnowledgeBaseSnapshot([+ document,+ ]),+ }+ : {}),+ };+ const first = await client.validate(request);+ expect(first.report).toBe("Original synthetic policy.");+ expect(fixture.captured.prompt).toContain(+ shellEnvironmentReference("CODEX_SECURITY_KNOWLEDGE_BASE"),+ );+ expect(fixture.captured.prompt).not.toContain(+ "Original synthetic policy.",+ );+ const movedDocument = join(fixture.root, "moved-policy.md");+ if (change === "modified")+ await writeFile(document, "Updated synthetic policy.");+ else if (change === "renamed") await rename(document, movedDocument);+ else if (change === "deleted") await rm(document);+ const second = await client.validate(request);+ expect(modelCalls).toBe(change === "modified" && !snapshot ? 2 : 1);+ expect(second.report).toBe(+ change === "modified" && !snapshot+ ? "Updated synthetic policy."+ : first.report,+ );+ if (change === "renamed" || change === "deleted") {+ const fresh = await client.validate({+ ...request,+ finding: "Another synthetic candidate.",+ });+ expect(fresh.report).toBe(first.report);+ expect(modelCalls).toBe(2);+ }+ expect(normalizedPaths.every((path) => !existsSync(path))).toBe(true);+ if (change === "deleted") expect(existsSync(document)).toBe(false);+ else+ expect(+ await readFile(+ change === "renamed" ? movedDocument : document,+ "utf8",+ ),+ ).toBe(+ change === "modified" ? "Updated synthetic policy." : first.report,+ );+ },+ );++ test+ .skipIf(process.platform === "win32")+ .each(["workflow", "file-backed workflow", "no workflow"] as const)(+ "validates an ignored FIFO without requiring a cache snapshot: %s",+ async (mode) => {+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const repository = fixture.options.repositoryPath;+ execFileSync("git", ["init", "-q", repository]);+ await writeFile(join(repository, ".gitignore"), "ignored-pipe\n");+ execFileSync(fixture.python, [+ "-c",+ "import os,sys; os.mkfifo(sys.argv[1])",+ join(repository, "ignored-pipe"),+ ]);+ const workflowId = await fixture.bindWorkflow(+ "unsupported-validation-snapshot",+ );+ if (mode === "file-backed workflow") {+ const instructions = join(fixture.root, "instructions.md");+ await writeFile(instructions, "Synthetic validation instructions.");+ client.config.codexOverrides!["model_instructions_file"] = instructions;+ }+ const request = {+ ...fixture.options,+ outputDir: undefined,+ ...(mode === "no workflow" ? {} : { workflowId }),+ };+ await client.validate(request);+ await client.validate(request);+ expect(modelCalls).toBe(2);+ expect(fixture.captured.prompt).toContain("Candidate finding");+ },+ );++ test+ .skipIf(process.platform === "win32" || process.geteuid?.() === 0)+ .each(["file", "directory"] as const)(+ "validates unreadable ignored cache entries: %s",+ async (kind) => {+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const repository = fixture.options.repositoryPath;+ execFileSync("git", ["init", "-q", repository]);+ await writeFile(join(repository, ".gitignore"), "ignored-cache\n");+ const entry = join(repository, "ignored-cache");+ if (kind === "directory") {+ await mkdir(entry);+ await writeFile(join(entry, "cache.txt"), "Synthetic cache data.");+ } else await writeFile(entry, "Synthetic cache data.");+ const mode = (await stat(entry)).mode;+ const workflowId = await fixture.bindWorkflow(+ "unreadable-validation-cache",+ );+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ await chmod(entry, 0);+ try {+ expect(await client.validate(request)).toMatchObject(assessment);+ expect(await client.validate(request)).toMatchObject(assessment);+ expect(modelCalls).toBe(2);+ } finally {+ await chmod(entry, mode);+ }+ expect(await client.validate(request)).toMatchObject(assessment);+ expect(await client.validate(request)).toMatchObject(assessment);+ expect(modelCalls).toBe(3);+ },+ );++ test.each([+ "home",+ "output worktree",+ "output project configuration",+ "none",+ ] as const)("uses current native validation guidance: %s", async (scope) => {+ const stateRoot = await temporaryDirectory("native-validation-guidance-");+ if (scope === "output worktree" || scope === "output project configuration")+ execFileSync("git", ["init", "-q", stateRoot]);+ let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;+ let guide: string | undefined;+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(+ fixture.codexHome,+ );+ if (+ scope === "output worktree" ||+ scope === "output project configuration"+ )+ expect(+ relative(+ stateRoot,+ fixture.captured.thread!.workingDirectory!,+ ).startsWith(".."),+ ).toBe(false);+ yield* validationEvents(+ JSON.stringify({+ ...assessment,+ report:+ guide === undefined+ ? "Synthetic assessment."+ : scope === "output project configuration"+ ? parseToml(await readFile(guide, "utf8"))[+ "developer_instructions"+ ]+ : await readFile(guide, "utf8"),+ }),+ );+ }+ fixture = await persistedValidationClient(events, PLUGIN_ROOT, undefined, {+ CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),+ });+ await using client = fixture.client;+ guide =+ scope === "home"+ ? join(fixture.codexHome, "AGENTS.md")+ : scope === "output worktree"+ ? join(stateRoot, "AGENTS.md")+ : scope === "output project configuration"+ ? join(stateRoot, ".codex", "config.toml")+ : undefined;+ if (guide !== undefined) {+ await mkdir(dirname(guide), { recursive: true });+ await writeFile(+ guide,+ scope === "output project configuration"+ ? 'developer_instructions = "Original synthetic guidance."\n'+ : "Original synthetic guidance.",+ );+ }+ const workflowId = await fixture.bindWorkflow("native-validation-guidance");+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect((await client.validate(request)).report).toBe(first.report);+ expect(modelCalls).toBe(guide === undefined ? 1 : 2);+ if (guide !== undefined) {+ await writeFile(+ guide,+ scope === "output project configuration"+ ? 'developer_instructions = "Updated synthetic guidance."\n'+ : "Updated synthetic guidance.",+ );+ expect((await client.validate(request)).report).toBe(+ "Updated synthetic guidance.",+ );+ expect(modelCalls).toBe(3);+ }+ });+});diff --git a/sdk/typescript/tests-ts/api-validation-persistence.test.ts b/sdk/typescript/tests-ts/api-validation-persistence.test.tsnew file mode 100644index 000000000..4ab4fb12f--- /dev/null+++ b/sdk/typescript/tests-ts/api-validation-persistence.test.ts@@ -0,0 +1,531 @@+import { Database } from "bun:sqlite";+import {+ chmod,+ mkdir,+ readFile,+ realpath,+ rm,+ writeFile,+} from "node:fs/promises";+import { join, relative } from "node:path";+import { type ThreadEvent } from "@openai/codex-sdk";+import { describe, expect, test } from "bun:test";+import { OutputDirectoryError, ScanInterruptedError } from "../src/index.js";+import { resolveCodexProfile, type JsonObject } from "../src/config.js";+import { runWorkbench } from "../src/runtime.js";++import { reviewSqliteHome } from "../src/deduplication/codex-review.js";+import { PLUGIN_ROOT } from "./plugin-root.js";+import { gitText } from "./support/shell.js";+import { createValidationFixtures } from "./support/validation.js";++describe("CodexSecurity finding validation", () => {+ const {+ validationEvents,+ validationClient,+ persistedValidationClient,+ temporaryDirectory,+ } = createValidationFixtures();++ test.each([+ "environment",+ "root configuration",+ "selected profile",+ "repository root",+ "repository ancestor",+ ] as const)(+ "keeps native SQLite state out of validation cache snapshots: %s",+ async (setting) => {+ const root = await temporaryDirectory();+ const repository = join(root, "repository");+ const coversRepository =+ setting === "repository root" || setting === "repository ancestor";+ const sqliteHome =+ setting === "repository root"+ ? repository+ : setting === "repository ancestor"+ ? root+ : join(repository, ".native-state", "nested");+ let modelCalls = 0;+ let changeSource = false;+ const writeNativeState = async () => {+ await mkdir(sqliteHome, { recursive: true, mode: 0o700 });+ const database = new Database(join(sqliteHome, "state.sqlite"));+ try {+ database.run("CREATE TABLE IF NOT EXISTS events (message TEXT)");+ database+ .query("INSERT INTO events VALUES (?)")+ .run("Synthetic validation thread");+ } finally {+ database.close();+ }+ };+ async function* events(): AsyncGenerator<ThreadEvent> {+ modelCalls++;+ expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(+ fixture.codexHome,+ );+ expect(+ reviewSqliteHome(+ fixture.captured.codex!.env!,+ resolveCodexProfile(fixture.captured.codex!.config as JsonObject),+ ),+ ).toBe(sqliteHome);+ await writeNativeState();+ if (changeSource)+ await writeFile(join(repository, "source.txt"), "Changed source.\n");+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(+ events,+ PLUGIN_ROOT,+ root,+ {+ CODEX_HOME: join(root, "ambient-home"),+ CODEX_SQLITE_HOME:+ setting === "root configuration" || setting === "selected profile"+ ? join(root, "unused-environment-storage")+ : sqliteHome,+ },+ );+ await using client = fixture.client;+ if (setting === "root configuration")+ client.config.codexOverrides!["sqlite_home"] = relative(+ fixture.codexHome,+ sqliteHome,+ );+ if (setting === "selected profile") {+ client.config.codexOverrides!["sqlite_home"] = join(+ root,+ "unused-root-storage",+ );+ client.config.codexOverrides!["profile"] = "selected";+ client.config.codexOverrides!["profiles"] = {+ selected: { sqlite_home: relative(fixture.codexHome, sqliteHome) },+ };+ }+ await writeFile(join(repository, "source.txt"), "Original source.\n");+ await writeFile(join(repository, ".gitignore"), ".native-state/\n");+ gitText(["-C", repository, "init", "-q"]);+ gitText(["-C", repository, "add", "."]);+ gitText([+ "-C",+ repository,+ "-c",+ "user.name=Fixture",+ "-c",+ "[email protected]",+ "commit",+ "-qm",+ "initial",+ ]);+ const snapshots: JsonObject[] = [];+ fixture.workbench.mockImplementation(async (options, args, input) => {+ if (args[0] === "finding-workflow" && input !== undefined) {+ const request = JSON.parse(input) as JsonObject;+ if (+ request["action"] === "source" &&+ request["repository"] === repository+ )+ snapshots.push(request);+ }+ return await runWorkbench(options, args, input);+ });+ const workflowId = await fixture.bindWorkflow("native-sqlite-validation");+ const request = { ...fixture.options, workflowId };+ const first = await client.validate(request);+ expect(first.disposition).toBe("reportable");+ expect(modelCalls).toBe(1);+ await writeNativeState();+ const second = await client.validate(request);+ if (coversRepository) {+ expect(second.disposition).toBe("reportable");+ expect(modelCalls).toBe(2);+ expect(snapshots).toEqual([]);+ } else {+ expect(second).toEqual(first);+ expect(modelCalls).toBe(1);+ expect(+ snapshots.map((snapshot) => snapshot["privateStatePaths"]),+ ).toEqual(Array(4).fill([sqliteHome]));+ changeSource = true;+ await expect(+ client.validate({ ...request, finding: "Another candidate" }),+ ).rejects.toThrow("Repository changed during validation.");+ expect(modelCalls).toBe(2);+ }+ },+ );++ test.each([+ "unchanged",+ "deleted",+ "modified",+ "git metadata",+ "unbound",+ ...(process.platform === "win32" ? [] : ["insecure permissions"]),+ ])(+ "checks saved validation evidence before cache reuse: %s",+ async (change) => {+ let modelCalls = 0;+ async function* events(): AsyncGenerator<ThreadEvent> {+ modelCalls++;+ const directory = fixture.captured.thread!.workingDirectory!;+ await writeFile(+ join(directory, "proof.txt"),+ `Synthetic evidence ${modelCalls}.\n`,+ );+ await mkdir(join(directory, ".git"));+ await writeFile(+ join(directory, ".git", "HEAD"),+ "Synthetic Git metadata.\n",+ );+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ let unbound = change === "unbound";+ fixture.workbench.mockImplementation(async (options, args, input) => {+ if (args[0] === "finding-workflow" && input !== undefined) {+ const payload = JSON.parse(input);+ if (unbound && payload.action === "save-review") {+ payload.result = payload.result.assessment ?? payload.result;+ input = JSON.stringify(payload);+ unbound = false;+ }+ }+ return await runWorkbench(options, args, input);+ });+ const workflowId = await fixture.bindWorkflow("validation-evidence");+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ };+ const first = await client.validate(request);+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(+ "Synthetic evidence 1.\n",+ );+ if (change === "deleted") await rm(first.outputDir, { recursive: true });+ if (change === "modified")+ await writeFile(+ join(first.outputDir, "proof.txt"),+ "Changed evidence.\n",+ );+ if (change === "git metadata")+ await writeFile(+ join(first.outputDir, ".git", "HEAD"),+ "Changed metadata.\n",+ );+ if (change === "insecure permissions")+ await chmod(first.outputDir, 0o777);+ const second = await client.validate(request);+ expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);+ expect(await readFile(join(second.outputDir, "proof.txt"), "utf8")).toBe(+ `Synthetic evidence ${modelCalls}.\n`,+ );+ expect(await client.validate(request)).toEqual(second);+ expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);+ if (change === "insecure permissions") {+ expect(second.outputDir).not.toBe(first.outputDir);+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(+ "Synthetic evidence 1.\n",+ );+ await chmod(first.outputDir, 0o700);+ }+ },+ );++ test("reuses explicit validation output without overwriting changed evidence", async () => {+ let modelCalls = 0;+ async function* events(): AsyncGenerator<ThreadEvent> {+ modelCalls++;+ await writeFile(+ join(fixture.captured.thread!.workingDirectory!, "proof.txt"),+ "Original evidence.\n",+ );+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const workflowId = await fixture.bindWorkflow(+ "explicit-validation-evidence",+ );+ const request = { ...fixture.options, workflowId };+ const first = await client.validate(request);+ expect(await client.validate(request)).toEqual(first);+ expect(modelCalls).toBe(1);+ if (process.platform !== "win32") {+ await chmod(first.outputDir, 0o770);+ await expect(client.validate(request)).rejects.toThrow(+ "must not be accessible to other users",+ );+ await chmod(first.outputDir, 0o700);+ }+ await writeFile(join(first.outputDir, "proof.txt"), "Changed evidence.\n");+ await expect(client.validate(request)).rejects.toBeInstanceOf(+ OutputDirectoryError,+ );+ expect(modelCalls).toBe(1);+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(+ "Changed evidence.\n",+ );+ });++ test.each(["empty", "nonempty"])(+ "respects a different explicit validation output directory that is %s",+ async (contents) => {+ let modelCalls = 0;+ async function* events(): AsyncGenerator<ThreadEvent> {+ modelCalls++;+ await writeFile(+ join(fixture.captured.thread!.workingDirectory!, "proof.txt"),+ `Synthetic evidence ${modelCalls}.\n`,+ );+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ const workflowId = await fixture.bindWorkflow(+ "validation-output-destination",+ );+ const request = { ...fixture.options, workflowId };+ const first = await client.validate(request);+ const outputDir = join(fixture.root, "other-validation");+ await mkdir(outputDir, { mode: 0o700 });+ if (contents === "nonempty") {+ await writeFile(join(outputDir, "proof.txt"), "Existing evidence.\n");+ await expect(+ client.validate({ ...request, outputDir }),+ ).rejects.toBeInstanceOf(OutputDirectoryError);+ expect(modelCalls).toBe(1);+ expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(+ "Existing evidence.\n",+ );+ } else {+ const second = await client.validate({ ...request, outputDir });+ expect(second.outputDir).toBe(await realpath(outputDir));+ expect(modelCalls).toBe(2);+ expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(+ "Synthetic evidence 2.\n",+ );+ expect(await client.validate({ ...request, outputDir })).toEqual(+ second,+ );+ expect(modelCalls).toBe(2);+ }+ expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(+ "Synthetic evidence 1.\n",+ );+ },+ );++ test.each(["unavailable", "aborted"] as const)(+ "handles %s evidence fingerprints without caching an invalid result",+ async (mode) => {+ const controller = new AbortController();+ let modelCalls = 0;+ let saved = 0;+ async function* events(): AsyncGenerator<ThreadEvent> {+ modelCalls++;+ yield* validationEvents();+ }+ const fixture = await persistedValidationClient(events);+ await using client = fixture.client;+ fixture.workbench.mockImplementation(async (options, args, input) => {+ if (args[0] === "finding-workflow" && input !== undefined) {+ const payload = JSON.parse(input);+ if (payload.action === "save-review") saved++;+ if (payload.action === "source" && payload.evidence === true) {+ if (mode === "unavailable") return { source: null };+ controller.abort();+ }+ }+ return await runWorkbench(options, args, input);+ });+ const workflowId = await fixture.bindWorkflow(+ "optional-validation-evidence",+ );+ const request = {+ ...fixture.options,+ outputDir: undefined,+ workflowId,+ signal: controller.signal,+ };+ if (mode === "aborted") {+ await expect(client.validate(request)).rejects.toBeInstanceOf(+ ScanInterruptedError,+ );+ expect(modelCalls).toBe(1);+ } else {+ expect((await client.validate(request)).disposition).toBe("reportable");+ expect((await client.validate(request)).disposition).toBe("reportable");+ expect(modelCalls).toBe(2);+ }+ expect(saved).toBe(0);+ },+ );++ function usePersistedValidationWorkbench(+ fixture: Awaited<ReturnType<typeof validationClient>>,+ targetChanged: () => boolean = () => false,+ ) {+ fixture.workbench.mockImplementation(async (options, args, input) => {+ if (args[0] === "get-scan") {+ if (targetChanged())+ throw new Error("The recorded scan target changed.");+ return { scan: { targetPath: fixture.options.repositoryPath } };+ }+ return await runWorkbench(options, args, input);+ });+ }++ test("resumes workflow validation from persisted assessments across clients", async () => {+ let modelCalls = 0;+ let targetChanged = false;+ async function* events() {+ modelCalls += 1;+ if (modelCalls === 2) throw new Error("Second assessment failed.");+ yield* validationEvents();+ }+ const original = await persistedValidationClient(events);+ usePersistedValidationWorkbench(original, () => targetChanged);+ await using originalClient = original.client;+ await original.bindWorkflow("validation-retry");+ const request = {+ ...original.options,+ outputDir: undefined,+ scanId: "recorded-scan",+ workflowId: "validation-retry",+ safetyIdentifier: "synthetic-validation-first",+ };+ const first = await originalClient.validate(request);+ await expect(+ originalClient.validate({ ...request, finding: "Second candidate" }),+ ).rejects.toThrow("Second assessment failed.");+ await originalClient.close();++ const resumed = await persistedValidationClient(+ events,+ PLUGIN_ROOT,+ original.root,+ );+ usePersistedValidationWorkbench(resumed, () => targetChanged);+ await using client = resumed.client;+ expect(await client.validate(request)).toEqual(first);+ expect(modelCalls).toBe(2);+ await client.validate({ ...request, finding: "Second candidate" });+ expect(modelCalls).toBe(3);+ targetChanged = true;+ await expect(client.validate(request)).rejects.toThrow(+ "scan target changed",+ );+ expect(modelCalls).toBe(3);+ });++ test("separates persisted validation by model and access program across clients", async () => {+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents();+ }+ const original = await persistedValidationClient(events);+ usePersistedValidationWorkbench(original);+ await using originalClient = original.client;+ await original.bindWorkflow("validation-settings");+ const request = {+ ...original.options,+ outputDir: undefined,+ scanId: "recorded-scan",+ workflowId: "validation-settings",+ safetyIdentifier: "synthetic-validation-first",+ };+ await originalClient.validate(request);+ await originalClient.close();++ const resumed = await persistedValidationClient(+ events,+ PLUGIN_ROOT,+ original.root,+ );+ usePersistedValidationWorkbench(resumed);+ await using client = resumed.client;+ await client.validate(request);+ expect(modelCalls).toBe(1);+ client.config.codexOverrides!["model"] = "changed-validation-model";+ const changedModel = await client.validate(request);+ expect(modelCalls).toBe(2);+ const blue = await client.validate({+ ...request,+ cyberAccessProgram: "daybreak_blue",+ });+ expect(modelCalls).toBe(3);+ const red = await client.validate({+ ...request,+ cyberAccessProgram: "daybreak_red",+ });+ expect(modelCalls).toBe(4);+ expect(+ await client.validate({+ ...request,+ cyberAccessProgram: "daybreak_blue",+ }),+ ).toEqual(blue);+ expect(+ await client.validate({ ...request, cyberAccessProgram: "daybreak_red" }),+ ).toEqual(red);+ expect(modelCalls).toBe(4);+ expect(await client.validate(request)).toEqual(changedModel);+ expect(modelCalls).toBe(4);+ });++ test("separates persisted validation by safety identifier across clients", async () => {+ let modelCalls = 0;+ async function* events() {+ modelCalls += 1;+ yield* validationEvents();+ }+ const original = await persistedValidationClient(events);+ original.client.config.codexOverrides!["model"] =+ "changed-validation-model";+ usePersistedValidationWorkbench(original);+ await using originalClient = original.client;+ await original.bindWorkflow("validation-safety-identifier");+ const request = {+ ...original.options,+ outputDir: undefined,+ scanId: "recorded-scan",+ workflowId: "validation-safety-identifier",+ safetyIdentifier: "synthetic-validation-first",+ };+ const first = await originalClient.validate(request);+ await originalClient.close();++ const resumed = await persistedValidationClient(+ events,+ PLUGIN_ROOT,+ original.root,+ );+ usePersistedValidationWorkbench(resumed);+ await using client = resumed.client;+ client.config.codexOverrides!["model"] = "changed-validation-model";+ await client.validate({ ...request, finding: "Second candidate" });+ expect(modelCalls).toBe(2);+ expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(+ request.safetyIdentifier,+ );+ for (const safetyIdentifier of ["synthetic-validation-second", undefined]) {+ const previousCalls = modelCalls;+ const changedRequest = { ...request, safetyIdentifier };+ const assessment = await client.validate(changedRequest);+ expect(modelCalls).toBe(previousCalls + 1);+ expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(+ safetyIdentifier,+ );+ expect(await client.validate(changedRequest)).toEqual(assessment);+ expect(modelCalls).toBe(previousCalls + 1);+ }+ expect(await client.validate(request)).toEqual(first);+ expect(modelCalls).toBe(4);+ });+});diff --git a/sdk/typescript/tests-ts/api-validation.test.ts b/sdk/typescript/tests-ts/api-validation.test.tsindex 5ad28ece7..e3af96221 100644--- a/sdk/typescript/tests-ts/api-validation.test.ts+++ b/sdk/typescript/tests-ts/api-validation.test.ts@@ -1,267 +1,37 @@import { requestMetadata } from "./support/request-metadata.js";import { once } from "node:events";-import { Database } from "bun:sqlite";-import {- chmod,- cp,- mkdir,- readFile,- readdir,- realpath,- rename,- rm,- stat,- symlink,- writeFile,-} from "node:fs/promises";-import * as fsPromises from "node:fs/promises";-import { execFileSync } from "node:child_process";+import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";import { existsSync } from "node:fs";-import { dirname, join, relative } from "node:path";+import { join } from "node:path";import { pathToFileURL } from "node:url";-import {- Codex,- type CodexOptions,- type ThreadEvent,- type ThreadOptions,- type TurnOptions,-} from "@openai/codex-sdk";-import {- afterEach,- beforeEach,- describe,- expect,- mock,- spyOn,- test,-} from "bun:test";+import { Codex } from "@openai/codex-sdk";+import { describe, expect, mock, test } from "bun:test";import { parse as parseToml } from "smol-toml";import {- OutputDirectoryError,OutputInsideProtectedRootError,ScanInterruptedError,} from "../src/index.js";-import { resolveCodexProfile, type JsonObject } from "../src/config.js";-import { resolvePluginPython, runWorkbench } from "../src/runtime.js";+import { type JsonObject } from "../src/config.js";-import { reviewSqliteHome } from "../src/deduplication/codex-review.js";import { PLUGIN_ROOT } from "./plugin-root.js";-import { shellEnvironmentReference, TestClient } from "./support/api-client.js";+import { TestClient } from "./support/api-client.js";import { preparedRuntime } from "./support/api-events.js";-import { createApiTestFixtures } from "./support/temporary-directories.js";-import { FindingWorkflow } from "../src/finding-workflow.js";import { readKnowledgeBaseSnapshot } from "../src/knowledge-base.js";import { VERSION } from "../src/version.js";-import { nodeCommand, gitText } from "./support/shell.js";+import { nodeCommand } from "./support/shell.js";import { fail, throwing } from "./support/errors.js";--const { cleanup, temporaryDirectory } = createApiTestFixtures();-afterEach(cleanup);--async function repositoryDirectories() {- const root = await temporaryDirectory();- const repository = join(root, "repository");- await mkdir(repository);- return { root, repository };-}--async function runtimeDirectories() {- const { root, repository } = await repositoryDirectories();- const codexHome = join(root, "codex-home");- await mkdir(codexHome);- return { root, repository, codexHome };-}--function runtimePreparer(- codexHome: string,- overrides: () => Partial<ReturnType<typeof preparedRuntime>>,-) {- return async () => ({- ...preparedRuntime(codexHome),- ...overrides(),- });-}+import { createValidationFixtures } from "./support/validation.js";describe("CodexSecurity finding validation", () => {- const actualLstat = fsPromises.lstat;- let systemConfigurationLookup: ReturnType<- typeof spyOn<typeof fsPromises, "lstat">- >;- beforeEach(() => {- const nativePath =- process.platform === "win32"- ? join("C:\\ProgramData", "OpenAI", "Codex", "config.toml")- : "/etc/codex/config.toml";- systemConfigurationLookup = spyOn(fsPromises, "lstat").mockImplementation(- (async (...args: Parameters<typeof actualLstat>) => {- if (args[0] === nativePath)- throw Object.assign(- new Error("Synthetic absent native system configuration"),- { code: "ENOENT" },- );- return await actualLstat(...args);- }) as typeof actualLstat,- );- });- afterEach(() => systemConfigurationLookup.mockRestore());-- const assessment = {- disposition: "reportable",- report: "Static trace reaches the SQL sink; runtime proof is still needed.",- } as const;-- async function* validationEvents(- response = JSON.stringify(assessment),- complete = true,- ): AsyncGenerator<ThreadEvent> {- yield { type: "thread.started", thread_id: "validation-thread" };- yield {- type: "item.completed",- item: { id: "result", type: "agent_message", text: response },- };- if (complete) {- yield {- type: "turn.completed",- usage: {- input_tokens: 10,- cached_input_tokens: 0,- cache_write_input_tokens: 0,- output_tokens: 3,- reasoning_output_tokens: 0,- },- };- }- }-- let fixturePython: Promise<string> | undefined;-- async function validationClient(- events: (signal: AbortSignal) => AsyncGenerator<ThreadEvent> = () =>- validationEvents(),- pluginRoot = PLUGIN_ROOT,- pythonPath = "/managed/python",- fixtureRoot?: string,- environmentOverrides: Record<string, string> = {},- ) {- const root = fixtureRoot ?? (await temporaryDirectory());- const repository = join(root, "repository");- const codexHome = await temporaryDirectory();- const stateDirectory =- environmentOverrides["CODEX_SECURITY_STATE_DIR"] ?? join(root, "state");- await mkdir(repository, { recursive: true });- const captured: {- codex?: CodexOptions;- thread?: ThreadOptions;- prompt?: string;- turn?: TurnOptions;- } = {};- const workbench = mock(- async (- _options: Parameters<typeof runWorkbench>[0],- _args: readonly string[],- _input?: string,- ): Promise<JsonObject> => ({- scan: { targetPath: repository },- }),- );- const environment = {- ...Object.fromEntries(- ["PATH", "SystemRoot", "TEMP", "TMP"].flatMap((name) =>- process.env[name] === undefined ? [] : [[name, process.env[name]!]],- ),- ),- CODEX_SECURITY_STATE_DIR: stateDirectory,- OPENAI_API_KEY: "synthetic-validation-key",- ...environmentOverrides,- };- const client = new TestClient(- {- pluginPath: pluginRoot === PLUGIN_ROOT ? undefined : pluginRoot,- pythonPath,- codexOverrides: {- model: "test-model",- model_reasoning_effort: "high",- approval_policy: "never",- analytics: { enabled: false },- },- },- {- environment,- prepareRuntime: runtimePreparer(codexHome, () => ({- plugin: { ...preparedRuntime(codexHome).plugin, pluginRoot },- environment,- })),- resolvePluginPython: async ({ configuredPath } = {}) => configuredPath!,- runWorkbench: workbench,- createCodex: (options) => {- captured.codex = options;- return {- startThread: (options) => {- captured.thread = options;- return {- id: null,- async runStreamed(prompt, options) {- captured.prompt = prompt;- captured.turn = options;- return { events: events(options.signal!) };- },- };- },- };- },- },- );- const options = {- repositoryPath: repository,- finding: "Candidate finding",- outputDir: join(root, "validation"),- };- return {- client,- options,- stateDirectory,- captured,- workbench,- root,- codexHome,- };- }-- async function persistedValidationClient(- events?: Parameters<typeof validationClient>[0],- pluginRoot = PLUGIN_ROOT,- fixtureRoot?: string,- environmentOverrides?: Record<string, string>,- ) {- const python = await (fixturePython ??= resolvePluginPython());- const fixture = await validationClient(- events,- pluginRoot,- python,- fixtureRoot,- environmentOverrides,- );- fixture.workbench.mockImplementation(runWorkbench);- async function bindWorkflow(- id: string,- environment: NodeJS.ProcessEnv = {- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,- },- ) {- const workflow = new FindingWorkflow(- id,- environment,- runWorkbench,- python,- {- workbenchOptions: { python, pluginRoot: PLUGIN_ROOT, environment },- },- );- await workflow.bind({ repositoryPath: fixture.options.repositoryPath });- return id;- }- return { ...fixture, python, bindWorkflow };- }+ const {+ assessment,+ validationEvents,+ validationClient,+ temporaryDirectory,+ repositoryDirectories,+ runtimeDirectories,+ runtimePreparer,+ } = createValidationFixtures();test.each(["output", "state", "runtime"].flatMap((destination) =>@@ -641,1431 +411,6 @@ describe("CodexSecurity finding validation", () => {expect(await readFile(evidence, "utf8")).toBe("synthetic evidence");});- test.each([- "environment",- "root configuration",- "selected profile",- "repository root",- "repository ancestor",- ] as const)(- "keeps native SQLite state out of validation cache snapshots: %s",- async (setting) => {- const root = await temporaryDirectory();- const repository = join(root, "repository");- const coversRepository =- setting === "repository root" || setting === "repository ancestor";- const sqliteHome =- setting === "repository root"- ? repository- : setting === "repository ancestor"- ? root- : join(repository, ".native-state", "nested");- let modelCalls = 0;- let changeSource = false;- const writeNativeState = async () => {- await mkdir(sqliteHome, { recursive: true, mode: 0o700 });- const database = new Database(join(sqliteHome, "state.sqlite"));- try {- database.run("CREATE TABLE IF NOT EXISTS events (message TEXT)");- database- .query("INSERT INTO events VALUES (?)")- .run("Synthetic validation thread");- } finally {- database.close();- }- };- async function* events(): AsyncGenerator<ThreadEvent> {- modelCalls++;- expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(- fixture.codexHome,- );- expect(- reviewSqliteHome(- fixture.captured.codex!.env!,- resolveCodexProfile(fixture.captured.codex!.config as JsonObject),- ),- ).toBe(sqliteHome);- await writeNativeState();- if (changeSource)- await writeFile(join(repository, "source.txt"), "Changed source.\n");- yield* validationEvents();- }- const fixture = await persistedValidationClient(- events,- PLUGIN_ROOT,- root,- {- CODEX_HOME: join(root, "ambient-home"),- CODEX_SQLITE_HOME:- setting === "root configuration" || setting === "selected profile"- ? join(root, "unused-environment-storage")- : sqliteHome,- },- );- await using client = fixture.client;- if (setting === "root configuration")- client.config.codexOverrides!["sqlite_home"] = relative(- fixture.codexHome,- sqliteHome,- );- if (setting === "selected profile") {- client.config.codexOverrides!["sqlite_home"] = join(- root,- "unused-root-storage",- );- client.config.codexOverrides!["profile"] = "selected";- client.config.codexOverrides!["profiles"] = {- selected: { sqlite_home: relative(fixture.codexHome, sqliteHome) },- };- }- await writeFile(join(repository, "source.txt"), "Original source.\n");- await writeFile(join(repository, ".gitignore"), ".native-state/\n");- gitText(["-C", repository, "init", "-q"]);- gitText(["-C", repository, "add", "."]);- gitText([- "-C",- repository,- "-c",- "user.name=Fixture",- "-c",- "[email protected]",- "commit",- "-qm",- "initial",- ]);- const snapshots: JsonObject[] = [];- fixture.workbench.mockImplementation(async (options, args, input) => {- if (args[0] === "finding-workflow" && input !== undefined) {- const request = JSON.parse(input) as JsonObject;- if (- request["action"] === "source" &&- request["repository"] === repository- )- snapshots.push(request);- }- return await runWorkbench(options, args, input);- });- const workflowId = await fixture.bindWorkflow("native-sqlite-validation");- const request = { ...fixture.options, workflowId };- const first = await client.validate(request);- expect(first.disposition).toBe("reportable");- expect(modelCalls).toBe(1);- await writeNativeState();- const second = await client.validate(request);- if (coversRepository) {- expect(second.disposition).toBe("reportable");- expect(modelCalls).toBe(2);- expect(snapshots).toEqual([]);- } else {- expect(second).toEqual(first);- expect(modelCalls).toBe(1);- expect(- snapshots.map((snapshot) => snapshot["privateStatePaths"]),- ).toEqual(Array(4).fill([sqliteHome]));- changeSource = true;- await expect(- client.validate({ ...request, finding: "Another candidate" }),- ).rejects.toThrow("Repository changed during validation.");- expect(modelCalls).toBe(2);- }- },- );-- test.each([- "unchanged",- "deleted",- "modified",- "git metadata",- "unbound",- ...(process.platform === "win32" ? [] : ["insecure permissions"]),- ])(- "checks saved validation evidence before cache reuse: %s",- async (change) => {- let modelCalls = 0;- async function* events(): AsyncGenerator<ThreadEvent> {- modelCalls++;- const directory = fixture.captured.thread!.workingDirectory!;- await writeFile(- join(directory, "proof.txt"),- `Synthetic evidence ${modelCalls}.\n`,- );- await mkdir(join(directory, ".git"));- await writeFile(- join(directory, ".git", "HEAD"),- "Synthetic Git metadata.\n",- );- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- let unbound = change === "unbound";- fixture.workbench.mockImplementation(async (options, args, input) => {- if (args[0] === "finding-workflow" && input !== undefined) {- const payload = JSON.parse(input);- if (unbound && payload.action === "save-review") {- payload.result = payload.result.assessment ?? payload.result;- input = JSON.stringify(payload);- unbound = false;- }- }- return await runWorkbench(options, args, input);- });- const workflowId = await fixture.bindWorkflow("validation-evidence");- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(- "Synthetic evidence 1.\n",- );- if (change === "deleted") await rm(first.outputDir, { recursive: true });- if (change === "modified")- await writeFile(- join(first.outputDir, "proof.txt"),- "Changed evidence.\n",- );- if (change === "git metadata")- await writeFile(- join(first.outputDir, ".git", "HEAD"),- "Changed metadata.\n",- );- if (change === "insecure permissions")- await chmod(first.outputDir, 0o777);- const second = await client.validate(request);- expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);- expect(await readFile(join(second.outputDir, "proof.txt"), "utf8")).toBe(- `Synthetic evidence ${modelCalls}.\n`,- );- expect(await client.validate(request)).toEqual(second);- expect(modelCalls).toBe(change === "unchanged" ? 1 : 2);- if (change === "insecure permissions") {- expect(second.outputDir).not.toBe(first.outputDir);- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(- "Synthetic evidence 1.\n",- );- await chmod(first.outputDir, 0o700);- }- },- );-- test("reuses explicit validation output without overwriting changed evidence", async () => {- let modelCalls = 0;- async function* events(): AsyncGenerator<ThreadEvent> {- modelCalls++;- await writeFile(- join(fixture.captured.thread!.workingDirectory!, "proof.txt"),- "Original evidence.\n",- );- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const workflowId = await fixture.bindWorkflow(- "explicit-validation-evidence",- );- const request = { ...fixture.options, workflowId };- const first = await client.validate(request);- expect(await client.validate(request)).toEqual(first);- expect(modelCalls).toBe(1);- if (process.platform !== "win32") {- await chmod(first.outputDir, 0o770);- await expect(client.validate(request)).rejects.toThrow(- "must not be accessible to other users",- );- await chmod(first.outputDir, 0o700);- }- await writeFile(join(first.outputDir, "proof.txt"), "Changed evidence.\n");- await expect(client.validate(request)).rejects.toBeInstanceOf(- OutputDirectoryError,- );- expect(modelCalls).toBe(1);- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(- "Changed evidence.\n",- );- });-- test.each(["empty", "nonempty"])(- "respects a different explicit validation output directory that is %s",- async (contents) => {- let modelCalls = 0;- async function* events(): AsyncGenerator<ThreadEvent> {- modelCalls++;- await writeFile(- join(fixture.captured.thread!.workingDirectory!, "proof.txt"),- `Synthetic evidence ${modelCalls}.\n`,- );- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const workflowId = await fixture.bindWorkflow(- "validation-output-destination",- );- const request = { ...fixture.options, workflowId };- const first = await client.validate(request);- const outputDir = join(fixture.root, "other-validation");- await mkdir(outputDir, { mode: 0o700 });- if (contents === "nonempty") {- await writeFile(join(outputDir, "proof.txt"), "Existing evidence.\n");- await expect(- client.validate({ ...request, outputDir }),- ).rejects.toBeInstanceOf(OutputDirectoryError);- expect(modelCalls).toBe(1);- expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(- "Existing evidence.\n",- );- } else {- const second = await client.validate({ ...request, outputDir });- expect(second.outputDir).toBe(await realpath(outputDir));- expect(modelCalls).toBe(2);- expect(await readFile(join(outputDir, "proof.txt"), "utf8")).toBe(- "Synthetic evidence 2.\n",- );- expect(await client.validate({ ...request, outputDir })).toEqual(- second,- );- expect(modelCalls).toBe(2);- }- expect(await readFile(join(first.outputDir, "proof.txt"), "utf8")).toBe(- "Synthetic evidence 1.\n",- );- },- );-- test.each(["unavailable", "aborted"] as const)(- "handles %s evidence fingerprints without caching an invalid result",- async (mode) => {- const controller = new AbortController();- let modelCalls = 0;- let saved = 0;- async function* events(): AsyncGenerator<ThreadEvent> {- modelCalls++;- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- fixture.workbench.mockImplementation(async (options, args, input) => {- if (args[0] === "finding-workflow" && input !== undefined) {- const payload = JSON.parse(input);- if (payload.action === "save-review") saved++;- if (payload.action === "source" && payload.evidence === true) {- if (mode === "unavailable") return { source: null };- controller.abort();- }- }- return await runWorkbench(options, args, input);- });- const workflowId = await fixture.bindWorkflow(- "optional-validation-evidence",- );- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- signal: controller.signal,- };- if (mode === "aborted") {- await expect(client.validate(request)).rejects.toBeInstanceOf(- ScanInterruptedError,- );- expect(modelCalls).toBe(1);- } else {- expect((await client.validate(request)).disposition).toBe("reportable");- expect((await client.validate(request)).disposition).toBe("reportable");- expect(modelCalls).toBe(2);- }- expect(saved).toBe(0);- },- );-- function usePersistedValidationWorkbench(- fixture: Awaited<ReturnType<typeof validationClient>>,- targetChanged: () => boolean = () => false,- ) {- fixture.workbench.mockImplementation(async (options, args, input) => {- if (args[0] === "get-scan") {- if (targetChanged())- throw new Error("The recorded scan target changed.");- return { scan: { targetPath: fixture.options.repositoryPath } };- }- return await runWorkbench(options, args, input);- });- }-- test("resumes workflow validation from persisted assessments across clients", async () => {- let modelCalls = 0;- let targetChanged = false;- async function* events() {- modelCalls += 1;- if (modelCalls === 2) throw new Error("Second assessment failed.");- yield* validationEvents();- }- const original = await persistedValidationClient(events);- usePersistedValidationWorkbench(original, () => targetChanged);- await using originalClient = original.client;- await original.bindWorkflow("validation-retry");- const request = {- ...original.options,- outputDir: undefined,- scanId: "recorded-scan",- workflowId: "validation-retry",- safetyIdentifier: "synthetic-validation-first",- };- const first = await originalClient.validate(request);- await expect(- originalClient.validate({ ...request, finding: "Second candidate" }),- ).rejects.toThrow("Second assessment failed.");- await originalClient.close();-- const resumed = await persistedValidationClient(- events,- PLUGIN_ROOT,- original.root,- );- usePersistedValidationWorkbench(resumed, () => targetChanged);- await using client = resumed.client;- expect(await client.validate(request)).toEqual(first);- expect(modelCalls).toBe(2);- await client.validate({ ...request, finding: "Second candidate" });- expect(modelCalls).toBe(3);- targetChanged = true;- await expect(client.validate(request)).rejects.toThrow(- "scan target changed",- );- expect(modelCalls).toBe(3);- });-- test("separates persisted validation by model and access program across clients", async () => {- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents();- }- const original = await persistedValidationClient(events);- usePersistedValidationWorkbench(original);- await using originalClient = original.client;- await original.bindWorkflow("validation-settings");- const request = {- ...original.options,- outputDir: undefined,- scanId: "recorded-scan",- workflowId: "validation-settings",- safetyIdentifier: "synthetic-validation-first",- };- await originalClient.validate(request);- await originalClient.close();-- const resumed = await persistedValidationClient(- events,- PLUGIN_ROOT,- original.root,- );- usePersistedValidationWorkbench(resumed);- await using client = resumed.client;- await client.validate(request);- expect(modelCalls).toBe(1);- client.config.codexOverrides!["model"] = "changed-validation-model";- const changedModel = await client.validate(request);- expect(modelCalls).toBe(2);- const blue = await client.validate({- ...request,- cyberAccessProgram: "daybreak_blue",- });- expect(modelCalls).toBe(3);- const red = await client.validate({- ...request,- cyberAccessProgram: "daybreak_red",- });- expect(modelCalls).toBe(4);- expect(- await client.validate({- ...request,- cyberAccessProgram: "daybreak_blue",- }),- ).toEqual(blue);- expect(- await client.validate({ ...request, cyberAccessProgram: "daybreak_red" }),- ).toEqual(red);- expect(modelCalls).toBe(4);- expect(await client.validate(request)).toEqual(changedModel);- expect(modelCalls).toBe(4);- });-- test("separates persisted validation by safety identifier across clients", async () => {- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents();- }- const original = await persistedValidationClient(events);- original.client.config.codexOverrides!["model"] =- "changed-validation-model";- usePersistedValidationWorkbench(original);- await using originalClient = original.client;- await original.bindWorkflow("validation-safety-identifier");- const request = {- ...original.options,- outputDir: undefined,- scanId: "recorded-scan",- workflowId: "validation-safety-identifier",- safetyIdentifier: "synthetic-validation-first",- };- const first = await originalClient.validate(request);- await originalClient.close();-- const resumed = await persistedValidationClient(- events,- PLUGIN_ROOT,- original.root,- );- usePersistedValidationWorkbench(resumed);- await using client = resumed.client;- client.config.codexOverrides!["model"] = "changed-validation-model";- await client.validate({ ...request, finding: "Second candidate" });- expect(modelCalls).toBe(2);- expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(- request.safetyIdentifier,- );- for (const safetyIdentifier of ["synthetic-validation-second", undefined]) {- const previousCalls = modelCalls;- const changedRequest = { ...request, safetyIdentifier };- const assessment = await client.validate(changedRequest);- expect(modelCalls).toBe(previousCalls + 1);- expect(resumed.captured.codex?.env?.["CODEX_SAFETY_IDENTIFIER"]).toBe(- safetyIdentifier,- );- expect(await client.validate(changedRequest)).toEqual(assessment);- expect(modelCalls).toBe(previousCalls + 1);- }- expect(await client.validate(request)).toEqual(first);- expect(modelCalls).toBe(4);- });-- test("invalidates cached validation when inline instructions change", async () => {- let modelCalls = 0;- let instructions = "Original standalone validation instructions.";- const instructionPath = join(- PLUGIN_ROOT,- "references",- "standalone-validation.md",- );- const actualReadFile = fsPromises.readFile;- const readInstructions = spyOn(fsPromises, "readFile").mockImplementation(- (async (...args: Parameters<typeof actualReadFile>) =>- args[0] === instructionPath- ? instructions- : await actualReadFile(...args)) as typeof actualReadFile,- );- try {- async function* events() {- modelCalls += 1;- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const workflowId = await fixture.bindWorkflow("validation-instructions");- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- expect(await client.validate(request)).toEqual(first);- expect(modelCalls).toBe(1);- instructions = "Updated standalone validation instructions.";- await client.validate(request);- expect(modelCalls).toBe(2);- expect(fixture.captured.prompt).toContain(instructions);- } finally {- readInstructions.mockRestore();- }- });-- test.each([- { custom: true, changed: true },- { custom: true, changed: false },- { custom: false, changed: false },- ] as const)(- "uses current custom validation references: %p",- async ({ custom, changed }) => {- const pluginRoot = custom- ? join(await temporaryDirectory(), "selected-plugin")- : PLUGIN_ROOT;- const reference = join(- await temporaryDirectory(),- "additional-guidance.md",- );- await writeFile(reference, "Original synthetic guidance.");- if (custom) {- await cp(PLUGIN_ROOT, pluginRoot, { recursive: true });- await writeFile(- join(pluginRoot, "references", "standalone-validation.md"),- `Read the validation guidance at ${JSON.stringify(reference)}.\n`,- );- }- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents(- JSON.stringify({- ...assessment,- report: await readFile(reference, "utf8"),- }),- );- }- const fixture = await persistedValidationClient(events, pluginRoot);- await using client = fixture.client;- const workflowId = await fixture.bindWorkflow(- "additional-validation-reference",- );- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- if (changed) await writeFile(reference, "Updated synthetic guidance.");- const second = await client.validate(request);- expect(second.report).toBe(- changed ? "Updated synthetic guidance." : first.report,- );- if (!custom) expect(modelCalls).toBe(1);- expect(fixture.captured.prompt).toContain(- await readFile(- join(pluginRoot, "references", "standalone-validation.md"),- "utf8",- ),- );- },- );-- for (const changed of [true, false]) {- test(`preserves system-configured validation instructions: changed=${changed}`, async () => {- const root = await temporaryDirectory("native-system-validation-");- const instructions = join(root, "instructions.md");- const nativeConfig = join(root, "config.toml");- await writeFile(instructions, "Original synthetic system guidance.");- await writeFile(- nativeConfig,- `model_instructions_file = ${JSON.stringify(instructions)}\n`,- );- let modelCalls = 0;- async function* events() {- modelCalls += 1;- const configuration = parseToml(await readFile(nativeConfig, "utf8"));- yield* validationEvents(- JSON.stringify({- ...assessment,- report: await readFile(- configuration["model_instructions_file"] as string,- "utf8",- ),- }),- );- }- const fixture = await persistedValidationClient(- events,- PLUGIN_ROOT,- undefined,- { ProgramData: root },- );- await using client = fixture.client;- const workflowId = await fixture.bindWorkflow(- "system-validation-instructions",- );- const nativePath =- process.platform === "win32"- ? join(root, "OpenAI", "Codex", "config.toml")- : "/etc/codex/config.toml";- const originalLstat = actualLstat;- const nativeLookup = spyOn(fsPromises, "lstat").mockImplementation(((- ...args: Parameters<typeof originalLstat>- ) =>- originalLstat(- args[0] === nativePath ? nativeConfig : args[0],- args[1],- )) as typeof originalLstat);- try {- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- if (changed)- await writeFile(instructions, "Updated synthetic system guidance.");- const second = await client.validate(request);- expect(second.report).toBe(- changed ? "Updated synthetic system guidance." : first.report,- );- } finally {- nativeLookup.mockRestore();- }- });- }-- test.each([- { setting: "top-level", changed: true },- { setting: "top-level", changed: false },- { setting: "selected profile", changed: true },- { setting: "selected profile", changed: false },- { setting: "none", changed: false },- ] as const)(- "runs file-backed validation instructions through Codex, %p",- async ({ setting, changed }) => {- const instructions = join(await temporaryDirectory(), "instructions.md");- await writeFile(instructions, "Original synthetic instructions.");- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents(- JSON.stringify({- ...assessment,- report: await readFile(instructions, "utf8"),- }),- );- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- if (setting === "top-level") {- client.config.codexOverrides!["model_instructions_file"] = instructions;- } else if (setting === "selected profile") {- client.config.codexOverrides!["profile"] = "selected";- client.config.codexOverrides!["profiles"] = {- selected: { model_instructions_file: instructions },- };- }- const workflowId = await fixture.bindWorkflow("file-backed-validation");- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- expect(modelCalls).toBe(1);- if (changed)- await writeFile(instructions, "Updated synthetic instructions.");- const second = await client.validate(request);- expect(modelCalls).toBe(setting === "none" ? 1 : 2);- expect(second.report).toBe(- changed ? "Updated synthetic instructions." : first.report,- );- expect(- resolveCodexProfile(fixture.captured.codex!.config as JsonObject)[- "model_instructions_file"- ],- ).toBe(setting === "none" ? undefined : instructions);- },- );-- test.each([- { modelFails: false, cleanupFails: true },- { modelFails: true, cleanupFails: true },- { modelFails: false, cleanupFails: false },- { modelFails: true, cleanupFails: false },- ])(- "preserves the validation outcome when temporary knowledge cleanup fails: %p",- async ({ modelFails, cleanupFails }) => {- const modelError = new Error("Synthetic validation failure.");- const cleanupError = new Error(- "Synthetic temporary knowledge cleanup failure.",- );- async function* events() {- if (modelFails) throw modelError;- yield* validationEvents();- }- const fixture = await validationClient(events);- await using client = fixture.client;- const document = join(fixture.root, "knowledge.md");- await writeFile(document, "Synthetic primary project guidance.");- const actualRm = fsPromises.rm;- let cleanupPath: string | undefined;- const cleanupSpy = spyOn(fsPromises, "rm").mockImplementation(- async (path, options) => {- if (- path ===- fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"]- ) {- cleanupPath = String(path);- if (cleanupFails) throw cleanupError;- }- return await actualRm(path, options);- },- );- try {- const result = client.validate({- ...fixture.options,- knowledgeBasePaths: [document],- });- if (modelFails) await expect(result).rejects.toBe(modelError);- else await expect(result).resolves.toMatchObject(assessment);- expect(cleanupPath).toBeDefined();- expect(await readFile(document, "utf8")).toBe(- "Synthetic primary project guidance.",- );- } finally {- cleanupSpy.mockRestore();- if (cleanupPath !== undefined)- await actualRm(cleanupPath, { recursive: true, force: true });- }- },- );-- test.each([- ["unchanged", false],- ["modified", false],- ["modified", true],- ["renamed", true],- ["deleted", true],- ] as const)(- "retains supplied knowledge context for workflow validation; change=%s, snapshot=%p",- async (change, snapshot) => {- let modelCalls = 0;- const normalizedPaths: string[] = [];- let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;- async function* events() {- modelCalls += 1;- const path =- fixture.captured.codex?.env?.["CODEX_SECURITY_KNOWLEDGE_BASE"];- expect(path).toBeDefined();- normalizedPaths.push(path!);- const files = await readdir(path!);- const context = await readFile(join(path!, files[0]!), "utf8");- yield* validationEvents(- JSON.stringify({ ...assessment, report: context }),- );- }- fixture = await persistedValidationClient(events);- await using client = fixture.client;- const document = join(fixture.root, "policy.md");- await writeFile(document, "Original synthetic policy.");- const workflowId = await fixture.bindWorkflow("knowledge-validation");- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- knowledgeBasePaths: [document],- ...(snapshot- ? {- knowledgeBaseSnapshot: await readKnowledgeBaseSnapshot([- document,- ]),- }- : {}),- };- const first = await client.validate(request);- expect(first.report).toBe("Original synthetic policy.");- expect(fixture.captured.prompt).toContain(- shellEnvironmentReference("CODEX_SECURITY_KNOWLEDGE_BASE"),- );- expect(fixture.captured.prompt).not.toContain(- "Original synthetic policy.",- );- const movedDocument = join(fixture.root, "moved-policy.md");- if (change === "modified")- await writeFile(document, "Updated synthetic policy.");- else if (change === "renamed") await rename(document, movedDocument);- else if (change === "deleted") await rm(document);- const second = await client.validate(request);- expect(modelCalls).toBe(change === "modified" && !snapshot ? 2 : 1);- expect(second.report).toBe(- change === "modified" && !snapshot- ? "Updated synthetic policy."- : first.report,- );- if (change === "renamed" || change === "deleted") {- const fresh = await client.validate({- ...request,- finding: "Another synthetic candidate.",- });- expect(fresh.report).toBe(first.report);- expect(modelCalls).toBe(2);- }- expect(normalizedPaths.every((path) => !existsSync(path))).toBe(true);- if (change === "deleted") expect(existsSync(document)).toBe(false);- else- expect(- await readFile(- change === "renamed" ? movedDocument : document,- "utf8",- ),- ).toBe(- change === "modified" ? "Updated synthetic policy." : first.report,- );- },- );-- test- .skipIf(process.platform === "win32")- .each(["workflow", "file-backed workflow", "no workflow"] as const)(- "validates an ignored FIFO without requiring a cache snapshot: %s",- async (mode) => {- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const repository = fixture.options.repositoryPath;- execFileSync("git", ["init", "-q", repository]);- await writeFile(join(repository, ".gitignore"), "ignored-pipe\n");- execFileSync(fixture.python, [- "-c",- "import os,sys; os.mkfifo(sys.argv[1])",- join(repository, "ignored-pipe"),- ]);- const workflowId = await fixture.bindWorkflow(- "unsupported-validation-snapshot",- );- if (mode === "file-backed workflow") {- const instructions = join(fixture.root, "instructions.md");- await writeFile(instructions, "Synthetic validation instructions.");- client.config.codexOverrides!["model_instructions_file"] = instructions;- }- const request = {- ...fixture.options,- outputDir: undefined,- ...(mode === "no workflow" ? {} : { workflowId }),- };- await client.validate(request);- await client.validate(request);- expect(modelCalls).toBe(2);- expect(fixture.captured.prompt).toContain("Candidate finding");- },- );-- test- .skipIf(process.platform === "win32" || process.geteuid?.() === 0)- .each(["file", "directory"] as const)(- "validates unreadable ignored cache entries: %s",- async (kind) => {- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const repository = fixture.options.repositoryPath;- execFileSync("git", ["init", "-q", repository]);- await writeFile(join(repository, ".gitignore"), "ignored-cache\n");- const entry = join(repository, "ignored-cache");- if (kind === "directory") {- await mkdir(entry);- await writeFile(join(entry, "cache.txt"), "Synthetic cache data.");- } else await writeFile(entry, "Synthetic cache data.");- const mode = (await stat(entry)).mode;- const workflowId = await fixture.bindWorkflow(- "unreadable-validation-cache",- );- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- await chmod(entry, 0);- try {- expect(await client.validate(request)).toMatchObject(assessment);- expect(await client.validate(request)).toMatchObject(assessment);- expect(modelCalls).toBe(2);- } finally {- await chmod(entry, mode);- }- expect(await client.validate(request)).toMatchObject(assessment);- expect(await client.validate(request)).toMatchObject(assessment);- expect(modelCalls).toBe(3);- },- );-- test.each([- "AGENTS.md",- "REVIEW.md",- ".codex/config.toml",- "unrelated.txt",- "unchanged",- ] as const)(- "rechecks native guidance created while cached evidence is fingerprinted: %s",- async (scope) => {- const stateRoot = await temporaryDirectory("validation-guidance-race-");- execFileSync("git", ["init", "-q", stateRoot]);- const guide = join(stateRoot, scope);- let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;- let modelCalls = 0;- async function* events() {- modelCalls++;- await writeFile(- join(fixture.captured.thread!.workingDirectory!, "proof.bin"),- Buffer.alloc(256 * 1024, 7),- );- yield* validationEvents(- JSON.stringify({- ...assessment,- report: existsSync(guide)- ? await readFile(guide, "utf8")- : "Original assessment.",- }),- );- }- fixture = await persistedValidationClient(- events,- PLUGIN_ROOT,- undefined,- {- CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),- },- );- await using client = fixture.client;- client.config.codexOverrides!["project_doc_fallback_filenames"] = [- "REVIEW.md",- ];- let changeDuringEvidence = false;- let changed = false;- fixture.workbench.mockImplementation(async (options, args, input) => {- const payload =- args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;- const pending = runWorkbench(options, args, input);- if (- changeDuringEvidence &&- !changed &&- payload?.action === "source" &&- payload.evidence === true- ) {- changed = true;- if (scope !== "unchanged") {- await mkdir(dirname(guide), { recursive: true });- await writeFile(- guide,- scope === ".codex/config.toml"- ? 'developer_instructions = "New native guidance."\n'- : "New native guidance.\n",- );- }- }- return await pending;- });- const workflowId = await fixture.bindWorkflow(- "native-guidance-during-evidence",- );- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- expect(await client.validate(request)).toEqual(first);- expect(modelCalls).toBe(1);- changeDuringEvidence = true;- const second = await client.validate(request);- const nativeGuidance = !["unrelated.txt", "unchanged"].includes(scope);- expect(changed).toBe(true);- expect(modelCalls).toBe(nativeGuidance ? 2 : 1);- expect(second.report).toBe(- nativeGuidance ? (await readFile(guide, "utf8")).trim() : first.report,- );- const sameGuidance = await client.validate(request);- expect(sameGuidance.report).toBe(second.report);- expect(modelCalls).toBe(nativeGuidance ? 3 : 1);- },- );-- test.each([- { scope: "home", changed: true },- { scope: "home", changed: false },- { scope: "output worktree", changed: true },- { scope: "output worktree", changed: false },- { scope: "output project configuration", changed: true },- { scope: "output project configuration", changed: false },- { scope: "none", changed: false },- ] as const)(- "preserves native validation guidance: %p",- async ({ scope, changed }) => {- const stateRoot = await temporaryDirectory("native-validation-guidance-");- if (- scope === "output worktree" ||- scope === "output project configuration"- )- execFileSync("git", ["init", "-q", stateRoot]);- let fixture: Awaited<ReturnType<typeof persistedValidationClient>>;- let guide: string | undefined;- let modelCalls = 0;- async function* events() {- modelCalls += 1;- expect(fixture.captured.codex?.env?.["CODEX_HOME"]).toBe(- fixture.codexHome,- );- if (- scope === "output worktree" ||- scope === "output project configuration"- )- expect(- relative(- stateRoot,- fixture.captured.thread!.workingDirectory!,- ).startsWith(".."),- ).toBe(false);- yield* validationEvents(- JSON.stringify({- ...assessment,- report:- guide === undefined- ? "Synthetic assessment."- : scope === "output project configuration"- ? parseToml(await readFile(guide, "utf8"))[- "developer_instructions"- ]- : await readFile(guide, "utf8"),- }),- );- }- fixture = await persistedValidationClient(- events,- PLUGIN_ROOT,- undefined,- {- CODEX_SECURITY_STATE_DIR: join(stateRoot, "state"),- },- );- await using client = fixture.client;- guide =- scope === "home"- ? join(fixture.codexHome, "AGENTS.md")- : scope === "output worktree"- ? join(stateRoot, "AGENTS.md")- : scope === "output project configuration"- ? join(stateRoot, ".codex", "config.toml")- : undefined;- if (guide !== undefined) {- await mkdir(dirname(guide), { recursive: true });- await writeFile(- guide,- scope === "output project configuration"- ? 'developer_instructions = "Original synthetic guidance."\n'- : "Original synthetic guidance.",- );- }- const workflowId = await fixture.bindWorkflow(- "native-validation-guidance",- );- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- };- const first = await client.validate(request);- if (changed)- await writeFile(- guide!,- scope === "output project configuration"- ? 'developer_instructions = "Updated synthetic guidance."\n'- : "Updated synthetic guidance.",- );- const second = await client.validate(request);- expect(second.report).toBe(- changed ? "Updated synthetic guidance." : first.report,- );- if (changed) expect(modelCalls).toBe(2);- if (scope === "none") expect(modelCalls).toBe(1);- },- );-- test.each([- "unchanged",- "source contents",- "recorded contents",- "recorded checkout",- ] as const)(- "rechecks %s after reading a saved validation assessment",- async (change) => {- let modelCalls = 0;- async function* events() {- modelCalls += 1;- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const repository = fixture.options.repositoryPath;- const source = join(repository, "source.ts");- await writeFile(source, "export const value = 'original';\n");- const environment = {- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,- };- const workbenchOptions = {- python: fixture.python,- pluginRoot: PLUGIN_ROOT,- environment,- };- const scanDir = join(fixture.root, "scan");- await mkdir(scanDir, { mode: 0o700 });- const registration = await runWorkbench(workbenchOptions, [- "register-cli-scan",- "--repository",- repository,- "--scan-dir",- scanDir,- "--recipe-json",- JSON.stringify({- repository,- mode: "standard",- target: { kind: "repository", paths: [] },- config: {},- }),- ]);- const workflowId = "cached-validation-recheck";- await fixture.bindWorkflow(workflowId, environment);- let readSavedAssessment = false;- fixture.workbench.mockImplementation(async (options, args, input) => {- const response = await runWorkbench(options, args, input);- if (- args[0] === "finding-workflow" &&- JSON.parse(input!)["action"] === "get-review" &&- response["review"] !== null- ) {- readSavedAssessment = true;- if (change === "recorded checkout") {- await rename(repository, join(fixture.root, "original-repository"));- await mkdir(repository);- await writeFile(source, "export const value = 'original';\n");- } else if (change !== "unchanged") {- await writeFile(source, "export const value = 'changed';\n");- }- }- return response;- });- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- ...(change === "source contents"- ? {}- : { scanId: registration["scanId"] as string }),- };- const first = await client.validate(request);- if (change === "unchanged") {- expect(await client.validate(request)).toEqual(first);- } else {- await expect(client.validate(request)).rejects.toThrow(- change === "source contents"- ? "Repository changed during validation"- : change === "recorded contents"- ? "Scan target contents changed"- : "checkout path was replaced",- );- }- expect(readSavedAssessment).toBe(true);- expect(modelCalls).toBe(1);- },- );-- test.each([- ["cached", "source contents", "unchanged"],- ["cached", "recorded contents", "unchanged"],- ["cached", "source contents", "changed"],- ["cached", "recorded contents", "missing"],- ["fresh", "source contents", "unchanged"],- ["fresh", "recorded contents", "missing"],- ] as const)(- "rechecks %s validation %s after reading %s evidence",- async (phase, change, evidence) => {- let modelCalls = 0;- async function* events() {- modelCalls++;- await writeFile(- join(fixture.captured.thread!.workingDirectory!, "proof.txt"),- "Synthetic evidence.\n",- );- yield* validationEvents();- }- const fixture = await persistedValidationClient(events);- await using client = fixture.client;- const repository = fixture.options.repositoryPath;- const source = join(repository, "source.ts");- await writeFile(source, "export const value = 'original';\n");- const environment = {- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,- };- const scanDir = join(fixture.root, "scan");- await mkdir(scanDir, { mode: 0o700 });- const registration = await runWorkbench(- { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },- [- "register-cli-scan",- "--repository",- repository,- "--scan-dir",- scanDir,- "--recipe-json",- JSON.stringify({- repository,- mode: "standard",- target: { kind: "repository", paths: [] },- config: {},- }),- ],- );- const workflowId = "evidence-validation-recheck";- await fixture.bindWorkflow(workflowId, environment);- let changeDuringEvidence = false;- let changed = false;- let savedReviews = 0;- fixture.workbench.mockImplementation(async (options, args, input) => {- const payload =- args[0] === "finding-workflow" ? JSON.parse(input!) : undefined;- const changeNow =- changeDuringEvidence &&- !changed &&- payload?.action === "source" &&- payload.evidence === true;- if (changeNow && evidence === "changed")- await writeFile(- join(payload.repository, "proof.txt"),- "Changed evidence.\n",- );- if (changeNow && evidence === "missing")- await rm(payload.repository, { recursive: true });- const response = await runWorkbench(options, args, input);- if (changeNow) {- await writeFile(source, "export const value = 'changed';\n");- changed = true;- }- if (payload?.action === "save-review") savedReviews++;- return response;- });- const request = {- ...fixture.options,- outputDir: undefined,- workflowId,- ...(change === "recorded contents"- ? { scanId: registration["scanId"] as string }- : {}),- };- const first = await client.validate(request);- expect(await client.validate(request)).toEqual(first);- expect(modelCalls).toBe(1);- changeDuringEvidence = true;- await expect(- client.validate({- ...request,- ...(phase === "fresh" ? { finding: "Another candidate" } : {}),- }),- ).rejects.toThrow(- change === "source contents"- ? "Repository changed during validation"- : "Scan target contents changed",- );- expect(changed).toBe(true);- expect(modelCalls).toBe(phase === "fresh" ? 2 : 1);- expect(savedReviews).toBe(1);- },- );-- test.each(["canonical", "repository alias"])(- "validates an unchanged recorded Git target with %s tool paths",- async (kind) => {- const root = await temporaryDirectory();- const repository = join(root, "repository");- await mkdir(repository);- const hostGit = Bun.which("git");- expect(hostGit).not.toBeNull();- const tools = join(root, "external-tools");- await mkdir(tools);- const selectedGit = join(- tools,- process.platform === "win32" ? "git.exe" : "git",- );- await symlink(await realpath(hostGit!), selectedGit);- const alias = join(repository, "tools");- await symlink(- tools,- alias,- process.platform === "win32" ? "junction" : "dir",- );- await writeFile(join(repository, ".gitignore"), "tools\n");- await writeFile(- join(repository, "source.ts"),- "export const value = 'original';\n",- );- gitText(["init", "--initial-branch=main"], { cwd: repository });- gitText(- [- "-c",- "user.name=Synthetic User",- "-c",- "[email protected]",- "add",- ".",- ],- { cwd: repository },- );- gitText(- [- "-c",- "user.name=Synthetic User",- "-c",- "[email protected]",- "commit",- "-m",- "Synthetic validation baseline",- ],- { cwd: repository },- );- const fixture = await persistedValidationClient(- undefined,- PLUGIN_ROOT,- root,- {- PATH: kind === "canonical" ? tools : alias,- },- );- await using client = fixture.client;- const environment = {- PATH: tools,- CODEX_SECURITY_GIT: selectedGit,- CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,- };- const scanDir = join(root, "scan");- await mkdir(scanDir, { mode: 0o700 });- const registration = await runWorkbench(- { python: fixture.python, pluginRoot: PLUGIN_ROOT, environment },- [- "register-cli-scan",- "--repository",- repository,- "--scan-dir",- scanDir,- "--recipe-json",- JSON.stringify({- repository,- mode: "standard",- target: { kind: "repository", paths: [] },- config: {},- }),- ],- );- const workflowId = "git-tool-validation";- await fixture.bindWorkflow(workflowId, environment);- const request = {- ...fixture.options,- outputDir: undefined,- scanId: registration["scanId"] as string,- workflowId,- };- const first = await client.validate(request);- expect(await client.validate(request)).toEqual(first);- expect(first.disposition).toBe(assessment.disposition);- for (const [options] of fixture.workbench.mock.calls) {- expect(options.environment?.["CODEX_SECURITY_GIT"]).toBe(selectedGit);- expect(options.environment?.["PATH"]).toBe(tools);- }- expect(fixture.captured.codex?.env?.["CODEX_SECURITY_GIT"]).toBe(- selectedGit,- );- expect(fixture.captured.codex?.env?.["PATH"]).toContain(tools);- },- );-test("rejects invalid inputs, unsafe output, and cancellation before preparing credentials", async () => {const repositoryPath = await temporaryDirectory();const prepareRuntime = mock(async () => fail("runtime must not start"));diff --git a/sdk/typescript/tests-ts/api-workflow.test.ts b/sdk/typescript/tests-ts/api-workflow.test.tsindex df0d8ae6a..2ea1a50ff 100644--- a/sdk/typescript/tests-ts/api-workflow.test.ts+++ b/sdk/typescript/tests-ts/api-workflow.test.ts@@ -194,9 +194,11 @@ test.each([(await new FindingWorkflow(workflowId, environment).get())?.stages.scan.status,).toBe("completed");- await expect(- resumed.run(repository, { workflowId, mode: "deep" }),- ).rejects.toThrow("already bound to a different");+ if (scenario === "completed") {+ await expect(+ resumed.run(repository, { workflowId, mode: "deep" }),+ ).rejects.toThrow("already bound to a different");+ }if (knowledge) {if (scenario === "knowledge-snapshot")await writeFile(promptFile, "Changed synthetic project guidance.");diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.tsindex c1b77c0c3..00de9e092 100644--- a/sdk/typescript/tests-ts/api.test.ts+++ b/sdk/typescript/tests-ts/api.test.ts@@ -68,11 +68,7 @@ import {matchScanFindingsInternal,} from "../src/scan-comparison.js";import { normalizeTarget } from "../src/targets.js";-import {- copyCompletedScan,- INTEGRATION_TARGET,- PLUGIN_ROOT,-} from "./plugin-root.js";+import { copyCompletedScan, PLUGIN_ROOT } from "./plugin-root.js";import {mockScanRegistration,mockWorkbench,@@ -102,7 +98,6 @@ import { pythonExecutable, nodeCommand, gitText } from "./support/shell.js";import { fail, rejecting, throwing } from "./support/errors.js";import { mockFs, restoreFs } from "./support/module-mocks.js";-const REPOSITORY_ROOT = fileURLToPath(new URL("../../..", import.meta.url));const EXAMPLE = join(PLUGIN_ROOT, "examples", "completed-scan");const { cleanup, temporaryDirectory } = createApiTestFixtures();afterEach(cleanup);@@ -539,12 +534,6 @@ describe("CodexSecurity orchestration", () => {},);- test("selects a real-scan target in the active repository layout", async () => {- await expect(- stat(join(REPOSITORY_ROOT, INTEGRATION_TARGET)),- ).resolves.toBeDefined();- });-test("validates local inputs before runtime or plugin Python discovery", async () => {const client = new CodexSecurity({pythonPath: "/definitely/missing/python",@@ -4836,14 +4825,6 @@ describe("CodexSecurity orchestration", () => {}});- const pricedModels = [- "gpt-5.5",- "gpt-6-astra",- "gpt-5.6-terra",- "gpt-5.6-cyber",- "gpt-daybreak-blue-latest",- "gpt-daybreak-red-latest",- ];test("reports unpriced model usage without letting its observer stop the scan", async () => {const { root, repository, codexHome, scanDir } = await scanDirectories();const usages: unknown[] = [];@@ -4874,20 +4855,14 @@ describe("CodexSecurity orchestration", () => {await client.close();}});- test.each(pricedModels)("tracks live and saved %s costs", async (model) => {+ const prices = [+ { model: "gpt-6-astra", min: 0.000232, max: 0.000389 },+ { model: "gpt-5.6-cyber", min: 0.0003275, max: null },+ ];+ test.each(prices)("tracks live and saved $model costs", async (price) => {+ const { model, min, max } = price;const { root, repository, codexHome, scanDir } = await scanDirectories();- const usage = {- input_tokens: 10,- cached_input_tokens: 2,- cache_write_input_tokens: 0,- output_tokens: 3,- reasoning_output_tokens: 1,- };- const expectedCost = estimateScanCost(model, usage);- expect(expectedCost).not.toBeNull();- if (expectedCost === null) throw new Error("Missing selected-model price");-const costs: ScanCost[] = [];const commands: Array<readonly string[]> = [];const client = new TestClient(@@ -4917,15 +4892,24 @@ describe("CodexSecurity orchestration", () => {});expect(result.turnResult.model).toBe(model);- expect(result.cost).toEqual(expectedCost);- expect(costs).toEqual([expectedCost]);+ expect(result.cost).toMatchObject({+ model,+ inputTokens: 10,+ cachedInputTokens: 2,+ cacheWriteInputTokens: 0,+ outputTokens: 3,+ estimatedUsd: min,+ estimatedUsdRange: { min, max, context: "unknown" },+ pricing: { serviceTier: "standard", context: "short" },+ });+ expect([result.cost]).toEqual(costs);const completion = commands.find((args) => args[0] === "complete-scan");expect(completion).toBeDefined();const costIndex = completion?.indexOf("--cost-json") ?? -1;expect(costIndex).toBeGreaterThan(0);expect(JSON.parse(completion?.[costIndex + 1] ?? "null")).toEqual(- expectedCost,+ result.cost,);await client.close();@@ -7629,8 +7613,10 @@ process.exit(2);},prepareRuntime: unauthenticatedRuntime(codexHome, () => ({CODEX_HOME: codexHome,- OpenAi_Api_Key: "forwarded-openai-key",- codex_api_key: "forwarded-codex-key",+ OPENAI_API_KEY: "forwarded-openai-key",+ OpenAi_Api_Key: "mixed-case-openai-key",+ CODEX_API_KEY: "forwarded-codex-key",+ codex_api_key: "lowercase-codex-key",...fakeCommand.environment,})),resolveCodexCommand: () => fakeCommand.command,@@ -8546,42 +8532,6 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login sawait expect(login.wait()).resolves.toMatchObject({ success: false });});- test("keeps ambient credentials available to scans", async () => {- const root = await temporaryDirectory();- const repository = join(root, "repository");- const codexHome = join(root, "codex-home");- const scanDir = join(root, "scan");- await mkdir(repository);- await mkdir(codexHome, { mode: 0o700 });- await mkdir(scanDir, { mode: 0o700 });- const createCodex = mock(completedCodex(root));- const client = TestClient.withDependencies({- environment: {- CODEX_SECURITY_STATE_DIR: join(root, "state"),- OPENAI_API_KEY: "ambient-key",- },- ...scanRuntimeDependencies(codexHome, scanDir),- prepareRuntime: unauthenticatedRuntime(codexHome, () => ({- ...process.env,- CODEX_HOME: codexHome,- OPENAI_API_KEY: "ambient-key",- CODEX_API_KEY: "secondary-ambient-key",- })),- createCodex,- });- try {- await client.run(repository);- expect(createCodex.mock.lastCall?.[0]?.apiKey).toBe("ambient-key");- expect(- Object.keys(createCodex.mock.lastCall?.[0]?.env ?? {}).some((name) =>- ["OPENAI_API_KEY", "CODEX_API_KEY"].includes(name.toUpperCase()),- ),- ).toBe(false);- } finally {- await client.close();- }- });-test("aborts and waits for an in-flight API-key login during close", async () => {const root = await temporaryDirectory();const codexHome = join(root, "codex-home");diff --git a/sdk/typescript/tests-ts/cli-authentication.test.ts b/sdk/typescript/tests-ts/cli-authentication.test.tsindex 103fcb8ef..0314343a4 100644--- a/sdk/typescript/tests-ts/cli-authentication.test.ts+++ b/sdk/typescript/tests-ts/cli-authentication.test.ts@@ -26,7 +26,6 @@ import {import {codexSecurityCredentialAllowsAmbientImport,prepareCodexSecurityCredentialHome,- resolveCodexCommand,setCodexSecurityCredentialLogout,} from "../src/runtime.js";import {@@ -929,25 +928,14 @@ describe("CLI authentication", () => {expect(run(["login", "--with-api-key"], "synthetic-key\n")).toBe(0);expect(await stat(join(credentialHome, "auth.json"))).toBeDefined();await expect(stat(join(repository, "auth.json"))).rejects.toThrow();- expect(run(["login", "status"])).toBe(0);- expect(run(["logout"])).toBe(0);+ if (configuredHome === mountedHome) {+ expect(run(["login", "status"])).toBe(0);+ expect(run(["logout"])).toBe(0);+ } else {+ // Empty and blank settings share the default. Each login must create its own file.+ await rm(join(credentialHome, "auth.json"));+ }}- expect(- spawnSync(- process.execPath,- [join(import.meta.dir, "../src/cli.ts"), "login", "--help"],- {- cwd: repository,- env: {- PATH: process.env["PATH"],- HOME: root,- USERPROFILE: root,- Codex_Home: " ",- },- encoding: "utf8",- },- ).status,- ).toBe(0);} finally {await rm(root, { recursive: true, force: true });}@@ -2246,27 +2234,40 @@ describe("skill authentication", () => {},);- test.each(["validate", "patch", "verify-fix"] as const)(- "%s stops before starting a model command without a stored login",- async (command) => {- for (const auth of ["auto", "chatgpt"] as const) {- await expect(- runCodexSkillCommand(- ["invalid-synthetic-command"],- {- command,- auth,- stdout: capture().stream,- stderr: capture().stream,- },- resolveCodexCommand({}),- {- CODEX_HOME: join(stateDirectory, "ambient"),- CODEX_SECURITY_STATE_DIR: stateDirectory,- },- ),- ).rejects.toThrow("No credentials were found");- }+ test.each(["auto", "chatgpt"] as const)(+ "%s auth stops after the native status check when no login is stored",+ async (auth) => {+ const directory = join(stateDirectory, "checkout");+ await mkdir(directory);+ const script = join(stateDirectory, "check-login.mjs");+ const calls = join(stateDirectory, "calls.jsonl");+ await writeFile(+ script,+ [+ 'import { appendFileSync } from "node:fs";',+ 'appendFileSync(process.argv[2], JSON.stringify(process.argv.slice(3)) + "\\n");',+ 'console.error("Not logged in");',+ "process.exitCode = 1;",+ ].join("\n"),+ );+ await expect(+ runCodexSkillCommand(+ [script, calls, "model"],+ {+ command: "validate",+ directory,+ auth,+ stdout: capture().stream,+ stderr: capture().stream,+ },+ { command: process.execPath, args: [script, calls] },+ {+ CODEX_HOME: join(stateDirectory, "ambient"),+ CODEX_SECURITY_STATE_DIR: stateDirectory,+ },+ ),+ ).rejects.toThrow("No credentials were found");+ expect(await readJsonLines(calls)).toEqual([["login", "status"]]);},);diff --git a/sdk/typescript/tests-ts/cli-patch-results.test.ts b/sdk/typescript/tests-ts/cli-patch-results.test.tsindex 4e85b18d1..97614afcb 100644--- a/sdk/typescript/tests-ts/cli-patch-results.test.ts+++ b/sdk/typescript/tests-ts/cli-patch-results.test.ts@@ -27,13 +27,21 @@ async function repositoryFixture({} = {}) {const directory = await mkdtemp(join(tmpdir(), "patch-results-"));const git = (...args: string[]) =>- runRepositoryCommand("git", args, directory);+ runRepositoryCommand(+ "git",+ [+ "-c",+ "user.name=Synthetic User",+ "-c",+ "[email protected]",+ ...args,+ ],+ directory,+ );try {await writeFile(join(directory, "app.ts"), "original\n");if (initializeGit) {await git("init", "--initial-branch=main");- await git("config", "user.name", "Synthetic User");- await git("config", "user.email", "[email protected]");await git("add", ".");if (initialCommit) await git("commit", "-m", "Synthetic fixture");}diff --git a/sdk/typescript/tests-ts/cli-scan-failures.test.ts b/sdk/typescript/tests-ts/cli-scan-failures.test.tsindex 568386935..ba66831e8 100644--- a/sdk/typescript/tests-ts/cli-scan-failures.test.ts+++ b/sdk/typescript/tests-ts/cli-scan-failures.test.ts@@ -55,8 +55,10 @@ function expectFailure() {const result = JSON.parse(text);if (fullOutput) {- expect(result).toMatchObject({ ok: false, error: { code, message } });- expect(result.data).toEqual(data);+ expect(result.ok).toBe(false);+ expect(result.error).toEqual({ code, message });+ if (data === undefined) expect(result).not.toHaveProperty("data");+ else expect(result.data).toEqual(data);} else {expect(result).toEqual({ status: "failed", code, message });}@@ -76,7 +78,8 @@ for (const formatArgs of [async (_name, command) => {const stdout = capture();const stderr = capture();- const message = "The scan could not save its results.";+ const message =+ "network failure ECONNRESET Bearer SYNTHETIC_TOKEN_123";expect(await main([...command, ...flags],@@ -169,26 +172,6 @@ for (const formatArgs of [expect(stderr.text()).toBe(`codex-security: ${message}\n`);},);-- test.each(commands)(- "preserves diagnostic text for %s failures",- async (_name, command) => {- const stdout = capture();- const stderr = capture();- const message =- "network failure ECONNRESET Bearer SYNTHETIC_TOKEN_123";- expect(- await main(- [...command, ...flags],- stdout.stream,- stderr.stream,- failingScan(message),- ),- ).toBe(2);- expectFailure(stdout.text(), fullOutput, "SCAN_FAILED", message);- expect(stderr.text()).toContain(message);- },- );});}}diff --git a/sdk/typescript/tests-ts/cli-scan-validation.test.ts b/sdk/typescript/tests-ts/cli-scan-validation.test.tsindex 064fdcbd9..9bea0bef5 100644--- a/sdk/typescript/tests-ts/cli-scan-validation.test.ts+++ b/sdk/typescript/tests-ts/cli-scan-validation.test.ts@@ -228,175 +228,168 @@ test.each([false, true])(},);-for (const changed of ["unchanged", "content", "head"] as const)- test(`cached empty scan validation checks current target: ${changed}`, async () => {- const root = await temporaryDirectory("empty-cached-scan-", true);- const repository = join(root, "repository"),- scanDir = join(root, "scan"),- home = join(root, "codex");- await mkdir(repository);- await mkdir(home);- await mkdir(scanDir, { mode: 0o700 });- git(repository, "init", "-q", "-b", "main");- await writeFile(join(repository, "source.ts"), "export const value = 1;\n");- git(repository, "add", ".");- git(repository, "commit", "-qm", "initial");- const environment = {- ...process.env,- CODEX_HOME: home,- CODEX_SECURITY_STATE_DIR: join(root, "state"),- OPENAI_API_KEY: "synthetic-cached-scan-key",- };- const python = await resolvePluginPython({- environment,- protectedRoot: repository,- });- let scanId = "",- modelCalls = 0;- let registration:- | {- contract: { target: { allowedKinds: string[] } };- targetId: string;- targetRevision?: string;- }- | undefined;- let selectedWorkbench: Parameters<typeof runWorkbench>[0] | undefined;- const workbench = async (- options: Parameters<typeof runWorkbench>[0],- args: readonly string[],- input?: string,- ) => {- const answer = await runWorkbench(- {- ...options,- python,- environment: { ...options.environment, ...environment },- },- args,- input,- );- if (args[0] === "register-cli-scan") {- scanId = String(answer["scanId"]);- registration = answer as unknown as NonNullable<typeof registration>;- selectedWorkbench = {- ...options,- python,- environment: { ...options.environment, ...environment },- };+test("cached empty scan validation rechecks an unchanged, edited, and committed target", async () => {+ const root = await temporaryDirectory("empty-cached-scan-", true);+ const repository = join(root, "repository"),+ scanDir = join(root, "scan"),+ home = join(root, "codex");+ await mkdir(repository);+ await mkdir(home);+ await mkdir(scanDir, { mode: 0o700 });+ git(repository, "init", "-q", "-b", "main");+ await writeFile(join(repository, "source.ts"), "export const value = 1;\n");+ git(repository, "add", ".");+ git(repository, "commit", "-qm", "initial");+ const environment = {+ ...process.env,+ CODEX_HOME: home,+ CODEX_SECURITY_STATE_DIR: join(root, "state"),+ OPENAI_API_KEY: "synthetic-cached-scan-key",+ };+ const python = await resolvePluginPython({+ environment,+ protectedRoot: repository,+ });+ let scanId = "",+ modelCalls = 0;+ let registration:+ | {+ contract: { target: { allowedKinds: string[] } };+ targetId: string;+ targetRevision?: string;}- return answer;- };- const createSecurity = (- config: ConstructorParameters<typeof TestClient>[0],- ) =>- new TestClient(- { ...config, pythonPath: python, pluginPath: PLUGIN_ROOT },- {- environment,- prepareRuntime: async () => {- const r = preparedRuntime(home);- r.environment = environment;- r.plugin.version = JSON.parse(- await readFile(- join(PLUGIN_ROOT, ".codex-plugin", "plugin.json"),- "utf8",- ),- ).version;- return r;- },- prepareOutputDir: async () => scanDir,- resolvePluginPython: async () => python,- runWorkbench: workbench,- createCodex: codexFactory(async () => {- modelCalls++;- await copyCompletedScan(root);- const manifest = JSON.parse(- await readFile(join(scanDir, "scan-manifest.json"), "utf8"),- );- manifest.scan.id = scanId;- manifest.scan.target.kind =- registration!.contract.target.allowedKinds[0];- manifest.scan.target.targetId = registration!.targetId;- manifest.scan.target.revision = registration!.targetRevision;- manifest.scan.target.displayName = "repository";- delete manifest.scan.target.snapshotDigest;- delete manifest.scan.sealedAt;- delete manifest.scan.artifacts;- await writeFile(- join(scanDir, "scan-manifest.json"),- JSON.stringify(manifest),- );- const findings = JSON.parse(- await readFile(join(scanDir, "findings.json"), "utf8"),- );- findings.scanId = scanId;- findings.findings = [];- await writeFile(- join(scanDir, "findings.json"),- JSON.stringify(findings),- );- const coverage = JSON.parse(- await readFile(join(scanDir, "coverage.json"), "utf8"),- );- coverage.scanId = scanId;- coverage.surfaces = [];- await writeFile(- join(scanDir, "coverage.json"),- JSON.stringify(coverage),- );- return { events: completedEvents() };- }, "empty-cached-conversation"),- },- );- const cliDependencies = {- ...dependencies({ currentDirectory: root, environment }),- createSecurity,- runWorkbench: async (- args: readonly string[],- input?: string,- signal?: AbortSignal,- configuredPython?: string,- protectedRoot?: string,- ) => {- expect(configuredPython).toBe(python);- expect(protectedRoot).toBe(repository);- expect(selectedWorkbench).toBeDefined();- return runWorkbench({ ...selectedWorkbench!, signal }, args, input);+ | undefined;+ let selectedWorkbench: Parameters<typeof runWorkbench>[0] | undefined;+ const workbench = async (+ options: Parameters<typeof runWorkbench>[0],+ args: readonly string[],+ input?: string,+ ) => {+ const answer = await runWorkbench(+ {+ ...options,+ python,+ environment: { ...options.environment, ...environment },},- };- try {- const first = createCliTest(main);- const args = [- "scan",- repository,- "--python",+ args,+ input,+ );+ if (args[0] === "register-cli-scan") {+ scanId = String(answer["scanId"]);+ registration = answer as unknown as NonNullable<typeof registration>;+ selectedWorkbench = {+ ...options,python,- "--plugin-path",- PLUGIN_ROOT,- "--workflow-id",- "empty-cached-workflow",- "--json",- ];- const initial = await first.runCli(args, cliDependencies);- expect(initial, first.stderr.text()).toBe(0);- expect(JSON.parse(first.stdout.text()).findings.findings).toEqual([]);- expect(modelCalls).toBe(1);- if (changed !== "unchanged")+ environment: { ...options.environment, ...environment },+ };+ }+ return answer;+ };+ const createSecurity = (+ config: ConstructorParameters<typeof TestClient>[0],+ ) =>+ new TestClient(+ { ...config, pythonPath: python, pluginPath: PLUGIN_ROOT },+ {+ environment,+ prepareRuntime: async () => {+ const r = preparedRuntime(home);+ r.environment = environment;+ r.plugin.version = JSON.parse(+ await readFile(+ join(PLUGIN_ROOT, ".codex-plugin", "plugin.json"),+ "utf8",+ ),+ ).version;+ return r;+ },+ prepareOutputDir: async () => scanDir,+ resolvePluginPython: async () => python,+ runWorkbench: workbench,+ createCodex: codexFactory(async () => {+ modelCalls++;+ await copyCompletedScan(root);+ const manifest = JSON.parse(+ await readFile(join(scanDir, "scan-manifest.json"), "utf8"),+ );+ manifest.scan.id = scanId;+ manifest.scan.target.kind =+ registration!.contract.target.allowedKinds[0];+ manifest.scan.target.targetId = registration!.targetId;+ manifest.scan.target.revision = registration!.targetRevision;+ manifest.scan.target.displayName = "repository";+ delete manifest.scan.target.snapshotDigest;+ delete manifest.scan.sealedAt;+ delete manifest.scan.artifacts;+ await writeFile(+ join(scanDir, "scan-manifest.json"),+ JSON.stringify(manifest),+ );+ const findings = JSON.parse(+ await readFile(join(scanDir, "findings.json"), "utf8"),+ );+ findings.scanId = scanId;+ findings.findings = [];+ await writeFile(+ join(scanDir, "findings.json"),+ JSON.stringify(findings),+ );+ const coverage = JSON.parse(+ await readFile(join(scanDir, "coverage.json"), "utf8"),+ );+ coverage.scanId = scanId;+ coverage.surfaces = [];+ await writeFile(+ join(scanDir, "coverage.json"),+ JSON.stringify(coverage),+ );+ return { events: completedEvents() };+ }, "empty-cached-conversation"),+ },+ );+ const targetChecks: string[] = [];+ const cliDependencies = {+ ...dependencies({ currentDirectory: root, environment }),+ createSecurity,+ runWorkbench: async (+ args: readonly string[],+ input?: string,+ signal?: AbortSignal,+ configuredPython?: string,+ protectedRoot?: string,+ ) => {+ expect(configuredPython).toBe(python);+ expect(protectedRoot).toBe(repository);+ expect(selectedWorkbench).toBeDefined();+ expect(args).toEqual(["get-scan", "--scan-id", scanId, "--check-target"]);+ targetChecks.push(scanId);+ return runWorkbench({ ...selectedWorkbench!, signal }, args, input);+ },+ };+ try {+ const first = createCliTest(main);+ const args = [+ "scan",+ repository,+ "--python",+ python,+ "--plugin-path",+ PLUGIN_ROOT,+ "--workflow-id",+ "empty-cached-workflow",+ "--json",+ ];+ const initial = await first.runCli(args, cliDependencies);+ expect(initial, first.stderr.text()).toBe(0);+ expect(JSON.parse(first.stdout.text()).findings.findings).toEqual([]);+ expect(modelCalls).toBe(1);+ for (const changed of ["unchanged", "content", "head"] as const) {+ if (changed === "content")await writeFile(join(repository, "source.ts"),"export const value = 2;\n",);if (changed === "head") git(repository, "commit", "-qam", "changed");- const recorded = await runWorkbench(selectedWorkbench!, [- "get-scan",- "--scan-id",- scanId,- "--check-target",- ]).then(- () => "unchanged",- (e) => (e as Error).message,- );- expect(recorded === "unchanged").toBe(changed === "unchanged");const second = createCliTest(main);const result = await second.runCli([...args, "--validate"],@@ -417,10 +410,12 @@ for (const changed of ["unchanged", "content", "head"] as const)findings: 0,message: expect.stringContaining("changed"),});- } finally {- await rm(root, { recursive: true, force: true });}- });+ expect(targetChecks).toEqual([scanId, scanId, scanId]);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});test("empty scan validation preserves configured Python and Git ignore rules", async () => {if (diff --git a/sdk/typescript/tests-ts/cli-skill-native-config.test.ts b/sdk/typescript/tests-ts/cli-skill-native-config.test.tsindex 59a63a8cf..84854ee1c 100644--- a/sdk/typescript/tests-ts/cli-skill-native-config.test.ts+++ b/sdk/typescript/tests-ts/cli-skill-native-config.test.ts@@ -361,11 +361,11 @@ async function withStoredApiKey(}}-test.each(- (["patch", "verify-fix"] as const).flatMap((command) =>- [true, false].map((daybreak) => [command, daybreak] as const),- ),-)(+test.each([+ ["patch", true],+ ["patch", false],+ ["verify-fix", true],+] as const)("%s preserves ambient Daybreak %p with a stored API key",async (command, daybreak) => {await withStoredApiKey(@@ -384,95 +384,80 @@ test.each(},);-test.each(["patch", "verify-fix"] as const)(- "%s preserves the Daybreak opt-out with an explicit toggle and a stored API key",- async (command) => {- await withStoredApiKey(command, async ({ ambientHome, run }) => {- await writeCodexConfig(join(ambientHome, "config.toml"), {- daybreak: false,- features: { cli_daybreak: false },- });- expect(await run({ daybreak: true })).toMatchObject({- daybreak: true,- daybreakEnabled: true,- daybreakFeatures: { cli_daybreak: false },- });+test("patch preserves the Daybreak opt-out with an explicit toggle and a stored API key", async () => {+ await withStoredApiKey("patch", async ({ ambientHome, run }) => {+ await writeCodexConfig(join(ambientHome, "config.toml"), {+ daybreak: false,+ features: { cli_daybreak: false },});- },-);+ expect(await run({ daybreak: true })).toMatchObject({+ daybreak: true,+ daybreakEnabled: true,+ daybreakFeatures: { cli_daybreak: false },+ });+ });+});-test.each(["patch", "verify-fix"] as const)(- "%s forwards wrapper-resolved ambient Daybreak profiles with a stored API key",- async (command) => {- await withStoredApiKey(command, async ({ ambientHome, run }) => {- await writeCodexConfig(join(ambientHome, "config.toml"), {- daybreak: false,- features: { cli_daybreak: true },- profile: "selected",- profiles: {- selected: { daybreak: true, features: disabledDaybreakFeatures },- },- });- expect(await run()).toMatchObject({- daybreak: true,- daybreakEnabled: true,- daybreakFeatures: disabledDaybreakFeatures,- });+test("patch forwards wrapper-resolved ambient Daybreak profiles with a stored API key", async () => {+ await withStoredApiKey("patch", async ({ ambientHome, run }) => {+ await writeCodexConfig(join(ambientHome, "config.toml"), {+ daybreak: false,+ features: { cli_daybreak: true },+ profile: "selected",+ profiles: {+ selected: { daybreak: true, features: disabledDaybreakFeatures },+ },});- },-);+ expect(await run()).toMatchObject({+ daybreak: true,+ daybreakEnabled: true,+ daybreakFeatures: disabledDaybreakFeatures,+ });+ });+});-test.each(["patch", "verify-fix"] as const)(- "%s clears removed ambient Daybreak settings on the next stored API-key launch",- async (command) => {- await withStoredApiKey(command, async ({ ambientHome, run }) => {- const configPath = join(ambientHome, "config.toml");- await writeCodexConfig(configPath, {});- const baseline = await run();- await writeCodexConfig(configPath, {- daybreak: true,- features: disabledDaybreakFeatures,- });- expect(await run()).toMatchObject({- daybreak: true,- daybreakEnabled: true,- daybreakFeatures: disabledDaybreakFeatures,- });- await writeCodexConfig(configPath, {});- expect(await run()).toEqual(baseline);+test("patch clears removed ambient Daybreak settings on the next stored API-key launch", async () => {+ await withStoredApiKey("patch", async ({ ambientHome, run }) => {+ const configPath = join(ambientHome, "config.toml");+ await writeCodexConfig(configPath, {});+ const baseline = await run();+ await writeCodexConfig(configPath, {+ daybreak: true,+ features: disabledDaybreakFeatures,});- },-);+ expect(await run()).toMatchObject({+ daybreak: true,+ daybreakEnabled: true,+ daybreakFeatures: disabledDaybreakFeatures,+ });+ await writeCodexConfig(configPath, {});+ expect(await run()).toEqual(baseline);+ });+});-test.each(["patch", "verify-fix"] as const)(- "%s preserves native project and explicit Daybreak precedence with a stored API key",- async (command) => {- await withStoredApiKey(- command,- async ({ ambientHome, repository, run }) => {- await writeCodexConfig(join(ambientHome, "config.toml"), {- daybreak: true,- features: { cli_daybreak: true },- projects: { [repository]: { trust_level: "trusted" } },- });- await writeCodexConfig(join(repository, ".codex", "config.toml"), {- daybreak: false,- features: { cli_daybreak: false },- });- expect(await run()).toMatchObject({- daybreak: false,- daybreakEnabled: null,- daybreakFeatures: { cli_daybreak: false },- });- expect(await run({ daybreak: true })).toMatchObject({- daybreak: true,- daybreakEnabled: true,- daybreakFeatures: { cli_daybreak: false },- });- },- );- },-);+test("patch preserves native project and explicit Daybreak precedence with a stored API key", async () => {+ await withStoredApiKey("patch", async ({ ambientHome, repository, run }) => {+ await writeCodexConfig(join(ambientHome, "config.toml"), {+ daybreak: true,+ features: { cli_daybreak: true },+ projects: { [repository]: { trust_level: "trusted" } },+ });+ await writeCodexConfig(join(repository, ".codex", "config.toml"), {+ daybreak: false,+ features: { cli_daybreak: false },+ });+ expect(await run()).toMatchObject({+ daybreak: false,+ daybreakEnabled: null,+ daybreakFeatures: { cli_daybreak: false },+ });+ expect(await run({ daybreak: true })).toMatchObject({+ daybreak: true,+ daybreakEnabled: true,+ daybreakFeatures: { cli_daybreak: false },+ });+ });+});test.each(["omitted", "omitted with changed gates", "explicit"] as const)("stored API-key patches preserve %s settings for an active scan's later workers",diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.tsindex 9acd9fbee..0daca1efd 100644--- a/sdk/typescript/tests-ts/cli.test.ts+++ b/sdk/typescript/tests-ts/cli.test.ts@@ -3778,16 +3778,6 @@ describe("CLI", () => {}});- test("honors Incur help before command validation", async () => {- const { stdout, stderr, runCli } = createCliTest(main);-- expect(- await runCli(["scan", "--mode", "bogus", "--help"], dependencies()),- ).toBe(0);- expect(stdout.text()).toContain("Usage: codex-security scan [repository]");- expect(stderr.text()).toBe("");- });-test("maps configuration and emits JSON only on stdout", async () => {const { stdout, stderr, runCli } = createCliTest(main);@@ -4629,49 +4619,18 @@ describe("CLI", () => {}});- test("emits structured failures for machine-readable scan output", async () => {+ test("preserves a model content refusal in structured scan failures", async () => {const message = "This content was flagged for possible cybersecurity risk.";- for (const formatArgs of [- ["--json"],- ["--format", "json"],- ["--format", "jsonl"],- ] as const) {- const { stdout, stderr, runCli } = createCliTest(main);-- const deps = dependencies();- deps.createSecurity = () => failingSecurity(message);-- expect(await runCli(["scan", ".", ...formatArgs], deps)).toBe(2);- expect(JSON.parse(stdout.text().trim())).toEqual({- status: "failed",- code: "SCAN_FAILED",- message,- });- expect(stderr.text()).toContain(`${message}\n`);- }- });-- test("reports full-output scan failures in an error envelope", async () => {- const message = "This content was flagged for possible cybersecurity risk.";- for (const formatArgs of [- ["--json"],- ["--format", "json"],- ["--format", "jsonl"],- ] as const) {- const { stdout, stderr, runCli } = createCliTest(main);-- const deps = dependencies();- deps.createSecurity = () => failingSecurity(message);-- expect(- await runCli(["scan", ".", ...formatArgs, "--full-output"], deps),- ).toBe(2);- const envelope = JSON.parse(stdout.text().trim());- expect(envelope.ok).toBe(false);- expect(envelope.error).toEqual({ code: "SCAN_FAILED", message });- expect(envelope).not.toHaveProperty("data");- expect(stderr.text()).toContain(`${message}\n`);- }+ const { stdout, stderr, runCli } = createCliTest(main);+ const deps = dependencies();+ deps.createSecurity = () => failingSecurity(message);+ expect(await runCli(["scan", ".", "--json"], deps)).toBe(2);+ expect(JSON.parse(stdout.text())).toEqual({+ status: "failed",+ code: "SCAN_FAILED",+ message,+ });+ expect(stderr.text()).toContain(message);});test("preserves diagnostic text in full-output scan failures", async () => {diff --git a/sdk/typescript/tests-ts/support/validation.ts b/sdk/typescript/tests-ts/support/validation.tsnew file mode 100644index 000000000..9dd1a1c88--- /dev/null+++ b/sdk/typescript/tests-ts/support/validation.ts@@ -0,0 +1,240 @@+import { mkdir } from "node:fs/promises";+import * as fsPromises from "node:fs/promises";+import { join } from "node:path";+import type {+ CodexOptions,+ ThreadEvent,+ ThreadOptions,+ TurnOptions,+} from "@openai/codex-sdk";+import { afterEach, beforeEach, mock, spyOn } from "bun:test";+import type { JsonObject } from "../../src/config.js";+import { resolvePluginPython, runWorkbench } from "../../src/runtime.js";++import { PLUGIN_ROOT } from "../plugin-root.js";+import { TestClient } from "./api-client.js";+import { preparedRuntime } from "./api-events.js";+import { createApiTestFixtures } from "./temporary-directories.js";+import { FindingWorkflow } from "../../src/finding-workflow.js";++export function createValidationFixtures() {+ const { cleanup, temporaryDirectory } = createApiTestFixtures();++ async function repositoryDirectories() {+ const root = await temporaryDirectory();+ const repository = join(root, "repository");+ await mkdir(repository);+ return { root, repository };+ }++ async function runtimeDirectories() {+ const { root, repository } = await repositoryDirectories();+ const codexHome = join(root, "codex-home");+ await mkdir(codexHome);+ return { root, repository, codexHome };+ }++ function runtimePreparer(+ codexHome: string,+ overrides: () => Partial<ReturnType<typeof preparedRuntime>>,+ ) {+ return async () => ({+ ...preparedRuntime(codexHome),+ ...overrides(),+ });+ }+ const actualLstat = fsPromises.lstat;+ let systemConfigurationLookup: ReturnType<+ typeof spyOn<typeof fsPromises, "lstat">+ >;+ beforeEach(() => {+ const nativePath =+ process.platform === "win32"+ ? join("C:\\ProgramData", "OpenAI", "Codex", "config.toml")+ : "/etc/codex/config.toml";+ systemConfigurationLookup = spyOn(fsPromises, "lstat").mockImplementation(+ (async (...args: Parameters<typeof actualLstat>) => {+ if (args[0] === nativePath)+ throw Object.assign(+ new Error("Synthetic absent native system configuration"),+ { code: "ENOENT" },+ );+ return await actualLstat(...args);+ }) as typeof actualLstat,+ );+ });+ afterEach(async () => {+ systemConfigurationLookup.mockRestore();+ await cleanup();+ });++ const assessment = {+ disposition: "reportable",+ report: "Static trace reaches the SQL sink; runtime proof is still needed.",+ } as const;++ async function* validationEvents(+ response = JSON.stringify(assessment),+ complete = true,+ ): AsyncGenerator<ThreadEvent> {+ yield { type: "thread.started", thread_id: "validation-thread" };+ yield {+ type: "item.completed",+ item: { id: "result", type: "agent_message", text: response },+ };+ if (complete) {+ yield {+ type: "turn.completed",+ usage: {+ input_tokens: 10,+ cached_input_tokens: 0,+ cache_write_input_tokens: 0,+ output_tokens: 3,+ reasoning_output_tokens: 0,+ },+ };+ }+ }++ let fixturePython: Promise<string> | undefined;++ async function validationClient(+ events: (signal: AbortSignal) => AsyncGenerator<ThreadEvent> = () =>+ validationEvents(),+ pluginRoot = PLUGIN_ROOT,+ pythonPath = "/managed/python",+ fixtureRoot?: string,+ environmentOverrides: Record<string, string> = {},+ ) {+ const root = fixtureRoot ?? (await temporaryDirectory());+ const repository = join(root, "repository");+ const codexHome = await temporaryDirectory();+ const stateDirectory =+ environmentOverrides["CODEX_SECURITY_STATE_DIR"] ?? join(root, "state");+ await mkdir(repository, { recursive: true });+ const captured: {+ codex?: CodexOptions;+ thread?: ThreadOptions;+ prompt?: string;+ turn?: TurnOptions;+ } = {};+ const workbench = mock(+ async (+ _options: Parameters<typeof runWorkbench>[0],+ _args: readonly string[],+ _input?: string,+ ): Promise<JsonObject> => ({+ scan: { targetPath: repository },+ }),+ );+ const environment = {+ ...Object.fromEntries(+ ["PATH", "SystemRoot", "TEMP", "TMP"].flatMap((name) =>+ process.env[name] === undefined ? [] : [[name, process.env[name]!]],+ ),+ ),+ CODEX_SECURITY_STATE_DIR: stateDirectory,+ OPENAI_API_KEY: "synthetic-validation-key",+ ...environmentOverrides,+ };+ const client = new TestClient(+ {+ pluginPath: pluginRoot === PLUGIN_ROOT ? undefined : pluginRoot,+ pythonPath,+ codexOverrides: {+ model: "test-model",+ model_reasoning_effort: "high",+ approval_policy: "never",+ analytics: { enabled: false },+ },+ },+ {+ environment,+ prepareRuntime: runtimePreparer(codexHome, () => ({+ plugin: { ...preparedRuntime(codexHome).plugin, pluginRoot },+ environment,+ })),+ resolvePluginPython: async ({ configuredPath } = {}) => configuredPath!,+ runWorkbench: workbench,+ createCodex: (options) => {+ captured.codex = options;+ return {+ startThread: (options) => {+ captured.thread = options;+ return {+ id: null,+ async runStreamed(prompt, options) {+ captured.prompt = prompt;+ captured.turn = options;+ return { events: events(options.signal!) };+ },+ };+ },+ };+ },+ },+ );+ const options = {+ repositoryPath: repository,+ finding: "Candidate finding",+ outputDir: join(root, "validation"),+ };+ return {+ client,+ options,+ stateDirectory,+ captured,+ workbench,+ root,+ codexHome,+ };+ }++ async function persistedValidationClient(+ events?: Parameters<typeof validationClient>[0],+ pluginRoot = PLUGIN_ROOT,+ fixtureRoot?: string,+ environmentOverrides?: Record<string, string>,+ ) {+ const python = await (fixturePython ??= resolvePluginPython());+ const fixture = await validationClient(+ events,+ pluginRoot,+ python,+ fixtureRoot,+ environmentOverrides,+ );+ fixture.workbench.mockImplementation(runWorkbench);+ async function bindWorkflow(+ id: string,+ environment: NodeJS.ProcessEnv = {+ CODEX_SECURITY_STATE_DIR: fixture.stateDirectory,+ },+ ) {+ const workflow = new FindingWorkflow(+ id,+ environment,+ runWorkbench,+ python,+ {+ workbenchOptions: { python, pluginRoot: PLUGIN_ROOT, environment },+ },+ );+ await workflow.bind({ repositoryPath: fixture.options.repositoryPath });+ return id;+ }+ return { ...fixture, python, bindWorkflow };+ }++ return {+ actualLstat,+ assessment,+ validationEvents,+ validationClient,+ persistedValidationClient,+ temporaryDirectory,+ repositoryDirectories,+ runtimeDirectories,+ runtimePreparer,+ };+}