Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(scan): report incomplete results as errors and deduplicate log warnings (#977)

mldangelo-oai3547e76d4e5f
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 603e74124..9508ca53b 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -301,6 +301,13 @@ execution failures with `--json`, JSON, or JSONL output produce:
```
With `--full-output`, the error appears under `error` in an `ok: false` envelope.
+A scan that returns partial or unknown coverage uses that failure envelope and
+keeps its available results under `data`. This also applies to `scans rerun` and
+`scans resume`. If the scan target also changed, the error explains that the
+results no longer represent the current checkout. For incomplete `scans rerun` and
+`scans resume` envelopes, `--filter-output` applies to `data`; selecting an
+unavailable field can return `null` or omit `data`.
+
Saved-scan setup failures use the same output shape with
`SCAN_REPLAY_UNAVAILABLE` for `scans rerun` (including when no completed scan is
available) or `SCAN_RESUME_UNAVAILABLE` for `scans resume`. Rerunning an imported
diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts
index 85f1f5ce3..6d192d602 100644
--- a/sdk/typescript/src/api.ts
+++ b/sdk/typescript/src/api.ts
@@ -1495,11 +1495,22 @@ export class CodexSecurity {
"onProgress",
)({ ...progress, filesTotal: scopeFileCount });
};
+ const reportedInaccessibleSessionLogs = new Set<string>();
const reportTrackingError = (error: unknown): void => {
if (options.maxCostUsd !== undefined) {
costAbortController.abort(error);
return;
}
+ if (
+ isRecord(error) &&
+ (error["code"] === "EACCES" || error["code"] === "EPERM") &&
+ error["syscall"] === "open" &&
+ typeof error["path"] === "string"
+ ) {
+ const path = error["path"];
+ if (reportedInaccessibleSessionLogs.has(path)) return;
+ reportedInaccessibleSessionLogs.add(path);
+ }
notifyObserver(
options,
"onWarning",
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 43a499bef..90909c50a 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -1031,12 +1031,28 @@ interface ScanOutcome {
exitCode: number;
data?: Record<string, unknown>;
error?: string;
+ coverageError?: string;
}
+const incompleteScanEnvelopeSchema = z
+ .looseObject({
+ ok: z.literal(false),
+ error: z.object({
+ code: z.literal("SCAN_FAILED"),
+ message: z.string(),
+ }),
+ data: z.unknown().optional(),
+ meta: z.record(z.string(), z.unknown()),
+ })
+ .describe(
+ "Incomplete full-output JSON or JSONL with available scan results.",
+ );
+
function scanOutputSchema(...codes: [string, ...string[]]) {
return z
.union([
z.record(z.string(), z.unknown()),
+ incompleteScanEnvelopeSchema,
z.object({
status: z.literal("failed"),
code: z.literal(codes),
@@ -1826,6 +1842,17 @@ export async function main(
let renderedPatch: string | undefined;
let patchStructuredError = false;
let scanStructuredError = false;
+ let incompleteScanOutput:
+ { format: "json" | "jsonl"; message: string } | undefined;
+ const recordIncompleteScanOutput = (outcome: ScanOutcome, format: string) => {
+ if (
+ outcome.coverageError !== undefined &&
+ (format === "json" || format === "jsonl") &&
+ argv.includes("--full-output")
+ ) {
+ incompleteScanOutput = { format, message: outcome.coverageError };
+ }
+ };
let filteredScanFailure:
{ format: string; data: Record<string, unknown> } | undefined;
let renderedScanFailure: string | undefined;
@@ -1840,7 +1867,10 @@ export async function main(
code = "SCAN_FAILED",
): Record<string, unknown> | undefined => {
exitCode = outcome.exitCode;
- if (outcome.error === undefined) return outcome.data;
+ if (outcome.error === undefined) {
+ recordIncompleteScanOutput(outcome, format);
+ return outcome.data;
+ }
if (format === "json" || format === "jsonl") {
const message = errorMessage(outcome.error);
if (!argv.includes("--full-output")) {
@@ -2256,6 +2286,7 @@ export async function main(
})
.command("resume", {
description: "Resume an interrupted Deep Scan in its original session.",
+ hint: "Incomplete full-output JSON or JSONL uses ok: false and keeps available scan results under data.",
mcp: false,
args: z.object({
scanId: z.string().min(1).describe("Interrupted Deep Scan identifier."),
@@ -2321,6 +2352,7 @@ export async function main(
})
.command("rerun", {
description: "Rerun a saved scan with its original configuration.",
+ hint: "Incomplete full-output JSON or JSONL uses ok: false and keeps available scan results under data.",
destructive: true,
mcp: false,
args: z.object({
@@ -3491,7 +3523,8 @@ export async function main(
"Import existing findings without security analysis:\n" +
" codex-security scan import --csv findings.csv\n" +
" codex-security scan import --json findings.json\n" +
- "Use ./import to scan a repository named import.",
+ "Use ./import to scan a repository named import.\n" +
+ "Incomplete JSON/JSONL --full-output returns ok: false and keeps scan results under data.",
destructive: true,
mcp: false,
alias: { config: "c" },
@@ -3712,6 +3745,7 @@ export async function main(
errorOutput.write(`${message}\n`);
outcome = { exitCode: 2, error: message };
}
+
if (
outcome.error === undefined &&
!options.dryRun &&
@@ -5933,7 +5967,7 @@ export async function main(
});
renderedScanFailure = failureOutput.text();
}
- const frameworkOutput = frameworkCapture.text();
+ let frameworkOutput = frameworkCapture.text();
if (notice !== undefined) errorOutput.write(formatUpdateNotice(notice));
if (frameworkExit !== undefined) {
if (policyFullOutput || patchStructuredError || scanStructuredError) {
@@ -5961,6 +5995,24 @@ export async function main(
streamedLogs,
frameworkOutput ? JSON.parse(frameworkOutput).cta : undefined,
);
+ if (incompleteScanOutput !== undefined) {
+ const envelope: JsonValue = JSON.parse(frameworkOutput);
+ // Token-count output is a number, not a full-output envelope.
+ if (isJsonObject(envelope) && envelope["ok"] === true) {
+ frameworkOutput = `${JSON.stringify(
+ {
+ ...envelope,
+ ok: false,
+ error: {
+ code: "SCAN_FAILED",
+ message: incompleteScanOutput.message,
+ },
+ },
+ null,
+ incompleteScanOutput.format === "json" ? 2 : undefined,
+ )}\n`;
+ }
+ }
await writeCliOutput(
output,
logOutput ??
@@ -8879,7 +8931,15 @@ async function executeScan(
showCost,
deepScanStop,
);
- const completedScan = (exitCode: number): ScanOutcome => {
+ const coverageError = incomplete
+ ? threshold === undefined
+ ? `Scan coverage is ${result.coverage.completeness}; results may be incomplete.`
+ : `Cannot evaluate the failure policy: coverage is ${result.coverage.completeness}.`
+ : undefined;
+ const completedScan = (
+ exitCode: number,
+ error = coverageError,
+ ): ScanOutcome => {
diagnostic("scan.completed", {
coverage: result.coverage.completeness,
findings: findings.length,
@@ -8892,20 +8952,20 @@ async function executeScan(
exit_code: exitCode,
});
progress?.stopTimer();
- return { exitCode, data: scanData };
+ return {
+ exitCode,
+ data: scanData,
+ ...(error === undefined ? {} : { coverageError: error }),
+ };
};
if (targetWarnings.length > 0) {
- errorOutput.write(
- "codex-security: Scan target changed during execution; results do not represent the current checkout.\n",
- );
- return completedScan(2);
+ const message =
+ "Scan target changed during execution; results do not represent the current checkout.";
+ errorOutput.write(`codex-security: ${message}\n`);
+ return completedScan(2, incomplete ? message : undefined);
}
- if (incomplete) {
- errorOutput.write(
- threshold === undefined
- ? `codex-security: Scan coverage is ${result.coverage.completeness}; results may be incomplete.\n`
- : `codex-security: Cannot evaluate the failure policy: coverage is ${result.coverage.completeness}.\n`,
- );
+ if (coverageError !== undefined) {
+ errorOutput.write(`codex-security: ${coverageError}\n`);
return completedScan(2);
}
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index fa9548f88..192d83500 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -3669,6 +3669,100 @@ describe("CodexSecurity orchestration", () => {
},
);
+ test.each(["EACCES", "EPERM", "EMFILE"])(
+ "retries session logs and limits repeated %s diagnostics appropriately",
+ async (code) => {
+ const { root, repository, codexHome, scanDir } = await scanDirectories();
+ const sessions = join(codexHome, "sessions");
+ await mkdir(sessions);
+ const logs = [
+ join(sessions, "a-synthetic.jsonl"),
+ join(sessions, "b-synthetic.jsonl"),
+ ];
+ await Promise.all(logs.map((path) => writeFile(path, "")));
+ const denied = new Set([logs[0]!]);
+ const attempts = new Map<string, number>();
+ let firstRepeated!: () => void;
+ let secondRepeated!: () => void;
+ const first = new Promise<void>((resolve) => {
+ firstRepeated = resolve;
+ });
+ const second = new Promise<void>((resolve) => {
+ secondRepeated = resolve;
+ });
+ const open = fsPromises.open;
+ const opening = spyOn(fsPromises, "open").mockImplementation(
+ async (...args: Parameters<typeof fsPromises.open>) => {
+ const path = String(args[0]);
+ if (denied.has(path)) {
+ const count = (attempts.get(path) ?? 0) + 1;
+ attempts.set(path, count);
+ if (count === 3)
+ (path === logs[0] ? firstRepeated : secondRepeated)();
+ throw Object.assign(
+ new Error(`Synthetic ${code} for ${basename(path)}`),
+ { code, syscall: "open", path },
+ );
+ }
+ return await open(...args);
+ },
+ );
+ const warnings: string[] = [];
+ const client = TestClient.withDependencies({
+ ...scanRuntimeDependencies(codexHome, scanDir),
+ createCodex: () => ({
+ startThread: () => ({
+ id: null,
+ async runStreamed() {
+ await copyCompletedScan(root);
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ yield { type: "thread.started", thread_id: "thread-1" };
+ await first;
+ denied.delete(logs[0]!);
+ denied.add(logs[1]!);
+ await second;
+ denied.delete(logs[1]!);
+ for await (const event of completedEvents()) {
+ if (event.type !== "thread.started") yield event;
+ }
+ }
+ return { events: events() };
+ },
+ }),
+ }),
+ });
+ // The fake stream has no process handle to keep the unref'ed poll alive.
+ const keepAlive = setTimeout(() => {}, 10_000);
+ const operation = client.run(repository, {
+ onActivity: () => {},
+ onWarning: (warning) => warnings.push(warning),
+ });
+ try {
+ expect(await operation).toMatchObject({
+ threadId: "thread-1",
+ });
+ await Promise.resolve();
+ for (const log of logs) {
+ expect(attempts.get(log)).toBeGreaterThanOrEqual(3);
+ const messages = warnings.filter((message) =>
+ message.includes(basename(log)),
+ );
+ if (code === "EMFILE")
+ expect(messages.length).toBeGreaterThanOrEqual(3);
+ else expect(messages).toHaveLength(1);
+ }
+ } finally {
+ clearTimeout(keepAlive);
+ denied.clear();
+ firstRepeated();
+ secondRepeated();
+ await operation.catch(() => {});
+ await client.close();
+ opening.mockRestore();
+ }
+ },
+ );
+
test.each(["agent_message", "command_execution"] as const)(
"uses the actual scanner inventory instead of a stale workbench estimate (%s)",
async (itemType) => {
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index 35566518e..1f544eff6 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -4152,6 +4152,58 @@ describe("CLI", () => {
}
});
+ test("reports a changed target before incomplete coverage in full-output errors", async () => {
+ const warning =
+ "Completed findings no longer describe the selected source tree.";
+ const message =
+ "Scan target changed during execution; results do not represent the current checkout.";
+ for (const completeness of ["partial", "unknown"] as const) {
+ for (const format of ["json", "jsonl"]) {
+ for (const paginated of [false, true]) {
+ const result = fakeResult(["high"], completeness);
+ const stdout = capture();
+ const stderr = capture();
+ const deps = dependencies();
+ deps.createSecurity = () => ({
+ run: async (_repository, options) => {
+ options?.onWarning?.(warning, { kind: "target_changed" });
+ return result;
+ },
+ close: async () => {},
+ preflight: async () => fakePreflight(),
+ });
+ expect(
+ await main(
+ [
+ "scan",
+ "--format",
+ format,
+ "--full-output",
+ "--fail-on-severity",
+ "high",
+ ...(paginated ? ["--token-limit", "1"] : []),
+ ],
+ stdout.stream,
+ stderr.stream,
+ deps,
+ ),
+ ).toBe(2);
+ expect(JSON.parse(stdout.text())).toMatchObject({
+ ok: false,
+ error: { code: "SCAN_FAILED", message },
+ data: paginated
+ ? expect.any(String)
+ : {
+ coverage: { completeness },
+ warnings: [warning],
+ },
+ });
+ expect(stderr.text()).toContain(message);
+ }
+ }
+ }
+ });
+
test("preserves non-target warnings without failing the scan", async () => {
for (const warning of [
"Recovered finding: normalized its semantic anchor.",
@@ -4905,6 +4957,227 @@ describe("CLI", () => {
}
});
+ test.each(["partial", "unknown"] as const)(
+ "marks %s coverage as a full-output error and keeps the scan results",
+ async (completeness) => {
+ const result = fakeResult(["high"], completeness);
+ for (const format of [
+ ["--json"],
+ ["--format", "json"],
+ ["--format", "jsonl"],
+ ]) {
+ for (const policy of [false, true]) {
+ const stdout = capture();
+ const stderr = capture();
+ expect(
+ await main(
+ [
+ "scan",
+ ".",
+ ...format,
+ "--full-output",
+ ...(policy ? ["--fail-on-severity", "critical"] : []),
+ ],
+ stdout.stream,
+ stderr.stream,
+ dependencies({ result }),
+ ),
+ ).toBe(2);
+ const envelope = JSON.parse(stdout.text());
+ const message = policy
+ ? `Cannot evaluate the failure policy: coverage is ${completeness}.`
+ : `Scan coverage is ${completeness}; results may be incomplete.`;
+ expect(envelope).toMatchObject({
+ ok: false,
+ error: { code: "SCAN_FAILED", message },
+ data: JSON.parse(JSON.stringify(result.toJSON())),
+ meta: { command: "scan" },
+ });
+ expect(stderr.text()).toContain(message);
+ if (format.includes("jsonl"))
+ expect(stdout.text().trim().split("\n")).toHaveLength(1);
+ }
+ }
+ },
+ );
+
+ test.each(["rerun", "resume"] as const)(
+ "reports incomplete %s coverage in the full-output error envelope",
+ async (command) => {
+ for (const completeness of ["partial", "unknown"] as const) {
+ for (const format of ["json", "jsonl"]) {
+ const result = fakeResult(["high"], completeness);
+ const stdout = capture();
+ const stderr = capture();
+ const recipe = {
+ repository: "/original/repository",
+ target: { kind: "repository", paths: [] },
+ mode: command === "resume" ? "deep" : "standard",
+ config: {},
+ };
+ expect(
+ await main(
+ [
+ "scans",
+ command,
+ "scan-original",
+ "--format",
+ format,
+ "--full-output",
+ ],
+ stdout.stream,
+ stderr.stream,
+ dependencies({
+ result,
+ onWorkbench: (args): JsonObject =>
+ args[0] === "get-cli-scan-resume"
+ ? {
+ scanId: "scan-original",
+ scanDir: "/tmp/saved-scan",
+ recipe,
+ }
+ : args[0] === "get-scan-recipe"
+ ? { recipe }
+ : {},
+ }),
+ ),
+ ).toBe(2);
+ const message = `Scan coverage is ${completeness}; results may be incomplete.`;
+ expect(JSON.parse(stdout.text())).toMatchObject({
+ ok: false,
+ error: { code: "SCAN_FAILED", message },
+ data: JSON.parse(JSON.stringify(result.toJSON())),
+ });
+ expect(stderr.text()).toContain(message);
+ if (format === "jsonl")
+ expect(stdout.text().trim().split("\n")).toHaveLength(1);
+ }
+ }
+ },
+ );
+
+ test.each(["rerun", "resume"] as const)(
+ "matches the %s full-output error schema to filtered data",
+ async (command) => {
+ const schemaOutput = capture();
+ expect(
+ await main(
+ ["scans", command, "--schema", "--format", "json"],
+ schemaOutput.stream,
+ capture().stream,
+ dependencies(),
+ ),
+ ).toBe(0);
+ const schema = JSON.parse(schemaOutput.text()).output.anyOf.find(
+ (variant: { properties?: { ok?: { const?: boolean } } }) =>
+ variant.properties?.ok?.const === false,
+ );
+ expect(schema.properties.data).toEqual({});
+ expect(schema.required).toEqual(["ok", "error", "meta"]);
+
+ const result = fakeResult(["high"], "partial");
+ const recipe = {
+ repository: "/original/repository",
+ target: { kind: "repository", paths: [] },
+ mode: command === "resume" ? "deep" : "standard",
+ config: {},
+ };
+ for (const format of ["json", "jsonl"]) {
+ for (const [filter, expected] of [
+ ["cost", null],
+ ["sarifPath", null],
+ ["warnings", undefined],
+ ["threadId", result.threadId],
+ [
+ "findings",
+ {
+ findings: JSON.parse(JSON.stringify(result.toJSON()["findings"])),
+ },
+ ],
+ ] as const) {
+ const output = capture();
+ expect(
+ await main(
+ [
+ "scans",
+ command,
+ "scan-original",
+ "--format",
+ format,
+ "--full-output",
+ "--filter-output",
+ filter,
+ ],
+ output.stream,
+ capture().stream,
+ dependencies({
+ result,
+ onWorkbench: () => ({
+ scanId: "scan-original",
+ scanDir: "/tmp/saved-scan",
+ recipe,
+ }),
+ }),
+ ),
+ ).toBe(2);
+ const envelope = JSON.parse(output.text());
+ expect(envelope).toMatchObject({
+ ok: false,
+ error: { code: "SCAN_FAILED" },
+ meta: expect.any(Object),
+ });
+ if (expected === undefined)
+ expect(envelope).not.toHaveProperty("data");
+ else expect(envelope.data).toEqual(expected);
+ }
+ }
+ },
+ );
+
+ test("preserves full-output token controls and the completed findings-policy envelope", async () => {
+ const partial = fakeResult(["high"], "partial");
+ const page = capture();
+ expect(
+ await main(
+ ["scan", "--json", "--full-output", "--token-limit", "1"],
+ page.stream,
+ capture().stream,
+ dependencies({ result: partial }),
+ ),
+ ).toBe(2);
+ expect(JSON.parse(page.text())).toMatchObject({
+ ok: false,
+ error: { code: "SCAN_FAILED" },
+ data: expect.any(String),
+ meta: { nextOffset: 1 },
+ });
+ const count = capture();
+ expect(
+ await main(
+ ["scan", "--json", "--full-output", "--token-count"],
+ count.stream,
+ capture().stream,
+ dependencies({ result: partial }),
+ ),
+ ).toBe(2);
+ expect(JSON.parse(count.text())).toBeNumber();
+ const complete = fakeResult(["high"]);
+ const blocked = capture();
+ expect(
+ await main(
+ ["scan", "--json", "--full-output", "--fail-on-severity", "high"],
+ blocked.stream,
+ capture().stream,
+ dependencies({ result: complete }),
+ ),
+ ).toBe(1);
+ expect(JSON.parse(blocked.text())).toMatchObject({
+ ok: true,
+ data: JSON.parse(JSON.stringify(complete.toJSON())),
+ });
+ expect(JSON.parse(blocked.text())).not.toHaveProperty("error");
+ });
+
test("reports SDK errors without a stack trace", async () => {
const { stdout, stderr, runCli } = createCliTest(main);
diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts
index 453fef682..49044739a 100644
--- a/sdk/typescript/tests-ts/cost.test.ts
+++ b/sdk/typescript/tests-ts/cost.test.ts
@@ -1,9 +1,10 @@
import { jsonLines } from "./support/json.js";
import { spawnSync } from "node:child_process";
+import * as filesystem from "node:fs/promises";
import { appendFile, writeFile } from "node:fs/promises";
import { join, parse, sep } from "node:path";
import { Codex } from "@openai/codex-sdk";
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterEach, describe, expect, spyOn, test } from "bun:test";
import {
estimateScanCost,
ScanCostTracker,
@@ -812,6 +813,77 @@ describe("live scan cost tracking", () => {
},
);
+ test.each([
+ ["EACCES", true],
+ ["EACCES", false],
+ ["EPERM", true],
+ ["EPERM", false],
+ ] as const)(
+ "retains and retries a %s worker when its parent appears later (already identified: %s)",
+ async (code, identified) => {
+ const home = await codexHome();
+ const usage = (input_tokens: number) => ({
+ input_tokens,
+ output_tokens: 0,
+ });
+ await writeSession(home, "scan-thread", usage(10));
+ const worker = await writeSession(home, "worker", usage(1), {
+ parent: "middle",
+ });
+ let denied = !identified;
+ let attempts = 0;
+ const opened = filesystem.open;
+ const opening = spyOn(filesystem, "open").mockImplementation(
+ async (...args: Parameters<typeof filesystem.open>) => {
+ if (String(args[0]) === worker && denied) {
+ attempts += 1;
+ throw Object.assign(new Error(`Synthetic ${code}`), {
+ code,
+ syscall: "open",
+ path: worker,
+ });
+ }
+ return await opened(...args);
+ },
+ );
+ const reported: number[] = [];
+ const tracker = new ScanCostTracker({
+ codexHome: home,
+ model: "gpt-5.6-sol",
+ maxCostUsd: 1,
+ onCost: (cost) => reported.push(cost.inputTokens),
+ });
+ tracker.start("scan-thread");
+ try {
+ if (identified) {
+ expect((await tracker.refresh()).cost?.inputTokens).toBe(10);
+ denied = true;
+ expect((await tracker.refresh()).cost?.inputTokens).toBe(10);
+ } else {
+ await expect(tracker.refresh()).rejects.toMatchObject({ code });
+ }
+ await appendFile(
+ worker,
+ `${JSON.stringify({ type: "event_msg", payload: { type: "token_count", info: { total_token_usage: usage(1_000_000) } } })}\n`,
+ );
+ await writeSession(home, "middle", usage(20), {
+ parent: "scan-thread",
+ });
+ await expect(tracker.refresh()).rejects.toMatchObject({ code });
+ denied = false;
+ const snapshot = await tracker.stop();
+ expect(snapshot.cost?.inputTokens).toBe(1_000_030);
+ expect(snapshot.cost!.estimatedUsd).toBeGreaterThan(1);
+ expect(reported).toContain(1_000_030);
+ expect(attempts).toBeGreaterThanOrEqual(2);
+ } finally {
+ denied = false;
+ await tracker.stop().catch(() => {});
+ opening.mockRestore();
+ }
+ },
+ );
+
test.each([...parentFields])(
"counts independent Deep workers and %s descendants",
async (parentField) => {