Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(cli): report verification failures in structured output (#1835)

Dipesh Babub97013c17bbd
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 7cd7ed1b2..744dd466a 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -2780,6 +2780,11 @@ for `patch`. Omit it or use `--verification-mode standard` for the default check
Results include evidence and `fixed`, `still_vulnerable`, or `inconclusive` status.
`--json` returns structured output. Exit codes are `0` when all findings are
fixed, `1` when any remain vulnerable, and `2` for inconclusive/failed checks.
+Authentication and execution failures print diagnostics on stderr and return
+`status: "failed"`, `code: "VERIFICATION_FAILED"`, and `message` with `--json`
+or `--format jsonl`. With JSON/JSONL `--full-output`, failures use an `ok: false`
+error envelope. Other formats retain stderr-only failures. Interruptions retain
+exit codes `130` and `143` without a result.
## Command discovery and integrations
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 893257e14..ceaecae11 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -5818,7 +5818,9 @@ export async function main(
},
],
output: z.record(z.string(), z.unknown()).optional(),
- async run({ format, options }) {
+ async run({ format, options, error: incurError }) {
+ const diagnostics = captureOutput();
+ let failureReported = false;
try {
if (options.host)
throw new CodexSecurityError("Use verify-fix --host alone.");
@@ -5906,13 +5908,23 @@ export async function main(
identifiers.length,
);
progress.startVerification();
+ const verificationErrors = new NodeWritable({
+ write(chunk, _encoding, callback) {
+ diagnostics.stream.write(chunk);
+ progress.stop();
+ void writeCliOutput(errorOutput, chunk).then(
+ () => callback(),
+ callback,
+ );
+ },
+ });
try {
exitCode = await runSkill(
"verify-fix",
selected === undefined ? [...positionals, ...imports] : [],
options,
response.stream,
- errorOutput,
+ verificationErrors,
dependencies,
{
directory: repository,
@@ -5927,9 +5939,15 @@ export async function main(
},
);
if (exitCode !== 0) {
- if (exitCode !== 130 && exitCode !== 143) exitCode = 2;
- frameworkExit = exitCode;
- return undefined;
+ if (exitCode === 130 || exitCode === 143) {
+ frameworkExit = exitCode;
+ return undefined;
+ }
+ failureReported = diagnostics.text().trim().length > 0;
+ throw new CodexSecurityError(
+ diagnostics.text().trim() ||
+ `Codex verification failed with exit code ${exitCode}.`,
+ );
}
results = parseFindingVerifications(response.text(), identifiers);
@@ -5960,8 +5978,15 @@ export async function main(
return undefined;
} catch (error) {
exitCode = 2;
- errorOutput.write(`codex-security: ${diagnosticValue(error)}\n`);
- throw error;
+ const message = errorMessage(error);
+ if (!failureReported)
+ errorOutput.write(`codex-security: ${diagnosticValue(message)}\n`);
+ return finishScan(
+ { exitCode, error: message },
+ format,
+ incurError,
+ "VERIFICATION_FAILED",
+ );
}
},
})
diff --git a/sdk/typescript/tests-ts/cli-verify-fix.test.ts b/sdk/typescript/tests-ts/cli-verify-fix.test.ts
index 2cd72e8d4..9d4b6e2d1 100644
--- a/sdk/typescript/tests-ts/cli-verify-fix.test.ts
+++ b/sdk/typescript/tests-ts/cli-verify-fix.test.ts
@@ -1,16 +1,20 @@
import { requestMetadata } from "./support/request-metadata.js";
-import { describe, expect, test, mock } from "bun:test";
+import { afterEach, describe, expect, test, mock } from "bun:test";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { Writable } from "node:stream";
import { stripVTControlCharacters } from "node:util";
-import { main } from "../src/cli.js";
+import { main, runCodexSkillCommand } from "../src/cli.js";
import { VERSION } from "../src/version.js";
import type { Finding, JsonObject } from "../src/index.js";
import type { LinearClientFactory } from "../src/linear.js";
import { capture, dependencies, fakeResult } from "./cli-fixtures.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { createCliTest, captureCli } from "./support/cli-run.js";
+import { createApiTestFixtures } from "./support/temporary-directories.js";
+
+const { temporaryDirectory, cleanup } = createApiTestFixtures("verify-fix-");
+afterEach(cleanup);
async function linearIssue(identifier: string) {
return {
@@ -430,18 +434,113 @@ describe("read-only finding verification", () => {
["--format", "yaml", "--full-output"],
["--token-count"],
]) {
- const stdout = captureCli(main, "stdout");
+ const { stdout, stderr, runCli } = createCliTest(main);
expect(
- await stdout.run(
+ await runCli(
["verify-fix", "Synthetic finding", ...flags],
dependencies({ onCodex: () => codexStatus }),
),
).toBe(expectedStatus);
- expect(stdout.text()).toBe("");
+ const message = `Codex verification failed with exit code ${codexStatus}.`;
+ if (expectedStatus === 2) {
+ expect(stderr.text()).toContain(`codex-security: ${message}\n`);
+ expect(stderr.text().split(message)).toHaveLength(2);
+ } else expect(stderr.text()).not.toContain(message);
+ if (expectedStatus === 2 && flags[0] === "--json") {
+ const failure = JSON.parse(stdout.text());
+ if (flags.includes("--full-output")) {
+ expect(failure.ok).toBe(false);
+ expect(failure.error).toMatchObject({
+ code: "VERIFICATION_FAILED",
+ message,
+ });
+ } else {
+ expect(failure).toMatchObject({
+ status: "failed",
+ code: "VERIFICATION_FAILED",
+ message,
+ });
+ }
+ } else expect(stdout.text()).toBe("");
+ }
+ },
+ );
+
+ test.each(["exit", "throw"] as const)(
+ "reports ChatGPT authentication diagnostics after a Codex %s failure",
+ async (transport) => {
+ const message =
+ "401 Unauthorized: stored ChatGPT credentials expired. Run codex-security login --auth chatgpt.";
+ for (const flags of [
+ [],
+ ["--json"],
+ ["--json", "--full-output"],
+ ["--format", "jsonl"],
+ ]) {
+ const { stdout, stderr, runCli } = createCliTest(main);
+ expect(
+ await runCli(
+ ["verify-fix", "Synthetic finding", "--auth", "chatgpt", ...flags],
+ dependencies({
+ onCodex: (_args, output) => {
+ if (transport === "throw") throw new Error(message);
+ output!.stderr.write(`${message}\n`);
+ return 1;
+ },
+ }),
+ ),
+ ).toBe(2);
+ if (flags.length === 0) expect(stdout.text()).toBe("");
+ else {
+ const failure = JSON.parse(stdout.text());
+ const details = flags.includes("--full-output")
+ ? failure.error
+ : failure;
+ expect(details).toMatchObject({
+ code: "VERIFICATION_FAILED",
+ message,
+ });
+ expect(failure.results).toBeUndefined();
+ }
+ expect(stderr.text()).toContain(message);
+ expect(stderr.text().split(message)).toHaveLength(2);
}
},
);
+ test("reports an authentication protocol error from a successfully exited Codex child", async () => {
+ const directory = await temporaryDirectory();
+ const message = "401 Unauthorized: synthetic ChatGPT token expired";
+ const source = `
+const lines = require("node:readline").createInterface({ input: process.stdin });
+lines.once("line", () => process.stdout.write(JSON.stringify({
+ id: 1, error: { code: -32001, message: ${JSON.stringify(message)} }
+}) + "\\n"));
+`;
+ const { stdout, stderr, runCli } = createCliTest(main);
+ expect(
+ await runCli(
+ ["verify-fix", "Synthetic finding", "--auth", "chatgpt", "--json"],
+ dependencies({
+ currentDirectory: directory,
+ onCodex: (_args, output) =>
+ runCodexSkillCommand(
+ ["-e", source],
+ { ...output!, auth: undefined },
+ { command: process.execPath },
+ {},
+ ),
+ }),
+ ),
+ ).toBe(2);
+ expect(JSON.parse(stdout.text())).toMatchObject({
+ status: "failed",
+ code: "VERIFICATION_FAILED",
+ message: expect.stringContaining(message),
+ });
+ expect(stderr.text()).toContain(message);
+ });
+
test.each([
["fixed", 0],
["still_vulnerable", 1],
@@ -658,7 +757,10 @@ describe("read-only finding verification", () => {
),
).toBe(2);
- expect(stdout.text()).toBe("");
+ expect(JSON.parse(stdout.text())).toMatchObject({
+ status: "failed",
+ code: "VERIFICATION_FAILED",
+ });
expect(stderr.text()).toContain(
"an evidence-backed verification result for every finding",
);