Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(cli): await required patch and verification output (#1777)

Michael D'Angelo6018441c8f29
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index e04df1748..91a8abf09 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -5362,7 +5362,8 @@ export async function main(
};
}
for (const result of results) {
- output.write(
+ await writeCliOutput(
+ output,
`${result.status.toUpperCase()} ${safePatchText(result.id)}: ${safePatchText(result.evidence)}\n`,
);
}
@@ -5661,7 +5662,7 @@ export async function main(
validationPrompt,
},
);
- if (!jsonOutput) output.write(report.text());
+ if (!jsonOutput) await writeCliOutput(output, report.text());
const files = await changedPatchFiles(
repository,
patchBase,
diff --git a/sdk/typescript/tests-ts/cli-patch.test.ts b/sdk/typescript/tests-ts/cli-patch.test.ts
index 20b9f2954..9150a58ed 100644
--- a/sdk/typescript/tests-ts/cli-patch.test.ts
+++ b/sdk/typescript/tests-ts/cli-patch.test.ts
@@ -428,6 +428,67 @@ describe("scan and patch workflow", () => {
).toMatchObject({ type: "string" });
});
+ test.each([
+ [0, false, 0],
+ [0, true, 2],
+ [130, true, 130],
+ [143, true, 143],
+ ] as const)(
+ "completes required patch output (status %i, failure %p)",
+ async (codexStatus, failOutput, expectedStatus) => {
+ const report = "Synthetic patch report: café → fixed.\n";
+ const started = Promise.withResolvers<(error?: Error) => void>();
+ const stdout = new Writable({
+ highWaterMark: 1,
+ write(_chunk, _encoding, callback) {
+ started.resolve(callback);
+ },
+ });
+ const failure = new Error("Synthetic patch output failure");
+ const emitted = failOutput
+ ? new Promise<Error>((resolve) => stdout.once("error", resolve))
+ : undefined;
+ const stderr = capture();
+ let patched = false;
+ let laterCommands = 0;
+ const operation = main(
+ ["patch", "Synthetic finding"],
+ stdout,
+ stderr.stream,
+ dependencies({
+ currentDirectory: CURRENT_REPOSITORY,
+ onCodex: (_args, output) => {
+ output?.stdout.write(report);
+ patched = true;
+ return codexStatus;
+ },
+ onRepositoryCommand: (_command, args) => {
+ if (patched) laterCommands++;
+ return args.includes("--name-only") ? "synthetic.ts\0" : "tree";
+ },
+ }),
+ );
+ const complete = await started.promise;
+ const commandsBeforeCompletion = laterCommands;
+ complete(failOutput ? failure : undefined);
+ expect(await operation).toBe(expectedStatus);
+ expect(commandsBeforeCompletion).toBe(0);
+ expect(patched).toBe(true);
+ if (failOutput) {
+ expect(await emitted).toBe(failure);
+ expect(laterCommands).toBe(0);
+ expect(stderr.text()).toContain(failure.message);
+ expect(stderr.text()).not.toContain("Patch applied.");
+ } else {
+ expect(laterCommands).toBeGreaterThan(0);
+ expect(stderr.text()).toContain("Patch applied. Files changed: 1.");
+ expect(stdout.writableEnded).toBe(false);
+ }
+ expect(stdout.listenerCount("error")).toBe(0);
+ stdout.destroy();
+ },
+ );
+
test.each(["literal", "file", "linear"])(
"passes custom validation instructions to the %s patch task",
async (source) => {
diff --git a/sdk/typescript/tests-ts/cli-verify-fix.test.ts b/sdk/typescript/tests-ts/cli-verify-fix.test.ts
index ac6e8915b..1a917572e 100644
--- a/sdk/typescript/tests-ts/cli-verify-fix.test.ts
+++ b/sdk/typescript/tests-ts/cli-verify-fix.test.ts
@@ -2,12 +2,13 @@ import { requestMetadata } from "./support/request-metadata.js";
import { describe, expect, test, mock } from "bun:test";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
+import { Writable } from "node:stream";
import { stripVTControlCharacters } from "node:util";
import { main } from "../src/cli.js";
import { VERSION } from "../src/version.js";
import type { Finding, JsonObject } from "../src/index.js";
import type { LinearClientFactory } from "../src/linear.js";
-import { dependencies, fakeResult } from "./cli-fixtures.js";
+import { capture, dependencies, fakeResult } from "./cli-fixtures.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { createCliTest, captureCli } from "./support/cli-run.js";
@@ -464,6 +465,105 @@ describe("read-only finding verification", () => {
);
});
+ test.each(["fixed", "still_vulnerable", "inconclusive"])(
+ "reports required output failure for a %s verification",
+ async (status) => {
+ const failure = new Error("Synthetic verification output failure");
+ const stdout = new Writable({
+ write(_chunk, _encoding, callback) {
+ queueMicrotask(() => callback(failure));
+ },
+ });
+ const emitted = new Promise<Error>((resolve) =>
+ stdout.once("error", resolve),
+ );
+ const stderr = capture();
+ const [exitCode, error] = await Promise.all([
+ main(
+ ["verify-fix", "Synthetic finding"],
+ stdout,
+ stderr.stream,
+ dependencies({
+ onCodex: (_args, output) => {
+ output?.stdout.write(
+ JSON.stringify({
+ results: [
+ {
+ id: "finding-1",
+ status,
+ evidence: "Synthetic evidence.",
+ },
+ ],
+ }),
+ );
+ return 0;
+ },
+ }),
+ ),
+ emitted,
+ ]);
+ expect(error).toBe(failure);
+ expect(exitCode).toBe(2);
+ expect(stderr.text()).toContain(failure.message);
+ expect(stdout.listenerCount("error")).toBe(0);
+ },
+ );
+
+ test("waits for each verification result before writing the next", async () => {
+ const writes = [
+ Promise.withResolvers<() => void>(),
+ Promise.withResolvers<() => void>(),
+ ];
+ const chunks: string[] = [];
+ const stdout = new Writable({
+ highWaterMark: 1,
+ write(chunk, _encoding, callback) {
+ chunks.push(chunk.toString());
+ writes[chunks.length - 1]!.resolve(callback);
+ },
+ });
+ const operation = main(
+ ["verify-fix", "First finding", "Second finding"],
+ stdout,
+ capture().stream,
+ dependencies({
+ onCodex: (_args, output) => {
+ output?.stdout.write(
+ JSON.stringify({
+ results: [
+ {
+ id: "finding-1",
+ status: "fixed",
+ evidence: "Café evidence.",
+ },
+ {
+ id: "finding-2",
+ status: "still_vulnerable",
+ evidence: "Exploit still succeeds.",
+ },
+ ],
+ }),
+ );
+ return 0;
+ },
+ }),
+ );
+ const first = await writes[0]!.promise;
+ const firstBufferedBytes = stdout.writableLength;
+ first();
+ const second = await writes[1]!.promise;
+ second();
+ expect(await operation).toBe(1);
+ expect(chunks).toEqual([
+ "FIXED finding-1: Café evidence.\n",
+ "STILL_VULNERABLE finding-2: Exploit still succeeds.\n",
+ ]);
+ expect(firstBufferedBytes).toBe(Buffer.byteLength(chunks[0]!));
+ expect(stdout.writableEnded).toBe(false);
+ expect(stdout.listenerCount("error")).toBe(0);
+ stdout.destroy();
+ });
+
test.each([
["missing result", { results: [] }],
[