Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(cli): share the host adapter for records commands (#2259)

Aismit46451d57e277
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs
index 219d3d356..4d3921cd2 100644
--- a/sdk/typescript/scripts/package-dist-files.mjs
+++ b/sdk/typescript/scripts/package-dist-files.mjs
@@ -142,7 +142,6 @@ const distModules = [
"deduplication/local",
"deduplication/finding-schema",
"deduplication/records",
- "deduplication/records-protocol",
"deduplication/review",
"saved-scan",
"saved-scan-bootstrap",
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index abda71ca0..ea24a38d1 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -134,15 +134,9 @@ import {
import { DEFAULT_DEDUPE_CONCURRENCY } from "./deduplication/deduplication.js";
import { savedScanWorkbench } from "./saved-scan-bootstrap.js";
import { deduplicateScanInternal } from "./deduplication/scan.js";
-import { runRecordsProtocol } from "./deduplication/records-protocol.js";
import { runHostRecordsProtocol } from "./host-records-protocol.js";
-import {
- classifySeverityRecords,
- type ClassifySeverityRecordsInput,
-} from "./classify-severity-records.js";
import { runHostedScanProtocol } from "./hosted-scan-protocol.js";
import { runHostedRemediationProtocol } from "./hosted-remediation-protocol.js";
-import { runValidationRecordsProtocol } from "./validation-records.js";
import {
FindingPatchSchema as findingPatchSchema,
parseFindingVerifications,
@@ -1977,13 +1971,16 @@ export async function main(
: argv[0] === "verify-fix"
? "verify"
: undefined;
+ const recordsCommand =
+ argv[0] === "dedupe" ||
+ argv[0] === "validate" ||
+ argv[0] === "classify-severity"
+ ? argv[0]
+ : undefined;
const hostedCommand =
argv[0] === "scan" || remediationOperation !== undefined;
if (
- (((argv[0] === "dedupe" ||
- argv[0] === "validate" ||
- argv[0] === "classify-severity") &&
- argv.includes("--records")) ||
+ ((recordsCommand !== undefined && argv.includes("--records")) ||
(hostedCommand &&
argv.some((arg) => arg === "--host" || arg.startsWith("--host=")))) &&
!argv.includes("--help") &&
@@ -2021,36 +2018,21 @@ export async function main(
const removeSignals = listenForAbort(dependencies, controller);
let exitCode: number;
try {
- const runProtocol =
- argv[0] === "scan"
- ? runHostedScanProtocol
- : argv[0] === "validate"
- ? runValidationRecordsProtocol
- : runRecordsProtocol;
+ const input = dependencies.recordsInput ?? process.stdin;
const code =
- argv[0] === "classify-severity"
- ? await runHostRecordsProtocol(
- dependencies.recordsInput ?? process.stdin,
- output,
- (value, options) =>
- classifySeverityRecords(value as ClassifySeverityRecordsInput, {
- ...options,
- classificationRuns,
- }),
- controller.signal,
- )
+ recordsCommand !== undefined
+ ? await runHostRecordsProtocol(recordsCommand, input, output, {
+ signal: controller.signal,
+ classificationRuns,
+ })
: remediationOperation !== undefined
? await runHostedRemediationProtocol(
- dependencies.recordsInput ?? process.stdin,
+ input,
output,
controller.signal,
remediationOperation,
)
- : await runProtocol(
- dependencies.recordsInput ?? process.stdin,
- output,
- controller.signal,
- );
+ : await runHostedScanProtocol(input, output, controller.signal);
exitCode =
controller.signal.reason === "SIGINT"
? 130
diff --git a/sdk/typescript/src/deduplication/records-protocol.ts b/sdk/typescript/src/deduplication/records-protocol.ts
deleted file mode 100644
index 7a2ad2e2e..000000000
--- a/sdk/typescript/src/deduplication/records-protocol.ts
+++ /dev/null
@@ -1,75 +0,0 @@
-import { randomUUID } from "node:crypto";
-import type { Readable } from "node:stream";
-import { z } from "zod";
-import { missingSubmissionPrompt } from "./deduplication-prompts.js";
-import { PendingReviewSubmission, runHostProtocol } from "../host-protocol.js";
-import { runRecords, type DeduplicateRecordsInput } from "./records.js";
-
-const missingSubmission = z.strictObject({
- status: z.literal("missing_submission"),
-});
-const released = z.strictObject({ status: z.literal("released") });
-
-export async function runRecordsProtocol(
- input: Readable,
- output: Parameters<typeof runHostProtocol>[1],
- signal?: AbortSignal,
-): Promise<number> {
- return await runHostProtocol(
- input,
- output,
- {
- label: "Deduplication",
- requestMethod: "review.run",
- run: (params, execute, signal) =>
- runRecords(
- params as DeduplicateRecordsInput,
- async (request, validateSubmission) => {
- const first = await execute(request.requestId, {
- ...request,
- supportsContinuation: true,
- supportsSubmissionValidation: true,
- });
- let rejection: string | undefined;
- if (first instanceof PendingReviewSubmission) {
- try {
- validateSubmission(first.submission);
- } catch (error) {
- rejection =
- error instanceof Error
- ? error.message
- : "Invalid submission.";
- }
- if (rejection === undefined) {
- const receipt = await execute(
- randomUUID(),
- { requestId: request.requestId },
- undefined,
- "review.release",
- );
- released.parse(receipt);
- return first.submission;
- }
- } else if (!missingSubmission.safeParse(first).success)
- return first;
- const result = await execute(
- randomUUID(),
- {
- requestId: request.requestId,
- prompt: missingSubmissionPrompt(rejection),
- },
- undefined,
- "review.continue",
- );
- if (missingSubmission.safeParse(result).success)
- throw new Error(
- "Review ended without a terminal submission after correction.",
- );
- return result;
- },
- signal,
- ),
- },
- signal,
- );
-}
diff --git a/sdk/typescript/src/host-protocol.ts b/sdk/typescript/src/host-protocol.ts
index 607f16e7b..c32f9ca81 100644
--- a/sdk/typescript/src/host-protocol.ts
+++ b/sdk/typescript/src/host-protocol.ts
@@ -41,7 +41,7 @@ type Output = Pick<NodeJS.WriteStream, "write"> & {
on?(event: "error", listener: (error: Error) => void): unknown;
off?(event: "error", listener: (error: Error) => void): unknown;
};
-type Execute = (
+export type HostExecute = (
id: string,
request: unknown,
onProgress?: (event: unknown) => void,
@@ -61,7 +61,7 @@ export async function runHostProtocol(
errorData?: (error: unknown) => object | undefined;
run(
params: unknown,
- execute: Execute,
+ execute: HostExecute,
signal: AbortSignal,
onProgress: (event: unknown) => void,
): Promise<{ status: string }>;
diff --git a/sdk/typescript/src/host-records-protocol.ts b/sdk/typescript/src/host-records-protocol.ts
index c33e85f0d..25c95700a 100644
--- a/sdk/typescript/src/host-records-protocol.ts
+++ b/sdk/typescript/src/host-records-protocol.ts
@@ -1,38 +1,125 @@
+import { randomUUID } from "node:crypto";
import type { Readable } from "node:stream";
-import { runHostProtocol } from "./host-protocol.js";
+import { z } from "zod";
+import {
+ classifySeverityRecords,
+ type ClassifySeverityRecordsInput,
+} from "./classify-severity-records.js";
+import { missingSubmissionPrompt } from "./deduplication/deduplication-prompts.js";
+import {
+ runRecords,
+ type DeduplicateRecordsInput,
+} from "./deduplication/records.js";
+import type { DeduplicationReviewRequest } from "./deduplication/review.js";
+import {
+ PendingReviewSubmission,
+ runHostProtocol,
+ type HostExecute,
+} from "./host-protocol.js";
+import {
+ validateRecords,
+ type ValidateRecordsInput,
+} from "./validation-records.js";
-export type RecordsProtocolOutput = Parameters<typeof runHostProtocol>[1];
+const missingSubmission = z.strictObject({
+ status: z.literal("missing_submission"),
+});
+const released = z.strictObject({ status: z.literal("released") });
+
+async function runDeduplicationReview(
+ request: DeduplicationReviewRequest,
+ validateSubmission: (value: unknown) => void,
+ execute: HostExecute,
+) {
+ const first = await execute(request.requestId, {
+ ...request,
+ supportsContinuation: true,
+ supportsSubmissionValidation: true,
+ });
+ let rejection: string | undefined;
+ if (first instanceof PendingReviewSubmission) {
+ try {
+ validateSubmission(first.submission);
+ } catch (error) {
+ rejection =
+ error instanceof Error ? error.message : "Invalid submission.";
+ }
+ if (rejection === undefined) {
+ const receipt = await execute(
+ randomUUID(),
+ { requestId: request.requestId },
+ undefined,
+ "review.release",
+ );
+ released.parse(receipt);
+ return first.submission;
+ }
+ } else if (!missingSubmission.safeParse(first).success) return first;
+ const result = await execute(
+ randomUUID(),
+ {
+ requestId: request.requestId,
+ prompt: missingSubmissionPrompt(rejection),
+ },
+ undefined,
+ "review.continue",
+ );
+ if (missingSubmission.safeParse(result).success)
+ throw new Error(
+ "Review ended without a terminal submission after correction.",
+ );
+ return result;
+}
+
+const labels = {
+ dedupe: "Deduplication",
+ validate: "Validation",
+ "classify-severity": "Records processing",
+};
/** Adapt record reviews to the shared host transport without replaying work. */
-export function runHostRecordsProtocol<
- Review extends { requestId: string },
- Result extends { status: "completed" | "unresolved" },
->(
+export function runHostRecordsProtocol(
+ command: keyof typeof labels,
input: Readable,
- output: RecordsProtocolOutput,
- run: (
- input: unknown,
- options: {
- signal: AbortSignal;
- reviewRunner: { run(review: Review): Promise<unknown> };
- },
- ) => Promise<Result>,
- signal?: AbortSignal,
+ output: Parameters<typeof runHostProtocol>[1],
+ options: {
+ signal?: AbortSignal;
+ classificationRuns?: number;
+ } = {},
): Promise<number> {
return runHostProtocol(
input,
output,
{
- label: "Records processing",
+ label: labels[command],
requestMethod: "review.run",
- run: (params, execute, signal) =>
- run(params, {
+ run: async (params, execute, signal) => {
+ if (command === "dedupe")
+ return runRecords(
+ params as DeduplicateRecordsInput,
+ (request, validate) =>
+ runDeduplicationReview(request, validate, execute),
+ signal,
+ );
+ const reviewRunner = {
+ run: (review: { requestId: string }) =>
+ execute(review.requestId, review),
+ };
+ if (command === "validate")
+ return {
+ status: "completed",
+ result: await validateRecords(params as ValidateRecordsInput, {
+ signal,
+ reviewRunner,
+ }),
+ };
+ return classifySeverityRecords(params as ClassifySeverityRecordsInput, {
+ classificationRuns: options.classificationRuns,
signal,
- reviewRunner: {
- run: (review) => execute(review.requestId, review),
- },
- }),
+ reviewRunner,
+ });
+ },
},
- signal,
+ options.signal,
);
}
diff --git a/sdk/typescript/src/validation-records.ts b/sdk/typescript/src/validation-records.ts
index c226e8a9f..708150a34 100644
--- a/sdk/typescript/src/validation-records.ts
+++ b/sdk/typescript/src/validation-records.ts
@@ -1,9 +1,7 @@
import { randomUUID } from "node:crypto";
-import type { Readable } from "node:stream";
import Ajv2020 from "ajv/dist/2020.js";
import { z } from "zod";
import { DEFAULT_CODEX_CONFIG, scanModelConfiguration } from "./config.js";
-import { runHostProtocol } from "./host-protocol.js";
import { bundledPluginRoot } from "./runtime.js";
import { standaloneValidationPrompt } from "./standalone-validation-prompt.js";
import { abortable } from "./targets.js";
@@ -94,28 +92,3 @@ export async function validateRecords(
);
return result;
}
-
-export function runValidationRecordsProtocol(
- input: Readable,
- output: Parameters<typeof runHostProtocol>[1],
- signal?: AbortSignal,
-): Promise<number> {
- return runHostProtocol(
- input,
- output,
- {
- label: "Validation",
- requestMethod: "review.run",
- run: async (params, execute, signal) => ({
- status: "completed",
- result: await validateRecords(params as ValidateRecordsInput, {
- signal,
- reviewRunner: {
- run: (request) => execute(request.requestId, request),
- },
- }),
- }),
- },
- signal,
- );
-}
diff --git a/sdk/typescript/tests-ts/dedupe-records.test.ts b/sdk/typescript/tests-ts/dedupe-records.test.ts
index f28f97e7d..28434c626 100644
--- a/sdk/typescript/tests-ts/dedupe-records.test.ts
+++ b/sdk/typescript/tests-ts/dedupe-records.test.ts
@@ -16,8 +16,7 @@ import {
type DeduplicateRecordsInput,
} from "../src/deduplication/records.js";
import type { DeduplicationReviewRequest } from "../src/deduplication/review.js";
-import { runRecordsProtocol } from "../src/deduplication/records-protocol.js";
-import { runValidationRecordsProtocol } from "../src/validation-records.js";
+import { runHostRecordsProtocol } from "../src/host-records-protocol.js";
import { main } from "../src/cli.js";
import { capture, dependencies, FakeSignals } from "./cli-fixtures.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
@@ -574,7 +573,7 @@ test.each(["corrected", "still-invalid", "release-failed"])(
);
}
});
- const done = runRecordsProtocol(host.stream, host.output);
+ const done = runHostRecordsProtocol("dedupe", host.stream, host.output);
host.send({ ...run, params });
expect(await done).toBe(outcome === "corrected" ? 0 : 1);
expect(originals).toHaveLength(outcome === "release-failed" ? 1 : 2);
@@ -608,7 +607,7 @@ test("a review that did not advertise pending submissions rejects the envelope f
result: { verdict: "example" },
});
});
- const done = runValidationRecordsProtocol(host.stream, host.output);
+ const done = runHostRecordsProtocol("validate", host.stream, host.output);
host.send({
...run,
params: {
@@ -635,7 +634,7 @@ test("raw tool arguments cannot reopen a legacy host's finished review", async (
},
});
});
- const done = runRecordsProtocol(host.stream, host.output);
+ const done = runHostRecordsProtocol("dedupe", host.stream, host.output);
host.send(run);
expect(await done).toBe(1);
expect(host.messages.map((m) => m.method)).toEqual(["review.run", undefined]);
@@ -665,7 +664,7 @@ test.each([false, true])(
: answer(original.params);
send({ jsonrpc: "2.0", id: message.id, result });
});
- const done = runRecordsProtocol(host.stream, host.output);
+ const done = runHostRecordsProtocol("dedupe", host.stream, host.output);
host.send(run);
expect(await done).toBe(missingAgain ? 1 : 0);
const originals = host.messages.filter((m) => m.method === "review.run");
@@ -777,7 +776,7 @@ test.each([
},
});
});
- const done = runRecordsProtocol(host.stream, host.output);
+ const done = runHostRecordsProtocol("dedupe", host.stream, host.output);
host.send(run);
expect(await done).toBe(scenario === "host-error" ? 1 : 2);
expect(
@@ -903,7 +902,11 @@ test("real CLI pipes exit after a fake-host run without local Codex or state wri
test("disconnect before run and a broken output pipe terminate cleanly", async () => {
const host = fakeHost(() => {});
- const disconnected = runRecordsProtocol(host.stream, host.output);
+ const disconnected = runHostRecordsProtocol(
+ "dedupe",
+ host.stream,
+ host.output,
+ );
host.stream.end();
expect(await disconnected).toBe(2);
expect(host.messages).toMatchObject([{ id: null, error: { code: -32000 } }]);
@@ -913,7 +916,7 @@ test("disconnect before run and a broken output pipe terminate cleanly", async (
callback(new Error("Broken pipe"));
},
});
- const failed = runRecordsProtocol(input, output);
+ const failed = runHostRecordsProtocol("dedupe", input, output);
input.write(`${JSON.stringify(run)}\n`);
expect(await failed).toBe(2);
input.destroy();
@@ -935,7 +938,7 @@ test("a disconnect while flushing the final result does not send a second respon
setImmediate(callback);
},
});
- const done = runRecordsProtocol(inputStream, output);
+ const done = runHostRecordsProtocol("dedupe", inputStream, output);
inputStream.write(`${JSON.stringify({ ...run, params: emptyRecords })}\n`);
expect(await done).toBe(0);
expect(messages).toMatchObject([
@@ -982,7 +985,9 @@ for (const trigger of ["signal", "notification"]) {
output.destroy();
inputStream.destroy();
});
- const done = runRecordsProtocol(inputStream, output, controller.signal);
+ const done = runHostRecordsProtocol("dedupe", inputStream, output, {
+ signal: controller.signal,
+ });
inputStream.write(`${JSON.stringify({ ...run, params })}\n`);
const flush = await writing.promise;
const signal = operation?.mock.calls[0]?.[2];