Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(dedupe): correct rejected decisions in the open records review (#2139)

Kyle Brownaa56acbffdcd
diff --git a/sdk/typescript/docs/dedupe-records.md b/sdk/typescript/docs/dedupe-records.md
index df890c3a9..fd3de1c54 100644
--- a/sdk/typescript/docs/dedupe-records.md
+++ b/sdk/typescript/docs/dedupe-records.md
@@ -190,8 +190,8 @@ For failure or an inconclusive review, send an explicit error:
}
```
-A host error ends the review without another submission. Invalid model
-answers, missing decisions, and host errors make the whole batch unresolved;
+A host error ends the review without another submission. Invalid final model
+answers, missing final decisions, and host errors make the whole batch unresolved;
previous successful reviews in that run do not become persistence decisions.
The existing algorithm treats a validated `DISTINCT` as a nonmatch, when
supplied finding content is insufficient to establish a shared correction. If the host considers the review
@@ -213,10 +213,23 @@ fresh JSON-RPC ID and `params: {requestId, prompt}`. The request ID in the
parameters identifies the original review. Submit the prompt in the same
session and reply on the new JSON-RPC ID. The correction omits
`supportsContinuation`, so the host releases the session after that turn. A second missing submission ends
-the review as unresolved. A pending tool call uses the original turn and
-requires no correction. An uncertain outcome or explicit blocker follows the
+the review as unresolved. A pending tool call stays in the original turn.
+An uncertain outcome or explicit blocker follows the
existing error path. Hosts without continuation support can ignore the field.
+When `review.run` also advertises `supportsSubmissionValidation: true`, a host
+can keep a pending decision tool call open by adding `reviewPending: true` to
+the JSON-RPC reply envelope, alongside `id` and `result`. Put the raw tool
+arguments in `result`; only the host sets the envelope flag. The CLI checks
+the arguments against the original assignment. If valid, it sends
+`review.release` with `{requestId}`. Release the session and reply
+`{"status":"released"}` only after cleanup succeeds. If invalid, the CLI sends
+`review.continue` with the validation error. Return that prompt to the pending
+tool call in the same session. Release the session and reply with the corrected
+decision. The CLI validates this final answer and allows no further correction.
+Either follow-up uses a fresh JSON-RPC ID. A host that has already closed the
+session returns its ordinary final answer as before.
+
## Final response and persistence
```json
diff --git a/sdk/typescript/src/deduplication/deduplication-reviewer.ts b/sdk/typescript/src/deduplication/deduplication-reviewer.ts
index cef2b0f15..6ae01ce4d 100644
--- a/sdk/typescript/src/deduplication/deduplication-reviewer.ts
+++ b/sdk/typescript/src/deduplication/deduplication-reviewer.ts
@@ -86,7 +86,9 @@ export function validateScreening(
submitted.length !== required.length ||
required.some((slot) => !Object.hasOwn(result.decisions, slot))
) {
- throw new Error("Submit exactly the assigned screening pair slots.");
+ throw new Error(
+ `Submit exactly the assigned screening pair slots: ${required.join(", ")}.`,
+ );
}
return result;
}
diff --git a/sdk/typescript/src/deduplication/records-protocol.ts b/sdk/typescript/src/deduplication/records-protocol.ts
index 9d7b5ccc2..7a2ad2e2e 100644
--- a/sdk/typescript/src/deduplication/records-protocol.ts
+++ b/sdk/typescript/src/deduplication/records-protocol.ts
@@ -2,12 +2,13 @@ import { randomUUID } from "node:crypto";
import type { Readable } from "node:stream";
import { z } from "zod";
import { missingSubmissionPrompt } from "./deduplication-prompts.js";
-import { runHostProtocol } from "../host-protocol.js";
-import { deduplicateRecords, type DeduplicateRecordsInput } from "./records.js";
+import { PendingReviewSubmission, runHostProtocol } from "../host-protocol.js";
+import { runRecords, type DeduplicateRecordsInput } from "./records.js";
const missingSubmission = z.strictObject({
status: z.literal("missing_submission"),
});
+const released = z.strictObject({ status: z.literal("released") });
export async function runRecordsProtocol(
input: Readable,
@@ -21,32 +22,53 @@ export async function runRecordsProtocol(
label: "Deduplication",
requestMethod: "review.run",
run: (params, execute, signal) =>
- deduplicateRecords(params as DeduplicateRecordsInput, {
- signal,
- reviewRunner: {
- async run(request) {
- const first = await execute(request.requestId, {
- ...request,
- supportsContinuation: true,
- });
- if (!missingSubmission.safeParse(first).success) return first;
- const result = await execute(
- randomUUID(),
- {
- requestId: request.requestId,
- prompt: missingSubmissionPrompt(),
- },
- undefined,
- "review.continue",
- );
- if (missingSubmission.safeParse(result).success)
- throw new Error(
- "Review ended without a terminal submission after correction.",
+ runRecords(
+ params as DeduplicateRecordsInput,
+ async (request, validateSubmission) => {
+ const first = await execute(request.requestId, {
+ ...request,
+ supportsContinuation: true,
+ supportsSubmissionValidation: true,
+ });
+ let rejection: string | undefined;
+ if (first instanceof PendingReviewSubmission) {
+ try {
+ validateSubmission(first.submission);
+ } catch (error) {
+ rejection =
+ error instanceof Error
+ ? error.message
+ : "Invalid submission.";
+ }
+ if (rejection === undefined) {
+ const receipt = await execute(
+ randomUUID(),
+ { requestId: request.requestId },
+ undefined,
+ "review.release",
);
- return result;
- },
+ released.parse(receipt);
+ return first.submission;
+ }
+ } else if (!missingSubmission.safeParse(first).success)
+ return first;
+ const result = await execute(
+ randomUUID(),
+ {
+ requestId: request.requestId,
+ prompt: missingSubmissionPrompt(rejection),
+ },
+ undefined,
+ "review.continue",
+ );
+ if (missingSubmission.safeParse(result).success)
+ throw new Error(
+ "Review ended without a terminal submission after correction.",
+ );
+ return result;
},
- }),
+ signal,
+ ),
},
signal,
);
diff --git a/sdk/typescript/src/deduplication/records.ts b/sdk/typescript/src/deduplication/records.ts
index 48145b771..b71564b10 100644
--- a/sdk/typescript/src/deduplication/records.ts
+++ b/sdk/typescript/src/deduplication/records.ts
@@ -16,6 +16,7 @@ import {
} from "./deduplication-prompts.js";
import type {
DeduplicationReviewAttribution,
+ DeduplicationReviewRequest,
DeduplicationReviewRunner,
} from "./review.js";
@@ -89,7 +90,23 @@ export async function deduplicateRecords(
input: DeduplicateRecordsInput,
options: DeduplicateRecordsOptions,
): Promise<DeduplicateRecordsResult> {
- options.signal?.throwIfAborted();
+ return await runRecords(
+ input,
+ (request) => options.reviewRunner.run(request, { signal: options.signal }),
+ options.signal,
+ );
+}
+
+/** The records transport can check a pending submission before releasing its host. */
+export async function runRecords(
+ input: DeduplicateRecordsInput,
+ runReview: (
+ request: DeduplicationReviewRequest,
+ validate: (value: unknown) => void,
+ ) => Promise<unknown>,
+ signal?: AbortSignal,
+): Promise<DeduplicateRecordsResult> {
+ signal?.throwIfAborted();
// Snapshot before yielding: a host cannot mutate evidence during the review.
const data = deduplicateRecordsInputSchema.parse(structuredClone(input));
const references = new Map<string, string>();
@@ -140,9 +157,9 @@ export async function deduplicateRecords(
}: CodexReview<T>): Promise<T> {
try {
return await abortable(async () => {
- options.signal?.throwIfAborted();
+ signal?.throwIfAborted();
return validate(
- await options.reviewRunner.run(
+ await runReview(
{
...review,
requestId: randomUUID(),
@@ -162,10 +179,10 @@ export async function deduplicateRecords(
].join("\n\n"),
findingSchema: structuredClone(findingSchema()),
},
- { signal: options.signal },
+ validate,
),
);
- }, options.signal);
+ }, signal);
} catch (error) {
// Records mode discards the batch after any failed review.
if (
@@ -190,16 +207,16 @@ export async function deduplicateRecords(
},
},
reviewer,
- options.signal,
+ signal,
1,
);
let groups: string[][];
try {
const decisions = await algorithm.run(sourceFindingIds);
- options.signal?.throwIfAborted();
+ signal?.throwIfAborted();
groups = decisions.duplicateGroups;
} catch (error) {
- options.signal?.throwIfAborted();
+ signal?.throwIfAborted();
result.status = "unresolved";
result.unresolved = [...relationships.keys()].map((id) => ({
observationId: id,
diff --git a/sdk/typescript/src/host-protocol.ts b/sdk/typescript/src/host-protocol.ts
index d08ebe3a1..607f16e7b 100644
--- a/sdk/typescript/src/host-protocol.ts
+++ b/sdk/typescript/src/host-protocol.ts
@@ -17,8 +17,13 @@ const progress = envelope.extend({
method: z.literal("execution.progress"),
params: z.strictObject({ requestId: z.string(), event: z.unknown() }),
});
+const success = envelope.extend({ id: rpcId, result: z.unknown() });
+const acceptsPendingReview = z.object({
+ supportsSubmissionValidation: z.literal(true),
+});
const response = z.union([
- envelope.extend({ id: rpcId, result: z.unknown() }),
+ success,
+ success.extend({ reviewPending: z.literal(true) }),
envelope.extend({
id: rpcId,
error: z.strictObject({
@@ -28,6 +33,10 @@ const response = z.union([
}),
}),
]);
+/** Only the host can mark the RPC envelope. Model-controlled results remain plain JSON. */
+export class PendingReviewSubmission {
+ constructor(readonly submission: unknown) {}
+}
type Output = Pick<NodeJS.WriteStream, "write"> & {
on?(event: "error", listener: (error: Error) => void): unknown;
off?(event: "error", listener: (error: Error) => void): unknown;
@@ -66,6 +75,7 @@ export async function runHostProtocol(
let pending:
| {
id: string;
+ acceptsPendingReview: boolean;
resolve(value: unknown): void;
reject(error: Error): void;
onProgress?: (event: unknown) => void;
@@ -223,7 +233,15 @@ export async function runHostProtocol(
if (pending)
return Promise.reject(new Error("Execution is already pending."));
return new Promise((resolve, reject) => {
- pending = { id, resolve, reject, onProgress };
+ pending = {
+ id,
+ resolve,
+ reject,
+ onProgress,
+ acceptsPendingReview:
+ method === "review.run" &&
+ acceptsPendingReview.safeParse(params).success,
+ };
void send({
jsonrpc: "2.0",
id,
@@ -282,7 +300,12 @@ export async function runHostProtocol(
return;
}
const reply = response.safeParse(message);
- if (!reply.success || !pending || reply.data.id !== pending.id) {
+ if (
+ !reply.success ||
+ !pending ||
+ reply.data.id !== pending.id ||
+ ("reviewPending" in reply.data && !pending.acceptsPendingReview)
+ ) {
fail(
-32600,
`Malformed, unexpected, or mismatched ${operation.requestMethod === "review.run" ? "review" : "execution"} response.`,
@@ -293,6 +316,8 @@ export async function runHostProtocol(
pending = undefined;
if ("error" in reply.data)
execution.reject(new Error(reply.data.error.message));
+ else if ("reviewPending" in reply.data)
+ execution.resolve(new PendingReviewSubmission(reply.data.result));
else execution.resolve(reply.data.result);
});
if (signal?.aborted) interrupted();
diff --git a/sdk/typescript/tests-ts/dedupe-records.test.ts b/sdk/typescript/tests-ts/dedupe-records.test.ts
index eeb5f0758..3d5a96d7b 100644
--- a/sdk/typescript/tests-ts/dedupe-records.test.ts
+++ b/sdk/typescript/tests-ts/dedupe-records.test.ts
@@ -17,6 +17,7 @@ import {
} from "../src/deduplication/records.js";
import type { DeduplicationReviewRequest } from "../src/deduplication/review.js";
import { runRecordsProtocol } from "../src/deduplication/records-protocol.js";
+import { runValidationRecordsProtocol } from "../src/validation-records.js";
import { main } from "../src/cli.js";
import { capture, dependencies, FakeSignals } from "./cli-fixtures.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
@@ -420,7 +421,8 @@ function fakeHost(
messages.push(message);
if (
message.method === "review.run" ||
- message.method === "review.continue"
+ message.method === "review.continue" ||
+ message.method === "review.release"
)
queueMicrotask(() => onReview(message, send, stream));
callback();
@@ -430,6 +432,125 @@ function fakeHost(
}
const run = { jsonrpc: "2.0", id: "run-1", method: "run", params: input() };
+test.each(["corrected", "still-invalid", "release-failed"])(
+ "validates pending screening decisions in the same review (%s)",
+ async (outcome) => {
+ const observations = [record("a"), record("b")];
+ const params = {
+ version: 1,
+ observations,
+ candidateRelationships: [
+ { observationId: "a", candidateObservationIds: ["b"] },
+ { observationId: "b", candidateObservationIds: ["a"] },
+ ],
+ };
+ const wrongSlot = {
+ decisions: { "pair-2": { decision: "DISTINCT", rationale: "Separate." } },
+ };
+ const originals: Message[] = [];
+ const host = fakeHost((message, send) => {
+ const reply = (result: unknown) =>
+ send({ jsonrpc: "2.0", id: message.id, result });
+ if (message.method === "review.run") {
+ originals.push(message);
+ expect(message.params).toHaveProperty(
+ "supportsSubmissionValidation",
+ true,
+ );
+ send({
+ jsonrpc: "2.0",
+ id: message.id,
+ reviewPending: true,
+ result: originals.length === 1 ? answer(message.params) : wrongSlot,
+ });
+ } else if (message.method === "review.release") {
+ expect(message.params.requestId).toBe(originals[0]!.params.requestId);
+ reply({
+ status: outcome === "release-failed" ? "unknown" : "released",
+ });
+ } else {
+ expect(message.params.requestId).toBe(originals[1]!.params.requestId);
+ expect(message.id).not.toBe(message.params.requestId);
+ expect(message.params.prompt).toContain(
+ "assigned screening pair slots: pair-1.",
+ );
+ reply(
+ outcome === "corrected" ? answer(originals[1]!.params) : wrongSlot,
+ );
+ }
+ });
+ const done = runRecordsProtocol(host.stream, host.output);
+ host.send({ ...run, params });
+ expect(await done).toBe(outcome === "corrected" ? 0 : 1);
+ expect(originals).toHaveLength(outcome === "release-failed" ? 1 : 2);
+ expect(
+ host.messages.filter((m) => m.method === "review.continue"),
+ ).toHaveLength(outcome === "release-failed" ? 0 : 1);
+ const result = host.messages.at(-1)!.result;
+ if (outcome === "corrected") {
+ expect(result).toMatchObject({
+ status: "completed",
+ groups: [{ observationIds: ["a"] }, { observationIds: ["b"] }],
+ unresolved: [],
+ });
+ } else {
+ expect(result).toMatchObject({
+ status: "unresolved",
+ groups: [],
+ unresolved: [{ observationId: "a" }, { observationId: "b" }],
+ });
+ }
+ host.stream.destroy();
+ },
+);
+
+test("a review that did not advertise pending submissions rejects the envelope flag", async () => {
+ const host = fakeHost((message, send) => {
+ send({
+ jsonrpc: "2.0",
+ id: message.id,
+ reviewPending: true,
+ result: { verdict: "example" },
+ });
+ });
+ const done = runValidationRecordsProtocol(host.stream, host.output);
+ host.send({
+ ...run,
+ params: {
+ version: 1,
+ findings: [{ id: "example", finding: { title: "Example" } }],
+ context: "Synthetic context.",
+ outputSchema: { type: "object" },
+ },
+ });
+ expect(await done).toBe(2);
+ expect(host.messages.map((m) => m.method)).toEqual(["review.run", undefined]);
+ expect(host.messages.at(-1)?.error?.code).toBe(-32600);
+ host.stream.destroy();
+});
+
+test("raw tool arguments cannot reopen a legacy host's finished review", async () => {
+ const host = fakeHost((message, send) => {
+ send({
+ jsonrpc: "2.0",
+ id: message.id,
+ result: {
+ status: "pending_submission",
+ submission: answer(message.params),
+ },
+ });
+ });
+ const done = runRecordsProtocol(host.stream, host.output);
+ host.send(run);
+ expect(await done).toBe(1);
+ expect(host.messages.map((m) => m.method)).toEqual(["review.run", undefined]);
+ expect(host.messages.at(-1)?.result).toMatchObject({
+ status: "unresolved",
+ groups: [],
+ });
+ host.stream.destroy();
+});
+
test.each([false, true])(
"a confirmed missing submission gets one correction (missing again: %s)",
async (missingAgain) => {
@@ -745,18 +866,16 @@ for (const trigger of ["signal", "notification"]) {
});
const writes = spyOn(output, "write");
let pendingReview: Promise<unknown> | undefined;
- const dedupe = records.deduplicateRecords;
+ const dedupe = records.runRecords;
const operation =
stage === "review request"
- ? spyOn(records, "deduplicateRecords").mockImplementationOnce(
- (input, options) =>
- dedupe(input, {
- ...options,
- reviewRunner: {
- run: (...args) =>
- (pendingReview = options.reviewRunner.run(...args)),
- },
- }),
+ ? spyOn(records, "runRecords").mockImplementationOnce(
+ (input, review, signal) =>
+ dedupe(
+ input,
+ (...args) => (pendingReview = review(...args)),
+ signal,
+ ),
)
: undefined;
const controller = new AbortController();
@@ -771,7 +890,7 @@ for (const trigger of ["signal", "notification"]) {
const done = runRecordsProtocol(inputStream, output, controller.signal);
inputStream.write(`${JSON.stringify({ ...run, params })}\n`);
const flush = await writing.promise;
- const signal = operation?.mock.calls[0]?.[1].signal;
+ const signal = operation?.mock.calls[0]?.[2];
const recordsResult = operation?.mock.results[0]?.value;
if (trigger === "signal") controller.abort("SIGTERM");
else