Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(validation): mark source review complete before custom validation (#1252)

Michael D'Angeloa255734a363b
diff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.ts
index 7914f633e..7ba8aa8c6 100644
--- a/sdk/typescript/src/custom-validation-prompt.ts
+++ b/sdk/typescript/src/custom-validation-prompt.ts
@@ -26,7 +26,7 @@ const DISABLED_TOOLS = [
];
const HANDOFF =
- "Preserve every surviving source-backed candidate as a provisional finding, including its identity, source locations, evidence, counterevidence, and preliminary assessments. Assemble unsealed scan-manifest.json, findings.json, and coverage.json using <plugin_dir>/examples/completed-scan/ and <plugin_dir>/schemas/ as shape references. Set scan.scope.validationMode to custom_pending and put each finding's matching coverage surface IDs in extensions.customValidationSurfaceIds. Every reported coverage surface must have a candidate. Do not claim that custom validation has occurred. Verify the three files exist, then return control to the SDK without invoking validation or attack-path skills, running application code, sealing files, generating a report, or calling a scan completion tool or helper.";
+ "Preserve every surviving source-backed candidate as a provisional finding, including its identity, source locations, evidence, counterevidence, and preliminary assessments. Assemble unsealed scan-manifest.json, findings.json, and coverage.json using <plugin_dir>/examples/completed-scan/ and <plugin_dir>/schemas/ as shape references. Set scan.scope.validationMode to custom_pending and put each finding's matching coverage surface IDs in extensions.customValidationSurfaceIds. Every reported coverage surface must have a candidate. When source discovery finishes and a bound semantic draft tool is available, submit an accepted final source-discovery draft with complete: true, preserving the provisional findings, coverage, threat model, and custom_pending validation mode. This marks source discovery finished; it does not claim custom validation has occurred or seal the scan. Keep complete: false when requested source review remains unfinished. Verify the three files exist, then return control to the SDK without invoking validation or attack-path skills, running application code, sealing files, generating a report, or calling a scan completion tool or helper.";
async function source(pluginRoot: string, name: keyof typeof SOURCES) {
const text = (await readFile(join(pluginRoot, name), "utf8")).replaceAll(
diff --git a/sdk/typescript/tests-ts/custom-validation.test.ts b/sdk/typescript/tests-ts/custom-validation.test.ts
index db3de27d5..0220c5e62 100644
--- a/sdk/typescript/tests-ts/custom-validation.test.ts
+++ b/sdk/typescript/tests-ts/custom-validation.test.ts
@@ -83,6 +83,7 @@ async function publishDraft(
scanDir: string,
scanId: string,
workbench: (args: readonly string[]) => Promise<Record<string, unknown>>,
+ complete?: boolean,
) {
const manifest = await json<ScanManifest>(
join(scanDir, "scan-manifest.json"),
@@ -94,6 +95,7 @@ async function publishDraft(
scan: {
target: manifest.scan.target,
scope: manifest.scan.scope,
+ ...(complete === undefined ? {} : { complete }),
},
},
findings: {
@@ -469,7 +471,15 @@ describe("custom validation", () => {
await expect(readFile(join(outside, "candidates.json"))).rejects.toThrow();
});
- test.each(["standard", "diff", "empty", "incomplete", "dismissed"])(
+ const completionScenarios = [
+ "standard",
+ "diff",
+ "empty",
+ "incomplete",
+ "dismissed",
+ "unfinished-source",
+ ];
+ test.each(completionScenarios)(
"SDK owns real workbench completion: %s",
async (scenario) => {
const diff = scenario === "diff";
@@ -601,8 +611,36 @@ describe("custom validation", () => {
);
expect(prompt).not.toContain("run `$validation` once");
expect(turnOptions.outputSchema).toBeUndefined();
+ await draft(scanDir, scanId, 0, diff);
+ const earlyCoverage = await json<CoverageDocument>(
+ join(scanDir, "coverage.json"),
+ );
+ earlyCoverage.completeness = "partial";
+ await save(join(scanDir, "coverage.json"), earlyCoverage);
+ await publishDraft(scanDir, scanId, workbench, false);
await draft(scanDir, scanId, count, diff);
- await publishDraft(scanDir, scanId, workbench);
+ if (scenario === "unfinished-source") {
+ const coverage = await json<CoverageDocument>(
+ join(scanDir, "coverage.json"),
+ );
+ coverage.completeness = "partial";
+ coverage.deferred = [
+ {
+ id: "unfinished-source",
+ reason:
+ "Requested source review remains unfinished.",
+ },
+ ];
+ await save(join(scanDir, "coverage.json"), coverage);
+ }
+ // Follow the adapter's terminal draft instruction at the real workbench boundary.
+ await publishDraft(
+ scanDir,
+ scanId,
+ workbench,
+ scenario !== "unfinished-source" &&
+ /\bcomplete:\s*true\b/u.test(prompt),
+ );
expect(commands).not.toContain("prepare-scan-completion");
expect(commands).not.toContain("complete-scan");
return { events: completedEvents() };
@@ -724,6 +762,22 @@ describe("custom validation", () => {
onActivity: (activity) => activities.push(activity),
...(diff ? { target: DiffTarget.workingTree({}) } : {}),
});
+ if (scenario === "unfinished-source") {
+ await expect(pending).rejects.toThrow(
+ "The latest saved scan draft is incomplete",
+ );
+ expect(turns).toBe(2);
+ expect(commands).not.toContain("complete-scan");
+ expect(commands).toContain("fail-scan");
+ expect(await json(join(scanDir, resultName))).toMatchObject({
+ status: "complete",
+ });
+ expect(
+ (await json<CoverageDocument>(join(scanDir, "coverage.json")))
+ .completeness,
+ ).toBe("partial");
+ return;
+ }
if (scenario === "incomplete") {
await expect(pending).rejects.toThrow(
"The validation environment did not start",