Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(scan): prevent scans from failing before custom validation (#1085)

ianw-oaie1253775d91f
diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py
index bad999a32..7b07e2d20 100644
--- a/plugins/codex-security/scripts/workbench_saved_results.py
+++ b/plugins/codex-security/scripts/workbench_saved_results.py
@@ -1438,15 +1438,10 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
binding = db.workbench_completion_binding(scan, db.now())
- # Validate on copies: saved canonical documents remain ordinary unsealed drafts.
- copied_manifest = copy.deepcopy(manifest)
- copied_findings = copy.deepcopy(findings)
- copied_coverage = copy.deepcopy(coverage)
- _populate_unsealed_manifest_envelope(copied_manifest, copied_manifest["scan"], binding)
- _populate_unsealed_artifact_envelope(
- copied_manifest, copied_findings, copied_coverage, binding
- )
- _validate_completion_binding(copied_manifest, copied_findings, copied_coverage, binding)
+ # Save scan IDs without sealing the draft.
+ _populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
+ _populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
+ _validate_completion_binding(manifest, findings, coverage, binding)
for filename, document in (
("findings.json", findings),
("coverage.json", coverage),
diff --git a/sdk/typescript/src/contract.ts b/sdk/typescript/src/contract.ts
index f88d88914..de90a7966 100644
--- a/sdk/typescript/src/contract.ts
+++ b/sdk/typescript/src/contract.ts
@@ -136,7 +136,7 @@ export async function loadContractWithScanDirectory(
}
valid = result;
if (!valid && filename === "findings.json") {
- payload = legacySealedFindingsForValidation(payload);
+ payload = normalizePersistedFindings(payload);
const compatibleResult = validate(payload);
if (typeof compatibleResult !== "boolean") {
throw new Error("asynchronous JSON Schema validation is unsupported");
@@ -217,7 +217,8 @@ export async function requireCanonicalScanDirectory(
type JsonRecord = Record<string, unknown>;
-function legacySealedFindingsForValidation(payload: unknown): unknown {
+/** Normalize optional legacy details on a copy, without changing saved artifacts. */
+export function normalizePersistedFindings(payload: unknown): unknown {
const compatible = structuredClone(payload);
if (!isJsonRecord(compatible) || !Array.isArray(compatible["findings"])) {
return compatible;
diff --git a/sdk/typescript/src/custom-validation.ts b/sdk/typescript/src/custom-validation.ts
index 4b630dd16..7e671e9dd 100644
--- a/sdk/typescript/src/custom-validation.ts
+++ b/sdk/typescript/src/custom-validation.ts
@@ -9,7 +9,7 @@ import {
} from "node:fs/promises";
import { basename, dirname, join } from "node:path";
import Ajv2020 from "ajv/dist/2020.js";
-import { requireScanFile } from "./contract.js";
+import { normalizePersistedFindings, requireScanFile } from "./contract.js";
import { IncompleteScanError, safeErrorMessage } from "./errors.js";
import type { CoverageDocument, FindingsDocument } from "./models.js";
import { requirePrivateOutputDirectory } from "./runtime.js";
@@ -226,7 +226,9 @@ export async function runCustomValidation(options: {
"The scan did not return an unsealed custom-validation draft.",
);
}
- const findings = findingsDocument.findings;
+ const { findings } = normalizePersistedFindings(
+ findingsDocument,
+ ) as typeof findingsDocument;
const [common, candidatesSchema, draft, coverageSchema] = await Promise.all([
readSchema(options.pluginRoot, "definitions/artifact-common.schema.json"),
readSchema(options.pluginRoot, "tools/candidate-validations.schema.json"),
@@ -239,26 +241,28 @@ export async function runCustomValidation(options: {
const validFinding = ajv.compile({ $ref: `${draft.$id}#/$defs/finding` });
if (!ajv.validate(coverageSchema, coverage))
throw new IncompleteScanError(
- "Custom validation requires valid provisional coverage.",
+ `Custom validation requires valid provisional coverage: ${ajv.errorsText(ajv.errors, { dataVar: "coverage" })}`,
);
const surfaceIds = new Set(coverage.surfaces.map((surface) => surface.id));
if (surfaceIds.size !== coverage.surfaces.length)
throw new IncompleteScanError(
"Provisional coverage contains duplicate surface IDs.",
);
- if (
- !Array.isArray(findings) ||
- findings.some((finding) => {
- const semantic = { ...finding };
- for (const key of ["findingId", "occurrenceId", "fingerprints"])
- delete semantic[key];
- return !validFinding(semantic);
- })
- ) {
+ if (!Array.isArray(findings)) {
throw new IncompleteScanError(
- "Custom validation requires a valid provisional finding set.",
+ "Custom validation requires a valid provisional finding set: findings must be an array.",
);
}
+ for (const [index, finding] of findings.entries()) {
+ const semantic = { ...finding };
+ for (const key of ["findingId", "occurrenceId", "fingerprints"])
+ delete semantic[key];
+ if (!validFinding(semantic)) {
+ throw new IncompleteScanError(
+ `Custom validation requires a valid provisional finding set: ${ajv.errorsText(validFinding.errors, { dataVar: `findings[${index}]` })}`,
+ );
+ }
+ }
const candidates = findings.map((finding, index) => {
const ids = finding.extensions?.["customValidationSurfaceIds"];
if (
diff --git a/sdk/typescript/tests-ts/custom-validation.test.ts b/sdk/typescript/tests-ts/custom-validation.test.ts
index c978e2bc7..95f4dad83 100644
--- a/sdk/typescript/tests-ts/custom-validation.test.ts
+++ b/sdk/typescript/tests-ts/custom-validation.test.ts
@@ -89,6 +89,55 @@ async function draft(scanDir: string, scanId: string, count = 1, diff = false) {
return findings;
}
+async function publishDraft(
+ scanDir: string,
+ scanId: string,
+ workbench: (args: readonly string[]) => Promise<Record<string, unknown>>,
+) {
+ const manifest = await json<ScanManifest>(
+ join(scanDir, "scan-manifest.json"),
+ );
+ const findings = await json<FindingsDocument>(join(scanDir, "findings.json"));
+ const coverage = await json<CoverageDocument>(join(scanDir, "coverage.json"));
+ const staged = {
+ manifest: {
+ scan: {
+ target: manifest.scan.target,
+ scope: manifest.scan.scope,
+ },
+ },
+ findings: {
+ findings: findings.findings.map(
+ ({
+ findingId: _id,
+ occurrenceId: _occurrence,
+ fingerprints: _fingerprints,
+ ...finding
+ }) => finding,
+ ),
+ },
+ coverage: {
+ completeness: coverage.completeness,
+ inventoryStrategy: coverage.inventoryStrategy,
+ surfaces: coverage.surfaces,
+ explicitExclusions: coverage.explicitExclusions,
+ deferred: coverage.deferred,
+ },
+ };
+ await mkdir(join(scanDir, "drafts"), { recursive: true });
+ const draftPath = join(scanDir, "drafts", `${scanId}.json`);
+ await save(draftPath, staged);
+ expect(
+ await workbench([
+ "write-scan-draft",
+ "--scan-id",
+ scanId,
+ "--draft-path",
+ draftPath,
+ ]),
+ ).toMatchObject({ scanId, status: "draft_written" });
+}
+
function result(
...dispositions: CustomValidationResult["validations"][number]["validation"]["disposition"][]
): CustomValidationResult {
@@ -246,12 +295,26 @@ describe("custom validation", () => {
});
});
- test("rejects a prematurely sealed discovery draft", async () => {
+ test.each([
+ "sealedAt",
+ "artifacts",
+ "manifest ID",
+ "findings ID",
+ "coverage ID",
+ ])("rejects an invalid discovery handoff: %s", async (kind) => {
const f = await fixture();
const manifest = await json<ScanManifest>(
join(f.scanDir, "scan-manifest.json"),
);
- manifest.scan.sealedAt = "2026-01-01T00:00:00Z";
+ if (kind === "sealedAt") manifest.scan.sealedAt = "2026-01-01T00:00:00Z";
+ if (kind === "artifacts") manifest.scan.artifacts = [];
+ if (kind === "manifest ID") manifest.scan.id = randomUUID();
+ if (kind === "findings ID" || kind === "coverage ID") {
+ const name = kind === "findings ID" ? "findings.json" : "coverage.json";
+ const document = await json<{ scanId: string }>(join(f.scanDir, name));
+ document.scanId = randomUUID();
+ await save(join(f.scanDir, name), document);
+ }
await save(join(f.scanDir, "scan-manifest.json"), manifest);
await expect(
runCustomValidation({
@@ -263,6 +326,101 @@ describe("custom validation", () => {
).rejects.toThrow("unsealed custom-validation draft");
});
+ test("validates persisted findings with empty optional dataflow details", async () => {
+ const f = await fixture(3);
+ for (const [index, finding] of f.findings.findings.entries()) {
+ finding.attackPath = {
+ dataflow: {
+ source: "Synthetic request input",
+ sink: index < 2 ? "" : "Synthetic output operation",
+ },
+ };
+ }
+ await save(join(f.scanDir, "findings.json"), f.findings);
+ let called = false;
+ await runCustomValidation({
+ ...f,
+ run: async () => {
+ called = true;
+ const candidates = await json<{
+ candidates: Array<{ finding: unknown }>;
+ }>(join(f.scanDir, "artifacts/custom-validation/candidates.json"));
+ expect(candidates.candidates).toHaveLength(3);
+ expect(candidates.candidates[0]!.finding).toHaveProperty(
+ "attackPath.dataflow",
+ { source: "Synthetic request input" },
+ );
+ return JSON.stringify(result("reportable", "reportable", "reportable"));
+ },
+ });
+ expect(called).toBe(true);
+ const saved = await json<FindingsDocument>(
+ join(f.scanDir, "findings.json"),
+ );
+ expect(saved.findings).toHaveLength(3);
+ for (const [index, finding] of saved.findings.entries()) {
+ expect(finding.identity).toEqual(f.findings.findings[index]!.identity);
+ expect(finding.locations).toEqual(f.findings.findings[index]!.locations);
+ expect(finding.attackPath).toEqual({
+ dataflow: {
+ source: "Synthetic request input",
+ ...(index < 2 ? {} : { sink: "Synthetic output operation" }),
+ },
+ });
+ }
+ });
+
+ test("retains original optional details when custom validation fails", async () => {
+ const f = await fixture();
+ f.findings.findings[0]!.attackPath = { dataflow: { sink: "" } };
+ await save(join(f.scanDir, "findings.json"), f.findings);
+ await expect(
+ runCustomValidation({
+ ...f,
+ run: async () => {
+ throw new Error("Synthetic validation failure");
+ },
+ }),
+ ).rejects.toThrow("Synthetic validation failure");
+ expect(
+ await json<FindingsDocument>(join(f.scanDir, "findings.json")),
+ ).toEqual(f.findings);
+ });
+
+ test("identifies an invalid provisional finding and field before validation", async () => {
+ const f = await fixture(2);
+ f.findings.findings[1]!.title = "";
+ await save(join(f.scanDir, "findings.json"), f.findings);
+ await expect(
+ runCustomValidation({
+ ...f,
+ run: async () => {
+ throw new Error("unexpected validation");
+ },
+ }),
+ ).rejects.toThrow(/findings\[1\].*title/);
+ expect(
+ await json<FindingsDocument>(join(f.scanDir, "findings.json")),
+ ).toEqual(f.findings);
+ });
+
+ test("identifies invalid provisional coverage before validation", async () => {
+ const f = await fixture();
+ const path = join(f.scanDir, "coverage.json");
+ const coverage = await json<CoverageDocument>(path);
+ coverage.surfaces[0]!.label = "";
+ await save(path, coverage);
+ await expect(
+ runCustomValidation({
+ ...f,
+ run: async () => {
+ throw new Error("unexpected validation");
+ },
+ }),
+ ).rejects.toThrow("coverage/surfaces/0/label");
+ expect(await json<CoverageDocument>(path)).toEqual(coverage);
+ });
+
test.each([
"missing",
"duplicate",
@@ -419,11 +577,23 @@ describe("custom validation", () => {
expect(prompt).not.toContain("run `$validation` once");
expect(turnOptions.outputSchema).toBeUndefined();
await draft(scanDir, scanId, count, diff);
+ await publishDraft(scanDir, scanId, workbench);
expect(commands).not.toContain("prepare-scan-completion");
expect(commands).not.toContain("complete-scan");
return { events: completedEvents() };
}
expect(prompt).toContain(workflow);
+ const pendingManifest = await json<ScanManifest>(
+ join(scanDir, "scan-manifest.json"),
+ );
+ expect(pendingManifest.scan.id).toBe(scanId);
+ expect(pendingManifest.scan.scope.validationMode).toBe(
+ "custom_pending",
+ );
+ expect(pendingManifest.scan).not.toHaveProperty("sealedAt");
+ expect(pendingManifest.scan).not.toHaveProperty(
+ "artifacts",
+ );
expect(turnOptions.outputSchema).toBeDefined();
if (scenario === "dismissed") {
expect(prompt).toContain("untrusted reviewer feedback");