Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: recover malformed security scan findings (#86)

ianw-oai150d6f6bba5a
diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py
index 5e5599cb2..d52d52dde 100644
--- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py
+++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py
@@ -767,6 +767,267 @@ def _populate_unsealed_finding_identities(
finding["fingerprints"] = fingerprints
+def _finding_strength(finding: dict[str, Any]) -> tuple[int, int, int]:
+ return (
+ ("informational", "low", "medium", "high", "critical").index(finding["severity"]["level"]),
+ ("low", "medium", "high").index(finding["confidence"]["level"]),
+ len(finding.get("codeEvidence") or []),
+ )
+
+
+def _recover_unsealed_findings(
+ manifest: dict[str, Any],
+ findings: dict[str, Any],
+ schema_dir: Path,
+ scan_dir: Path,
+ warnings: list[str],
+) -> list[str]:
+ schema = _read_json(schema_dir / "findings.schema.json")
+ _require_safe_schema(schema, "findings.schema.json")
+ properties = _require_dict(schema, "properties", "findings.schema")
+ finding_array = _require_dict(properties, "findings", "findings.schema.properties")
+ finding_schema = _require_dict(finding_array, "items", "findings.schema.properties.findings")
+ finding_properties = _require_dict(
+ finding_schema, "properties", "findings.schema.properties.findings.items"
+ )
+ writeup_schema = _require_dict(
+ finding_properties, "writeup", "findings.schema.properties.findings.items.properties"
+ )
+ scan = _require_dict(manifest, "scan", "manifest")
+ scan_id = _require_str(scan, "id", "manifest.scan")
+ if findings.get("scanId") != scan_id:
+ raise ContractError("findings.scanId: must match manifest scan id")
+
+ recovered: list[dict[str, Any]] = []
+ discarded: list[str] = []
+ finding_positions: dict[str, int] = {}
+ writeup_paths: set[str] = set()
+ for index, finding in enumerate(_require_list(findings, "findings", "findings")):
+ context = f"findings.findings[{index}]"
+ try:
+ if not isinstance(finding, dict):
+ raise ContractError(f"{context}: expected an object")
+ identity = _require_dict(finding, "identity", context)
+ fields: list[tuple[dict[str, Any], str, str, str]] = [
+ (finding, "ruleId", context, "rule identifier"),
+ (identity, "anchor", f"{context}.identity", "semantic anchor"),
+ ]
+ if "instance" in identity:
+ fields.append((identity, "instance", f"{context}.identity", "instance"))
+ normalized_fields = []
+ for parent, field, field_context, label in fields:
+ value = _require_str(parent, field, field_context)
+ if SLUG_RE.fullmatch(value):
+ continue
+ normalized = re.sub(r"[^a-z0-9._/-]+", "-", value.lower()).strip("._/-")
+ if not SLUG_RE.fullmatch(normalized):
+ raise ContractError(
+ f"{field_context}.{field}: expected a stable lowercase semantic slug"
+ )
+ parent[field] = normalized
+ normalized_fields.append(label)
+
+ _populate_unsealed_finding_identities(
+ manifest,
+ {"scanId": scan_id, "findings": [finding]},
+ )
+ finding_id = finding["findingId"]
+ previous_position = finding_positions.get(finding_id)
+ _validate_finding(finding, context)
+ if "writeup" in finding:
+ try:
+ _validate_schema_node(finding["writeup"], writeup_schema, f"{context}.writeup")
+ report_path = finding["writeup"]["reportPath"]
+ previous_writeup = (
+ recovered[previous_position].get("writeup")
+ if previous_position is not None
+ else None
+ )
+ if report_path in writeup_paths and (
+ previous_writeup is None or previous_writeup["reportPath"] != report_path
+ ):
+ raise ContractError(f"{context}.writeup.reportPath: duplicate report path")
+ _require_scan_local_file(scan_dir, report_path, f"{context}.writeup.reportPath")
+ except ContractError as exc:
+ finding.pop("writeup")
+ warnings.append(f"Skipped malformed writeup for finding {index + 1}: {exc}.")
+ _validate_schema_node(finding, finding_schema, context)
+ except ContractError as exc:
+ warning = f"Skipped malformed finding {index + 1}: {exc}."
+ warnings.append(warning)
+ discarded.append(warning)
+ continue
+
+ if previous_position is not None:
+ previous = recovered[previous_position]
+ if _finding_strength(finding) <= _finding_strength(previous):
+ warnings.append(
+ f"Skipped malformed finding {index + 1}: duplicate logical finding."
+ )
+ continue
+ previous_writeup = previous.get("writeup")
+ if previous_writeup is not None:
+ writeup_paths.discard(previous_writeup["reportPath"])
+ recovered[previous_position] = finding
+ warnings.append(
+ f"Recovered finding {index + 1}: retained stronger duplicate logical finding."
+ )
+ else:
+ finding_positions[finding_id] = len(recovered)
+ recovered.append(finding)
+
+ if "writeup" in finding:
+ writeup_paths.add(finding["writeup"]["reportPath"])
+ if normalized_fields:
+ warnings.append(
+ f"Recovered finding {index + 1}: normalized {', '.join(normalized_fields)}."
+ )
+
+ findings["findings"] = recovered
+ return discarded
+
+
+def _recover_unsealed_coverage(
+ coverage: dict[str, Any],
+ schema_dir: Path,
+ scan_dir: Path,
+ warnings: list[str],
+ discarded_findings: list[str],
+) -> None:
+ schema = _read_json(schema_dir / "coverage.schema.json")
+ _require_safe_schema(schema, "coverage.schema.json")
+ properties = _require_dict(schema, "properties", "coverage.schema")
+ completeness = coverage.get("completeness")
+ partial = completeness not in ("complete", "partial", "unknown")
+ if partial:
+ warnings.append("Recovered malformed coverage completeness; marked coverage as partial.")
+
+ surface_ids: set[str] = set()
+ for field, label in (
+ ("surfaces", "coverage surface"),
+ ("explicitExclusions", "coverage exclusion"),
+ ("deferred", "deferred coverage item"),
+ ):
+ array_schema = _require_dict(properties, field, "coverage.schema.properties")
+ item_schema = _require_dict(array_schema, "items", f"coverage.schema.properties.{field}")
+ items = coverage.get(field)
+ if not isinstance(items, list):
+ warnings.append(f"Skipped malformed {label} records: expected an array.")
+ coverage[field] = []
+ partial = True
+ continue
+
+ recovered: list[dict[str, Any]] = []
+ for index, item in enumerate(items):
+ context = f"coverage.{field}[{index}]"
+ try:
+ if not isinstance(item, dict):
+ raise ContractError(f"{context}: expected an object")
+ if field == "surfaces":
+ surface_id = _require_str(item, "id", context)
+ if surface_id in surface_ids:
+ raise ContractError(f"{context}.id: duplicate surface id")
+ disposition = item.get("disposition")
+ surface_recovered = False
+ if not isinstance(disposition, str) or disposition not in DISPOSITIONS:
+ warnings.append(
+ f"Recovered coverage surface {index + 1}: "
+ "the review disposition could not be verified."
+ )
+ item["disposition"] = "needs_follow_up"
+ surface_recovered = True
+
+ receipt_refs = item.get("receiptRefs")
+ if not isinstance(receipt_refs, list):
+ warnings.append(
+ f"Skipped malformed receipt references for coverage surface "
+ f"{index + 1}: expected an array."
+ )
+ receipt_refs = []
+ surface_recovered = True
+
+ recovered_receipts: list[str] = []
+ for ref_index, ref in enumerate(receipt_refs):
+ ref_context = f"{context}.receiptRefs[{ref_index}]"
+ try:
+ if not isinstance(ref, str):
+ raise ContractError(f"{ref_context}: expected a string")
+ normalized_ref = _require_safe_relative_path(ref, ref_context)
+ if not normalized_ref.startswith("artifacts/"):
+ raise ContractError(
+ f"{ref_context}: expected a file under artifacts/"
+ )
+ _require_scan_local_file(scan_dir, normalized_ref, ref_context)
+ except ContractError as exc:
+ warnings.append(
+ f"Skipped malformed coverage receipt "
+ f"{index + 1}.{ref_index + 1}: {exc}."
+ )
+ surface_recovered = True
+ continue
+ recovered_receipts.append(normalized_ref)
+
+ item["receiptRefs"] = recovered_receipts
+ if surface_recovered or item["disposition"] == "needs_follow_up":
+ if not surface_recovered and completeness != "partial":
+ warnings.append(
+ f"Coverage surface {index + 1} requires follow-up; "
+ "marked coverage as partial."
+ )
+ item["disposition"] = "needs_follow_up"
+ partial = True
+
+ _validate_schema_node(item, item_schema, context)
+ except ContractError as exc:
+ warnings.append(f"Skipped malformed {label} {index + 1}: {exc}.")
+ partial = True
+ continue
+
+ if field == "surfaces":
+ surface_ids.add(surface_id)
+ recovered.append(item)
+
+ coverage[field] = recovered
+
+ if discarded_findings:
+ for surface in coverage["surfaces"]:
+ surface["disposition"] = "needs_follow_up"
+ coverage["deferred"].extend(
+ {"id": f"discarded-finding-{index}", "reason": warning}
+ for index, warning in enumerate(discarded_findings, 1)
+ )
+ partial = True
+
+ if coverage["deferred"] and completeness != "partial":
+ if not discarded_findings:
+ warnings.append("Coverage has deferred review work; marked coverage as partial.")
+ partial = True
+ if partial:
+ coverage["completeness"] = "partial"
+
+
+def _recover_unsealed_hardening(
+ manifest: dict[str, Any],
+ scan_dir: Path,
+ warnings: list[str],
+) -> None:
+ scan = _require_dict(manifest, "scan", "manifest")
+ if "hardening" not in scan:
+ return
+
+ try:
+ hardening = _require_dict(scan, "hardening", "manifest.scan")
+ portfolio_path = _require_str(hardening, "portfolioPath", "manifest.scan.hardening")
+ if portfolio_path != "hardening/hardening.md":
+ raise ContractError(
+ "manifest.scan.hardening.portfolioPath: expected hardening/hardening.md"
+ )
+ _require_hardening_portfolio_file(scan_dir, scan)
+ except ContractError as exc:
+ scan.pop("hardening")
+ warnings.append(f"Skipped malformed hardening portfolio: {exc}.")
+
+
def _validate_derived_finding_identities(
manifest: dict[str, Any],
findings: dict[str, Any],
@@ -1792,8 +2053,21 @@ def build_sarif_projection(
source_root_is_directory = False
if not source_root_is_directory:
raise ContractError("source root: expected an existing directory")
- manifest, findings, _, _ = _read_sealed_scan(scan_dir, schema_dir, "SARIF projection")
+ manifest, findings, coverage, _ = _read_sealed_scan(scan_dir, schema_dir, "SARIF projection")
sarif = build_sarif(manifest, findings, source_root)
+ if coverage["completeness"] != "complete":
+ run = sarif["runs"][0]
+ run["properties"]["codexSecurityCoverageCompleteness"] = coverage["completeness"]
+ if coverage["deferred"]:
+ run["invocations"] = [
+ {
+ "executionSuccessful": True,
+ "toolExecutionNotifications": [
+ {"level": "warning", "message": {"text": item["reason"]}}
+ for item in coverage["deferred"]
+ ],
+ }
+ ]
_validate_sarif(sarif)
return sarif
@@ -2009,6 +2283,7 @@ def _prepare_scan_finalization(
*,
expected_coverage_mode: str | None = None,
completion_binding: dict[str, Any] | None = None,
+ completion_warnings: list[str] | None = None,
) -> PreparedScanFinalization:
"""Read, populate, and validate a scan without writing any output files."""
@@ -2061,8 +2336,15 @@ def _prepare_scan_finalization(
_validate_completion_binding(manifest, findings, coverage, completion_binding)
if was_sealed:
_validate_findings(manifest, findings)
- if was_sealed:
_validate_derived_finding_identities(manifest, findings)
+ elif completion_warnings is not None:
+ discarded_findings = _recover_unsealed_findings(
+ manifest, findings, schema_dir, scan_dir, completion_warnings
+ )
+ _recover_unsealed_coverage(
+ coverage, schema_dir, scan_dir, completion_warnings, discarded_findings
+ )
+ _recover_unsealed_hardening(manifest, scan_dir, completion_warnings)
else:
_populate_unsealed_finding_identities(manifest, findings)
_validate_findings(manifest, findings)
diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py
index d142da291..8faa6d9d4 100644
--- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py
+++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py
@@ -1430,6 +1430,7 @@ def complete_scan_locked(
scan_dir,
expected_coverage_mode=expected_coverage_mode(scan),
completion_binding=completion_binding,
+ completion_warnings=warnings,
)
warning = scan_target_warning(scan)
if warning is not None and warning not in warnings:
diff --git a/sdk/typescript/tests-ts/scan-recovery.test.ts b/sdk/typescript/tests-ts/scan-recovery.test.ts
new file mode 100644
index 000000000..014fcd033
--- /dev/null
+++ b/sdk/typescript/tests-ts/scan-recovery.test.ts
@@ -0,0 +1,609 @@
+import { spawnSync } from "node:child_process";
+import {
+ cp,
+ mkdir,
+ mkdtemp,
+ readFile,
+ realpath,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, test } from "bun:test";
+import { runWorkbench } from "../src/runtime.js";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+
+type Finding = Record<string, unknown> & {
+ ruleId: string;
+ identity: { anchor: string; instance?: string };
+ summary: string;
+ severity: { level: string };
+ confidence: { level: string };
+ locations: Array<{ path: string }>;
+ codeEvidence?: Array<{
+ id: string;
+ label: string;
+ path: string;
+ startLine: number;
+ code: string;
+ explanation: string;
+ }>;
+ writeup?: unknown;
+};
+
+type FindingsDocument = {
+ scanId: string;
+ findings: Array<Finding | null>;
+};
+
+type CoverageSurface = Record<string, unknown> & {
+ id: string;
+ label: string;
+ disposition: string;
+ receiptRefs: unknown[];
+};
+
+type CoverageDocument = Record<string, unknown> & {
+ scanId: string;
+ completeness: string;
+ inventoryStrategy: string;
+ surfaces: CoverageSurface[] | Record<string, unknown>;
+ explicitExclusions: unknown;
+ deferred: unknown;
+};
+
+type ScanSummary = {
+ findingCount: number;
+ progress: { status: string };
+ warnings: string[];
+};
+
+type SarifDocument = {
+ runs: Array<{
+ properties: { codexSecurityCoverageCompleteness?: string };
+ results: Array<{ properties: { severity: string } }>;
+ invocations?: Array<{
+ executionSuccessful: boolean;
+ toolExecutionNotifications: Array<{
+ level: string;
+ message: { text: string };
+ }>;
+ }>;
+ }>;
+};
+
+type ScanFixture = {
+ python: string;
+ stateDir: string;
+ scanDir: string;
+ scanId: string;
+};
+
+const temporaryDirectories: string[] = [];
+
+afterEach(async () => {
+ await Promise.all(
+ temporaryDirectories
+ .splice(0)
+ .map((path) => rm(path, { recursive: true, force: true })),
+ );
+});
+
+async function readJson<T>(path: string): Promise<T> {
+ return JSON.parse(await readFile(path, "utf8")) as T;
+}
+
+async function writeJson(path: string, value: unknown): Promise<void> {
+ await writeFile(path, `${JSON.stringify(value)}\n`);
+}
+
+async function workbench(fixture: ScanFixture, args: readonly string[]) {
+ return runWorkbench(
+ {
+ python: fixture.python,
+ pluginRoot: PLUGIN_ROOT,
+ environment: {
+ PATH: process.env["PATH"],
+ CODEX_SECURITY_STATE_DIR: fixture.stateDir,
+ },
+ },
+ args,
+ );
+}
+
+async function startDraftScan(): Promise<ScanFixture> {
+ const root = await realpath(
+ await mkdtemp(join(tmpdir(), "codex-security-scan-recovery-")),
+ );
+ temporaryDirectories.push(root);
+ const python = Bun.which("python3") ?? Bun.which("python");
+ expect(python).not.toBeNull();
+
+ const target = join(root, "repository");
+ const scanDir = join(root, "scan");
+ await mkdir(join(target, "src"), { recursive: true });
+ await writeFile(join(target, "src", "extract.py"), "# fixture\n");
+ await mkdir(scanDir, { mode: 0o700 });
+
+ const fixture: ScanFixture = {
+ python: python!,
+ stateDir: join(root, "state"),
+ scanDir,
+ scanId: "",
+ };
+ const registration = await workbench(fixture, [
+ "register-cli-scan",
+ "--repository",
+ target,
+ "--scan-dir",
+ scanDir,
+ "--recipe-json",
+ JSON.stringify({
+ config: {},
+ mode: "standard",
+ repository: target,
+ target: { kind: "repository", paths: [] },
+ }),
+ ]);
+ fixture.scanId = String(registration["scanId"]);
+
+ await cp(join(PLUGIN_ROOT, "examples", "completed-scan"), scanDir, {
+ recursive: true,
+ });
+ const manifestPath = join(scanDir, "scan-manifest.json");
+ const manifest = await readJson<{
+ scan: {
+ id: string;
+ target: { kind: string };
+ sealedAt?: string;
+ artifacts?: unknown[];
+ };
+ }>(manifestPath);
+ manifest.scan.id = fixture.scanId;
+ manifest.scan.target.kind = "directory_snapshot";
+ delete manifest.scan.sealedAt;
+ delete manifest.scan.artifacts;
+ await writeJson(manifestPath, manifest);
+
+ for (const name of ["findings.json", "coverage.json"] as const) {
+ const path = join(scanDir, name);
+ const document = await readJson<{ scanId: string }>(path);
+ document.scanId = fixture.scanId;
+ await writeJson(path, document);
+ }
+ await writeFile(join(scanDir, "report.md"), "# Draft report\n");
+ return fixture;
+}
+
+async function completeScan(fixture: ScanFixture): Promise<ScanSummary> {
+ const result = await workbench(fixture, [
+ "complete-scan",
+ "--scan-id",
+ fixture.scanId,
+ ]);
+ return result["scan"] as unknown as ScanSummary;
+}
+
+describe("malformed scan artifact recovery", () => {
+ test("normalizes finding identities and persists recovery warnings", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "findings.json");
+ const document = await readJson<FindingsDocument>(path);
+ const finding = document.findings[0]!;
+ finding.ruleId = "Path Traversal: Archive Extraction";
+ finding.identity.anchor = "Archive Entry Write Without Containment";
+ finding.identity.instance = "User Input #1";
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.findingCount).toBe(1);
+ expect(completed.warnings).toEqual([
+ "Recovered finding 1: normalized rule identifier, semantic anchor, instance.",
+ ]);
+ const recovered = (await readJson<FindingsDocument>(path)).findings[0]!;
+ expect(recovered.ruleId).toBe("path-traversal-archive-extraction");
+ expect(recovered.identity).toEqual({
+ anchor: "archive-entry-write-without-containment",
+ instance: "user-input-1",
+ });
+ const saved = await workbench(fixture, [
+ "get-scan",
+ "--scan-id",
+ fixture.scanId,
+ ]);
+ expect((saved["scan"] as unknown as ScanSummary).warnings).toEqual(
+ completed.warnings,
+ );
+ });
+
+ test("keeps valid findings and skips malformed or duplicate findings", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "findings.json");
+ const document = await readJson<FindingsDocument>(path);
+ const valid = document.findings[0]!;
+ const missingSummary = structuredClone(valid);
+ missingSummary.identity.anchor = "missing-summary";
+ missingSummary.summary = "";
+ const unsafeLocation = structuredClone(valid);
+ unsafeLocation.identity.anchor = "unsafe-location";
+ unsafeLocation.locations[0]!.path = "../outside.py";
+ const missingIdentity = structuredClone(valid);
+ delete (missingIdentity as Partial<Finding>).identity;
+ document.findings.push(
+ missingSummary,
+ unsafeLocation,
+ missingIdentity,
+ structuredClone(valid),
+ null,
+ );
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.findingCount).toBe(1);
+ expect(completed.warnings).toHaveLength(5);
+ expect(
+ completed.warnings.every((warning) =>
+ warning.startsWith("Skipped malformed finding"),
+ ),
+ ).toBe(true);
+ for (const reason of [
+ "summary",
+ "safe repository-relative",
+ "identity",
+ "duplicate logical finding",
+ "expected an object",
+ ]) {
+ expect(
+ completed.warnings.some((warning) => warning.includes(reason)),
+ ).toBe(true);
+ }
+ expect((await readJson<FindingsDocument>(path)).findings).toHaveLength(1);
+ const coverage = await readJson<CoverageDocument>(
+ join(fixture.scanDir, "coverage.json"),
+ );
+ expect(coverage.completeness).toBe("partial");
+ expect((coverage.surfaces as CoverageSurface[])[0]?.disposition).toBe(
+ "needs_follow_up",
+ );
+ expect(coverage.deferred).toHaveLength(4);
+ });
+
+ test("retains the strongest duplicate finding regardless of input order", async () => {
+ const cases = [
+ {
+ name: "severity ascending",
+ candidates: [
+ ["informational", "high", 1],
+ ["critical", "high", 1],
+ ],
+ expected: ["critical", "high", 1],
+ },
+ {
+ name: "severity descending",
+ candidates: [
+ ["critical", "high", 1],
+ ["informational", "high", 1],
+ ],
+ expected: ["critical", "high", 1],
+ },
+ {
+ name: "confidence ascending",
+ candidates: [
+ ["critical", "low", 1],
+ ["critical", "high", 1],
+ ],
+ expected: ["critical", "high", 1],
+ },
+ {
+ name: "confidence descending",
+ candidates: [
+ ["critical", "high", 1],
+ ["critical", "low", 1],
+ ],
+ expected: ["critical", "high", 1],
+ },
+ {
+ name: "evidence ascending",
+ candidates: [
+ ["critical", "high", 1],
+ ["critical", "high", 2],
+ ],
+ expected: ["critical", "high", 2],
+ },
+ {
+ name: "evidence descending",
+ candidates: [
+ ["critical", "high", 2],
+ ["critical", "high", 1],
+ ],
+ expected: ["critical", "high", 2],
+ },
+ ] as const;
+
+ for (const { name, candidates, expected } of cases) {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "findings.json");
+ const document = await readJson<FindingsDocument>(path);
+ const baseline = document.findings[0]!;
+ document.findings = candidates.map(([severity, confidence, count]) => {
+ const finding = structuredClone(baseline);
+ finding.severity.level = severity;
+ finding.confidence.level = confidence;
+ finding.codeEvidence = Array.from({ length: count }, (_, index) => ({
+ id: `evidence-${index + 1}`,
+ label: "Archive extraction",
+ path: "src/extract.py",
+ startLine: 1,
+ code: "# fixture",
+ explanation: "The archive entry reaches a filesystem write.",
+ }));
+ return finding;
+ });
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status, name).toBe("complete");
+ expect(completed.findingCount, name).toBe(1);
+ expect(completed.warnings, name).toHaveLength(1);
+ expect(completed.warnings[0], name).toContain(
+ "duplicate logical finding",
+ );
+ const recovered = (await readJson<FindingsDocument>(path)).findings[0]!;
+ expect(
+ [
+ recovered.severity.level,
+ recovered.confidence.level,
+ recovered.codeEvidence?.length,
+ ],
+ name,
+ ).toEqual([...expected]);
+ const coverage = await readJson<CoverageDocument>(
+ join(fixture.scanDir, "coverage.json"),
+ );
+ expect(coverage.completeness, name).toBe("complete");
+ expect(
+ await readFile(join(fixture.scanDir, "report.md"), "utf8"),
+ name,
+ ).not.toContain("### No findings");
+ const sarif = await readJson<SarifDocument>(
+ join(fixture.scanDir, "exports", "results.sarif"),
+ );
+ expect(sarif.runs[0]?.results[0]?.properties.severity, name).toBe(
+ "critical",
+ );
+ }
+ });
+
+ test("completes scans when every draft finding is malformed", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "findings.json");
+ const document = await readJson<FindingsDocument>(path);
+ document.findings[0]!.summary = "";
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.findingCount).toBe(0);
+ expect(completed.warnings).toHaveLength(1);
+ expect(completed.warnings[0]).toContain("summary");
+ expect((await readJson<FindingsDocument>(path)).findings).toEqual([]);
+ const coverage = await readJson<CoverageDocument>(
+ join(fixture.scanDir, "coverage.json"),
+ );
+ expect(coverage.completeness).toBe("partial");
+ expect((coverage.surfaces as CoverageSurface[])[0]?.disposition).toBe(
+ "needs_follow_up",
+ );
+ expect(coverage.deferred).toEqual([
+ { id: "discarded-finding-1", reason: completed.warnings[0] },
+ ]);
+ const report = await readFile(join(fixture.scanDir, "report.md"), "utf8");
+ expect(report).toContain("| Coverage | partial |");
+ expect(report).toContain("Skipped malformed finding 1");
+ const sarif = await readJson<SarifDocument>(
+ join(fixture.scanDir, "exports", "results.sarif"),
+ );
+ expect(sarif.runs[0]?.properties.codexSecurityCoverageCompleteness).toBe(
+ "partial",
+ );
+ expect(sarif.runs[0]?.invocations).toEqual([
+ {
+ executionSuccessful: true,
+ toolExecutionNotifications: [
+ { level: "warning", message: { text: completed.warnings[0]! } },
+ ],
+ },
+ ]);
+ });
+
+ test("keeps findings while removing invalid or duplicate writeups", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "findings.json");
+ const document = await readJson<FindingsDocument>(path);
+ const valid = document.findings[0]!;
+ const reportPath = "findings/linked-writeup/linked-writeup.md";
+ await mkdir(join(fixture.scanDir, "findings", "linked-writeup"), {
+ recursive: true,
+ });
+ await writeFile(join(fixture.scanDir, reportPath), "# Verified finding\n");
+
+ for (const [anchor, writeup] of [
+ ["linked-writeup", { reportPath }],
+ ["duplicate-writeup", { reportPath }],
+ ["missing-writeup", { reportPath: "findings/missing/missing.md" }],
+ ["unsafe-writeup", { reportPath: "../outside.md" }],
+ ["invalid-writeup", "not an object"],
+ ] as const) {
+ const finding = structuredClone(valid);
+ finding.identity.anchor = anchor;
+ finding.writeup = writeup;
+ document.findings.push(finding);
+ }
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.findingCount).toBe(6);
+ expect(completed.warnings).toHaveLength(4);
+ expect(
+ completed.warnings.every((warning) =>
+ warning.startsWith("Skipped malformed writeup for finding"),
+ ),
+ ).toBe(true);
+ expect(completed.warnings.join("\n")).not.toContain("../outside.md");
+ const recovered = (await readJson<FindingsDocument>(path)).findings;
+ expect(
+ recovered.find((finding) => finding?.identity.anchor === "linked-writeup")
+ ?.writeup,
+ ).toEqual({ reportPath });
+ for (const anchor of [
+ "duplicate-writeup",
+ "missing-writeup",
+ "unsafe-writeup",
+ "invalid-writeup",
+ ]) {
+ expect(
+ recovered.find((finding) => finding?.identity.anchor === anchor),
+ ).not.toHaveProperty("writeup");
+ }
+ });
+
+ test("keeps verified coverage receipts and downgrades invalid coverage", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "coverage.json");
+ const document = await readJson<CoverageDocument>(path);
+ const receipt = "artifacts/02_discovery/work_ledger.jsonl";
+ await mkdir(join(fixture.scanDir, "artifacts", "02_discovery"), {
+ recursive: true,
+ });
+ await writeFile(join(fixture.scanDir, receipt), '{"status":"reviewed"}\n');
+ const surface = (document.surfaces as CoverageSurface[])[0]!;
+ surface.receiptRefs = [
+ receipt,
+ "report.md",
+ "../outside.json",
+ "artifacts/02_discovery/missing.jsonl",
+ null,
+ ];
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.warnings).toHaveLength(4);
+ expect(
+ completed.warnings.every((warning) =>
+ warning.startsWith("Skipped malformed coverage receipt"),
+ ),
+ ).toBe(true);
+ expect(completed.warnings.join("\n")).not.toContain("../outside.json");
+ const recovered = await readJson<CoverageDocument>(path);
+ expect(recovered.completeness).toBe("partial");
+ expect((recovered.surfaces as CoverageSurface[])[0]).toMatchObject({
+ disposition: "needs_follow_up",
+ receiptRefs: [receipt],
+ });
+ const manifest = await readJson<{
+ scan: { artifacts: Array<{ path: string }> };
+ }>(join(fixture.scanDir, "scan-manifest.json"));
+ expect(manifest.scan.artifacts.map((artifact) => artifact.path)).toContain(
+ receipt,
+ );
+ });
+
+ test("downgrades malformed coverage collections without claiming completeness", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "coverage.json");
+ const document = await readJson<CoverageDocument>(path);
+ document.completeness = "finished";
+ document.surfaces = { id: "not-an-array" };
+ document.explicitExclusions = null;
+ document.deferred = "later";
+ await writeJson(path, document);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.warnings).toHaveLength(4);
+ const recovered = await readJson<CoverageDocument>(path);
+ expect(recovered).toMatchObject({
+ completeness: "partial",
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [],
+ });
+ });
+
+ test("discards unsafe hardening portfolios without discarding findings", async () => {
+ for (const hardening of [
+ "not an object",
+ { portfolioPath: "../outside.md" },
+ { portfolioPath: "hardening/hardening.md" },
+ ]) {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "scan-manifest.json");
+ const manifest = await readJson<{
+ scan: { hardening?: unknown };
+ }>(path);
+ manifest.scan.hardening = hardening;
+ await writeJson(path, manifest);
+
+ const completed = await completeScan(fixture);
+
+ expect(completed.progress.status).toBe("complete");
+ expect(completed.findingCount).toBe(1);
+ expect(completed.warnings).toHaveLength(1);
+ expect(completed.warnings[0]).toContain(
+ "Skipped malformed hardening portfolio:",
+ );
+ expect(completed.warnings[0]).not.toContain("../outside.md");
+ expect(
+ (await readJson<{ scan: { hardening?: unknown } }>(path)).scan,
+ ).not.toHaveProperty("hardening");
+ }
+ });
+
+ test("keeps direct finalization strict unless recovery is explicitly enabled", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "findings.json");
+ const document = await readJson<FindingsDocument>(path);
+ document.findings[0]!.identity.anchor = "Invalid Anchor";
+ await writeJson(path, document);
+
+ const strict = spawnSync(
+ fixture.python,
+ [
+ "-I",
+ "-B",
+ join(PLUGIN_ROOT, "scripts", "finalize_scan_contract.py"),
+ "--scan-dir",
+ fixture.scanDir,
+ ],
+ { encoding: "utf8" },
+ );
+
+ expect(strict.status).not.toBe(0);
+ expect(strict.stderr).toContain("stable lowercase semantic slug");
+ expect((await completeScan(fixture)).findingCount).toBe(1);
+ });
+
+ test("refuses to repair scan-wide coverage contract violations", async () => {
+ const fixture = await startDraftScan();
+ const path = join(fixture.scanDir, "coverage.json");
+ const document = await readJson<CoverageDocument>(path);
+ document.inventoryStrategy = "";
+ await writeJson(path, document);
+ const original = await readFile(path, "utf8");
+
+ await expect(completeScan(fixture)).rejects.toThrow("inventoryStrategy");
+ expect(await readFile(path, "utf8")).toBe(original);
+ });
+});