Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): recover checkpoints at discovery deadlines (#1594)

Michael D'Angelo57f698ebc115
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts
index 839bf2211..301bac885 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts
@@ -6,7 +6,14 @@ import { mock } from "node:test";
import { execFile, spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { once } from "node:events";
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import {
+ mkdir,
+ readFile,
+ readdir,
+ rm,
+ stat,
+ writeFile,
+} from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { importModule } from "./import-module.ts";
@@ -1180,20 +1187,106 @@ async function testDeadlinePreservesValidWorkerCheckpoints() {
scanId: run.scanId,
threadId,
});
- const worker = await createWorkerFixture(run, "saved", "discovery");
- await store.updateWorker({ ...worker, status: "running" });
- await saveScanDraftCheckpoint(
- { root: worker.artifactDir, layout: "worker", repoRoot: targetPath },
- {
+ const workers: { id: string }[] = [];
+ for (const label of [
+ "valid",
+ "second-valid",
+ "anonymous",
+ "mismatch",
+ "corrupt",
+ "missing",
+ ]) {
+ const worker = await createWorkerFixture(run, label, "discovery");
+ await store.updateWorker({ ...worker, status: "running" });
+ const context = {
+ root: worker.artifactDir,
+ layout: "worker",
+ repoRoot: targetPath,
+ };
+ const saved = {
...checkpoint(
run.scanId,
"pending-review",
"Worker did not finish reviewing this source.",
),
- findings: [finding("saved", "fixture.py")],
- },
- );
- await store.updateWorker({ ...worker, status: "canceled" });
+ findings: [finding(label, "fixture.py")],
+ };
+ if (label === "valid" || label === "second-valid") {
+ if (label === "valid") {
+ saved.findings.push(finding("out-of-scope", "../outside.py"));
+ saved.findings.push({
+ title: "Unvalidated, incomplete finding",
+ } as (typeof saved.findings)[number]);
+ } else {
+ saved.findings = [];
+ }
+ await mkdir(path.join(worker.artifactDir, "artifacts"));
+ await writeFile(
+ path.join(worker.artifactDir, "artifacts/review.md"),
+ `Source evidence for ${label}.\n`,
+ );
+ await saveScanDraftCheckpoint(context, {
+ ...saved,
+ coverage: {
+ ...saved.coverage,
+ surfaces: [
+ {
+ id: "api",
+ label: `${label} API`,
+ disposition: "no_issue_found",
+ receiptRefs: ["artifacts/review.md"],
+ },
+ ],
+ deferred: [
+ ...saved.coverage.deferred,
+ {
+ id: "surface-review",
+ reason: "Review remains.",
+ surfaceIds: ["api"],
+ },
+ ],
+ },
+ });
+ } else if (label === "anonymous") {
+ for (const reason of [
+ "Review authentication.",
+ "Review authorization.",
+ ]) {
+ await saveScanDraftCheckpoint(context, {
+ scanId: run.scanId,
+ complete: false,
+ findings: [],
+ threatModel: {},
+ coverage: {
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [{ reason }],
+ },
+ });
+ }
+ } else if (label === "mismatch") {
+ await saveScanDraftCheckpoint(context, {
+ ...saved,
+ scanId: randomUUID(),
+ });
+ } else {
+ await mkdir(path.join(worker.artifactDir, "checkpoints"), {
+ recursive: true,
+ });
+ const name = "a".repeat(64) + ".json";
+ await writeFile(
+ path.join(worker.artifactDir, "checkpoint-head.json"),
+ JSON.stringify({ checkpoint: name }),
+ );
+ if (label === "corrupt")
+ await writeFile(
+ path.join(worker.artifactDir, "checkpoints", name),
+ '{"scanId":',
+ );
+ }
+ await store.updateWorker({ ...worker, status: "canceled" });
+ workers.push(worker);
+ }
const context = await createScanArtifactContext(run.scanId, runWorkbench, {
requireRunning: true,
});
@@ -1229,6 +1322,26 @@ async function testDeadlinePreservesValidWorkerCheckpoints() {
const beforeFailure = await Promise.all(
draftNames.map((name) => readFile(path.join(run.scanDir, name), "utf8")),
);
+ const readObservationTimes = () =>
+ Promise.all(
+ draftNames.map(
+ async (name) =>
+ (await stat(path.join(run.scanDir, name), { bigint: true }))
+ .mtimeNs,
+ ),
+ );
+ const observationTimesBeforeFailure = await readObservationTimes();
+ const checkpointDirectory = path.join(run.scanDir, "checkpoints");
+ const readParentCheckpoints = async () =>
+ Promise.all(
+ (await readdir(checkpointDirectory))
+ .sort()
+ .map(async (name) => [
+ name,
+ await readFile(path.join(checkpointDirectory, name), "utf8"),
+ ]),
+ );
+ const checkpointsBeforeFailure = await readParentCheckpoints();
const database = path.join(
environment.CODEX_SECURITY_STATE_DIR,
"workbench.sqlite3",
@@ -1265,6 +1378,16 @@ async function testDeadlinePreservesValidWorkerCheckpoints() {
beforeFailure,
"a failed finish commit must restore the unsealed parent and accepted checkpoint head",
);
+ assert.deepEqual(
+ await readParentCheckpoints(),
+ checkpointsBeforeFailure,
+ "rollback must not publish an aggregate checkpoint as new parent evidence",
+ );
+ assert.deepEqual(
+ await readObservationTimes(),
+ observationTimesBeforeFailure,
+ "rollback must preserve parent observation order",
+ );
assert.equal((await store.get(run.scanId, threadId)).status, "running");
await runSql("DROP TRIGGER reject_deadline_commit;");
const finished = await store.finish({
@@ -1283,11 +1406,27 @@ async function testDeadlinePreservesValidWorkerCheckpoints() {
const recovered = await readJson(run.scanDir, "findings.json");
assert.deepEqual(
recovered.findings.map((item: { title: string }) => item.title),
- ["Unsafe request output saved"],
+ ["Unsafe request output valid"],
);
- assert.equal(recovered.findings[0].provenance.workerId, worker.id);
+ assert.equal(recovered.findings[0].provenance.workerId, workers[0].id);
const coverage = await readJson(run.scanDir, "coverage.json");
-
+ assert.equal(coverage.surfaces.length, 2);
+ for (const [index, label] of ["valid", "second-valid"].entries()) {
+ const workerId: string = workers[index].id;
+ const surface = coverage.surfaces.find(
+ (row: { workerId: string }) => row.workerId === workerId,
+ );
+ const task = coverage.deferred.find(
+ (row: { id: string }) => row.id === `${workerId}/surface-review`,
+ );
+ assert.equal(surface.id, `${workerId}/api`);
+ assert.equal(surface.disposition, "no_issue_found");
+ assert.deepEqual(task.surfaceIds, [surface.id]);
+ assert.equal(
+ await readFile(path.join(run.scanDir, surface.receiptRefs[0]), "utf8"),
+ `Source evidence for ${label}.\n`,
+ );
+ }
assert.equal(
coverage.deferred.some((item: { reason: string }) =>
/time limit elapsed/.test(item.reason),
@@ -1301,12 +1440,25 @@ async function testDeadlinePreservesValidWorkerCheckpoints() {
),
true,
);
+ const anonymousTasks = coverage.deferred.filter(
+ (row: { workerId?: string }) => row.workerId === workers[2].id,
+ );
+ assert.deepEqual(
+ new Set(anonymousTasks.map((row: { reason: string }) => row.reason)),
+ new Set(["Review authentication.", "Review authorization."]),
+ );
const manifest = await readJson(run.scanDir, "scan-manifest.json");
+ assert.equal(manifest.scan.threatModel, undefined);
+ const savedScan = await runWorkbench(["get-scan", "--scan-id", run.scanId]);
+ assert.match(
+ JSON.stringify(savedScan),
+ /Skipped malformed saved threat model.*missing required schema property/,
+ );
assert.equal(manifest.scan.sealedAt, undefined);
assert.equal(manifest.scan.complete, undefined);
assert.equal(
Object.keys(manifest.scan.preservedSources).some((source) =>
- source.includes("saved/output/checkpoints/"),
+ source.includes("valid/output/checkpoints/"),
),
true,
);
@@ -1325,6 +1477,14 @@ async function testDeadlinePreservesValidWorkerCheckpoints() {
beforeReplay,
);
await runWorkbench(["complete-scan", "--scan-id", run.scanId]);
+ assert.deepEqual(
+ (await readJson(run.scanDir, "coverage.json")).surfaces,
+ coverage.surfaces,
+ );
+ assert.deepEqual(
+ (await readJson(run.scanDir, "coverage.json")).deferred,
+ coverage.deferred,
+ );
const sealed = await readJson(run.scanDir, "scan-manifest.json");
assert.equal(typeof sealed.scan.sealedAt, "string");
assert.deepEqual(
diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py
index c15d48b5b..8ae81df50 100644
--- a/plugins/codex-security/scripts/deep_scan_workbench.py
+++ b/plugins/codex-security/scripts/deep_scan_workbench.py
@@ -9,7 +9,6 @@ import shutil
import sqlite3
import sys
import uuid
-from contextlib import nullcontext
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Callable
@@ -1656,6 +1655,9 @@ def finish_deep_scan_locked(
if len(set(omitted_worker_ids)) != len(omitted_worker_ids):
raise SystemExit("Omitted Deep Scan worker IDs must be unique.")
promotion: tuple[Path, Path, Path | None] | None = None
+ deadline_publication: (
+ tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None
+ ) = None
connection.execute("BEGIN IMMEDIATE")
try:
run = require_deep_scan_run(connection, scan_id)
@@ -1779,25 +1781,33 @@ def finish_deep_scan_locked(
kind="file",
)
promotion = promote_staged_file(staged_manifest_path, manifest_path)
- with (
- saved_results.preserve_deep_deadline_draft(dependencies(), connection, scan)
- if zero_discovery_deadline and standard_scan_manifest
- else nullcontext()
- ):
- timestamp = dependencies().now()
- connection.execute(
- """
- UPDATE deep_scan_runs
- SET status = 'succeeded', phase = 'terminal', terminal_reason = ?,
- manifest_path = ?, completed_at = ?, updated_at = ?
- WHERE scan_id = ?
- """,
- (args.terminal_reason, manifest_path, timestamp, timestamp, scan_id),
+ if zero_discovery_deadline:
+ deadline_publication = saved_results.preserve_deep_deadline_draft(
+ dependencies(), connection, scan, allow_empty=not standard_scan_manifest
)
- cancel_active_workers(connection, scan_id, timestamp)
- connection.commit()
+ timestamp = dependencies().now()
+ connection.execute(
+ """
+ UPDATE deep_scan_runs
+ SET status = 'succeeded', phase = 'terminal', terminal_reason = ?,
+ manifest_path = ?, completed_at = ?, updated_at = ?
+ WHERE scan_id = ?
+ """,
+ (args.terminal_reason, manifest_path, timestamp, timestamp, scan_id),
+ )
+ cancel_active_workers(connection, scan_id, timestamp)
+ connection.commit()
except BaseException:
connection.rollback()
+ if deadline_publication is not None or promotion is not None:
+ # A signal can arrive after SQLite commits but before commit returns.
+ # The completion lock makes persisted success authoritative here.
+ if require_deep_scan_run(connection, scan_id)["status"] == "succeeded":
+ if promotion is not None:
+ finish_staged_file(promotion)
+ raise
+ if deadline_publication is not None:
+ saved_results._restore_published_outputs(*deadline_publication)
if promotion is not None:
rollback_staged_file(promotion)
raise
diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py
index 1287f6777..8442910fa 100644
--- a/plugins/codex-security/scripts/finalize_scan_contract.py
+++ b/plugins/codex-security/scripts/finalize_scan_contract.py
@@ -1094,6 +1094,8 @@ def _recover_unsealed_findings(
schema_dir: Path,
scan_dir: Path,
warnings: list[str],
+ *,
+ current_contract: bool = False,
) -> list[str]:
schema = _read_json(schema_dir / "findings.schema.json")
properties = _require_dict(schema, "properties", "findings.schema")
@@ -1209,6 +1211,8 @@ def _recover_unsealed_findings(
f"Skipped malformed {auxiliary} for finding {index + 1}: {exc}."
)
_validate_schema_node(finding, finding_schema, context, schema)
+ if current_contract:
+ _validate_current_findings({"findings": [finding]}, schema_dir)
except ContractError as exc:
warning = f"Skipped malformed finding {index + 1}: {exc}."
warnings.append(warning)
diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py
index 6d3abc7f8..052b8d1db 100644
--- a/plugins/codex-security/scripts/workbench_saved_results.py
+++ b/plugins/codex-security/scripts/workbench_saved_results.py
@@ -30,9 +30,11 @@ from finalize_scan_contract import (
_read_scan_local_json,
_read_scan_local_json_bytes,
_read_scan_local_json_with_metadata,
+ _recover_unsealed_coverage,
_recover_unsealed_findings,
_remove_scan_local_file_if_exists,
_require_portable_relative_path,
+ _require_safe_json_value,
_require_str,
_validate_completion_binding,
_validate_resolved_deferred,
@@ -238,6 +240,7 @@ def _capture_saved_source(
kind: str | None = None,
snapshot_head: bool = True,
write: bool = True,
+ created_paths: set[str] | None = None,
) -> dict[str, tuple[str, int]]:
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
@@ -251,6 +254,8 @@ def _capture_saved_source(
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
# Capture the selected file even if the worker created it after directory enumeration.
if write and not (scan_dir / snapshot).exists():
+ if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
+ created_paths.add(snapshot)
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
return {
snapshot: (digest, int(observation["observedAtNs"])),
@@ -319,7 +324,12 @@ def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str])
def _freeze_source_times(
- scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
+ scan_dir: Path,
+ scan_id: str,
+ sources: dict[str, str],
+ times: dict[str, int],
+ *,
+ created_paths: set[str] | None = None,
) -> None:
# Identical result rewrites must not change the order of frozen review evidence.
observations = {
@@ -333,6 +343,8 @@ def _freeze_source_times(
digest = _digest(record)
path = f"source-order/{digest}.json"
if not (scan_dir / path).exists():
+ if created_paths is not None and not os.path.lexists(scan_dir / path):
+ created_paths.add(path)
write_scan_local_bytes(scan_dir, path, _encoded(record))
if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
raise ContractError("saved source ordering does not match its digest")
@@ -1047,6 +1059,8 @@ def merge_saved_results(
frozen_model_source: str | None = None,
selected_model_source: list[str] | None = None,
project_worker_coverage: bool = False,
+ created_paths: set[str] | None = None,
+ current_contract: bool = False,
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
initial_warnings = set(warnings)
@@ -1102,6 +1116,8 @@ def merge_saved_results(
parent_checkpoint = f"checkpoints/{parent_digest}.json"
checkpoint_path = scan_dir / parent_checkpoint
if not checkpoint_path.exists():
+ if created_paths is not None and not os.path.lexists(checkpoint_path):
+ created_paths.add(parent_checkpoint)
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
# A recovery copy must not appear newer than the review it copies.
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
@@ -1113,7 +1129,9 @@ def merge_saved_results(
)
os.utime(head_path, ns=(parent_modified, parent_modified))
if frozen_source_digests is not None:
- captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
+ captured = _capture_saved_source(
+ scan_dir, "checkpoint-head.json", scan_id, created_paths=created_paths
+ )
frozen_source_digests = {
**frozen_source_digests,
parent_checkpoint: parent_digest,
@@ -1198,7 +1216,9 @@ def merge_saved_results(
continue
del paths[relative]
try:
- captured = _capture_saved_source(scan_dir, relative, scan_id)
+ captured = _capture_saved_source(
+ scan_dir, relative, scan_id, created_paths=created_paths
+ )
paths.update({path: worker_id for path in captured})
except (ContractError, OSError, ValueError) as exc:
if (scan_dir / relative).exists():
@@ -1437,7 +1457,9 @@ def merge_saved_results(
or any(_is_source_order_snapshot(path) for path in source_digests)
or allow_frozen_legacy_parent
):
- _freeze_source_times(scan_dir, scan_id, source_digests, source_times)
+ _freeze_source_times(
+ scan_dir, scan_id, source_digests, source_times, created_paths=created_paths
+ )
target_kind = binding["allowedTargetKinds"][0]
if (
@@ -1471,16 +1493,51 @@ def merge_saved_results(
manifest["scan"][key] = copy.deepcopy(parent[key])
if isinstance(manifest["scan"].get("scope"), dict):
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
+ model_checks: dict[str, dict[str, Any] | None] = {}
+
+ def recovered_model(
+ relative: str, draft: dict[str, Any], owner: str | None
+ ) -> dict[str, Any] | None:
+ if relative not in model_checks:
+ model = copy.deepcopy(draft.get("threatModel"))
+ if isinstance(model, dict) and owner is not None:
+ model["origin"] = "recovered"
+ schema = _read_json(
+ Path(__file__).resolve().parent.parent / "schemas/scan-manifest.schema.json"
+ )
+ try:
+ _require_safe_json_value(model, f"Saved threat model {relative}")
+ _validate_schema_node(
+ model,
+ schema["properties"]["scan"]["properties"]["threatModel"],
+ f"Saved threat model {relative}",
+ schema,
+ )
+ except ContractError as exc:
+ warnings.append(f"Skipped malformed saved threat model {relative}: {exc}")
+ model = None
+ model_checks[relative] = model
+ return model_checks[relative]
+
+ if "threatModel" in manifest["scan"]:
+ model = recovered_model("parent", manifest["scan"], None)
+ if model is None:
+ manifest["scan"].pop("threatModel")
+ else:
+ manifest["scan"]["threatModel"] = model
+
if frozen_model_source is not None:
- # Publication retries retain the choice made with the frozen source set.
+ # Publication retries retain a valid choice from the frozen source set.
model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
if not isinstance(model, dict):
raise ContractError("Frozen stopped-scan model source is unavailable.")
- manifest["scan"]["threatModel"] = copy.deepcopy(model)
- if paths[frozen_model_source] is not None:
- manifest["scan"]["threatModel"]["origin"] = "recovered"
+ model = recovered_model(
+ frozen_model_source, drafts_by_path[frozen_model_source], paths[frozen_model_source]
+ )
+ if model is not None:
+ manifest["scan"]["threatModel"] = model
if selected_model_source is not None:
- selected_model_source[:] = [frozen_model_source]
+ selected_model_source[:] = [] if model is None else [frozen_model_source]
coverage = (
copy.deepcopy(parent["coverage"])
if parent and parent["coverage"]
@@ -1534,6 +1591,7 @@ def merge_saved_results(
Path(__file__).resolve().parent.parent / "schemas",
scan_dir,
[],
+ current_contract=current_contract,
)
return bool(document["findings"])
@@ -2038,7 +2096,7 @@ def merge_saved_results(
and "threatModel" not in manifest["scan"]
and isinstance(draft.get("threatModel"), dict)
):
- model = draft["threatModel"]
+ model = recovered_model(relative, draft, worker_id)
model_path = relative
checkpoint_dir = Path(relative).parent
prefer_worker_head = worker_id is not None and (
@@ -2052,6 +2110,7 @@ def merge_saved_results(
for path in selected_observations
if Path(path).parent == checkpoint_dir
and isinstance(drafts_by_path[path].get("threatModel"), dict)
+ and recovered_model(path, drafts_by_path[path], worker_id) is not None
]
if selected_models:
head_path = max(selected_models, key=source_order.__getitem__)
@@ -2061,13 +2120,12 @@ def merge_saved_results(
if not prefer_worker_head and current.get("complete") is not False:
prefer_worker_head = source_order[head_path] >= source_order[relative]
if prefer_worker_head:
- model = current["threatModel"]
+ model = recovered_model(head_path, current, worker_id)
model_path = head_path
- manifest["scan"]["threatModel"] = copy.deepcopy(model)
- if worker_id is not None:
- manifest["scan"]["threatModel"]["origin"] = "recovered"
- if selected_model_source is not None and worker_id is not None:
- selected_model_source[:] = [model_path]
+ if model is not None:
+ manifest["scan"]["threatModel"] = model
+ if selected_model_source is not None and worker_id is not None:
+ selected_model_source[:] = [model_path]
for value in draft["findings"]:
if skip_superseded_findings and not (
isinstance(value, dict)
@@ -2414,9 +2472,11 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
return prepared.coverage
-def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.stat_result] | None]:
+def _snapshot_published_outputs(
+ scan_dir: Path, relative_paths: tuple[str, ...] = _PUBLISHED_OUTPUTS
+) -> dict[str, tuple[bytes, os.stat_result] | None]:
snapshots: dict[str, tuple[bytes, os.stat_result] | None] = {}
- for relative in _PUBLISHED_OUTPUTS:
+ for relative in relative_paths:
descriptor = -1
try:
descriptor = open_scan_local_file_descriptor(
@@ -2441,7 +2501,9 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.sta
def _restore_published_outputs(
- scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
+ scan_dir: Path,
+ snapshots: dict[str, tuple[bytes, os.stat_result] | None],
+ created_paths: set[str] | None = None,
) -> None:
for relative, snapshot in snapshots.items():
if snapshot is None:
@@ -2451,70 +2513,105 @@ def _restore_published_outputs(
else:
contents, metadata = snapshot
write_scan_local_bytes(scan_dir, relative, contents)
- # Saved-result selection uses mtime; rollback must preserve its ordering.
os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
+ for relative in sorted(created_paths or ()):
+ # A journaled write can fail before creating its parent directory.
+ path = scan_dir / relative
+ if path.exists() or path.is_symlink():
+ _remove_scan_local_file_if_exists(scan_dir, relative)
-@contextmanager
-def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
+def preserve_deep_deadline_draft(
+ db: Any, connection: Any, scan: Any, *, allow_empty: bool = False
+) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None:
"""Recover a settled, zero-accepted Deep deadline without sealing the parent."""
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
- snapshots = _snapshot_published_outputs(scan_dir)
+ snapshots = _snapshot_published_outputs(
+ scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json")
+ )
warnings = json.loads(scan["completion_warnings_json"])
binding = db.workbench_completion_binding(scan, db.now())
- with preserve_parent_head_on_error(scan_dir):
- try:
- documents = merge_saved_results(
- scan_dir,
- scan["id"],
- binding,
- connection.execute(
- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
- (scan["id"],),
- ).fetchall(),
- warnings,
- stopped=True,
- reason=(
- "The configured discovery time limit elapsed before any source review completed. "
- "Saved findings and pending review were preserved without a completed reduction."
- ),
- project_worker_coverage=True,
- )
- if documents is None:
+ created_paths: set[str] = set()
+ try:
+ documents = merge_saved_results(
+ scan_dir,
+ scan["id"],
+ binding,
+ connection.execute(
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
+ (scan["id"],),
+ ).fetchall(),
+ warnings,
+ stopped=True,
+ project_worker_coverage=True,
+ created_paths=created_paths,
+ current_contract=True,
+ reason=(
+ "The configured discovery time limit elapsed before any source review completed. "
+ "Saved findings and pending review were preserved without a completed reduction."
+ ),
+ )
+ if documents is None:
+ if not allow_empty:
raise ContractError("Deep Scan deadline has no saved parent draft.")
- documents[0]["scan"].pop("complete", None)
- # Validate recovered evidence against the current finding contract,
- # scope, source lineage, and receipts before admitting this draft.
- prepared = _prepare_scan_finalization(
- scan_dir,
- expected_coverage_mode=db.expected_coverage_mode(scan),
- completion_binding=binding,
- recover_unsealed=True,
- draft_documents=documents,
- )
- warnings.extend(prepared.warnings)
- manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
- manifest["scan"].pop("sealedAt", None)
- manifest["scan"].pop("artifacts", None)
- checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
- contents = _encoded(checkpoint)
- name = f"{hashlib.sha256(contents).hexdigest()}.json"
- write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
- write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
- for filename, document in (
- ("findings.json", findings),
- ("coverage.json", coverage),
- ("scan-manifest.json", manifest),
- ):
- write_scan_local_bytes(scan_dir, filename, _encoded(document))
+ _restore_published_outputs(scan_dir, snapshots, created_paths)
connection.execute(
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
(json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
)
- yield
- except BaseException:
- _restore_published_outputs(scan_dir, snapshots)
- raise
+ return None
+ documents[0]["scan"].pop("complete", None)
+ # A capped deadline can retain interrupted records even though its
+ # completion binding requires strict validation of completed findings.
+ expected_coverage_mode = db.expected_coverage_mode(scan)
+ _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)
+ _populate_unsealed_artifact_envelope(*documents, binding)
+ if expected_coverage_mode == "deep_repository":
+ documents[2]["inventoryStrategy"] = "repository"
+ elif expected_coverage_mode == "scoped_path":
+ documents[2]["inventoryStrategy"] = "scoped_path"
+ schema_dir = Path(__file__).resolve().parent.parent / "schemas"
+ discarded_findings = _recover_unsealed_findings(
+ documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True
+ )
+ _recover_unsealed_coverage(documents[2], schema_dir, scan_dir, warnings, discarded_findings)
+ # Reuse stopped-result recovery's per-record validation, scope checks,
+ # source lineage, and receipt validation before admitting this draft.
+ prepared = _prepare_scan_finalization(
+ scan_dir,
+ expected_coverage_mode=expected_coverage_mode,
+ completion_binding=binding,
+ recover_unsealed=True,
+ draft_documents=documents,
+ finding_scope=db.requested_finding_scope(scan),
+ )
+ warnings.extend(prepared.warnings)
+ manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
+ manifest["scan"].pop("sealedAt", None)
+ manifest["scan"].pop("artifacts", None)
+ checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
+ contents = _encoded(checkpoint)
+ name = f"{hashlib.sha256(contents).hexdigest()}.json"
+ checkpoint_path = f"checkpoints/{name}"
+ # This draft is recoverable parent evidence too. Restore its prior state
+ # on local publication failure or a later caller transaction rollback.
+ snapshots.update(_snapshot_published_outputs(scan_dir, (checkpoint_path,)))
+ write_scan_local_bytes(scan_dir, checkpoint_path, contents)
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
+ for filename, document in (
+ ("findings.json", findings),
+ ("coverage.json", coverage),
+ ("scan-manifest.json", manifest),
+ ):
+ write_scan_local_bytes(scan_dir, filename, _encoded(document))
+ connection.execute(
+ "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
+ (json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
+ )
+ except BaseException:
+ _restore_published_outputs(scan_dir, snapshots, created_paths)
+ raise
+ return scan_dir, snapshots, created_paths
def preserve_scan_results_locked(
@@ -2688,7 +2785,8 @@ def preserve_scan_results_locked(
)
return False
if frozen_source_digests is None or (
- recovery_source_digests is None and saved_model_source is None and model_source
+ recovery_source_digests is None
+ and model_source != ([saved_model_source] if saved_model_source is not None else [])
):
retained_sources = _source_digests(
documents[0].get("scan", {}).get("preservedSources"),
diff --git a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
new file mode 100644
index 000000000..7f494f9b4
--- /dev/null
+++ b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
@@ -0,0 +1,578 @@
+from __future__ import annotations
+
+import copy
+import json
+import os
+import signal
+import sqlite3
+import uuid
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from test_workbench_standard_deep_results import (
+ deep_scan_fixture,
+ standard_finding,
+ write_saved_parent,
+)
+from workbench_test_support import (
+ finish_deep_scan,
+ get_scan,
+ run_workbench,
+ saved_draft,
+ worker_paths,
+ write_checkpoint,
+)
+
+
+def deadline_fixture(
+ tmp_path: Path, checkpoint_kind: str = "empty", *, include_paths: list[str] | None = None
+) -> SimpleNamespace:
+ state, home, target, directory, scan_id = deep_scan_fixture(
+ tmp_path, include_paths=include_paths
+ )
+ environment = {"CODEX_HOME": str(home)}
+ claimed = run_workbench(
+ state,
+ "claim-deep-scan-coordinator",
+ "--scan-id",
+ scan_id,
+ "--thread-id",
+ "standard-worker-thread",
+ environment=environment,
+ )
+ generation = str(claimed["deepScan"]["coordinatorGeneration"])
+ finding = standard_finding(tmp_path, scan_id, target)
+ if checkpoint_kind != "empty":
+ prompt, output, _ = worker_paths(directory, "deadline-worker")
+ args = (
+ "upsert-deep-scan-worker",
+ "--scan-id",
+ scan_id,
+ "--worker-id",
+ str(uuid.uuid4()),
+ "--kind",
+ "discovery",
+ "--prompt-path",
+ str(prompt),
+ "--artifact-dir",
+ str(output),
+ "--attempt",
+ "1",
+ "--coordinator-generation",
+ generation,
+ )
+ run_workbench(state, *args, "--status", "running", environment=environment)
+ draft = saved_draft(
+ scan_id if checkpoint_kind == "valid" else str(uuid.uuid4()),
+ findings=[finding],
+ deferred=[{"id": "review", "reason": "Synthetic review remains."}],
+ )
+ draft["threatModel"] = {"summary": "Synthetic worker model."}
+ checkpoint = write_checkpoint(output / "checkpoints", draft)
+ (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name}))
+ run_workbench(state, *args, "--status", "canceled", environment=environment)
+ artifacts = directory / "artifacts/02_discovery"
+ artifacts.mkdir(parents=True, exist_ok=True)
+ (artifacts / "in_scope_files.txt").write_text("app.py\n")
+ (artifacts / "candidate_ledger.jsonl").write_text("")
+ manifest = directory / "artifacts/deep_discovery/coordinator-manifest.json"
+ manifest.parent.mkdir(parents=True, exist_ok=True)
+ manifest.write_text('{"status":"succeeded","discoveryCount":0}\n')
+ database = state / "workbench.sqlite3"
+ with sqlite3.connect(database) as connection:
+ connection.execute(
+ "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00Z' WHERE scan_id = ?",
+ (scan_id,),
+ )
+ return SimpleNamespace(
+ state=state,
+ directory=directory,
+ scan_id=scan_id,
+ generation=generation,
+ environment=environment,
+ manifest=manifest,
+ database=database,
+ finding=finding,
+ )
+
+
+def finish(scan: SimpleNamespace, *, canonical: bool = False, check: bool = True):
+ return finish_deep_scan(
+ scan.state,
+ scan.scan_id,
+ "capped",
+ str(scan.directory / "scan-manifest.json" if canonical else scan.manifest),
+ "--coordinator-generation",
+ scan.generation,
+ environment=scan.environment,
+ check=check,
+ )
+
+
+def canonical_parent(scan: SimpleNamespace, draft: dict, timestamp: int = 100):
+ write_saved_parent(scan.directory, draft, timestamp)
+ path = scan.directory / "scan-manifest.json"
+ manifest = json.loads(path.read_text())
+ manifest["scan"]["id"] = scan.scan_id
+ if "threatModel" in draft:
+ manifest["scan"]["threatModel"] = draft["threatModel"]
+ path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.directory / name, ns=(timestamp, timestamp))
+
+
+def assert_retained_finding(scan: SimpleNamespace):
+ findings = json.loads((scan.directory / "findings.json").read_text())["findings"]
+ assert len(findings) == 1
+ assert findings[0]["ruleId"] == scan.finding["ruleId"]
+ assert findings[0]["identity"] == scan.finding["identity"]
+ assert findings[0]["severity"] == scan.finding["severity"]
+
+
[email protected]("canonical", [False, True])
[email protected]("same_identity", [False, True])
+ ("field", "value"), [("severity", None), ("code_evidence", []), ("root_cause", None)]
+)
+def test_deadline_recovers_partial_worker_findings_before_completion(
+ tmp_path, canonical, same_identity, field, value
+):
+ scan = deadline_fixture(tmp_path, "valid")
+ if canonical:
+ canonical_parent(scan, saved_draft(scan.scan_id))
+ head = next(scan.directory.glob("artifacts/**/checkpoint-head.json"))
+ selected = head.parent / "checkpoints" / json.loads(head.read_text())["checkpoint"]
+ draft = json.loads(selected.read_text())
+ partial = copy.deepcopy(scan.finding)
+ if not same_identity:
+ partial["ruleId"] = "synthetic.partial"
+ partial["identity"] = {"anchor": "partial"}
+ partial["provenance"]["candidateId"] = "partial-candidate"
+ if field == "severity":
+ partial.pop(field)
+ else:
+ partial[field] = value
+ partial["severity"]["level"] = "critical"
+ draft["findings"].append(partial)
+ checkpoint = write_checkpoint(head.parent / "checkpoints", draft)
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ source_bytes = {path: path.read_bytes() for path in (selected, checkpoint, head)}
+
+ terminal = finish(scan, canonical=canonical)["deepScan"]
+ assert terminal["status"] == "succeeded"
+ assert terminal["terminalReason"] == "capped"
+ assert_retained_finding(scan)
+ warnings = get_scan(scan.state, scan.scan_id)["scan"]["warnings"]
+ assert any(field in warning for warning in warnings)
+ coverage = json.loads((scan.directory / "coverage.json").read_text())
+ assert any(field in row["reason"] for row in coverage["deferred"])
+ assert all(path.read_bytes() == contents for path, contents in source_bytes.items())
+ run_workbench(
+ scan.state,
+ "complete-scan",
+ "--scan-id",
+ scan.scan_id,
+ environment=scan.environment,
+ )
+ manifest = json.loads((scan.directory / "scan-manifest.json").read_text())
+ assert manifest["scan"]["status"] == "completed"
+ assert "sealedAt" in manifest["scan"]
+ assert_retained_finding(scan)
+
+
[email protected]("checkpoint_kind", ["empty", "valid", "invalid"])
+def test_coordinator_deadline_preserves_only_usable_worker_checkpoints(
+ tmp_path: Path, checkpoint_kind: str
+):
+ scan = deadline_fixture(tmp_path, checkpoint_kind)
+ terminal = finish(scan)["deepScan"]
+ assert terminal["status"] == "succeeded"
+ assert terminal["terminalReason"] == "capped"
+ findings = scan.directory / "findings.json"
+ if checkpoint_kind == "valid":
+ assert_retained_finding(scan)
+ coverage = json.loads((scan.directory / "coverage.json").read_text())
+ assert "completeness" not in coverage
+ assert any(row.get("id", "").endswith("/review") for row in coverage["deferred"])
+ manifest = json.loads((scan.directory / "scan-manifest.json").read_text())
+ assert "sealedAt" not in manifest["scan"]
+ else:
+ assert not findings.exists()
+ warnings = get_scan(scan.state, scan.scan_id)["scan"]["warnings"]
+ assert bool(warnings) is (checkpoint_kind == "invalid")
+
+
[email protected]("valid_parent", [False, True])
+def test_deadline_validates_optional_parent_model_before_worker_fallback(
+ tmp_path: Path, valid_parent: bool
+):
+ scan = deadline_fixture(tmp_path, "valid")
+ draft = saved_draft(scan.scan_id)
+ draft["threatModel"] = {"summary": "Synthetic parent model." if valid_parent else 7}
+ canonical_parent(scan, draft)
+ assert finish(scan, canonical=True)["deepScan"]["status"] == "succeeded"
+ assert_retained_finding(scan)
+ model = json.loads((scan.directory / "scan-manifest.json").read_text())["scan"]["threatModel"]
+ assert model["summary"] == (
+ "Synthetic parent model." if valid_parent else "Synthetic worker model."
+ )
+ assert bool(get_scan(scan.state, scan.scan_id)["scan"]["warnings"]) is not valid_parent
+
+
+def test_deadline_recovery_does_not_snapshot_untouched_optional_sarif(tmp_path):
+ scan = deadline_fixture(tmp_path, "valid")
+ external = tmp_path / "optional-export.sarif"
+ external.write_text("Unchanged optional export.\n")
+ sarif = scan.directory / "exports/results.sarif"
+ sarif.parent.mkdir()
+ try:
+ sarif.symlink_to(external)
+ except OSError:
+ if os.name == "nt":
+ pytest.skip("Symbolic links are unavailable")
+ raise
+ assert finish(scan)["deepScan"]["status"] == "succeeded"
+ assert_retained_finding(scan)
+ completion = run_workbench(
+ scan.state,
+ "complete-scan",
+ "--scan-id",
+ scan.scan_id,
+ check=False,
+ environment=scan.environment,
+ )
+ assert completion["returncode"] == 0
+ assert "automatic SARIF export failed" in completion["stderr"]
+ assert sarif.is_symlink()
+ assert os.readlink(sarif) == str(external)
+ assert external.read_text() == "Unchanged optional export.\n"
+
+
[email protected]("include_paths", [None, ["app.py"], ["unrelated.py"]])
[email protected]("source", ["parent", "worker", "both"])
+def test_deadline_recovery_applies_requested_scope_to_parent_and_worker_findings(
+ tmp_path, include_paths, source
+):
+ scan = deadline_fixture(
+ tmp_path, "empty" if source == "parent" else "valid", include_paths=include_paths
+ )
+ if source != "worker":
+ draft = saved_draft(scan.scan_id, findings=[scan.finding])
+ draft["coverage"].update(
+ mode="scoped_path" if include_paths is not None else "deep_repository",
+ inventoryStrategy="scoped_path" if include_paths is not None else "repository",
+ includePaths=include_paths or ["."],
+ excludePaths=[],
+ )
+ canonical_parent(scan, draft)
+ workers = scan.directory / "artifacts/deep_discovery"
+ original = {p: p.read_bytes() for p in workers.rglob("*.json") if p != scan.manifest}
+ assert finish(scan)["deepScan"]["status"] == "succeeded"
+ findings = json.loads((scan.directory / "findings.json").read_text())["findings"]
+ head = json.loads((scan.directory / "checkpoint-head.json").read_text())["checkpoint"]
+ checkpoint = json.loads((scan.directory / "checkpoints" / head).read_text())
+ expected = 0 if include_paths == ["unrelated.py"] else 1
+ assert len(findings) == len(checkpoint["findings"]) == expected
+ assert all(path.read_bytes() == contents for path, contents in original.items())
+
+
[email protected]("with_worker", [False, True])
[email protected]("inventory", [None, "", "invalid_strategy", 7, "scoped_path"])
+def test_scoped_deadline_recovers_interrupted_inventory_before_final_publication(
+ tmp_path: Path, with_worker: bool, inventory: object
+):
+ scan = deadline_fixture(tmp_path, "valid" if with_worker else "empty", include_paths=["app.py"])
+ draft = saved_draft(scan.scan_id)
+ draft["coverage"].update(mode="scoped_path", includePaths=["app.py"], excludePaths=[])
+ if inventory is None:
+ draft["coverage"].pop("inventoryStrategy", None)
+ else:
+ draft["coverage"]["inventoryStrategy"] = inventory
+ canonical_parent(scan, draft)
+ worker_sources = {
+ path: path.read_bytes() for path in scan.directory.glob("artifacts/**/checkpoints/*.json")
+ }
+
+ terminal = finish(scan)["deepScan"]
+ assert terminal["status"] == "succeeded"
+ assert terminal["terminalReason"] == "capped"
+ assert terminal["completionSequence"] == 0
+ recovered = json.loads((scan.directory / "coverage.json").read_text())
+ assert recovered["mode"] == "scoped_path"
+ assert recovered["inventoryStrategy"] == "scoped_path"
+ assert recovered["includePaths"] == ["app.py"]
+ assert "sealedAt" not in json.loads((scan.directory / "scan-manifest.json").read_text())["scan"]
+
+ completed = run_workbench(
+ scan.state,
+ "complete-scan",
+ "--scan-id",
+ scan.scan_id,
+ environment=scan.environment,
+ )["scan"]
+ assert completed["progress"]["status"] == "complete"
+ assert completed["findingCount"] == int(with_worker)
+ if with_worker:
+ assert_retained_finding(scan)
+ manifest = json.loads((scan.directory / "scan-manifest.json").read_text())
+ assert manifest["scan"]["status"] == "completed"
+ assert "sealedAt" in manifest["scan"]
+ assert all(path.read_bytes() == contents for path, contents in worker_sources.items())
+
+
[email protected]("failure", ["publication", "commit"])
+def test_failed_deadline_without_parent_head_restores_all_recovery_evidence(
+ tmp_path: Path, failure: str
+):
+ scan = deadline_fixture(tmp_path, "valid")
+ canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))
+ assert not (scan.directory / "checkpoint-head.json").exists()
+ unrelated = scan.directory / "artifacts/unrelated-worker/checkpoints/keep.json"
+ unrelated.parent.mkdir(parents=True)
+ unrelated.write_text("Synthetic unrelated evidence.\n")
+ before = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in scan.directory.rglob("*")
+ if path.is_file()
+ }
+ trigger = (
+ "BEFORE UPDATE OF completion_warnings_json ON scans"
+ if failure == "publication"
+ else "BEFORE UPDATE OF status ON deep_scan_runs WHEN NEW.status = 'succeeded'"
+ )
+ with sqlite3.connect(scan.database) as connection:
+ connection.execute(
+ f"CREATE TRIGGER reject_deadline {trigger} "
+ "BEGIN SELECT RAISE(ABORT, 'synthetic deadline failure'); END"
+ )
+ failed = finish(scan, canonical=True, check=False)
+ assert failed["returncode"] != 0
+ assert "synthetic deadline failure" in failed["stderr"]
+ after = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in scan.directory.rglob("*")
+ if path.is_file()
+ }
+ assert after == before
+ with sqlite3.connect(scan.database) as connection:
+ connection.execute("DROP TRIGGER reject_deadline")
+ assert finish(scan, canonical=True)["deepScan"]["status"] == "succeeded"
+ assert_retained_finding(scan)
+
+
[email protected]("committed", [False, True])
+ "checkpoint_kind,staged,existing_target",
+ [("valid", False, True), ("valid", True, True), ("empty", True, False), ("empty", True, True)],
+)
+def test_deadline_interrupt_preserves_only_committed_publication(
+ tmp_path, monkeypatch, workbench_api, committed, checkpoint_kind, staged, existing_target
+):
+ scan = deadline_fixture(tmp_path, checkpoint_kind)
+ monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(scan.state))
+ monkeypatch.setenv("CODEX_HOME", scan.environment["CODEX_HOME"])
+ deep_scan = workbench_api["deep_scan"]
+ deep_scan.configure(SimpleNamespace(**workbench_api))
+ original_manifest = scan.manifest.read_bytes()
+ if not existing_target:
+ scan.manifest.unlink()
+ staged_path = scan.manifest.with_name("staged-manifest.json") if staged else None
+ if staged_path:
+ staged_path.write_bytes(original_manifest + b"\n")
+ arguments = SimpleNamespace(
+ omitted_worker_id=[],
+ manifest_path=str(scan.manifest),
+ staged_manifest_path=str(staged_path) if staged_path else None,
+ terminal_reason="capped",
+ coordinator_generation=int(scan.generation),
+ scan_id=scan.scan_id,
+ )
+ with sqlite3.connect(scan.database) as connection:
+ connection.row_factory = sqlite3.Row
+
+ class InterruptedCommit:
+ def __getattr__(self, name):
+ return getattr(connection, name)
+
+ def commit(self):
+ # Both success and SQLite auto-rollback end the transaction.
+ (connection.commit if committed else connection.rollback)()
+ signal.raise_signal(signal.SIGINT)
+
+ with pytest.raises(KeyboardInterrupt):
+ deep_scan.finish_deep_scan(InterruptedCommit(), arguments)
+ status = connection.execute(
+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ assert status == ("succeeded" if committed else "running")
+ published = committed and checkpoint_kind == "valid"
+ assert (scan.directory / "findings.json").exists() is published
+ assert (scan.directory / "checkpoint-head.json").exists() is published
+ if committed or existing_target:
+ assert scan.manifest.read_bytes() == original_manifest + (
+ b"\n" if staged and committed else b""
+ )
+ else:
+ assert not scan.manifest.exists()
+ if staged_path:
+ assert staged_path.exists() is not committed
+ with sqlite3.connect(scan.database) as connection:
+ connection.row_factory = sqlite3.Row
+ assert (
+ deep_scan.finish_deep_scan(connection, arguments)["deepScan"]["status"] == "succeeded"
+ )
+ assert scan.manifest.read_bytes() == original_manifest + (b"\n" if staged else b"")
+ if checkpoint_kind == "valid":
+ assert_retained_finding(scan)
+
+
[email protected]("write_kind", ["parent-checkpoint", "head-observation", "source-order"])
+def test_mid_merge_failure_removes_only_created_recovery_evidence(
+ tmp_path: Path, monkeypatch, workbench_api, write_kind: str
+):
+ scan = deadline_fixture(tmp_path, "valid")
+ canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))
+ before = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in scan.directory.rglob("*")
+ if path.is_file()
+ }
+ saved = workbench_api["saved_results"]
+ write = saved.write_scan_local_bytes
+ unrelated = scan.directory / "artifacts/unrelated-worker/checkpoints/keep.json"
+ failed = False
+
+ def fail_during_merge(directory, relative, contents):
+ nonlocal failed
+ write(directory, relative, contents)
+ matches = {
+ "parent-checkpoint": relative.startswith("checkpoints/"),
+ "head-observation": "/checkpoint-heads/" in "/" + relative,
+ "source-order": relative.startswith("source-order/"),
+ }
+ if not failed and matches[write_kind]:
+ failed = True
+ unrelated.parent.mkdir(parents=True)
+ unrelated.write_text("Unrelated worker evidence.\n")
+ raise RuntimeError("Synthetic merge failure")
+
+ monkeypatch.setattr(saved, "write_scan_local_bytes", fail_during_merge)
+ with sqlite3.connect(scan.database) as connection:
+ connection.row_factory = sqlite3.Row
+ record = workbench_api["require_scan"](connection, scan.scan_id)
+ with pytest.raises(RuntimeError, match="Synthetic merge failure"):
+ saved.preserve_deep_deadline_draft(SimpleNamespace(**workbench_api), connection, record)
+ after = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in scan.directory.rglob("*")
+ if path.is_file() and path != unrelated
+ }
+ assert after == before
+ assert unrelated.read_text() == "Unrelated worker evidence.\n"
+
+
[email protected]("snapshot_failure", [False, True])
+def test_failed_head_snapshot_does_not_interrupt_publication_rollback(
+ tmp_path, monkeypatch, workbench_api, snapshot_failure
+):
+ scan = deadline_fixture(tmp_path, "valid")
+ canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))
+ before = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in scan.directory.rglob("*")
+ if path.is_file()
+ }
+ saved = workbench_api["saved_results"]
+ write = saved.write_scan_local_bytes
+ failed_snapshots = []
+
+ def fail_before_snapshot_directory(directory, relative, contents):
+ if snapshot_failure and "/checkpoint-heads/" in relative:
+ assert not (directory / relative).parent.exists()
+ failed_snapshots.append(relative)
+ raise OSError("Synthetic head snapshot write failure")
+ write(directory, relative, contents)
+
+ monkeypatch.setattr(saved, "write_scan_local_bytes", fail_before_snapshot_directory)
+ with sqlite3.connect(scan.database) as connection:
+ connection.row_factory = sqlite3.Row
+ connection.execute(
+ "CREATE TRIGGER reject_deadline BEFORE UPDATE OF completion_warnings_json ON scans "
+ "BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END"
+ )
+ record = workbench_api["require_scan"](connection, scan.scan_id)
+ with pytest.raises(sqlite3.IntegrityError, match="synthetic publication failure"):
+ saved.preserve_deep_deadline_draft(SimpleNamespace(**workbench_api), connection, record)
+ assert bool(failed_snapshots) is snapshot_failure
+ assert all(not (scan.directory / path).exists() for path in failed_snapshots)
+ after = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in scan.directory.rglob("*")
+ if path.is_file()
+ }
+ assert after == before
+
+
[email protected]("directory", ["checkpoints", "checkpoint-heads", "source-order"])
+def test_deadline_rollback_preserves_preexisting_dangling_recovery_entry(
+ tmp_path: Path, workbench_api, directory: str
+):
+ scan = deadline_fixture(tmp_path)
+ canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))
+ saved = workbench_api["saved_results"]
+ original = {
+ path.relative_to(scan.directory): (path.read_bytes(), path.stat())
+ for path in scan.directory.rglob("*")
+ if path.is_file()
+ }
+ with sqlite3.connect(scan.database) as connection:
+ connection.row_factory = sqlite3.Row
+ record = workbench_api["require_scan"](connection, scan.scan_id)
+ binding = workbench_api["workbench_completion_binding"](record, workbench_api["now"]())
+ saved.merge_saved_results(
+ scan.directory, scan.scan_id, binding, [], [], stopped=True, reason="Synthetic stop."
+ )
+ created = {
+ path.relative_to(scan.directory)
+ for path in scan.directory.rglob("*")
+ if path.is_file() and path.relative_to(scan.directory) not in original
+ }
+ relative = next(path for path in created if path.parent == Path(directory))
+ for path in created:
+ (scan.directory / path).unlink()
+ for path, (contents, metadata) in original.items():
+ target = scan.directory / path
+ target.write_bytes(contents)
+ os.utime(target, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
+ target = scan.directory / relative
+ try:
+ target.symlink_to("missing-target")
+ except OSError:
+ if os.name == "nt":
+ pytest.skip("Symbolic links are unavailable")
+ raise
+ with pytest.raises(saved.ContractError) as rejected:
+ saved.write_scan_local_bytes(scan.directory, relative.as_posix(), b"{}")
+ try:
+ publication = saved.preserve_deep_deadline_draft(
+ SimpleNamespace(**workbench_api), connection, record
+ )
+ except saved.ContractError as error:
+ assert str(error) == str(rejected.value)
+ else:
+ warnings = json.loads(
+ connection.execute(
+ "SELECT completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert any(str(rejected.value) in warning for warning in warnings)
+ connection.rollback()
+ saved._restore_published_outputs(*publication)
+ assert target.is_symlink()
+ assert os.readlink(target) == "missing-target"
+ assert {
+ path.relative_to(scan.directory)
+ for path in scan.directory.rglob("*")
+ if path.is_file() or path.is_symlink()
+ } == {*original, relative}
diff --git a/plugins/codex-security/tests/test_deep_deadline_rollback.py b/plugins/codex-security/tests/test_deep_deadline_rollback.py
new file mode 100644
index 000000000..8e0a517a7
--- /dev/null
+++ b/plugins/codex-security/tests/test_deep_deadline_rollback.py
@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import copy
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from test_deep_coverage_recovery import worker
+from test_workbench_standard_deep_results import write_saved_parent
+from workbench_test_support import saved_binding, saved_draft, write_checkpoint
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import finalize_scan_contract as finalizer
+import workbench_saved_results as saved
+
+
+def publication_context(tmp_path, scan_id):
+ binding = {
+ **saved_binding("deep_repository", repository="synthetic", status="completed"),
+ "scanId": scan_id,
+ "startedAt": "2026-01-01T00:00:00Z",
+ "completedAt": "2026-01-01T00:01:00Z",
+ "producer": {"name": "codex-security", "version": "0.0.0"},
+ }
+ binding["target"].update(
+ revision="a" * 40, targetId="synthetic", displayName="Synthetic fixture"
+ )
+ db = SimpleNamespace(
+ require_canonical_scan_directory=finalizer._require_scan_directory,
+ now=lambda: binding["completedAt"],
+ workbench_completion_binding=lambda *_args: copy.deepcopy(binding),
+ expected_coverage_mode=lambda _scan: "deep_repository",
+ requested_finding_scope=lambda _scan: None,
+ )
+ scan = {"id": scan_id, "scan_dir": str(tmp_path), "completion_warnings_json": "[]"}
+ return binding, db, scan
+
+
[email protected]("local_failure", [False, True], ids=["caller-rollback", "local-failure"])
+def test_deadline_rollback_does_not_reintroduce_a_later_rejected_worker_finding(
+ tmp_path, local_failure
+):
+ scan_id = "00000000-0000-4000-8000-000000000001"
+ finding = {
+ "ruleId": "synthetic.input",
+ "identity": {"anchor": "fixture"},
+ "title": "Synthetic candidate",
+ "summary": "A synthetic source-backed result.",
+ "severity": {"level": "low"},
+ "confidence": {"level": "high", "rationale": "Synthetic fixture evidence."},
+ "taxonomy": {"category": "other", "cwe": []},
+ "locations": [{"path": "fixture.py", "startLine": 1}],
+ "remediation": "Apply the synthetic correction.",
+ "provenance": {"source": "local_plugin", "candidateId": "candidate-one"},
+ }
+ draft = saved_draft(scan_id, findings=[finding])
+ source = worker(tmp_path, "reviewer", draft)
+ output = Path(source["artifact_dir"])
+ worker_checkpoint = write_checkpoint(output / "checkpoints", draft)
+ worker_bytes = worker_checkpoint.read_bytes()
+ previous = write_checkpoint(tmp_path / "checkpoints", saved_draft(scan_id))
+ previous_bytes = previous.read_bytes()
+ head = tmp_path / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": previous.name}))
+ original_head = head.read_bytes()
+ binding, db, scan = publication_context(tmp_path, scan_id)
+
+ class Connection:
+ def execute(self, sql, _args):
+ if sql.startswith("UPDATE") and local_failure:
+ raise RuntimeError("Synthetic publication failure")
+ return SimpleNamespace(fetchall=lambda: [source])
+
+ if local_failure:
+ with pytest.raises(RuntimeError, match="Synthetic publication failure"):
+ saved.preserve_deep_deadline_draft(db, Connection(), scan)
+ else:
+ publication = saved.preserve_deep_deadline_draft(db, Connection(), scan)
+ # The caller uses this same restoration path when its transaction fails.
+ saved._restore_published_outputs(*publication)
+
+ rejected = saved_draft(
+ scan_id,
+ complete=True,
+ surfaces=[
+ {
+ "id": "candidate-one",
+ "candidateId": "candidate-one",
+ "label": "Reviewed candidate",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ (output / "result.json").write_text(json.dumps(rejected))
+ warnings = []
+ recovered = saved.merge_saved_results(
+ tmp_path,
+ scan_id,
+ binding,
+ [source],
+ warnings,
+ stopped=True,
+ reason="Synthetic stop after rejection.",
+ project_worker_coverage=True,
+ )
+ assert recovered is not None
+ assert recovered[1]["findings"] == []
+ assert worker_checkpoint.read_bytes() == worker_bytes
+ assert previous.read_bytes() == previous_bytes
+ assert head.read_bytes() == original_head
+ assert list((tmp_path / "checkpoints").iterdir()) == [previous]
+
+
[email protected]("local_failure", [False, True], ids=["caller-rollback", "local-failure"])
[email protected]("newer_source", ["canonical", "unselected-checkpoint"])
+def test_deadline_rollback_preserves_parent_observation_order(
+ tmp_path, local_failure, newer_source
+):
+ scan_id = "00000000-0000-4000-8000-000000000001"
+ pending = saved_draft(
+ scan_id, deferred=[{"id": "review", "reason": "Review remains."}], complete=True
+ )
+ closed = saved_draft(
+ scan_id, closures=[{"id": "review", "reason": "Review completed."}], complete=True
+ )
+ older, newer = (pending, closed) if newer_source == "canonical" else (closed, pending)
+ older["threatModel"] = {"summary": "The earlier model."}
+ newer["threatModel"] = {"summary": "The current model."}
+ selected = write_checkpoint(tmp_path / "checkpoints", older)
+ os.utime(selected, ns=(100, 100))
+ head = tmp_path / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(100, 100))
+ canonical = newer if newer_source == "canonical" else older
+ modified = 200 if newer_source == "canonical" else 100
+ write_saved_parent(tmp_path, canonical, modified)
+ manifest_path = tmp_path / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"]["threatModel"] = canonical["threatModel"]
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(tmp_path / name, ns=(modified, modified))
+ if newer_source == "unselected-checkpoint":
+ unselected = write_checkpoint(tmp_path / "checkpoints", newer)
+ os.utime(unselected, ns=(200, 200))
+ binding, db, scan = publication_context(tmp_path, scan_id)
+
+ class Connection:
+ def execute(self, sql, _args):
+ if sql.startswith("UPDATE") and local_failure:
+ raise RuntimeError("Synthetic publication failure")
+ return SimpleNamespace(fetchall=list)
+
+ def recover(directory):
+ result = saved.merge_saved_results(
+ directory, scan_id, binding, [], [], stopped=True, reason="Synthetic interruption."
+ )
+ assert result is not None
+ return result
+
+ def pending_ids(result):
+ return {row.get("id") for row in result[2]["deferred"] if row.get("id") == "review"}
+
+ baseline = tmp_path.with_name(tmp_path.name + "-baseline")
+ shutil.copytree(tmp_path, baseline)
+ before = recover(baseline)
+ assert pending_ids(before) == ({"review"} if newer_source == "unselected-checkpoint" else set())
+ assert before[0]["scan"]["threatModel"] == canonical["threatModel"]
+ original = {
+ path: (path.read_bytes(), path.stat().st_mtime_ns)
+ for path in [
+ head,
+ manifest_path,
+ tmp_path / "findings.json",
+ tmp_path / "coverage.json",
+ *(tmp_path / "checkpoints").glob("*.json"),
+ ]
+ }
+ if local_failure:
+ with pytest.raises(RuntimeError, match="Synthetic publication failure"):
+ saved.preserve_deep_deadline_draft(db, Connection(), scan)
+ else:
+ publication = saved.preserve_deep_deadline_draft(db, Connection(), scan)
+ saved._restore_published_outputs(*publication)
+
+ restored = {path: (path.read_bytes(), path.stat().st_mtime_ns) for path in original}
+ after = recover(tmp_path)
+ assert pending_ids(after) == pending_ids(before)
+ assert after[0]["scan"]["threatModel"] == before[0]["scan"]["threatModel"]
+ assert restored == original
diff --git a/plugins/codex-security/tests/test_saved_threat_model_recovery.py b/plugins/codex-security/tests/test_saved_threat_model_recovery.py
new file mode 100644
index 000000000..477c1e8b2
--- /dev/null
+++ b/plugins/codex-security/tests/test_saved_threat_model_recovery.py
@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import copy
+import sys
+from pathlib import Path
+
+import pytest
+from test_deep_coverage_recovery import worker
+from workbench_test_support import saved_binding, saved_draft
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import finalize_scan_contract as finalizer
+import workbench_saved_results as saved
+
+
+ "invalid_model",
+ [
+ {},
+ {"summary": ""},
+ {"summary": "Review", "assets": [42]},
+ pytest.param({"summary": "\ud800"}, id="lone-surrogate"),
+ ],
+)
[email protected]("with_fallback", [False, True])
+def test_invalid_optional_context_does_not_block_findings_or_frozen_replay(
+ tmp_path, invalid_model, with_fallback
+):
+ scan_id = "00000000-0000-4000-8000-000000000001"
+ finding = {
+ "ruleId": "synthetic.input",
+ "identity": {"anchor": "fixture"},
+ "title": "Synthetic finding",
+ "summary": "A synthetic source-backed result.",
+ "severity": {"level": "low"},
+ "confidence": {"level": "high", "rationale": "Synthetic fixture evidence."},
+ "taxonomy": {"category": "other", "cwe": []},
+ "locations": [{"path": "fixture.py", "startLine": 1}],
+ "remediation": "Apply the synthetic correction.",
+ "provenance": {"source": "local_plugin"},
+ }
+ draft = saved_draft(scan_id, findings=[finding])
+ draft["threatModel"] = copy.deepcopy(invalid_model)
+ sources = [worker(tmp_path, "invalid-context", draft)]
+ fallback = {"summary": "A valid saved worker context.", "assets": ["Synthetic records"]}
+ if with_fallback:
+ valid = saved_draft(scan_id)
+ valid["threatModel"] = fallback
+ sources.append(worker(tmp_path, "valid-context", valid))
+ originals = {path: path.read_bytes() for path in tmp_path.rglob("result.json")}
+ binding = {
+ **saved_binding("deep_repository", repository="synthetic", status="completed"),
+ "scanId": scan_id,
+ "startedAt": "2026-01-01T00:00:00Z",
+ "completedAt": "2026-01-01T00:01:00Z",
+ "producer": {"name": "codex-security", "version": "0.0.0"},
+ }
+ binding["target"].update(
+ revision="a" * 40, targetId="synthetic", displayName="Synthetic fixture"
+ )
+ frozen = None
+ invalid_source = "artifacts/deep_discovery/workers/invalid-context/output/result.json"
+ for replay in (False, True):
+ warnings = []
+ selected = [invalid_source] if replay else []
+ documents = saved.merge_saved_results(
+ tmp_path,
+ scan_id,
+ binding,
+ sources,
+ warnings,
+ stopped=True,
+ reason="Time limit.",
+ project_worker_coverage=True,
+ frozen_source_digests=frozen,
+ frozen_model_source=invalid_source if replay else None,
+ selected_model_source=selected,
+ )
+ assert documents is not None
+ frozen = documents[0]["scan"]["preservedSources"]
+ documents[0]["scan"].pop("complete", None)
+ prepared = finalizer._prepare_scan_finalization(
+ tmp_path,
+ expected_coverage_mode="deep_repository",
+ completion_binding=binding,
+ recover_unsealed=True,
+ draft_documents=documents,
+ )
+ assert len(prepared.findings["findings"]) == 1
+ assert prepared.findings["findings"][0]["provenance"]["workerId"] == "invalid-context"
+ assert any("Skipped malformed saved threat model" in warning for warning in warnings)
+ if with_fallback:
+ assert prepared.manifest["scan"]["threatModel"] == {**fallback, "origin": "recovered"}
+ assert selected == ["artifacts/deep_discovery/workers/valid-context/output/result.json"]
+ else:
+ assert "threatModel" not in prepared.manifest["scan"]
+ assert selected == []
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
index 2b6717589..45016127b 100644
--- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py
+++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
@@ -1040,19 +1040,27 @@ def standard_finding(tmp_path: Path, scan_id: str, target: Path) -> dict[str, ob
def deep_scan_fixture(
- tmp_path: Path, *, budget: bool = False, workers: int = 1
+ tmp_path: Path,
+ *,
+ budget: bool = False,
+ workers: int = 1,
+ include_paths: list[str] | None = None,
) -> tuple[Path, Path, Path, Path, str]:
state_dir = tmp_path / "state"
codex_home = tmp_path / "codex-home"
target = tmp_path / "target"
target.mkdir()
(target / "app.py").write_text("value = request.args['value']\n")
+ for relative in include_paths or []:
+ included = target / relative
+ if not included.exists():
+ included.write_text("value = 1\n")
config_path = codex_home / "codex-security" / "config.toml"
config_path.parent.mkdir(parents=True)
config_path.write_text(f"[deep_scan]\nworkers = {workers}\nmax_discovery_runs = {workers}\n")
environment = {"CODEX_HOME": str(codex_home)}
- if budget:
+ if budget or include_paths is not None:
scan_dir = tmp_path / "scan"
scan_dir.mkdir(mode=0o700)
registered = run_workbench(
@@ -1068,8 +1076,11 @@ def deep_scan_fixture(
"config": {},
"mode": "deep",
"repository": str(target),
- "target": {"kind": "repository", "paths": []},
- "maxCostUsd": 0.005,
+ "target": {
+ "kind": "paths" if include_paths is not None else "repository",
+ "paths": include_paths or [],
+ },
+ **({"maxCostUsd": 0.005} if budget else {}),
}
),
)