Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): keep selected results stable during retries (#915)

Michael D'Angelo07d84b7a5943
diff --git a/plugins/codex-security/mcp-app/tests/scan-identity-recovery-fixture.ts b/plugins/codex-security/mcp-app/tests/scan-identity-recovery-fixture.ts
index 192ab9715..5c53b5127 100644
--- a/plugins/codex-security/mcp-app/tests/scan-identity-recovery-fixture.ts
+++ b/plugins/codex-security/mcp-app/tests/scan-identity-recovery-fixture.ts
@@ -71,7 +71,12 @@ binding.update(scanId=scan_id,producer={'name':'codex-security-plugin','version'
warnings=[]
recovery=merge_saved_results(recovered,scan_id,binding,json.loads(sys.argv[5]),warnings,stopped=True,reason='Synthetic interruption',scan_mode=sys.argv[10]) if sys.argv[9]=='saved' else tuple(json.loads((recovered/name).read_text()) for name in ['scan-manifest.json','findings.json','coverage.json'])
if json.loads(sys.argv[7]):
- replayed=merge_saved_results(recovered,scan_id,binding,json.loads(sys.argv[5]),[],stopped=True,reason='Synthetic interruption',frozen_source_digests=recovery[0]['scan']['preservedSources'],scan_mode=sys.argv[10])
+ frozen=recovery[0]['scan']
+ replayed=merge_saved_results(
+ recovered,scan_id,binding,json.loads(sys.argv[5]),[],stopped=True,reason='Synthetic interruption',
+ frozen_source_digests=frozen['preservedSources'],checkpoint_heads=frozen['preservedCheckpointHeads'].copy(),
+ scan_mode=sys.argv[10],
+ )
assert replayed == recovery, 'Frozen recovery changed the retained documents'
ordinary=(manifest,json.loads((normal/'findings.json').read_text()),coverage)
results=[]
diff --git a/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts b/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts
index c5460540c..86a2954f4 100644
--- a/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts
+++ b/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts
@@ -133,24 +133,26 @@ def capture(*args, **kwargs):
return captured
saved._capture_saved_source = capture
retained._capture_saved_source = capture
-frozen = None
+frozen = heads = None
if sys.argv[9] == "true":
db = SimpleNamespace(require_canonical_scan_directory=lambda path: path, artifact_path=lambda *args, **kwargs: None, ARTIFACTS={"manifest": "scan-manifest.json"})
connection = SimpleNamespace(execute=lambda *args: SimpleNamespace(fetchall=lambda: [worker]))
- scan = {"scan_dir": str(root), "id": scan_id, "retained_source_digests_json": None, "seal_manifest_digest": None}
- frozen, _ = saved._recovery_source_digests(db, connection, scan)
-merged = merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=frozen)
+ scan = {"scan_dir": str(root), "id": scan_id, "retained_source_digests_json": None, "retained_checkpoint_heads_json": None, "seal_manifest_digest": None}
+ frozen, _, heads = saved._recovery_source_digests(db, connection, scan)
+merged = merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=frozen, checkpoint_heads=heads)
saved._capture_saved_source = original_capture
retained._capture_saved_source = original_retained_capture
-replay = merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=merged[0]["scan"]["preservedSources"])
-assert merged[2] == replay[2]
+snapshot = merged[0]["scan"]
+def replay():
+ return merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=snapshot["preservedSources"], checkpoint_heads=snapshot["preservedCheckpointHeads"])
+assert merged[2] == replay()[2]
rejects_changed_checkpoint = None
if published:
checkpoint = root / published[-1]
original = checkpoint.read_bytes()
checkpoint.write_text("{}")
try:
- merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=merged[0]["scan"]["preservedSources"])
+ replay()
except saved.ContractError:
rejects_changed_checkpoint = True
else:
@@ -161,7 +163,7 @@ refs = merged[2]["surfaces"][0]["receiptRefs"]
warnings = []
coverage = merged[2]
_recover_unsealed_coverage(coverage, plugin / "schemas", root, warnings, [])
-print(json.dumps({"refs": refs, "surfaces": coverage["surfaces"], "warnings": warnings, "published": published, "captured": merged[0]["scan"]["preservedSources"], "rejectsChangedCheckpoint": rejects_changed_checkpoint}))
+print(json.dumps({"refs": refs, "surfaces": coverage["surfaces"], "warnings": warnings, "published": published, "captured": snapshot["preservedSources"], "rejectsChangedCheckpoint": rejects_changed_checkpoint}))
`,
path.resolve(import.meta.dirname, "../.."),
f.root,
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts
index af466ebda..6104edc13 100644
--- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts
@@ -168,6 +168,180 @@ test("every released schema upgrades to the same current schema and remains idem
}
});
+for (const { missingVersions, legacyCheckpoint, mainReader } of [
+ { missingVersions: [57], legacyCheckpoint: 0, mainReader: false },
+ { missingVersions: [], legacyCheckpoint: 47, mainReader: false },
+ { missingVersions: [], legacyCheckpoint: 47, mainReader: true },
+ { missingVersions: [], legacyCheckpoint: 48, mainReader: false },
+ { missingVersions: [], legacyCheckpoint: 50, mainReader: false },
+ { missingVersions: [], legacyCheckpoint: 52, mainReader: false },
+ { missingVersions: [], legacyCheckpoint: 53, mainReader: false },
+ { missingVersions: [], legacyCheckpoint: 54, mainReader: false },
+ { missingVersions: [43], legacyCheckpoint: 47, mainReader: false },
+ { missingVersions: [44, 45, 46], legacyCheckpoint: 47, mainReader: false },
+]) {
+ test(`database-info retains stopped-scan checkpoints when adding migration ${missingVersions.join(",")} (legacy checkpoint: ${legacyCheckpoint}, main reader: ${mainReader})`, async () => {
+ const directory = await temporary.create("workbench-checkpoint-upgrade-");
+ const databasePath = join(directory, "workbench.sqlite3");
+ const sources = JSON.stringify({ "results.json": "retained-digest" });
+ const heads = JSON.stringify({ worker: "accepted-checkpoint" });
+ const context = JSON.stringify("Original discovery context");
+ let originalDiscoveryMigration: Record<string, unknown> | undefined;
+ let originalCheckpointMigration: Record<string, unknown> | undefined;
+ const old = new DatabaseSync(databasePath);
+ try {
+ applyMigrations(
+ old,
+ migrations
+ .filter(
+ (item) =>
+ !missingVersions.includes(item.version) &&
+ !(legacyCheckpoint && [54, 55, 56].includes(item.version)) &&
+ !(
+ [47, 48, 50, 52, 53].includes(legacyCheckpoint) &&
+ [52, 53].includes(item.version)
+ ) &&
+ !(
+ [47, 48, 50].includes(legacyCheckpoint) &&
+ [50, 51].includes(item.version)
+ ) &&
+ !(
+ [47, 48].includes(legacyCheckpoint) &&
+ [48, 49].includes(item.version)
+ ) &&
+ !(legacyCheckpoint === 47 && item.version === 47),
+ )
+ .map((item) =>
+ legacyCheckpoint && item.version === 57
+ ? { ...item, version: legacyCheckpoint }
+ : item,
+ ),
+ );
+ if (mainReader)
+ old.exec(
+ migrations.find((item) => item.version === 47)!.statements.join("\n"),
+ );
+ insertScan(old);
+ old.exec(`INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, status, phase,
+ workers, subagents, stop_after_no_new, max_discovery_runs, created_at, updated_at)
+ VALUES ('scan', 1, 'v1', 'failed', 'terminal', 1, 0, 7, 10, 'created', 'updated')`);
+ if (legacyCheckpoint !== 47 || mainReader) {
+ old
+ .prepare("UPDATE deep_scan_runs SET discovery_user_context_json = ?")
+ .run(context);
+ originalDiscoveryMigration = old
+ .prepare("SELECT * FROM schema_migrations WHERE version = 47")
+ .get();
+ }
+ old
+ .prepare(
+ "UPDATE scans SET status = 'failed', retained_source_digests_json = ?",
+ )
+ .run(sources);
+ if (!missingVersions.includes(57)) {
+ old
+ .prepare("UPDATE scans SET retained_checkpoint_heads_json = ?")
+ .run(heads);
+ }
+ originalCheckpointMigration = old
+ .prepare(
+ "SELECT name, applied_at FROM schema_migrations WHERE version = ?",
+ )
+ .get(legacyCheckpoint || 57);
+ } finally {
+ old.close();
+ }
+ await databaseInfo(directory);
+ const upgraded = new DatabaseSync(databasePath);
+ try {
+ assert.equal(
+ upgraded
+ .prepare("SELECT retained_checkpoint_heads_json FROM scans")
+ .get()?.retained_checkpoint_heads_json,
+ !missingVersions.includes(57) ? heads : null,
+ );
+ upgraded
+ .prepare("UPDATE scans SET retained_checkpoint_heads_json = ?")
+ .run(heads);
+ } finally {
+ upgraded.close();
+ }
+ await databaseInfo(directory);
+ const reopened = new DatabaseSync(databasePath);
+ try {
+ const scan = reopened.prepare("SELECT * FROM scans").get()!;
+ assert.equal(scan.status, "failed");
+ assert.equal(scan.retained_source_digests_json, sources);
+ assert.equal(scan.retained_checkpoint_heads_json, heads);
+ if (originalCheckpointMigration !== undefined) {
+ assert.deepEqual(
+ reopened
+ .prepare(
+ "SELECT name, applied_at FROM schema_migrations WHERE version = 57",
+ )
+ .get(),
+ originalCheckpointMigration,
+ );
+ }
+ assert.equal(
+ reopened
+ .prepare("SELECT COUNT(*) AS count FROM schema_migrations")
+ .get()?.count,
+ migrations.length,
+ );
+ assert.equal(
+ reopened
+ .prepare("SELECT MAX(version) AS version FROM schema_migrations")
+ .get()?.version,
+ 57,
+ );
+ for (const { version } of migrations.filter(
+ (item) => item.version >= 43,
+ )) {
+ assert.equal(
+ reopened
+ .prepare(
+ "SELECT COUNT(*) AS count FROM schema_migrations WHERE version = ?",
+ )
+ .get(version)?.count,
+ 1,
+ );
+ }
+ assert.deepEqual(reopened.prepare("PRAGMA foreign_key_check").all(), []);
+ assert.equal(
+ reopened
+ .prepare("SELECT discovery_user_context_json FROM deep_scan_runs")
+ .get()?.discovery_user_context_json,
+ legacyCheckpoint !== 47 || mainReader ? context : null,
+ );
+ assertMigrationNames(
+ reopened,
+ 47,
+ 48,
+ 49,
+ 50,
+ 51,
+ 52,
+ 53,
+ 54,
+ 55,
+ 56,
+ 57,
+ );
+ if (legacyCheckpoint !== 47) {
+ assert.deepEqual(
+ reopened
+ .prepare("SELECT * FROM schema_migrations WHERE version = 47")
+ .get(),
+ originalDiscoveryMigration,
+ );
+ }
+ } finally {
+ reopened.close();
+ }
+ });
+}
+
test("worker thread history survives retry, replay, and worker retirement", (t) => {
const database = memory(t, 49);
insertScan(database);
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index 3501bfa12..748822711 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -82,6 +82,7 @@
"scripts/workbench/__init__.py",
"scripts/workbench/handoff.py",
"scripts/workbench/retained_coverage.py",
+ "scripts/workbench/retained_records.py",
"scripts/workbench/runtime_toml.py",
"scripts/workbench/saved_coverage.py",
"scripts/workbench/saved_outputs.py",
diff --git a/plugins/codex-security/scripts/workbench/retained_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.py
index a45a125ab..0bb6fd9ae 100644
--- a/plugins/codex-security/scripts/workbench/retained_coverage.py
+++ b/plugins/codex-security/scripts/workbench/retained_coverage.py
@@ -5,10 +5,7 @@ from __future__ import annotations
import copy
import hashlib
import os
-import re
-import unicodedata
from collections import Counter, deque
-from collections.abc import Iterator
from functools import partial
from pathlib import Path, PurePosixPath
from typing import Any
@@ -16,23 +13,47 @@ from typing import Any
from finalize_scan_contract import (
ContractError,
_finding_strength,
- _has_identity_text,
_read_json,
- _recover_unsealed_coverage,
_recover_unsealed_findings,
_require_safe_json_value,
- _validate_current_findings,
_validate_schema_node,
finding_candidate_id,
write_scan_local_bytes,
)
+from workbench.retained_records import (
+ _append_previous_finding,
+ _competing_rejection_receipts,
+ _conflicting_outcome_receipts,
+ _ensure_finding_identities,
+ _ensure_finding_identity,
+ _finding_content,
+ _finding_identity,
+ _finding_key,
+ _identity_candidate,
+ _recover_preserved_worker_coverage,
+ _recover_preserved_worker_source,
+ _retained_findings,
+ _saved_candidate_outcomes,
+ _semantic_identity_key,
+ _worker_candidate_key,
+ contains_raw_value,
+ distinct_reports,
+ identifiers,
+ raw_owner,
+ raw_scope,
+ same_raw_content,
+ same_raw_finding,
+ same_raw_value,
+)
from workbench.saved_coverage import (
+ _coverage_rows,
+ _has_candidate_metadata,
+ _inherit_coverage_sources,
_merge_tied_parent_observations,
_parent_observation_content,
_project_worker_closure,
_projected_worker_closure_sources,
_resolved_deferred_rows,
- _saved_coverage_id,
prepare_saved_coverage_projection,
)
from workbench.saved_sources import (
@@ -40,15 +61,19 @@ from workbench.saved_sources import (
_capture_saved_source,
_checkpoint_head_directory,
_checkpoint_paths,
+ _CheckpointSnapshotError,
_digest,
_encoded,
_freeze_source_times,
+ _frozen_checkpoint_heads,
_frozen_source_times,
_is_source_order_snapshot,
_latest_successful_reducer,
+ _model_capture,
_parent_publication_ancestors,
_read_saved_parent_result,
_read_saved_result,
+ _unselected_root_order,
_worker_outputs,
)
from workbench_coverage_receipts import (
@@ -61,366 +86,6 @@ from workbench_coverage_receipts import (
from workbench_target import committed_diff_snapshot_digest
from workbench_validation import javascript_utf8, path_within_scope
-_FINDING_IDENTITY_SCHEMA = _read_json(
- Path(__file__).resolve().parents[2] / "schemas" / "findings.schema.json"
-)["properties"]["findings"]["items"]["properties"]["identity"]
-
-
-def _finding_identity(finding: dict[str, Any]) -> Any:
- provenance = finding.get("provenance")
- if isinstance(provenance, dict) and "preservedIdentity" in provenance:
- try:
- _validate_schema_node(
- provenance["preservedIdentity"], _FINDING_IDENTITY_SCHEMA, "finding.identity"
- )
- except ContractError:
- pass
- else:
- return provenance["preservedIdentity"]
- return finding.get("identity")
-
-
-def _finding_key(finding: dict[str, Any], owner: str | None = None) -> str:
- # Wording and evidence may improve between checkpoints; distinct source locations
- # must not collide merely because two workers chose the same semantic identity.
- provenance = finding.get("provenance")
- identity = _finding_identity(finding)
- if not isinstance(identity, dict):
- normalized = dict(finding)
- normalized.pop("identity", None)
- _ensure_finding_identity(normalized)
- identity = normalized["identity"]
- owner = provenance.get("workerId", owner) if isinstance(provenance, dict) else owner
- parts = [finding.get("ruleId"), identity, _finding_locations(finding)]
- if owner is not None and finding_candidate_id(finding):
- parts.append(owner)
- return _digest(parts)
-
-
-def _finding_locations(finding: dict[str, Any]) -> list[tuple[Any, Any, Any]]:
- locations = finding.get("locations", [])
- return sorted(
- (
- (
- location.get("path"),
- location.get("startLine"),
- location.get("endLine", location.get("startLine")),
- )
- for location in (locations if isinstance(locations, list) else [])
- if isinstance(location, dict)
- ),
- key=_encoded,
- )
-
-
-def _worker_candidate_key(
- worker_id: str, candidate_id: str, finding: dict[str, Any]
-) -> tuple[str, str, Any, Any, Any]:
- """Identify one worker-local candidate without merging unrelated locations."""
- identity = _finding_identity(finding)
- if not isinstance(identity, dict):
- normalized = dict(finding)
- _ensure_finding_identity(normalized)
- identity = normalized.get("identity")
- anchor = identity.get("anchor") if isinstance(identity, dict) else None
- instance = identity.get("instance") if isinstance(identity, dict) else None
- return worker_id, candidate_id, finding.get("ruleId"), anchor, instance
-
-
-def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
- """Return substantive finding content without generated identity or provenance."""
- return {
- key: value
- for key, value in finding.items()
- if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
- }
-
-
-def _identity_candidate(finding: dict[str, Any]) -> str | None:
- return next(
- (
- value
- for section in (finding.get("extensions"), finding.get("provenance"))
- if isinstance(section, dict) and _has_identity_text(value := section.get("candidateId"))
- ),
- None,
- )
-
-
-def _ensure_finding_identity(finding: Any) -> None:
- _ensure_finding_identities([finding])
-
-
-def _semantic_identifier(value: str, fallback: str) -> str:
- normalized = re.sub(r"[\u0300-\u036f]", "", unicodedata.normalize("NFKD", value))
- identifier = re.sub(r"[^a-z0-9._/-]+", "-", normalized.lower())
- return re.sub(r"^[^a-z0-9]+|-+$", "", identifier) or fallback
-
-
-def _semantic_identity_key(finding: dict[str, Any], identity: Any = None) -> bytes:
- identity = finding.get("identity") if identity is None else identity
- if isinstance(identity, dict):
- identity = {key: identity.get(key) for key in ("anchor", "instance")}
- return _encoded([finding.get("ruleId"), identity])
-
-
-def _ensure_finding_identities(findings: list[Any], *, deep: bool = False) -> None:
- """Use the same candidate, anchor and sibling rules as draft publication."""
- anchors: dict[int, Any] = {}
- for index, finding in enumerate(findings):
- if not isinstance(finding, dict):
- continue
- identity = finding.get("identity")
- if "identity" in finding:
- anchors[index] = identity.get("anchor") if isinstance(identity, dict) else None
- continue
- candidate = _identity_candidate(finding)
- anchors[index] = _semantic_identifier(
- candidate or str(finding.get("title") or "finding"), f"finding-{index + 1}"
- )
- counts = Counter(
- _digest([findings[index].get("ruleId"), anchor]) for index, anchor in anchors.items()
- )
-
- authored = {
- _semantic_identity_key(finding)
- for finding in findings
- if isinstance(finding, dict) and "identity" in finding
- }
- generated = []
- for index, anchor in anchors.items():
- finding = findings[index]
- if "identity" in finding:
- continue
- identity = {"anchor": anchor}
- extensions = finding.get("extensions")
- sibling = next(
- (
- value
- for field in ("reportId", "ledgerRowId")
- if isinstance(extensions, dict)
- and _has_identity_text(value := extensions.get(field))
- ),
- None,
- )
- if sibling or counts[_digest([finding.get("ruleId"), anchor])] > 1:
- identity["instance"] = _semantic_identifier(
- sibling or str(finding.get("title") or "finding"), f"finding-{index + 1}"
- )
- finding["identity"] = identity
- generated.append(finding)
-
- reserved = {
- _semantic_identity_key(finding) for finding in findings if isinstance(finding, dict)
- }
- used = set() if deep else authored.copy()
- generated_ids = {id(finding) for finding in generated}
- for finding in findings if deep else generated:
- key = _semantic_identity_key(finding)
- if key in used or (id(finding) in generated_ids and key in authored):
- identity = finding["identity"]
- if id(finding) not in generated_ids:
- finding.setdefault("provenance", {}).setdefault(
- "preservedIdentity", copy.deepcopy(identity)
- )
- base_instance = identity.get("instance", "saved")
- suffix = 2
- while True:
- identity["instance"] = f"{base_instance}-{suffix}"
- key = _semantic_identity_key(finding)
- if key not in reserved:
- reserved.add(key)
- break
- suffix += 1
- used.add(key)
-
-
-def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
- """Yield canonical and historical findings without trusting candidate IDs."""
- pending = [finding]
- seen: set[int] = set()
- while pending:
- current = pending.pop()
- marker = id(current)
- if marker in seen:
- continue
- seen.add(marker)
- yield current
- provenance = current.get("provenance")
- if not isinstance(provenance, dict):
- continue
- previous = provenance.get("previousFindings")
- if isinstance(previous, list):
- pending.extend(item for item in reversed(previous) if isinstance(item, dict))
- sources = provenance.get("sourceFindings")
- if isinstance(sources, list):
- pending.extend(
- source["finding"]
- for source in reversed(sources)
- if isinstance(source, dict) and isinstance(source.get("finding"), dict)
- )
-
-
-def _append_previous_finding(
- item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False
-) -> None:
- if not isinstance(item.get("previousFindings"), list):
- item["previousFindings"] = []
- history = item["previousFindings"]
- if not any(
- isinstance(previous, dict)
- and _finding_key(previous) == _finding_key(finding)
- and _finding_content(previous) == _finding_content(finding)
- for previous in history
- ):
- history.append(copy.deepcopy(finding) if clone else finding)
-
-
-def _has_candidate_metadata(row: dict[str, Any]) -> bool:
- return any(key in row for key in ("candidateId", "candidate", "finding"))
-
-
-def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]:
- rows = coverage.get(field, [])
- return rows if isinstance(rows, list) else []
-
-
-def _inherit_coverage_sources(
- row: Any,
- originals: list[Any],
- sources: dict[int, tuple[dict[str, Any], list[str]]],
-) -> None:
- if not isinstance(row, dict):
- return
- for original in originals:
- saved = sources.get(id(original))
- if saved is None:
- continue
- # Retain the object as well as its ID while copies are reconciled.
- paths = sources.setdefault(id(row), (row, []))[1]
- for relative in saved[1]:
- if relative not in paths:
- paths.append(relative)
-
-
-def _recover_preserved_worker_source(
- scan_dir, relative, scan_id, binding, worker_id, worker, draft, warnings
-):
- discarded_findings = []
- # Recover imported records without relaxing canonical parent validation.
- if "complete" in draft and not isinstance(draft["complete"], bool):
- draft["complete"] = False
- warnings.append(f"{relative}: Invalid complete marker; retained as a progress checkpoint.")
- schema_dir = Path(__file__).resolve().parents[2] / "schemas"
- if "threatModel" in draft:
- try:
- model = draft["threatModel"]
- if isinstance(model, dict):
- model = {**model, "origin": "recovered"}
- schema = _read_json(schema_dir / "scan-manifest.schema.json")
- _validate_schema_node(
- model,
- schema["properties"]["scan"]["properties"]["threatModel"],
- "threatModel",
- schema,
- )
- _require_safe_json_value(model, "threatModel")
- except ContractError as exc:
- draft.pop("threatModel")
- warnings.append(f"{relative}: Skipped malformed threat model: {exc}.")
- directory = Path(relative).parent
- if directory.name == "checkpoints":
- directory = directory.parent
- recovered_findings = []
- for index, finding in enumerate(draft["findings"]):
- needs_identity = isinstance(finding, dict) and "identity" not in finding
- _ensure_finding_identity(finding)
- recovered = {"scanId": scan_id, "findings": [finding]}
- source_warnings: list[str] = []
- discarded = _recover_unsealed_findings(
- {"scan": {"id": scan_id, "target": binding["target"]}},
- recovered,
- schema_dir,
- scan_dir / directory,
- source_warnings,
- )
- if recovered["findings"]:
- try:
- _validate_current_findings(recovered, schema_dir)
- except ContractError as exc:
- warning = f"Skipped malformed finding: {exc}."
- source_warnings.append(warning)
- discarded.append(warning)
- recovered["findings"] = []
- if discarded:
- discarded_findings.extend(discarded)
- for recovered_finding in recovered["findings"]:
- if needs_identity:
- # Reconcile anonymous revisions before assigning sibling identities.
- recovered_finding.pop("identity", None)
- writeup = recovered_finding.pop("writeup", None)
- if writeup is not None:
- source_warnings.append(
- "Retained worker writeup at "
- f"{directory.as_posix()}/{writeup['reportPath']}; "
- "the parent report renders this finding inline."
- )
- recovered_findings.extend(recovered["findings"])
- warnings.extend(f"{relative} finding {index + 1}: {warning}" for warning in source_warnings)
- draft["findings"] = recovered_findings
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
- rows = draft["coverage"].get(field)
- if not isinstance(rows, list):
- continue
- for index, row in enumerate(rows):
- if field == "openQuestions" and isinstance(row, str):
- row = rows[index] = {"question": row}
- if not isinstance(row, dict):
- continue
- row["workerId"] = worker_id
- if field == "surfaces":
- row["receiptRefs"] = resolve_coverage_receipts(
- scan_dir,
- row,
- worker,
- relative,
- )
- return discarded_findings
-
-
-def _recover_preserved_worker_coverage(
- scan_dir, relative, worker_id, draft, discarded_findings, warnings
-):
- discarded_coverage = False
- for field in ("surfaces", "explicitExclusions", "deferred"):
- rows = draft["coverage"].get(field)
- for row in rows if isinstance(rows, list) else []:
- if isinstance(row, dict):
- row.setdefault("id", _saved_coverage_id(row))
- if field == "surfaces":
- row.setdefault("receiptRefs", [])
- source_warnings = []
- discarded_fields = _recover_unsealed_coverage(
- draft["coverage"],
- Path(__file__).resolve().parents[2] / "schemas",
- scan_dir,
- source_warnings,
- discarded_findings,
- # Reconcile local IDs and valid outcomes against worker history below.
- worker_draft=True,
- )
- if discarded_fields & {
- "surfaces",
- "explicitExclusions",
- "deferred",
- "resolvedDeferred",
- }:
- discarded_coverage = True
- for row in draft["coverage"]["deferred"]:
- row["workerId"] = worker_id
- warnings.extend(f"{relative}: {warning}" for warning in source_warnings)
- return discarded_coverage
-
def merge_saved_results(
scan_dir: Path,
@@ -433,22 +98,65 @@ def merge_saved_results(
reason: str,
scan_mode: str | None = None,
frozen_source_digests: dict[str, str] | None = None,
+ checkpoint_heads: dict[str, str | None] | None = None,
allow_frozen_legacy_parent: bool = False,
+ parent_already_captured: bool = False,
frozen_model_source: str | None = None,
selected_model_source: list[str] | None = None,
+ canonical_model: dict[str, Any] | None = None,
+ canonical_model_source: str | None = None,
+ canonical_sources: dict[str, str] | None = None,
finding_scope: list[tuple[str, bool]] | None = None,
preserve_canonical_parent: bool = False,
project_worker_coverage: bool = False,
created_paths: set[str] | None = None,
current_contract: bool = False,
+ recovery_snapshot_error: _CheckpointSnapshotError | None = None,
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
initial_warnings = set(warnings)
- parent_context: dict[str, str] = {}
+ if frozen_source_digests is None:
+ checkpoint_heads = {}
+ snapshot_error = recovery_snapshot_error
+ captured_records = dict(snapshot_error.records) if snapshot_error is not None else {}
+
+ def capture_head(relative: str, canonical: Path | None = None) -> dict[str, tuple[str, int]]:
+ nonlocal snapshot_error
+ try:
+ if canonical is not None:
+ return _capture_parent_observation(
+ scan_dir,
+ scan_id,
+ canonical.name,
+ max(parent_modified, canonical.lstat().st_mtime_ns),
+ allow_frozen_legacy_parent,
+ created_paths=created_paths,
+ )
+ return _capture_saved_source(
+ scan_dir,
+ relative,
+ scan_id,
+ checkpoint_heads=checkpoint_heads,
+ created_paths=created_paths,
+ )
+ except _CheckpointSnapshotError as exc:
+ if "status" not in binding:
+ # A running coordinator can continue from its remaining evidence.
+ if checkpoint_heads is not None:
+ checkpoint_heads.pop(Path(relative).parent.as_posix(), None)
+ raise
+ # Finish selection from captured bytes before freezing an I/O failure.
+ snapshot_error = snapshot_error or exc
+ captured_records.update(exc.records)
+ return exc.captured
+
+ parent_context = dict(recovery_snapshot_error.parent_context) if recovery_snapshot_error else {}
try:
source_times = _frozen_source_times(
scan_dir, scan_id, frozen_source_digests or {}, parent_context
)
+ if snapshot_error is not None:
+ source_times.update({path: item[1] for path, item in snapshot_error.captured.items()})
except (ContractError, OSError, ValueError) as exc:
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
parent: dict[str, Any] | None = None
@@ -456,7 +164,12 @@ def merge_saved_results(
parent_manifest: dict[str, Any] | None = None
parent_is_canonical = False
parent_modified = 0
- if frozen_source_digests is None or allow_frozen_legacy_parent:
+ parent_model_source: str | None = None
+ canonical_model_order: tuple[int, int] | None = None
+ canonical_model_digest: str | None = None
+ if frozen_source_digests is None or (
+ allow_frozen_legacy_parent and not parent_already_captured
+ ):
try:
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
# Without an accepted head, file-authored coverage is a full replacement.
@@ -485,6 +198,11 @@ def merge_saved_results(
payload = _encoded(parent)
parent_digest = hashlib.sha256(payload).hexdigest()
parent_checkpoint = f"checkpoints/{parent_digest}.json"
+ if isinstance(parent.get("threatModel"), dict):
+ canonical_model_source = parent_checkpoint
+ parent_model_source = parent_checkpoint
+ canonical_model_order = (0, parent_modified)
+ canonical_model_digest = parent_digest
parent_context["canonical"] = parent_checkpoint
parent_context["canonicalObservedAtNs"] = str(parent_modified)
checkpoint_path = scan_dir / parent_checkpoint
@@ -494,20 +212,12 @@ def merge_saved_results(
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
# A recovery copy must not appear newer than the review it copies.
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
- captured = (
- _capture_saved_source(
- scan_dir, "checkpoint-head.json", scan_id, created_paths=created_paths
- )
- if head_modified is not None
- else _capture_parent_observation(
- scan_dir,
- checkpoint_path.name,
- parent_modified,
- allow_frozen_legacy_parent,
- created_paths=created_paths,
- )
+ captured = capture_head(
+ "checkpoint-head.json", checkpoint_path if head_modified is None else None
)
- if head_modified is not None or allow_frozen_legacy_parent:
+ if captured and (head_modified is not None or allow_frozen_legacy_parent):
+ if checkpoint_heads is not None:
+ checkpoint_heads["."] = next(iter(captured))
parent_context["head"] = next(iter(captured))
if frozen_source_digests is not None:
frozen_source_digests = {
@@ -596,9 +306,7 @@ def merge_saved_results(
continue
del paths[relative]
try:
- captured = _capture_saved_source(
- scan_dir, relative, scan_id, created_paths=created_paths
- )
+ captured = capture_head(relative)
if relative == "checkpoint-head.json" and frozen_source_digests is None:
parent_context["head"] = next(iter(captured))
paths.update({path: worker_id for path in captured})
@@ -630,7 +338,7 @@ def merge_saved_results(
while pending:
relative, worker_id = pending.popleft()
try:
- draft, digest, observed = _read_saved_result(
+ draft, digest, observed = captured_records.get(relative) or _read_saved_result(
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
)
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
@@ -638,11 +346,19 @@ def merge_saved_results(
source_digests[relative] = digest
source_times.setdefault(relative, observed)
source_order[relative] = (0, source_times[relative])
+ directory = Path(relative).parent
+ if directory.name in {"checkpoints", "checkpoint-heads"}:
+ directory = directory.parent
if _checkpoint_head_directory(relative) is not None:
saved_heads[relative] = draft["checkpoint"]
if isinstance(draft.get("inputCheckpoint"), str):
saved_inputs[relative] = draft["inputCheckpoint"]
continue
+ if (
+ checkpoint_heads is not None
+ and checkpoint_heads.get(directory.as_posix(), relative) is None
+ ):
+ continue
if preserve_canonical_parent and worker_id is not None:
discarded = _recover_preserved_worker_source(
scan_dir,
@@ -672,7 +388,45 @@ def merge_saved_results(
if frozen_source_digests.keys() - source_digests.keys():
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
- # Frozen observations keep checkpoint selection after a worker moves its head.
+ if checkpoint_heads is None:
+ checkpoint_heads = _frozen_checkpoint_heads(scan_dir, scan_id, source_digests)
+ heads_snapshotted = False
+ if frozen_source_digests is not None and snapshot_error is None:
+ # Raw heads are a temporary fallback, not permanent publication sources.
+ for relative in list(saved_heads):
+ if Path(relative).name != "checkpoint-head.json":
+ continue
+ captured = _capture_saved_source(
+ scan_dir,
+ relative,
+ scan_id,
+ expected_digest=source_digests[relative],
+ created_paths=created_paths,
+ )
+ for path, (digest, observed) in captured.items():
+ if path in source_digests and source_digests[path] != digest:
+ raise ContractError("checkpoint changed after the scan stopped")
+ source_digests[path] = digest
+ source_times.setdefault(path, observed)
+ if _checkpoint_head_directory(path) is not None:
+ saved_heads[path] = saved_heads[relative]
+ if relative in saved_inputs:
+ saved_inputs[path] = saved_inputs[relative]
+ if parent_context.get("head") == relative:
+ parent_context["head"] = path
+ source_order[path] = (0, observed)
+ key = Path(relative).parent.as_posix()
+ if checkpoint_heads.get(key) == relative:
+ checkpoint_heads[key] = path
+ saved_inputs.pop(relative, None)
+ del saved_heads[relative], source_digests[relative]
+ heads_snapshotted = True
+ if heads_snapshotted:
+ source_digests = {
+ path: digest
+ for path, digest in source_digests.items()
+ if not _is_source_order_snapshot(path)
+ }
headed_workers = {
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
for head in saved_heads
@@ -687,21 +441,36 @@ def merge_saved_results(
_, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
source_order[relative] = (attempt, source_order[relative][1])
selected_observations: dict[str, tuple[int, int]] = {}
+ authoritative_heads: set[str] = set()
+ authoritative_observations: dict[str, tuple[int, int]] = {}
parent_heads: list[tuple[int, str]] = []
for head, checkpoint in saved_heads.items():
directory = _checkpoint_head_directory(head)
selected = (directory / "checkpoints" / checkpoint).as_posix()
if selected not in source_order:
raise ContractError("Checkpoint head is outside the saved source set.")
+ key = directory.as_posix()
+ if key in checkpoint_heads and checkpoint_heads[key] is None:
+ continue
observed = source_order[head][1]
- if directory == Path("."):
- order = (0, max(source_order[selected][1], observed))
- parent_heads.append((observed, selected))
- else:
- _, attempt = worker_attempts[directory.as_posix()]
- order = (attempt, observed)
+ attempt = 0 if directory == Path(".") else worker_attempts[key][1]
+ order = (attempt, observed)
+ if key not in checkpoint_heads or checkpoint_heads[key] in (head, selected):
+ authoritative_heads.add(selected)
+ if checkpoint_heads.get(key) == head:
+ authoritative_observations[selected] = order
+ if directory == Path("."):
+ parent_heads.append((observed, selected))
selected_observations[selected] = max(selected_observations.get(selected, order), order)
+ # A registered result independently retains its latest accepted observation.
+ # Restoring the head does not withdraw that result's publication authority.
+ registered_result_orders = {
+ path: order for path, order in selected_observations.items() if path in current_results
+ }
+ # A restored head selects one observation, not the newest time for its payload.
+ selected_observations.update(authoritative_observations)
source_order.update(selected_observations)
+ source_order.update(registered_result_orders)
# Keep frozen source digests over the original data, before discarding markers
# that never passed through the worker draft writer.
@@ -709,9 +478,21 @@ def merge_saved_results(
sources, _digest, binding["scope"]["includePaths"], finding_scope
)
drafts_by_path = {relative: draft for relative, draft, _ in sources}
+ # A frozen model capture supplies context without replacing earlier findings.
+ model_path = frozen_model_source or canonical_model_source
+ model_draft = drafts_by_path.get(model_path)
+ if (
+ model_draft is not None
+ and model_path not in selected_observations
+ and model_path != parent_context.get("canonical")
+ and isinstance(model := model_draft.get("threatModel"), dict)
+ and model_draft == _model_capture(scan_id, model)
+ ):
+ model_draft["complete"] = False
if frozen_source_digests is not None and "canonical" in parent_context:
canonical = parent_context["canonical"]
parent = drafts_by_path[canonical]
+ parent_model_source = canonical
parent_modified = int(
parent_context.get("canonicalObservedAtNs", source_order[canonical][1])
)
@@ -737,6 +518,7 @@ def merge_saved_results(
for relative, draft in saved_parents.items():
if relative not in previous_parents:
frozen_parent_path, parent = relative, draft
+ parent_model_source = relative
parent_is_canonical = draft.get("canonicalParent") is True
break
projected_coverage = parent["coverage"] if parent else {}
@@ -912,7 +694,12 @@ def merge_saved_results(
draft = drafts_by_path[parent_path]
modified = source_order[parent_path][1]
if parent is None or modified > parent_modified:
+ # Only a replacement head can replay without the canonical draft.
+ if parent_context.get("canonical") not in source_digests:
+ parent_context.pop("canonical", None)
+ parent_context.pop("canonicalObservedAtNs", None)
parent = draft
+ parent_model_source = parent_path
frozen_parent_path = (
parent_path
if isinstance(draft.get("previousParentCheckpoints"), list)
@@ -930,6 +717,7 @@ def merge_saved_results(
parent_is_root = parent is not None
if parent is None and latest_reducer is not None:
parent = drafts_by_path[latest_reducer]
+ parent_model_source = latest_reducer
projected_coverage = saved_parent["coverage"] if saved_parent else {}
reviews = projected_coverage.get("reviews")
@@ -1138,24 +926,11 @@ def merge_saved_results(
and parent_manifest["scan"].get("sealedAt")
and parent_manifest["scan"].get("status") == binding["status"]
and parent_manifest["scan"].get("preservedSources") == source_digests
+ and parent_manifest["scan"].get("preservedCheckpointHeads", {}) == checkpoint_heads
and all(warning in initial_warnings for warning in warnings)
):
return None
- if (
- frozen_source_digests is None
- or any(_is_source_order_snapshot(path) for path in source_digests)
- or allow_frozen_legacy_parent
- ):
- _freeze_source_times(
- scan_dir,
- scan_id,
- source_digests,
- source_times,
- parent_context,
- created_paths=created_paths,
- )
-
target_kind = binding["allowedTargetKinds"][0]
if (
target_kind == "git_worktree"
@@ -1174,18 +949,20 @@ def merge_saved_results(
if parent_manifest
else {"scan": {"target": target, "scope": binding["scope"]}}
)
+ if allow_frozen_legacy_parent:
+ manifest["scan"]["target"] = target
for key in ("sealedAt", "artifacts"):
manifest["scan"].pop(key, None)
manifest["scan"]["preservedSources"] = source_digests
+ manifest["scan"]["preservedCheckpointHeads"] = checkpoint_heads
# Completion follows the selected parent, including legacy terminal drafts
# that omit the optional marker, rather than an older canonical manifest.
if parent is not None:
manifest["scan"].pop("complete", None)
if "complete" in parent:
manifest["scan"]["complete"] = parent["complete"]
- for key in ("scope", "threatModel"):
- if isinstance(parent.get(key), dict):
- manifest["scan"][key] = copy.deepcopy(parent[key])
+ if isinstance(parent.get("scope"), dict):
+ manifest["scan"]["scope"] = copy.deepcopy(parent["scope"])
if isinstance(manifest["scan"].get("scope"), dict):
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
model_checks: dict[str, dict[str, Any] | None] = {}
@@ -1214,12 +991,121 @@ def merge_saved_results(
model_checks[relative] = model
return model_checks[relative]
- if "threatModel" in manifest["scan"]:
- model = recovered_model("parent", manifest["scan"], None)
- if model is None:
- manifest["scan"].pop("threatModel")
+ parent_model = (
+ recovered_model(parent_model_source or "parent", parent, None)
+ if parent is not None and "threatModel" in parent
+ else None
+ )
+ canonical_model = manifest["scan"].pop("threatModel", canonical_model)
+ if parent_model is not None:
+ manifest["scan"]["threatModel"] = parent_model
+ if (
+ selected_model_source is not None
+ and parent_model_source in drafts_by_path
+ and (
+ parent_model_source != canonical_model_source
+ or canonical_model_digest is None
+ or source_digests[parent_model_source] == canonical_model_digest
+ )
+ ):
+ selected_model_source[:] = [parent_model_source]
+ elif frozen_model_source is None:
+ # Model fallback is independent of whether older findings are superseded.
+ if canonical_model is not None:
+ canonical_model = recovered_model("canonical", {"threatModel": canonical_model}, None)
+ if (
+ canonical_model_source is not None
+ and Path(canonical_model_source).parent != Path("checkpoints")
+ ) or (
+ canonical_model_source is None
+ and isinstance(canonical_model, dict)
+ and canonical_model.get("origin") == "recovered"
+ ):
+ # Worker models follow their own selected head and independent result.
+ canonical_model = None
+ canonical_model_source = None
+ if (
+ canonical_model_source in drafts_by_path
+ and recovered_model(
+ canonical_model_source, drafts_by_path[canonical_model_source], None
+ )
+ is not None
+ and (
+ canonical_model_digest is None
+ or source_digests[canonical_model_source] == canonical_model_digest
+ )
+ ):
+ canonical_model = drafts_by_path[canonical_model_source].get("threatModel")
+ canonical_model_order = max(
+ canonical_model_order or source_order[canonical_model_source],
+ source_order[canonical_model_source],
+ )
else:
- manifest["scan"]["threatModel"] = model
+ # A skipped checkpoint cannot supply provenance for valid canonical data.
+ canonical_model_source = None
+ root_models = [
+ path
+ for path in drafts_by_path
+ if path in selected_observations
+ and Path(path).parent == Path("checkpoints")
+ and isinstance(drafts_by_path[path].get("threatModel"), dict)
+ and recovered_model(path, drafts_by_path[path], None) is not None
+ ]
+ selected_root_models = [path for path in root_models if path in authoritative_heads]
+ if isinstance(canonical_model, dict) and canonical_model_order is None:
+ # Legacy publications have no model-source identity. Only newly admitted
+ # observations can supersede that fallback without an explicit selection.
+ new_models = {
+ f"checkpoints/{checkpoint}"
+ for head, checkpoint in saved_heads.items()
+ if _checkpoint_head_directory(head) == Path(".")
+ and head not in (canonical_sources or {})
+ }
+ root_models = [path for path in root_models if path in new_models]
+ model_source = max(
+ selected_root_models or root_models,
+ key=lambda path: source_order[path],
+ default=None,
+ )
+ if model_source is not None and (
+ selected_root_models
+ or not isinstance(canonical_model, dict)
+ or canonical_model_order is None
+ or source_order[model_source] > canonical_model_order
+ ):
+ manifest["scan"]["threatModel"] = copy.deepcopy(
+ drafts_by_path[model_source]["threatModel"]
+ )
+ if selected_model_source is not None:
+ selected_model_source[:] = [model_source]
+ elif isinstance(canonical_model, dict):
+ manifest["scan"]["threatModel"] = copy.deepcopy(canonical_model)
+ if stopped and frozen_source_digests is None and canonical_model_source is None:
+ # An unreadable canonical checkpoint must not leave a valid
+ # inherited model dependent on live files after the first stop.
+ model_draft = _model_capture(scan_id, canonical_model)
+ digest = _digest(model_draft)
+ canonical_model_source = f"checkpoints/{digest}.json"
+ observed = (canonical_model_order or (0, parent_modified))[1]
+ source_digests[canonical_model_source] = digest
+ source_times[canonical_model_source] = observed
+ try:
+ if not (scan_dir / canonical_model_source).exists():
+ if created_paths is not None and not os.path.lexists(
+ scan_dir / canonical_model_source
+ ):
+ created_paths.add(canonical_model_source)
+ write_scan_local_bytes(
+ scan_dir, canonical_model_source, _encoded(model_draft)
+ )
+ os.utime(scan_dir / canonical_model_source, ns=(observed, observed))
+ _, actual, _ = _read_saved_result(scan_dir, canonical_model_source, scan_id)
+ if actual != digest:
+ raise ContractError("Saved inherited model checkpoint changed.")
+ except (ContractError, OSError) as exc:
+ snapshot_error = snapshot_error or exc
+ if selected_model_source is not None and canonical_model_source is not None:
+ selected_model_source[:] = [canonical_model_source]
if frozen_model_source is not None:
# Publication retries retain a valid choice from the frozen source set.
@@ -1609,7 +1495,9 @@ def merge_saved_results(
try:
# A malformed update cannot discard valid saved evidence.
_validate_schema_node(
- item, coverage_schema["deferred"]["items"], "coverage.deferred"
+ {"id": identity, **item},
+ coverage_schema["deferred"]["items"],
+ "coverage.deferred",
)
except ContractError:
continue
@@ -1676,7 +1564,6 @@ def merge_saved_results(
or not isinstance(row.get("id"), str)
or (None, row["id"]) not in closed_deferred
]
- resolved: dict[tuple[str | None, str], str] = {}
ordered_candidates = {
(owner, identity)
@@ -1684,91 +1571,91 @@ def merge_saved_results(
if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
and (owner, identity) in candidate_ids
}
- ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
- pending_candidate_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
- outcomes: list[tuple[str, str | None, str, str, tuple[int, int]]] = []
- for relative, draft, owner in current_drafts:
- reported_candidates = {
- finding_candidate_id(finding)
- for finding in draft["findings"]
- if isinstance(finding, dict) and valid_finding(finding)
+ outcomes, pending_candidate_orders = _saved_candidate_outcomes(
+ scan_dir,
+ current_drafts,
+ valid_finding,
+ coverage_candidate,
+ candidate_order,
+ workers_by_id,
+ coverage_schema,
+ )
+ ordered_candidates.update(pending_candidate_orders)
+
+ def select_outcomes(unusable_receipts):
+ permitted = [entry for entry in outcomes if not unusable_receipts.intersection(entry[-1])]
+ resolved, ordered_outcomes = {}, {}
+ candidates = ordered_candidates.union(
+ (owner, candidate_id)
+ for relative, owner, candidate_id, _, _, _ in permitted
+ if relative in selected_observations
+ )
+ selected_head_candidates = {
+ (Path(relative).parent, owner, candidate_id)
+ for relative, owner, candidate_id, _, _, _ in permitted
+ if relative in authoritative_heads
}
- pending_candidates = set()
- for field in ("deferred", "surfaces"):
- items = draft["coverage"].get(field, [])
- for item in items if isinstance(items, list) else []:
- if isinstance(item, dict) and (
- field == "deferred" or item.get("disposition") == "needs_follow_up"
- ):
- if (pending := coverage_candidate(owner, item)) is not None:
- if (
- owner is not None
- and pending[0] == owner
- and pending[1] in reported_candidates
- ):
- continue
- pending_candidates.add(pending)
- order = candidate_order(relative, pending[0], item)
- pending_candidate_orders[pending] = max(
- pending_candidate_orders.get(pending, order), order
- )
- for finding in draft["findings"]:
+ other_root_order = _unselected_root_order(saved_heads, checkpoint_heads, source_order)
+ for relative, owner, candidate_id, disposition, order, _ in permitted:
if (
- isinstance(finding, dict)
- and valid_finding(finding)
- and (candidate_id := finding_candidate_id(finding))
+ relative in selected_observations
+ and relative not in authoritative_heads
+ and relative not in current_results
+ and (Path(relative).parent, owner, candidate_id) in selected_head_candidates
):
- outcomes.append(
- (relative, owner, candidate_id, "reported", candidate_order(relative, owner))
+ continue
+ key = (owner, candidate_id)
+ if pending_surface_orders.get(key, (-1, -1)) >= order:
+ continue
+ if key not in candidates:
+ if relative == "parent" or relative in current_results:
+ resolved.setdefault(key, disposition)
+ continue
+ root_terminal = (
+ owner is None
+ and (relative == "parent" or relative in authoritative_heads)
+ and (parent if relative == "parent" else drafts_by_path[relative]).get("complete")
+ is not False
+ and other_root_order < order
+ )
+ pending_order = pending_candidate_orders.get(key, (-1, -1))
+ if (pending_order > order or (pending_order == order and not root_terminal)) or any(
+ (
+ coverage_candidate(saved_owner, row)
+ or (saved_owner, _deferred_candidate_id(row, saved_owner, ambiguous_deferred))
)
- for field in ("surfaces", "explicitExclusions"):
- for item in _coverage_rows(draft["coverage"], field):
- if (
- isinstance(item, dict)
- and isinstance(item.get("candidateId"), str)
- and item.get("disposition") in ("reported", "rejected", "not_applicable")
- and (candidate := coverage_candidate(owner, item)) is not None
- and (item["disposition"] != "reported" or candidate[1] in reported_candidates)
- and (candidate not in pending_candidates or item["disposition"] == "reported")
- ):
- outcomes.append(
- (
- relative,
- candidate[0],
- candidate[1],
- item["disposition"],
- candidate_order(relative, candidate[0], item),
- )
+ == key
+ and (
+ saved_order > order
+ or saved_order == order
+ and (
+ not root_terminal
+ or accepted_deferred_orders.get((saved_owner, identity)) == saved_order
)
- ordered_candidates.update(pending_candidate_orders)
- ordered_candidates.update(
- (owner, candidate_id)
- for relative, owner, candidate_id, _, _ in outcomes
- if owner is not None and relative in selected_observations
- )
- # Reopened work and selected checkpoint outcomes follow the saved source order.
- for relative, owner, candidate_id, disposition, order in outcomes:
- key = (owner, candidate_id)
- if pending_surface_orders.get(key, (-1, -1)) >= order:
- continue
- if key not in ordered_candidates:
- if relative == "parent" or relative in current_results:
- resolved.setdefault(key, disposition)
- continue
- if pending_candidate_orders.get(key, (-1, -1)) >= order or any(
- (
- coverage_candidate(saved_owner, row)
- or (saved_owner, _deferred_candidate_id(row, saved_owner, ambiguous_deferred))
- )
- == key
- and saved_order >= order
- for (saved_owner, _), (saved_order, row, _) in active_deferred.items()
- ):
- continue
- if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
- resolved[key] = disposition
- ordered_outcomes[key] = (order, relative)
+ )
+ for (saved_owner, identity), (saved_order, row, _) in active_deferred.items()
+ ):
+ continue
+ previous = ordered_outcomes.get(key)
+ if (
+ previous is None
+ or order > previous[0]
+ or (
+ order == previous[0]
+ and previous[1] != relative
+ and previous[1] in current_results
+ and (
+ relative in checkpoint_heads.values()
+ or relative in authoritative_observations
+ )
+ )
+ ):
+ resolved[key] = disposition
+ ordered_outcomes[key] = (order, relative)
+ return resolved, ordered_outcomes
+
+ resolved, ordered_outcomes = select_outcomes(set())
# Only the current parent may claim that another worker finding was absorbed.
# A superseded checkpoint must not suppress a newer independent result.
if parent:
@@ -1811,30 +1698,10 @@ def merge_saved_results(
represented_candidate_history.setdefault(candidate_key, set()).add(
_digest(_finding_content(original["finding"]))
)
- # Apply reconciled outcomes to coverage already copied from the parent too.
- for field in ("surfaces", "deferred"):
- rows = coverage.get(field)
- if isinstance(rows, list):
- coverage[field] = [
- row
- for row in rows
- if not (
- isinstance(row, dict)
- and (
- (field == "deferred" and record_key(field, None, row) in closed_deferred)
- or (
- coverage_candidate(None, row) in resolved
- and (field == "deferred" or row.get("disposition") == "needs_follow_up")
- )
- )
- )
- ]
-
- selected_terminal_orders: dict[str, tuple[int, int]] = {}
+ selected_terminal_orders: dict[str | None, tuple[int, int]] = {}
for relative, draft, worker_id in sources:
if (
- worker_id is not None
- and relative in selected_observations
+ relative in authoritative_heads
and draft.get("complete") is not False
and relative not in discarded_by_source
and relative not in discarded_coverage_sources
@@ -1855,117 +1722,44 @@ def merge_saved_results(
terminal_worker_orders[worker_id] = max(
terminal_worker_orders.get(worker_id, order), order
)
- if relative not in discarded_coverage_sources:
+ if relative not in reducer_paths and relative not in discarded_coverage_sources:
terminal_coverage_orders[worker_id] = max(
terminal_coverage_orders.get(worker_id, order), order
)
+ prior_terminal_orders: dict[str | None, tuple[int, int]] = {}
+ partial_head_orders = {
+ Path(path).parent: selected_observations[path]
+ for path in authoritative_heads
+ if drafts_by_path[path].get("complete") is False
+ }
+ for head, checkpoint in saved_heads.items():
+ directory = _checkpoint_head_directory(head)
+ relative = (directory / "checkpoints" / checkpoint).as_posix()
+ partial_order = partial_head_orders.get(Path(relative).parent)
+ if partial_order is None:
+ continue
+ order = (partial_order[0], source_order[head][1])
+ if (
+ drafts_by_path[relative].get("complete") is not False
+ and relative not in discarded_by_source
+ and relative not in discarded_coverage_sources
+ and order <= partial_order
+ ):
+ # A partial head adds coverage after the preceding terminal replacement.
+ worker_id = paths[relative]
+ prior_terminal_orders[worker_id] = max(
+ prior_terminal_orders.get(worker_id, order), order
+ )
+
# Reconcile raw records before generating context-dependent publication IDs.
logical_rows: dict[tuple[Any, ...], dict[str, dict[str, Any]]] = {}
row_groups: dict[tuple[str, int], dict[str, Any]] = {}
published_groups: list[dict[str, Any]] = []
- def identifiers(value: dict[str, Any]) -> tuple[Any, Any]:
- extensions = value.get("extensions")
- return tuple(
- identifier if _has_identity_text(identifier) else None
- for identifier in (
- extensions.get(field) if isinstance(extensions, dict) else None
- for field in ("reportId", "ledgerRowId")
- )
- )
-
- def distinct_reports(left: dict[str, Any] | None, right: dict[str, Any] | None) -> bool:
- return (
- left is not None
- and right is not None
- and any(
- a is not None and b is not None and a != b
- for a, b in zip(
- identifiers(left["latest"]), identifiers(right["latest"]), strict=True
- )
- )
- )
-
- def raw_owner(value: dict[str, Any], source_owner: str | None) -> str | None:
- if source_owner is not None:
- return source_owner
- provenance = value.get("provenance", {})
- owner = provenance.get("workerId")
- return owner if isinstance(owner, str) else source_owner
-
- def raw_scope(value: dict[str, Any]) -> tuple[Any, ...]:
- # Match scanFindingIdentity's primary location for anonymous reports.
- location = value["locations"][0]
- return (
- value.get("ruleId"),
- location["path"],
- location["startLine"],
- location.get("endLine"),
- )
-
def same_owner(left: str | None, right: str | None) -> bool:
# Canonical recovery needs identity or recorded lineage to absorb worker evidence.
return left == right or (not preserve_canonical_parent and (left is None or right is None))
- def same_raw_finding(left: dict[str, Any], right: dict[str, Any]) -> bool:
- if isinstance(left.get("identity"), dict) and isinstance(right.get("identity"), dict):
- return _semantic_identity_key(left, _finding_identity(left)) == _semantic_identity_key(
- right, _finding_identity(right)
- )
- candidates = (_identity_candidate(left), _identity_candidate(right))
- if all(candidates) and candidates[0] != candidates[1]:
- return False
- pairs = list(zip(identifiers(left), identifiers(right), strict=True))
- if any(a is not None for a, _ in pairs) and any(b is not None for _, b in pairs):
- if not any(a is not None and b is not None for a, b in pairs) or any(
- a is not None and b is not None and a != b for a, b in pairs
- ):
- return False
- return True
-
- def same_raw_value(left: Any, right: Any) -> bool:
- if isinstance(left, dict):
- return (
- isinstance(right, dict)
- and left.keys() == right.keys()
- and all(same_raw_value(value, right[key]) for key, value in left.items())
- )
- if isinstance(left, list):
- return (
- isinstance(right, list)
- and len(left) == len(right)
- and all(same_raw_value(a, b) for a, b in zip(left, right, strict=True))
- )
- # JSON numbers compare by value, but booleans are a distinct scalar type.
- return isinstance(left, bool) == isinstance(right, bool) and left == right
-
- def same_raw_content(left: dict[str, Any], right: dict[str, Any]) -> bool:
- def content(value: dict[str, Any]) -> dict[str, Any]:
- result = _finding_content(value)
- if isinstance(result.get("extensions"), dict):
- result["extensions"] = {
- key: value
- for key, value in result["extensions"].items()
- if key not in {"candidateId", "reportId", "ledgerRowId"}
- }
- if not result["extensions"]:
- result.pop("extensions")
- return result
-
- return same_raw_value(content(left), content(right))
-
- def contains_raw_value(current: Any, previous: Any) -> bool:
- if isinstance(previous, dict):
- return isinstance(current, dict) and all(
- key in current and contains_raw_value(current[key], value)
- for key, value in previous.items()
- )
- if isinstance(previous, list):
- return isinstance(current, list) and all(
- any(contains_raw_value(item, value) for item in current) for value in previous
- )
- return isinstance(current, bool) == isinstance(previous, bool) and current == previous
-
def finding_source_order(source, within_worker: bool = False):
relative, draft, _ = source
attempt, modified = source_order[relative]
@@ -2407,11 +2201,33 @@ def merge_saved_results(
worker_id: str | None,
) -> tuple[bool, set[str], bool, bool, bool]:
worker_result_order = terminal_worker_orders.get(worker_id)
- selected_coverage_superseded = worker_id in selected_terminal_orders and (
- source_order[relative] < selected_terminal_orders[worker_id]
- or (
- relative in current_results
- and source_order[relative] <= selected_terminal_orders[worker_id]
+ selected_terminal_order = selected_terminal_orders.get(worker_id)
+ terminal_coverage_order = max(
+ (
+ order
+ for order in (
+ terminal_coverage_orders.get(worker_id),
+ selected_terminal_order,
+ prior_terminal_orders.get(worker_id),
+ )
+ if order is not None
+ ),
+ default=None,
+ )
+ selected_coverage_superseded = (
+ terminal_coverage_order is not None and source_order[relative] < terminal_coverage_order
+ ) or (
+ selected_terminal_order is not None
+ and source_order[relative][0] == selected_terminal_order[0]
+ and (
+ relative in selected_observations
+ and relative not in authoritative_heads
+ and relative not in current_results
+ or (
+ relative in current_results
+ and relative not in authoritative_heads
+ and source_order[relative] <= selected_terminal_order
+ )
)
)
selected_candidates = {
@@ -2567,193 +2383,7 @@ def merge_saved_results(
and not _has_candidate_metadata(row)
)
- fallback_surfaces = projection.fallback_surfaces(
- coverage,
- all_sources,
- deferred_rows,
- reopened_rows,
- selection_for_source=source_coverage_selection,
- row_is_retained=coverage_row_is_retained,
- replaced_deferred=replaced_deferred,
- closed_deferred=closed_deferred,
- ambiguous_surface_ids=ambiguous_surface_ids,
- deferred_is_resolved=lambda owner, row: (
- (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) in resolved
- ),
- )
-
- replaced_surfaces, surface_updates = _generic_surface_updates(
- all_sources,
- closed_deferred,
- active_deferred,
- resolved,
- reopened_generic,
- coverage_schema["surfaces"]["items"],
- deferred_rows,
- ambiguous_deferred,
- record_source,
- candidate_order,
- source_surface_receipts,
- canonical_rows=canonical_rows,
- coverage_sources=coverage_sources,
- receipt_root=scan_dir if project_worker_coverage else None,
- receipt_warnings=warnings,
- fallback_surfaces=fallback_surfaces,
- surface_is_retained=lambda relative, owner, row: coverage_row_is_retained(
- "surfaces", row, owner, source_coverage_selection(relative, owner)
- ),
- eligible_receipt_sources={
- relative
- for relative, draft, _ in all_sources
- if draft.get("complete") is not False or relative in selected_observations
- },
- )
- replaced_rows = {
- "surfaces": replaced_surfaces,
- "deferred": {
- id(row)
- for relative, _, owner in all_sources
- for row in deferred_rows[relative]
- if isinstance(row, dict)
- and isinstance(identity := row.get("id"), str)
- and (key := record_key("deferred", owner, row)) not in candidate_ids
- and key not in ambiguous_deferred
- and (updated := accepted_deferred_orders.get(key)) is not None
- and updated > candidate_order(relative, key[0], row)
- },
- }
- for field, replaced in replaced_rows.items():
- if parent and isinstance(coverage.get(field), list):
- previous = parent["coverage"].get(field, [])
- if isinstance(previous, list):
- removed_parent = [row for row in previous if id(row) in replaced]
- coverage[field] = [row for row in coverage[field] if row not in removed_parent]
- if isinstance(coverage.get("surfaces"), list):
- for relative, owner, source_surface, surface in surface_updates:
- original_surface = surface
- # Generic updates already resolve each receipt at its saved source.
- surface = projection.repair_reviewed_surface(
- coverage,
- projected_coverage if parent_is_canonical else {},
- workers_by_id.get(owner),
- source_surface,
- surface,
- ) or project_record("surfaces", surface, source_surface, receipts_resolved=True)
- reconciled_surface_ids[record_key("surfaces", owner, source_surface)] = surface["id"]
- if surface not in coverage["surfaces"] and (
- surface is original_surface or retained_coverage_record("surfaces", surface) is None
- ):
- retain_coverage_row(
- coverage["surfaces"],
- surface,
- owner,
- "surfaces",
- source_surface,
- surface is not original_surface or relative == "parent",
- )
- for surface in coverage["surfaces"]:
- if isinstance(surface, dict):
- surface.setdefault("receiptRefs", [])
-
- # Reopened work survives a superseded checkpoint, but current candidate
- # outcomes still apply. Parent closures cannot remove another worker's row.
- for owner, item in reopened_rows:
- source_owner, original = record_source("deferred", owner, item)
- if (
- identity := _deferred_candidate_id(original, source_owner, ambiguous_deferred)
- ) is not None and (
- source_owner,
- identity,
- ) in resolved:
- continue
- original_item = item
- item = project_record("deferred", item)
- pending = coverage.setdefault("deferred", [])
- if (
- isinstance(pending, list)
- and item not in pending
- and (item is original_item or retained_coverage_record("deferred", item) is None)
- ):
- retain_coverage_row(
- pending, item, owner, "deferred", original_item, item is not original_item
- )
- for _, draft, owner in all_sources:
- for surface in _coverage_rows(draft["coverage"], "surfaces"):
- if (
- isinstance(surface, dict)
- and ambiguous_pending_surface(surface, owner)
- and isinstance(coverage.get("surfaces"), list)
- ):
- original_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
- retained_surface = project_record("surfaces", original_surface, surface)
- if retained_surface not in coverage["surfaces"] and (
- retained_surface is original_surface
- or retained_coverage_record("surfaces", retained_surface) is None
- ):
- retain_coverage_row(
- coverage["surfaces"],
- retained_surface,
- owner,
- "surfaces",
- surface,
- retained_surface is not original_surface,
- )
- for relative, draft, worker_id in all_sources:
- selection = source_coverage_selection(relative, worker_id)
- (
- selected_coverage_superseded,
- selected_candidates,
- superseded,
- retain_pending,
- coverage_superseded,
- ) = selection
- skip_superseded_findings = (
- superseded
- and not stopped
- and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
- )
- if (
- skip_superseded_findings
- and coverage_superseded
- and not selected_candidates
- and not retain_pending
- ):
- continue
- if (
- not skip_superseded_findings
- and "threatModel" not in manifest["scan"]
- and isinstance(draft.get("threatModel"), dict)
- ):
- model = recovered_model(relative, draft, worker_id)
- model_path = relative
- checkpoint_dir = Path(relative).parent
- prefer_worker_head = worker_id is not None and (
- checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
- )
- if worker_id is not None:
- if checkpoint_dir.name != "checkpoints":
- checkpoint_dir /= "checkpoints"
- selected_models = [
- path
- for path in selected_observations
- if Path(path).parent == checkpoint_dir
- and isinstance(drafts_by_path[path].get("threatModel"), dict)
- and recovered_model(path, drafts_by_path[path], worker_id) is not None
- ]
- if selected_models:
- head_path = max(selected_models, key=source_order.__getitem__)
- current = drafts_by_path[head_path]
- # A terminal checkpoint is committed before result.json is replaced.
- # Use the admitted observation, including its frozen ordering on retries.
- if not prefer_worker_head and current.get("complete") is not False:
- prefer_worker_head = source_order[head_path] >= source_order[relative]
- if prefer_worker_head:
- model = recovered_model(head_path, current, worker_id)
- model_path = head_path
- if model is not None:
- manifest["scan"]["threatModel"] = model
- if selected_model_source is not None and worker_id is not None:
- selected_model_source[:] = [model_path]
+ def retain_findings(relative, draft, worker_id, selected_candidates, skip_superseded_findings):
normalized_findings = copy.deepcopy(draft["findings"])
_ensure_finding_identities(
[finding for finding in normalized_findings if valid_finding(finding)]
@@ -2979,118 +2609,493 @@ def merge_saved_results(
continue
finding_positions[key] = (len(findings), group)
findings.append(finding)
- worker = workers_by_id.get(worker_id)
- # Only the accepted current output belongs to this worker attempt. An
- # unsuperseded archive can remain evidence after that output is lost.
- project_coverage = (
- worker is not None
- and worker["status"] == "succeeded"
- and worker["result_manifest_path"] == str(scan_dir / relative)
- and draft.get("complete") is not False
+
+ competing_receipts = _competing_rejection_receipts(
+ outcomes, all_sources, source_surface_receipts
+ )
+ check_outcome_receipts = bool(competing_receipts)
+ dry_run = check_outcome_receipts
+ unavailable = set()
+ if dry_run:
+ initial_coverage = coverage
+ initial_canonical_rows = canonical_rows
+ initial_sources = coverage_sources
+ initial_surface_rows = retained_surface_rows
+ initial_ids = surface_ids_by_source
+ initial_projection = projection
+ output_warnings = warnings
+
+ def retain_surface_links(owner, row):
+ if isinstance(row, dict):
+ links = row.get("surfaceIds", [])
+ for identity in [row.get("id"), *(links if isinstance(links, list) else [])]:
+ if isinstance(identity, str):
+ pending_surface_ids.add((owner, identity))
+
+ def surface_is_retained(relative: str, owner: str | None, row: Any) -> bool:
+ selection = source_coverage_selection(relative, owner)
+ if (
+ isinstance(row, dict)
+ and isinstance(row.get("id"), str)
+ and (owner, row["id"]) in pending_surface_ids
+ ):
+ selection = (False, *selection[1:])
+ return coverage_row_is_retained("surfaces", row, owner, selection)
+
+ while True:
+ if check_outcome_receipts:
+ warnings = [] if dry_run else output_warnings
+ coverage = copy.deepcopy(initial_coverage)
+ canonical_rows = initial_canonical_rows.copy()
+ coverage_sources = {
+ identity: (row, paths.copy()) for identity, (row, paths) in initial_sources.items()
+ }
+ for field in ("surfaces", "deferred", "explicitExclusions"):
+ for initial, current in zip(
+ _coverage_rows(initial_coverage, field),
+ _coverage_rows(coverage, field),
+ strict=True,
+ ):
+ if id(initial) in initial_canonical_rows:
+ canonical_rows.add(id(current))
+ _inherit_coverage_sources(current, [initial], coverage_sources)
+ retained_surface_rows = initial_surface_rows.copy()
+ surface_ids_by_source = {key: ids.copy() for key, ids in initial_ids.items()}
+ reconciled_surface_ids.clear()
+ projection = type(initial_projection)(
+ initial_projection.project_worker_coverage,
+ initial_projection.legacy_surface_links,
+ sources,
+ )
+ # Apply reconciled outcomes to coverage already copied from the parent too.
+ for field in ("surfaces", "deferred"):
+ rows = coverage.get(field)
+ if isinstance(rows, list):
+ coverage[field] = [
+ row
+ for row in rows
+ if not (
+ isinstance(row, dict)
+ and (
+ (
+ field == "deferred"
+ and record_key(field, None, row) in closed_deferred
+ )
+ or (
+ coverage_candidate(None, row) in resolved
+ and (
+ field == "deferred"
+ or row.get("disposition") == "needs_follow_up"
+ )
+ )
+ )
+ )
+ ]
+
+ # Reserve receipts for supporting surfaces before repairing historical refs.
+ prospective_tasks = []
+ for relative, _, owner in all_sources:
+ selection = source_coverage_selection(relative, owner)
+ for row in deferred_rows[relative]:
+ if (
+ coverage_row_is_retained("deferred", row, owner, selection)
+ and id(row) not in replaced_deferred
+ and not (
+ isinstance(row, dict)
+ and isinstance(row.get("id"), str)
+ and record_key("deferred", owner, row) in closed_deferred
+ )
+ ):
+ prospective_tasks.append((owner, row))
+ prospective_tasks.extend(
+ (owner, row)
+ for owner, row in reopened_rows
+ if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
)
- if coverage_superseded and not selected_candidates and not retain_pending:
- continue
- if project_coverage:
- reviewed_attempts.add((worker_id, worker["attempt"]))
- reviews = coverage.setdefault("reviews", [])
- if isinstance(reviews, list) and not any(
- isinstance(review, dict)
- and review.get("workerId") == worker_id
- and review.get("attempt") == worker["attempt"]
- for review in reviews
+ pending_surface_ids = set()
+
+ for owner, row in prospective_tasks:
+ retain_surface_links(owner, row)
+ if parent and isinstance(coverage.get("deferred"), list):
+ previous = parent["coverage"].get("deferred", [])
+ removed_parent = (
+ [row for row in previous if id(row) in replaced_deferred]
+ if isinstance(previous, list)
+ else []
+ )
+ for row in coverage["deferred"]:
+ if row not in removed_parent:
+ retain_surface_links(None, row)
+
+ fallback_surfaces = projection.fallback_surfaces(
+ coverage,
+ all_sources,
+ prospective_tasks,
+ surface_is_retained=surface_is_retained,
+ ambiguous_surface_ids=ambiguous_surface_ids,
+ )
+
+ replaced_surfaces, surface_updates = _generic_surface_updates(
+ all_sources,
+ closed_deferred,
+ active_deferred,
+ resolved,
+ reopened_generic,
+ coverage_schema["surfaces"]["items"],
+ deferred_rows,
+ ambiguous_deferred,
+ record_source,
+ candidate_order,
+ source_surface_receipts,
+ canonical_rows=canonical_rows,
+ coverage_sources=coverage_sources,
+ receipt_root=scan_dir if project_worker_coverage else None,
+ receipt_warnings=warnings,
+ fallback_surfaces=fallback_surfaces,
+ surface_is_retained=surface_is_retained,
+ eligible_receipt_sources={
+ relative
+ for relative, draft, _ in all_sources
+ if draft.get("complete") is not False or relative in selected_observations
+ },
+ )
+ replaced_rows = {
+ "surfaces": replaced_surfaces,
+ "deferred": {
+ id(row)
+ for relative, _, owner in all_sources
+ for row in deferred_rows[relative]
+ if isinstance(row, dict)
+ and isinstance(identity := row.get("id"), str)
+ and (key := record_key("deferred", owner, row)) not in candidate_ids
+ and key not in ambiguous_deferred
+ and (updated := accepted_deferred_orders.get(key)) is not None
+ and updated > candidate_order(relative, key[0], row)
+ },
+ }
+ for field, replaced in replaced_rows.items():
+ if parent and isinstance(coverage.get(field), list):
+ previous = parent["coverage"].get(field, [])
+ if isinstance(previous, list):
+ removed_parent = [row for row in previous if id(row) in replaced]
+ coverage[field] = [row for row in coverage[field] if row not in removed_parent]
+ if isinstance(coverage.get("surfaces"), list):
+ for relative, owner, source_surface, surface in surface_updates:
+ original_surface = surface
+ # Generic updates already resolve each receipt at its saved source.
+ surface = projection.repair_reviewed_surface(
+ coverage,
+ projected_coverage if parent_is_canonical else {},
+ workers_by_id.get(owner),
+ source_surface,
+ surface,
+ ) or project_record("surfaces", surface, source_surface, receipts_resolved=True)
+ reconciled_surface_ids[record_key("surfaces", owner, source_surface)] = surface[
+ "id"
+ ]
+ if surface not in coverage["surfaces"] and (
+ surface is original_surface
+ or retained_coverage_record("surfaces", surface) is None
+ ):
+ retain_coverage_row(
+ coverage["surfaces"],
+ surface,
+ owner,
+ "surfaces",
+ source_surface,
+ surface is not original_surface or relative == "parent",
+ )
+ for surface in coverage["surfaces"]:
+ if isinstance(surface, dict):
+ surface.setdefault("receiptRefs", [])
+
+ # Reopened work survives a superseded checkpoint, but current candidate
+ # outcomes still apply. Parent closures cannot remove another worker's row.
+ for owner, item in reopened_rows:
+ source_owner, original = record_source("deferred", owner, item)
+ if (
+ identity := _deferred_candidate_id(original, source_owner, ambiguous_deferred)
+ ) is not None and (
+ source_owner,
+ identity,
+ ) in resolved:
+ continue
+ original_item = item
+ item = project_record("deferred", item)
+ pending = coverage.setdefault("deferred", [])
+ if (
+ isinstance(pending, list)
+ and item not in pending
+ and (item is original_item or retained_coverage_record("deferred", item) is None)
):
- reviews.append(
- {
- "workerId": worker_id,
- "attempt": worker["attempt"],
- "completeness": draft["coverage"].get("completeness", "partial"),
- }
+ retain_coverage_row(
+ pending, item, owner, "deferred", original_item, item is not original_item
)
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"):
- if field == "reviews" and (worker_id is not None or field not in draft["coverage"]):
- continue
- if coverage_superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
- continue
- items = draft["coverage"].get(field, [])
- if field == "deferred":
- items = deferred_rows[relative]
- if not isinstance(items, list):
+ for _, draft, owner in all_sources:
+ for surface in _coverage_rows(draft["coverage"], "surfaces"):
+ if (
+ isinstance(surface, dict)
+ and ambiguous_pending_surface(surface, owner)
+ and isinstance(coverage.get("surfaces"), list)
+ ):
+ original_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
+ retained_surface = project_record("surfaces", original_surface, surface)
+ if retained_surface not in coverage["surfaces"] and (
+ retained_surface is original_surface
+ or retained_coverage_record("surfaces", retained_surface) is None
+ ):
+ retain_coverage_row(
+ coverage["surfaces"],
+ retained_surface,
+ owner,
+ "surfaces",
+ surface,
+ retained_surface is not original_surface,
+ )
+ for relative, draft, worker_id in all_sources:
+ selection = source_coverage_selection(relative, worker_id)
+ (
+ selected_coverage_superseded,
+ selected_candidates,
+ superseded,
+ retain_pending,
+ coverage_superseded,
+ ) = selection
+ skip_superseded_findings = (
+ superseded
+ and not stopped
+ and all(
+ valid_finding(finding) for finding in (parent["findings"] if parent else [])
+ )
+ )
+ if (
+ skip_superseded_findings
+ and coverage_superseded
+ and not selected_candidates
+ and not retain_pending
+ ):
continue
- output = coverage.setdefault(field, [])
- if not isinstance(output, list):
- # Keep malformed canonical collections for the existing finalizer's
- # recovery and warnings rather than silently changing its contract.
+ if (
+ not dry_run
+ and not skip_superseded_findings
+ and "threatModel" not in manifest["scan"]
+ and isinstance(draft.get("threatModel"), dict)
+ ):
+ model = recovered_model(relative, draft, worker_id)
+ model_path = relative
+ checkpoint_dir = Path(relative).parent
+ root_checkpoint = checkpoint_dir == Path("checkpoints")
+ prefer_checkpoint_head = root_checkpoint or (
+ worker_id is not None
+ and (checkpoint_dir.name == "checkpoints" or draft.get("complete") is False)
+ )
+ if worker_id is not None or root_checkpoint:
+ if checkpoint_dir.name != "checkpoints":
+ checkpoint_dir /= "checkpoints"
+ selected_models = [
+ path
+ for path in drafts_by_path
+ if (
+ path in selected_observations
+ or (
+ path in current_results
+ and drafts_by_path[path].get("complete") is not False
+ )
+ )
+ and (
+ Path(path).parent == checkpoint_dir
+ or (
+ path in current_results
+ and Path(path).parent / "checkpoints" == checkpoint_dir
+ )
+ )
+ and isinstance(drafts_by_path[path].get("threatModel"), dict)
+ and recovered_model(path, drafts_by_path[path], worker_id) is not None
+ ]
+ if selected_models:
+ authoritative_models = [
+ path
+ for path in selected_models
+ if path in authoritative_heads or path in current_results
+ ]
+ if not any(path in authoritative_heads for path in selected_models):
+ authoritative_models = selected_models
+ head_path = max(
+ authoritative_models or selected_models,
+ key=lambda path: (source_order[path], path in authoritative_heads),
+ )
+ current = drafts_by_path[head_path]
+ # A terminal checkpoint is committed before result.json is replaced.
+ # Use the admitted observation, including its frozen ordering on retries.
+ if not prefer_checkpoint_head and current.get("complete") is not False:
+ prefer_checkpoint_head = (
+ source_order[head_path] >= source_order[relative]
+ )
+ if prefer_checkpoint_head:
+ model = recovered_model(head_path, current, worker_id)
+ model_path = head_path
+ if model is not None:
+ manifest["scan"]["threatModel"] = model
+ if selected_model_source is not None and model_path in source_digests:
+ selected_model_source[:] = [model_path]
+ if not dry_run:
+ retain_findings(
+ relative, draft, worker_id, selected_candidates, skip_superseded_findings
+ )
+ worker = workers_by_id.get(worker_id)
+ # Only the accepted current output belongs to this worker attempt. An
+ # unsuperseded archive can remain evidence after that output is lost.
+ project_coverage = (
+ worker is not None
+ and worker["status"] == "succeeded"
+ and worker["result_manifest_path"] == str(scan_dir / relative)
+ and draft.get("complete") is not False
+ )
+ if coverage_superseded and not selected_candidates and not retain_pending:
continue
- for index, item in enumerate(items, 1):
- source_item = item
- if not coverage_row_is_retained(field, item, worker_id, selection):
+ if project_coverage:
+ reviewed_attempts.add((worker_id, worker["attempt"]))
+ reviews = coverage.setdefault("reviews", [])
+ if isinstance(reviews, list) and not any(
+ isinstance(review, dict)
+ and review.get("workerId") == worker_id
+ and review.get("attempt") == worker["attempt"]
+ for review in reviews
+ ):
+ reviews.append(
+ {
+ "workerId": worker_id,
+ "attempt": worker["attempt"],
+ "completeness": draft["coverage"].get("completeness", "partial"),
+ }
+ )
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"):
+ if field == "reviews" and (worker_id is not None or field not in draft["coverage"]):
continue
- if id(item) in replaced_rows.get(field, ()):
+ if coverage_superseded and field not in {
+ "surfaces",
+ "explicitExclusions",
+ "deferred",
+ }:
continue
- if (
- field == "deferred"
- and isinstance(item, dict)
- and isinstance(item.get("id"), str)
- and record_key("deferred", worker_id, item) in closed_deferred
- ):
+ items = draft["coverage"].get(field, [])
+ if field == "deferred":
+ items = deferred_rows[relative]
+ if not isinstance(items, list):
continue
- if field == "openQuestions" and isinstance(item, str):
- item = {"question": item.strip()}
- if (
- field == "surfaces"
- and isinstance(item, dict)
- and item.get("disposition") in ("rejected", "not_applicable")
- and isinstance(item.get("candidateId"), str)
- and (history_findings := rejected_history.get((worker_id, item["candidateId"])))
- ):
- item = copy.deepcopy(item)
- for finding in history_findings:
- _append_previous_finding(item, finding, clone=True)
- projected_row = False
- if isinstance(item, dict):
- original_item = item
- item = (
- project_missing_record(field, item, index, worker, draft["coverage"])
- if project_coverage
- else project_record(field, item, source_item)
- )
- projected_row = item is not original_item
- if field == "surfaces" and isinstance(item, dict):
- item = {**item, "receiptRefs": item.get("receiptRefs", [])}
- if (
- field == "surfaces"
- and worker is not None
- and isinstance(item, dict)
- and (project_coverage or not projected_row)
- ):
- item = copy.deepcopy(item)
- item["receiptRefs"] = coverage_receipts(item, worker, relative)
- if (
- projected_row
- and isinstance(item, dict)
- and retained_coverage_record(field, item) is not None
- ):
+ output = coverage.setdefault(field, [])
+ if not isinstance(output, list):
+ # Keep malformed canonical collections for the existing finalizer's
+ # recovery and warnings rather than silently changing its contract.
continue
- if isinstance(item, dict) and "id" not in item:
- semantic_item = dict(item)
- existing = next(
- (
- existing
- for existing in output
- if isinstance(existing, dict)
- and {key: value for key, value in existing.items() if key != "id"}
- == semantic_item
- ),
- None,
- )
- if existing is not None:
- _inherit_coverage_sources(existing, [source_item], coverage_sources)
- if field == "surfaces":
- retained_surface_rows.add(id(source_item))
+ for index, item in enumerate(items, 1):
+ source_item = item
+ if not (
+ surface_is_retained(relative, worker_id, item)
+ if field == "surfaces"
+ else coverage_row_is_retained(field, item, worker_id, selection)
+ ):
continue
- retain_coverage_row(
- output, item, worker_id, field, source_item, projected_row or project_coverage
- )
+ if id(item) in replaced_rows.get(field, ()):
+ continue
+ if (
+ field == "deferred"
+ and isinstance(item, dict)
+ and isinstance(item.get("id"), str)
+ and record_key("deferred", worker_id, item) in closed_deferred
+ ):
+ continue
+ if field == "openQuestions" and isinstance(item, str):
+ item = {"question": item.strip()}
+ if (
+ field == "surfaces"
+ and isinstance(item, dict)
+ and item.get("disposition") in ("rejected", "not_applicable")
+ and isinstance(item.get("candidateId"), str)
+ and (
+ history_findings := rejected_history.get(
+ (worker_id, item["candidateId"])
+ )
+ )
+ ):
+ item = copy.deepcopy(item)
+ for finding in history_findings:
+ _append_previous_finding(item, finding, clone=True)
+ projected_row = False
+ if isinstance(item, dict):
+ original_item = item
+ item = (
+ project_missing_record(field, item, index, worker, draft["coverage"])
+ if project_coverage
+ else project_record(field, item, source_item)
+ )
+ projected_row = item is not original_item
+ if field == "surfaces" and isinstance(item, dict):
+ item = {**item, "receiptRefs": item.get("receiptRefs", [])}
+ if (
+ field == "surfaces"
+ and worker is not None
+ and isinstance(item, dict)
+ and (project_coverage or not projected_row)
+ ):
+ item = copy.deepcopy(item)
+ item["receiptRefs"] = coverage_receipts(item, worker, relative)
+ if (
+ projected_row
+ and isinstance(item, dict)
+ and retained_coverage_record(field, item) is not None
+ ):
+ continue
+ if isinstance(item, dict) and "id" not in item:
+ semantic_item = dict(item)
+ existing = next(
+ (
+ existing
+ for existing in output
+ if isinstance(existing, dict)
+ and {key: value for key, value in existing.items() if key != "id"}
+ == semantic_item
+ ),
+ None,
+ )
+ if existing is not None:
+ _inherit_coverage_sources(existing, [source_item], coverage_sources)
+ if field == "surfaces":
+ retained_surface_rows.add(id(source_item))
+ continue
+ retain_coverage_row(
+ output,
+ item,
+ worker_id,
+ field,
+ source_item,
+ projected_row or project_coverage,
+ )
+
+ projection.finalize(
+ coverage,
+ canonical_rows,
+ _digest,
+ preserve_canonical_parent=preserve_canonical_parent,
+ all_sources=all_sources,
+ source_order=source_order,
+ coverage_sources=coverage_sources,
+ surface_ids_by_source=surface_ids_by_source,
+ retained_surface_rows=retained_surface_rows,
+ resolved=resolved,
+ )
+ if not dry_run:
+ break
+ invalid = _conflicting_outcome_receipts(
+ coverage,
+ scan_dir,
+ competing_receipts - unavailable,
+ )
+ if invalid:
+ unavailable.update(invalid)
+ resolved, ordered_outcomes = select_outcomes(unavailable)
+ else:
+ dry_run = False
# Generate new worker identities from the retained collection, as a normal
# Deep parent does. Authored, restored and canonical identities stay stable.
@@ -3165,18 +3170,6 @@ def merge_saved_results(
break
suffix += 1
identities.setdefault(key, []).append((variant, group, identity.get("instance")))
- projection.finalize(
- coverage,
- canonical_rows,
- _digest,
- preserve_canonical_parent=preserve_canonical_parent,
- all_sources=all_sources,
- source_order=source_order,
- coverage_sources=coverage_sources,
- surface_ids_by_source=surface_ids_by_source,
- retained_surface_rows=retained_surface_rows,
- resolved=resolved,
- )
coverage.pop("completeness", None)
if stopped:
if not isinstance(coverage.get("deferred"), list):
@@ -3188,4 +3181,38 @@ def merge_saved_results(
for surface in surfaces if isinstance(surfaces, list) else []:
if isinstance(surface, dict) and isinstance(surface.get("provenance"), dict):
surface["provenance"].pop("scanReceiptRefs", None)
+ if (
+ frozen_source_digests is None
+ or heads_snapshotted
+ or snapshot_error is not None
+ or any(_is_source_order_snapshot(path) for path in source_digests)
+ or allow_frozen_legacy_parent
+ ):
+ try:
+ if snapshot_error is not None:
+ raise snapshot_error
+ _freeze_source_times(
+ scan_dir,
+ scan_id,
+ source_digests,
+ source_times,
+ parent_context,
+ created_paths=created_paths,
+ )
+ except (ContractError, OSError) as exc:
+ failure = _CheckpointSnapshotError(
+ str(exc),
+ {
+ path: (digest, source_times[path])
+ for path, digest in source_digests.items()
+ if not _is_source_order_snapshot(path)
+ },
+ )
+ failure.parent_context = parent_context
+ failure.heads = checkpoint_heads
+ failure.documents = (manifest, {"findings": findings}, coverage)
+ if isinstance(manifest["scan"].get("threatModel"), dict):
+ failure.model_source = list(selected_model_source or [])
+ raise failure from exc
+
return manifest, {"findings": findings}, coverage
diff --git a/plugins/codex-security/scripts/workbench/retained_records.py b/plugins/codex-security/scripts/workbench/retained_records.py
new file mode 100644
index 000000000..562139eb3
--- /dev/null
+++ b/plugins/codex-security/scripts/workbench/retained_records.py
@@ -0,0 +1,627 @@
+"""Identify and recover records retained from saved scan evidence."""
+
+from __future__ import annotations
+
+import copy
+import re
+import unicodedata
+from collections import Counter
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from finalize_scan_contract import (
+ ContractError,
+ _has_identity_text,
+ _read_json,
+ _recover_unsealed_coverage,
+ _recover_unsealed_findings,
+ _require_safe_json_value,
+ _validate_current_findings,
+ _validate_schema_node,
+ finding_candidate_id,
+)
+from workbench.saved_coverage import _coverage_rows, _saved_coverage_id
+from workbench.saved_sources import _digest, _encoded
+from workbench_coverage_receipts import resolve_coverage_receipts
+
+_FINDING_IDENTITY_SCHEMA = _read_json(
+ Path(__file__).resolve().parents[2] / "schemas" / "findings.schema.json"
+)["properties"]["findings"]["items"]["properties"]["identity"]
+
+
+def _finding_identity(finding: dict[str, Any]) -> Any:
+ provenance = finding.get("provenance")
+ if isinstance(provenance, dict) and "preservedIdentity" in provenance:
+ try:
+ _validate_schema_node(
+ provenance["preservedIdentity"], _FINDING_IDENTITY_SCHEMA, "finding.identity"
+ )
+ except ContractError:
+ pass
+ else:
+ return provenance["preservedIdentity"]
+ return finding.get("identity")
+
+
+def _finding_key(finding: dict[str, Any], owner: str | None = None) -> str:
+ # Wording and evidence may improve between checkpoints; distinct source locations
+ # must not collide merely because two workers chose the same semantic identity.
+ provenance = finding.get("provenance")
+ identity = _finding_identity(finding)
+ if not isinstance(identity, dict):
+ normalized = dict(finding)
+ normalized.pop("identity", None)
+ _ensure_finding_identity(normalized)
+ identity = normalized["identity"]
+ owner = provenance.get("workerId", owner) if isinstance(provenance, dict) else owner
+ parts = [finding.get("ruleId"), identity, _finding_locations(finding)]
+ if owner is not None and finding_candidate_id(finding):
+ parts.append(owner)
+ return _digest(parts)
+
+
+def _finding_locations(finding: dict[str, Any]) -> list[tuple[Any, Any, Any]]:
+ locations = finding.get("locations", [])
+ return sorted(
+ (
+ (
+ location.get("path"),
+ location.get("startLine"),
+ location.get("endLine", location.get("startLine")),
+ )
+ for location in (locations if isinstance(locations, list) else [])
+ if isinstance(location, dict)
+ ),
+ key=_encoded,
+ )
+
+
+def _worker_candidate_key(
+ worker_id: str, candidate_id: str, finding: dict[str, Any]
+) -> tuple[str, str, Any, Any, Any]:
+ """Identify one worker-local candidate without merging unrelated locations."""
+ identity = _finding_identity(finding)
+ if not isinstance(identity, dict):
+ normalized = dict(finding)
+ _ensure_finding_identity(normalized)
+ identity = normalized.get("identity")
+ anchor = identity.get("anchor") if isinstance(identity, dict) else None
+ instance = identity.get("instance") if isinstance(identity, dict) else None
+ return worker_id, candidate_id, finding.get("ruleId"), anchor, instance
+
+
+def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
+ """Return substantive finding content without generated identity or provenance."""
+ return {
+ key: value
+ for key, value in finding.items()
+ if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
+ }
+
+
+def _identity_candidate(finding: dict[str, Any]) -> str | None:
+ return next(
+ (
+ value
+ for section in (finding.get("extensions"), finding.get("provenance"))
+ if isinstance(section, dict) and _has_identity_text(value := section.get("candidateId"))
+ ),
+ None,
+ )
+
+
+def _ensure_finding_identity(finding: Any) -> None:
+ _ensure_finding_identities([finding])
+
+
+def _semantic_identifier(value: str, fallback: str) -> str:
+ normalized = re.sub(r"[\u0300-\u036f]", "", unicodedata.normalize("NFKD", value))
+ identifier = re.sub(r"[^a-z0-9._/-]+", "-", normalized.lower())
+ return re.sub(r"^[^a-z0-9]+|-+$", "", identifier) or fallback
+
+
+def _semantic_identity_key(finding: dict[str, Any], identity: Any = None) -> bytes:
+ identity = finding.get("identity") if identity is None else identity
+ if isinstance(identity, dict):
+ identity = {key: identity.get(key) for key in ("anchor", "instance")}
+ return _encoded([finding.get("ruleId"), identity])
+
+
+def _ensure_finding_identities(findings: list[Any], *, deep: bool = False) -> None:
+ """Use the same candidate, anchor and sibling rules as draft publication."""
+ anchors: dict[int, Any] = {}
+ for index, finding in enumerate(findings):
+ if not isinstance(finding, dict):
+ continue
+ identity = finding.get("identity")
+ if "identity" in finding:
+ anchors[index] = identity.get("anchor") if isinstance(identity, dict) else None
+ continue
+ candidate = _identity_candidate(finding)
+ anchors[index] = _semantic_identifier(
+ candidate or str(finding.get("title") or "finding"), f"finding-{index + 1}"
+ )
+ counts = Counter(
+ _digest([findings[index].get("ruleId"), anchor]) for index, anchor in anchors.items()
+ )
+
+ authored = {
+ _semantic_identity_key(finding)
+ for finding in findings
+ if isinstance(finding, dict) and "identity" in finding
+ }
+ generated = []
+ for index, anchor in anchors.items():
+ finding = findings[index]
+ if "identity" in finding:
+ continue
+ identity = {"anchor": anchor}
+ extensions = finding.get("extensions")
+ sibling = next(
+ (
+ value
+ for field in ("reportId", "ledgerRowId")
+ if isinstance(extensions, dict)
+ and _has_identity_text(value := extensions.get(field))
+ ),
+ None,
+ )
+ if sibling or counts[_digest([finding.get("ruleId"), anchor])] > 1:
+ identity["instance"] = _semantic_identifier(
+ sibling or str(finding.get("title") or "finding"), f"finding-{index + 1}"
+ )
+ finding["identity"] = identity
+ generated.append(finding)
+
+ reserved = {
+ _semantic_identity_key(finding) for finding in findings if isinstance(finding, dict)
+ }
+ used = set() if deep else authored.copy()
+ generated_ids = {id(finding) for finding in generated}
+ for finding in findings if deep else generated:
+ key = _semantic_identity_key(finding)
+ if key in used or (id(finding) in generated_ids and key in authored):
+ identity = finding["identity"]
+ if id(finding) not in generated_ids:
+ finding.setdefault("provenance", {}).setdefault(
+ "preservedIdentity", copy.deepcopy(identity)
+ )
+ base_instance = identity.get("instance", "saved")
+ suffix = 2
+ while True:
+ identity["instance"] = f"{base_instance}-{suffix}"
+ key = _semantic_identity_key(finding)
+ if key not in reserved:
+ reserved.add(key)
+ break
+ suffix += 1
+ used.add(key)
+
+
+def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
+ """Yield canonical and historical findings without trusting candidate IDs."""
+ pending = [finding]
+ seen: set[int] = set()
+ while pending:
+ current = pending.pop()
+ marker = id(current)
+ if marker in seen:
+ continue
+ seen.add(marker)
+ yield current
+ provenance = current.get("provenance")
+ if not isinstance(provenance, dict):
+ continue
+ previous = provenance.get("previousFindings")
+ if isinstance(previous, list):
+ pending.extend(item for item in reversed(previous) if isinstance(item, dict))
+ sources = provenance.get("sourceFindings")
+ if isinstance(sources, list):
+ pending.extend(
+ source["finding"]
+ for source in reversed(sources)
+ if isinstance(source, dict) and isinstance(source.get("finding"), dict)
+ )
+
+
+def _append_previous_finding(
+ item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False
+) -> None:
+ if not isinstance(item.get("previousFindings"), list):
+ item["previousFindings"] = []
+ history = item["previousFindings"]
+ if not any(
+ isinstance(previous, dict)
+ and _finding_key(previous) == _finding_key(finding)
+ and _finding_content(previous) == _finding_content(finding)
+ for previous in history
+ ):
+ history.append(copy.deepcopy(finding) if clone else finding)
+
+
+def _recover_preserved_worker_source(
+ scan_dir, relative, scan_id, binding, worker_id, worker, draft, warnings
+):
+ discarded_findings = []
+ # Recover imported records without relaxing canonical parent validation.
+ if "complete" in draft and not isinstance(draft["complete"], bool):
+ draft["complete"] = False
+ warnings.append(f"{relative}: Invalid complete marker; retained as a progress checkpoint.")
+ schema_dir = Path(__file__).resolve().parents[2] / "schemas"
+ if "threatModel" in draft:
+ try:
+ model = draft["threatModel"]
+ if isinstance(model, dict):
+ model = {**model, "origin": "recovered"}
+ schema = _read_json(schema_dir / "scan-manifest.schema.json")
+ _validate_schema_node(
+ model,
+ schema["properties"]["scan"]["properties"]["threatModel"],
+ "threatModel",
+ schema,
+ )
+ _require_safe_json_value(model, "threatModel")
+ except ContractError as exc:
+ draft.pop("threatModel")
+ warnings.append(f"{relative}: Skipped malformed threat model: {exc}.")
+ directory = Path(relative).parent
+ if directory.name == "checkpoints":
+ directory = directory.parent
+ recovered_findings = []
+ for index, finding in enumerate(draft["findings"]):
+ needs_identity = isinstance(finding, dict) and "identity" not in finding
+ _ensure_finding_identity(finding)
+ recovered = {"scanId": scan_id, "findings": [finding]}
+ source_warnings: list[str] = []
+ discarded = _recover_unsealed_findings(
+ {"scan": {"id": scan_id, "target": binding["target"]}},
+ recovered,
+ schema_dir,
+ scan_dir / directory,
+ source_warnings,
+ )
+ if recovered["findings"]:
+ try:
+ _validate_current_findings(recovered, schema_dir)
+ except ContractError as exc:
+ warning = f"Skipped malformed finding: {exc}."
+ source_warnings.append(warning)
+ discarded.append(warning)
+ recovered["findings"] = []
+ if discarded:
+ discarded_findings.extend(discarded)
+ for recovered_finding in recovered["findings"]:
+ if needs_identity:
+ # Reconcile anonymous revisions before assigning sibling identities.
+ recovered_finding.pop("identity", None)
+ writeup = recovered_finding.pop("writeup", None)
+ if writeup is not None:
+ source_warnings.append(
+ "Retained worker writeup at "
+ f"{directory.as_posix()}/{writeup['reportPath']}; "
+ "the parent report renders this finding inline."
+ )
+ recovered_findings.extend(recovered["findings"])
+ warnings.extend(f"{relative} finding {index + 1}: {warning}" for warning in source_warnings)
+ draft["findings"] = recovered_findings
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
+ rows = draft["coverage"].get(field)
+ if not isinstance(rows, list):
+ continue
+ for index, row in enumerate(rows):
+ if field == "openQuestions" and isinstance(row, str):
+ row = rows[index] = {"question": row}
+ if not isinstance(row, dict):
+ continue
+ row["workerId"] = worker_id
+ if field == "surfaces":
+ row["receiptRefs"] = resolve_coverage_receipts(
+ scan_dir,
+ row,
+ worker,
+ relative,
+ )
+ return discarded_findings
+
+
+def _recover_preserved_worker_coverage(
+ scan_dir, relative, worker_id, draft, discarded_findings, warnings
+):
+ discarded_coverage = False
+ for field in ("surfaces", "explicitExclusions", "deferred"):
+ rows = draft["coverage"].get(field)
+ for row in rows if isinstance(rows, list) else []:
+ if isinstance(row, dict):
+ row.setdefault("id", _saved_coverage_id(row))
+ if field == "surfaces":
+ row.setdefault("receiptRefs", [])
+ source_warnings = []
+ discarded_fields = _recover_unsealed_coverage(
+ draft["coverage"],
+ Path(__file__).resolve().parents[2] / "schemas",
+ scan_dir,
+ source_warnings,
+ discarded_findings,
+ # Reconcile local IDs and valid outcomes against worker history below.
+ worker_draft=True,
+ )
+ if discarded_fields & {
+ "surfaces",
+ "explicitExclusions",
+ "deferred",
+ "resolvedDeferred",
+ }:
+ discarded_coverage = True
+ for row in draft["coverage"]["deferred"]:
+ row["workerId"] = worker_id
+ warnings.extend(f"{relative}: {warning}" for warning in source_warnings)
+ return discarded_coverage
+
+
+def identifiers(value: dict[str, Any]) -> tuple[Any, Any]:
+ extensions = value.get("extensions")
+ return tuple(
+ identifier if _has_identity_text(identifier) else None
+ for identifier in (
+ extensions.get(field) if isinstance(extensions, dict) else None
+ for field in ("reportId", "ledgerRowId")
+ )
+ )
+
+
+def distinct_reports(left: dict[str, Any] | None, right: dict[str, Any] | None) -> bool:
+ return (
+ left is not None
+ and right is not None
+ and any(
+ a is not None and b is not None and a != b
+ for a, b in zip(identifiers(left["latest"]), identifiers(right["latest"]), strict=True)
+ )
+ )
+
+
+def raw_owner(value: dict[str, Any], source_owner: str | None) -> str | None:
+ if source_owner is not None:
+ return source_owner
+ provenance = value.get("provenance", {})
+ owner = provenance.get("workerId")
+ return owner if isinstance(owner, str) else source_owner
+
+
+def raw_scope(value: dict[str, Any]) -> tuple[Any, ...]:
+ # Match scanFindingIdentity's primary location for anonymous reports.
+ location = value["locations"][0]
+ return (
+ value.get("ruleId"),
+ location["path"],
+ location["startLine"],
+ location.get("endLine"),
+ )
+
+
+def same_raw_finding(left: dict[str, Any], right: dict[str, Any]) -> bool:
+ if isinstance(left.get("identity"), dict) and isinstance(right.get("identity"), dict):
+ return _semantic_identity_key(left, _finding_identity(left)) == _semantic_identity_key(
+ right, _finding_identity(right)
+ )
+ candidates = (_identity_candidate(left), _identity_candidate(right))
+ if all(candidates) and candidates[0] != candidates[1]:
+ return False
+ pairs = list(zip(identifiers(left), identifiers(right), strict=True))
+ if any(a is not None for a, _ in pairs) and any(b is not None for _, b in pairs):
+ if not any(a is not None and b is not None for a, b in pairs) or any(
+ a is not None and b is not None and a != b for a, b in pairs
+ ):
+ return False
+ return True
+
+
+def same_raw_value(left: Any, right: Any) -> bool:
+ if isinstance(left, dict):
+ return (
+ isinstance(right, dict)
+ and left.keys() == right.keys()
+ and all(same_raw_value(value, right[key]) for key, value in left.items())
+ )
+ if isinstance(left, list):
+ return (
+ isinstance(right, list)
+ and len(left) == len(right)
+ and all(same_raw_value(a, b) for a, b in zip(left, right, strict=True))
+ )
+ # JSON numbers compare by value, but booleans are a distinct scalar type.
+ return isinstance(left, bool) == isinstance(right, bool) and left == right
+
+
+def same_raw_content(left: dict[str, Any], right: dict[str, Any]) -> bool:
+ def content(value: dict[str, Any]) -> dict[str, Any]:
+ result = _finding_content(value)
+ if isinstance(result.get("extensions"), dict):
+ result["extensions"] = {
+ key: value
+ for key, value in result["extensions"].items()
+ if key not in {"candidateId", "reportId", "ledgerRowId"}
+ }
+ if not result["extensions"]:
+ result.pop("extensions")
+ return result
+
+ return same_raw_value(content(left), content(right))
+
+
+def contains_raw_value(current: Any, previous: Any) -> bool:
+ if isinstance(previous, dict):
+ return isinstance(current, dict) and all(
+ key in current and contains_raw_value(current[key], value)
+ for key, value in previous.items()
+ )
+ if isinstance(previous, list):
+ return isinstance(current, list) and all(
+ any(contains_raw_value(item, value) for item in current) for value in previous
+ )
+ return isinstance(current, bool) == isinstance(previous, bool) and current == previous
+
+
+def _saved_outcome_receipts(scan_dir, relative, worker, field, item, coverage_schema):
+ normalized = dict(item)
+ if field == "surfaces":
+ normalized.setdefault("id", _saved_coverage_id(item))
+ normalized.setdefault("receiptRefs", [])
+ if worker is not None:
+ normalized["receiptRefs"] = resolve_coverage_receipts(
+ scan_dir, normalized, worker, relative
+ )
+ verified = {
+ "completeness": "partial",
+ "surfaces": [normalized],
+ "explicitExclusions": [],
+ "deferred": [],
+ }
+ _recover_unsealed_coverage(
+ verified, Path(__file__).resolve().parents[2] / "schemas", scan_dir, [], []
+ )
+ if (
+ not verified["surfaces"]
+ or verified["surfaces"][0]["disposition"] != item["disposition"]
+ ):
+ return None
+ normalized = verified["surfaces"][0]
+ try:
+ _validate_schema_node(normalized, coverage_schema[field]["items"], f"coverage.{field}")
+ except ContractError:
+ return None
+ return tuple(normalized.get("receiptRefs", [])) if field == "surfaces" else ()
+
+
+def _conflicting_outcome_receipts(coverage, scan_dir, receipts):
+ # Only a recovered surface can reserve the spelling that invalidates a second row.
+ surfaces = _coverage_rows(coverage, "surfaces")
+ projected = {
+ "surfaces": [dict(row) if isinstance(row, dict) else row for row in surfaces],
+ "deferred": [],
+ "explicitExclusions": [],
+ }
+ _recover_unsealed_coverage(
+ projected, Path(__file__).resolve().parents[2] / "schemas", scan_dir, [], []
+ )
+ kept = {ref.lower(): ref for row in projected["surfaces"] for ref in row["receiptRefs"]}
+ for row in surfaces:
+ if not (
+ isinstance(row, dict)
+ and isinstance(row.get("candidateId"), str)
+ and row.get("disposition") in ("rejected", "not_applicable")
+ and isinstance(row.get("receiptRefs"), list)
+ ):
+ continue
+ normalized = {Path(ref).as_posix() for ref in row["receiptRefs"] if isinstance(ref, str)}
+ if losing := {ref for ref in normalized & receipts if kept.get(ref.lower(), ref) != ref}:
+ return losing
+ return set()
+
+
+def _saved_candidate_outcomes(
+ scan_dir,
+ current_drafts,
+ valid_finding,
+ coverage_candidate,
+ candidate_order,
+ workers_by_id,
+ coverage_schema,
+):
+ pending_candidate_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
+ outcomes = []
+ for relative, draft, owner in current_drafts:
+ reported_candidates = {
+ finding_candidate_id(finding)
+ for finding in draft["findings"]
+ if isinstance(finding, dict) and valid_finding(finding)
+ }
+ pending_candidates = set()
+ for field in ("deferred", "surfaces"):
+ items = draft["coverage"].get(field, [])
+ for item in items if isinstance(items, list) else []:
+ if isinstance(item, dict) and (
+ field == "deferred" or item.get("disposition") == "needs_follow_up"
+ ):
+ if (pending := coverage_candidate(owner, item)) is not None:
+ if (
+ owner is not None
+ and pending[0] == owner
+ and pending[1] in reported_candidates
+ ):
+ continue
+ pending_candidates.add(pending)
+ order = candidate_order(relative, pending[0], item)
+ pending_candidate_orders[pending] = max(
+ pending_candidate_orders.get(pending, order), order
+ )
+ for finding in draft["findings"]:
+ if (
+ isinstance(finding, dict)
+ and valid_finding(finding)
+ and (candidate_id := finding_candidate_id(finding))
+ ):
+ outcomes.append(
+ (
+ relative,
+ owner,
+ candidate_id,
+ "reported",
+ candidate_order(relative, owner),
+ (),
+ )
+ )
+ for field in ("surfaces", "explicitExclusions"):
+ for item in _coverage_rows(draft["coverage"], field):
+ if (
+ isinstance(item, dict)
+ and isinstance(item.get("candidateId"), str)
+ and item.get("disposition") in ("reported", "rejected", "not_applicable")
+ and (candidate := coverage_candidate(owner, item)) is not None
+ and (item["disposition"] != "reported" or candidate[1] in reported_candidates)
+ and (candidate not in pending_candidates or item["disposition"] == "reported")
+ ):
+ receipts = ()
+ if item["disposition"] != "reported":
+ receipts = _saved_outcome_receipts(
+ scan_dir,
+ relative,
+ workers_by_id.get(owner),
+ field,
+ item,
+ coverage_schema,
+ )
+ if receipts is None:
+ continue
+ outcomes.append(
+ (
+ relative,
+ candidate[0],
+ candidate[1],
+ item["disposition"],
+ candidate_order(relative, candidate[0], item),
+ receipts,
+ )
+ )
+ return outcomes, pending_candidate_orders
+
+
+def _competing_rejection_receipts(outcomes, all_sources, source_surface_receipts):
+ receipt_spellings: dict[str, set[str]] = {}
+ for _, _, _, _, _, receipts in outcomes:
+ for ref in receipts:
+ receipt_spellings.setdefault(ref.lower(), set()).add(ref)
+ if receipt_spellings and any(
+ isinstance(ref, str)
+ and (spellings := receipt_spellings.get(Path(ref).as_posix().lower())) is not None
+ and spellings != {Path(ref).as_posix()}
+ for relative, draft, owner in all_sources
+ for row in _coverage_rows(draft["coverage"], "surfaces")
+ if isinstance(row, dict)
+ for refs in (source_surface_receipts(relative, row),)
+ if isinstance(refs, list)
+ for ref in refs
+ ):
+ return {ref for spellings in receipt_spellings.values() for ref in spellings}
+ return set()
diff --git a/plugins/codex-security/scripts/workbench/saved_coverage.py b/plugins/codex-security/scripts/workbench/saved_coverage.py
index f83ebbfb4..c2cbf8b78 100644
--- a/plugins/codex-security/scripts/workbench/saved_coverage.py
+++ b/plugins/codex-security/scripts/workbench/saved_coverage.py
@@ -18,6 +18,33 @@ from finalize_scan_contract import (
)
+def _has_candidate_metadata(row: dict[str, Any]) -> bool:
+ return any(key in row for key in ("candidateId", "candidate", "finding"))
+
+
+def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]:
+ rows = coverage.get(field, [])
+ return rows if isinstance(rows, list) else []
+
+
+def _inherit_coverage_sources(
+ row: Any,
+ originals: list[Any],
+ sources: dict[int, tuple[dict[str, Any], list[str]]],
+) -> None:
+ if not isinstance(row, dict):
+ return
+ for original in originals:
+ saved = sources.get(id(original))
+ if saved is None:
+ continue
+ # Retain the object as well as its ID while copies are reconciled.
+ paths = sources.setdefault(id(row), (row, []))[1]
+ for relative in saved[1]:
+ if relative not in paths:
+ paths.append(relative)
+
+
def _saved_coverage_id(item: dict[str, Any]) -> str:
if candidate_id := item.get("candidateId"):
return candidate_id
@@ -409,7 +436,6 @@ class SavedCoverageProjection:
retained_surface_rows,
resolved,
):
- from workbench.retained_coverage import _inherit_coverage_sources
source_owners = {relative: owner for relative, _, owner in all_sources}
recovered_worker_ids = {
@@ -644,43 +670,27 @@ class SavedCoverageProjection:
self,
coverage,
all_sources,
- deferred_rows,
- reopened_rows,
+ prospective_tasks,
*,
- selection_for_source,
- row_is_retained,
- replaced_deferred,
- closed_deferred,
+ surface_is_retained,
ambiguous_surface_ids,
- deferred_is_resolved,
):
fallback_surfaces = []
if self.legacy_surface_links and isinstance(coverage.get("surfaces"), list):
# Plan the surviving associations before historical receipts can claim
# a spelling needed by a surface that must be restored for its task.
- prospective_tasks = []
+ prospective_tasks = [
+ (owner, _project_worker_coverage_row(row, owner, "deferred"))
+ for owner, row in prospective_tasks
+ ]
prospective_surfaces = {}
prospective_named = {}
for relative, draft, owner in all_sources:
- selection = selection_for_source(relative, owner)
- for row in deferred_rows[relative]:
- if (
- row_is_retained("deferred", row, owner, selection)
- and id(row) not in replaced_deferred
- and not (
- isinstance(row, dict)
- and isinstance(row.get("id"), str)
- and (owner, row["id"]) in closed_deferred
- )
- ):
- prospective_tasks.append(
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
- )
rows = draft["coverage"].get("surfaces", [])
for row in rows if isinstance(rows, list) else []:
if owner is None or not isinstance(row, dict):
continue
- emitted = row_is_retained("surfaces", row, owner, selection)
+ emitted = surface_is_retained(relative, owner, row)
ambiguous = (
isinstance(row.get("id"), str)
and (owner, row["id"]) in ambiguous_surface_ids
@@ -699,11 +709,6 @@ class SavedCoverageProjection:
named[id(projected)] = projected
if emitted:
prospective_surfaces[id(row)] = projected
- prospective_tasks.extend(
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
- for owner, row in reopened_rows
- if not deferred_is_resolved(owner, row)
- )
bound = {
(id(retained), published_id)
for task, published_id, surface, _, task_source in self.legacy_surface_links
diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py
index 70e4ca149..93926d8f9 100644
--- a/plugins/codex-security/scripts/workbench/saved_sources.py
+++ b/plugins/codex-security/scripts/workbench/saved_sources.py
@@ -23,6 +23,20 @@ from finalize_scan_contract import (
from workbench_coverage_receipts import capture_receipt_checkpoints
+class _CheckpointSnapshotError(ContractError):
+ """A captured source observation could not be persisted for publication replay."""
+
+ def __init__(self, message: str, captured: dict[str, tuple[str, int]]) -> None:
+ super().__init__(message)
+ self.captured = captured
+ self.records: dict[str, tuple[dict[str, Any], str, int]] = {}
+ self.heads: dict[str, str | None] = {}
+ self.model_source: list[str] = []
+ self.parent_context: dict[str, str] = {}
+ self.include_parent = False
+ self.documents: tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None = None
+
+
def _encoded(value: Any) -> bytes:
return json.dumps(
value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
@@ -47,14 +61,43 @@ def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
value["sources"], "Saved stopped-scan source digests are malformed."
)
model_source = value.get("threatModelSource")
- if not isinstance(model_source, str) or model_source not in sources:
+ if "threatModelSource" in value and (
+ not isinstance(model_source, str) or model_source not in sources
+ ):
raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
return sources, model_source
return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
-def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
- state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
+def _retained_checkpoint_state(
+ scan: Any,
+) -> tuple[dict[str, str | None] | None, dict[str, Any] | None]:
+ raw = scan["retained_checkpoint_heads_json"]
+ if raw is None:
+ return None, None
+ saved = json.loads(raw)
+ if not isinstance(saved.get("heads"), dict):
+ return saved, None
+ if saved["recoveryBaseDigest"] != scan["seal_manifest_digest"]:
+ raise ContractError("Stopped scan recovery publication changed after selection.")
+ return saved["heads"], saved
+
+
+def _encode_retained_sources(
+ sources: dict[str, str],
+ model_source: list[str],
+ source_times: dict[str, int] | None = None,
+ parent_context: dict[str, str] | None = None,
+) -> str:
+ state: dict[str, Any] = sources
+ if model_source or source_times is not None:
+ state = {"sources": sources}
+ if model_source:
+ state["threatModelSource"] = model_source[0]
+ if source_times is not None:
+ state["sourceTimes"] = source_times
+ if parent_context:
+ state["parentContext"] = parent_context
return json.dumps(state, sort_keys=True)
@@ -89,6 +132,23 @@ def _checkpoint_head_directory(relative: str) -> Path | None:
return None
+def _unselected_root_order(
+ saved_heads: dict[str, str],
+ checkpoint_heads: dict[str, str | None],
+ source_order: dict[str, tuple[int, int]],
+) -> tuple[int, int]:
+ # One payload can have several accepted observations; retain their order.
+ return max(
+ (
+ source_order[head]
+ for head, checkpoint in saved_heads.items()
+ if _checkpoint_head_directory(head) == Path(".")
+ and checkpoint_heads.get(".") not in (None, head, f"checkpoints/{checkpoint}")
+ ),
+ default=(-1, 0),
+ )
+
+
def _capture_saved_source(
scan_dir: Path,
relative: str,
@@ -98,6 +158,8 @@ def _capture_saved_source(
snapshot_head: bool = True,
write: bool = True,
created_paths: set[str] | None = None,
+ checkpoint_heads: dict[str, str | None] | None = None,
+ expected_digest: str | None = None,
) -> dict[str, tuple[str, int]]:
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
@@ -108,27 +170,42 @@ def _capture_saved_source(
partial(_checkpoint_paths, scan_dir),
partial(_read_saved_result, scan_dir, scan_id=scan_id),
)
- head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
+ head, head_digest, observed = _read_saved_result(scan_dir, relative, scan_id)
+ if expected_digest is not None and head_digest != expected_digest:
+ raise ContractError("checkpoint changed after the scan stopped")
observation = {
**{key: head[key] for key in ("checkpoint", "inputCheckpoint") if key in head},
"observedAtNs": str(observed),
}
directory = Path(relative).parent
- selected_sources = {}
+ selected_records = {}
for key in ("checkpoint", "inputCheckpoint"):
if key in head:
selected = (directory / "checkpoints" / head[key]).as_posix()
- _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
- selected_sources[selected] = (selected_digest, selected_time)
+ selected_records[selected] = _read_saved_result(scan_dir, selected, scan_id)
+ selected_sources = {
+ path: (digest, seen) for path, (_, digest, seen) in selected_records.items()
+ }
digest = _digest(observation)
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
# Capture the selected file even if the worker created it after directory enumeration.
+ if checkpoint_heads is not None:
+ checkpoint_heads[directory.as_posix()] = snapshot
if write and not (scan_dir / snapshot).exists():
if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
created_paths.add(snapshot)
- write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
+ try:
+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
+ except (ContractError, OSError) as exc:
+ if checkpoint_heads is not None:
+ checkpoint_heads[directory.as_posix()] = relative
+ failure = _CheckpointSnapshotError(
+ str(exc), {relative: (head_digest, observed), **selected_sources}
+ )
+ failure.records = {relative: (head, head_digest, observed), **selected_records}
+ raise failure from exc
return {
- snapshot: (digest, int(observation["observedAtNs"])),
+ snapshot: (digest, observed),
**selected_sources,
}
@@ -228,6 +305,91 @@ def _frozen_source_times(
return times
+def _frozen_checkpoint_heads(
+ scan_dir: Path, scan_id: str, sources: dict[str, str], *, skip_unreadable: bool = False
+) -> dict[str, str]:
+ parent_context: dict[str, str] = {}
+ try:
+ times = _frozen_source_times(scan_dir, scan_id, sources, parent_context)
+ except (ContractError, OSError, ValueError):
+ if not skip_unreadable:
+ raise
+ times = {}
+ parent_context.clear()
+ # A frozen parent can identify its accepted head even when observation times tie.
+ accepted_root = None
+ heads: dict[str, tuple[int, str]] = {}
+ ambiguous: set[str] = set()
+ for relative, expected_digest in sources.items():
+ directory = _checkpoint_head_directory(relative)
+ if directory is None:
+ continue
+ try:
+ head, digest, observed = _read_saved_result(scan_dir, relative, scan_id)
+ if digest != expected_digest:
+ raise ContractError("checkpoint changed after the scan stopped")
+ except (ContractError, OSError, ValueError):
+ if skip_unreadable:
+ continue
+ raise
+ selected = (directory / "checkpoints" / head["checkpoint"]).as_posix()
+ observation = (times.get(relative, observed), selected)
+ key = directory.as_posix()
+ if relative == parent_context.get("head"):
+ accepted_root = relative
+ if key not in heads or observation[0] > heads[key][0]:
+ heads[key] = observation
+ ambiguous.discard(key)
+ elif observation[0] == heads[key][0] and selected != heads[key][1]:
+ ambiguous.add(key)
+ # A legacy tie retains both observations until recovery selects a head.
+ frozen = {
+ directory: selected
+ for directory, (_, selected) in heads.items()
+ if directory not in ambiguous
+ }
+ if accepted_root is not None:
+ frozen["."] = accepted_root
+ return frozen
+
+
+def _legacy_checkpoint_observed_at(
+ scan_dir: Path, scan_id: str, sources: dict[str, str], selected: str
+) -> int | None:
+ times = _frozen_source_times(scan_dir, scan_id, sources)
+ checkpoint = Path(selected)
+ observations = []
+ for relative, expected_digest in sources.items():
+ if _checkpoint_head_directory(relative) != checkpoint.parent.parent:
+ continue
+ head, digest, observed = _read_saved_result(scan_dir, relative, scan_id)
+ if digest != expected_digest:
+ raise ContractError("checkpoint changed after the scan stopped")
+ if head["checkpoint"] == checkpoint.name:
+ observations.append(times.get(relative, observed))
+ # Payload-only selections used the newest retained observation for that payload.
+ return max(observations, default=None)
+
+
+def _restore_checkpoint_head(
+ heads: dict[str, str | None],
+ previous_heads: dict[str, str | None],
+ sources: dict[str, str],
+ relative: str,
+) -> None:
+ path = Path(relative)
+ if path.name != "checkpoint-head.json":
+ return
+ directory = path.parent.as_posix()
+ if directory in previous_heads:
+ heads[directory] = previous_heads[directory]
+ return
+ directories = {path.parent, path.parent / "checkpoints", path.parent / "checkpoint-heads"}
+ if any(Path(source).parent in directories for source in sources):
+ # Legacy evidence can remain accepted without one selected head.
+ heads.pop(directory, None)
+
+
def _freeze_source_times(
scan_dir: Path,
scan_id: str,
@@ -326,6 +488,7 @@ def _admit_parent_checkpoint(
def _capture_parent_observation(
scan_dir: Path,
+ scan_id: str,
checkpoint: str,
observed: int,
legacy_head: bool = False,
@@ -338,13 +501,20 @@ def _capture_parent_observation(
scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint})
)
os.utime(scan_dir / "checkpoint-head.json", ns=(observed, observed))
+ return _capture_saved_source(
+ scan_dir, "checkpoint-head.json", scan_id, created_paths=created_paths
+ )
observation = {"checkpoint": checkpoint, "observedAtNs": str(observed)}
digest = _digest(observation)
snapshot = f"checkpoint-heads/{digest}.json"
if not (scan_dir / snapshot).exists():
if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
created_paths.add(snapshot)
- write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
+ try:
+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
+ except (ContractError, OSError) as exc:
+ # The canonical checkpoint carries this observation without a live head.
+ raise _CheckpointSnapshotError(str(exc), {}) from exc
return {snapshot: (digest, observed)}
@@ -369,6 +539,33 @@ def _parent_scan_draft(
return parent
+def _model_capture(scan_id: str, model: dict[str, Any]) -> dict[str, Any]:
+ return _parent_scan_draft(scan_id, {"threatModel": model}, {"findings": []}, {})
+
+
+def _restore_frozen_model_capture(
+ scan_dir: Path,
+ scan_id: str,
+ sources: dict[str, str],
+ times: dict[str, int] | None,
+ model_source: str | None,
+) -> None:
+ if model_source not in (times or {}) or (scan_dir / model_source).exists():
+ return
+ manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved scan manifest")
+ scan = manifest.get("scan")
+ if not isinstance(scan, dict) or not isinstance(scan.get("threatModel"), dict):
+ return
+ draft = _model_capture(scan_id, scan["threatModel"])
+ digest = _digest(draft)
+ if model_source != f"checkpoints/{digest}.json" or sources.get(model_source) != digest:
+ return
+ # A failed first capture can retry only the model and time bound by that stop.
+ write_scan_local_bytes(scan_dir, model_source, _encoded(draft))
+ observed = times[model_source]
+ os.utime(scan_dir / model_source, ns=(observed, observed))
+
+
def _read_saved_parent_result(
scan_dir: Path, scan_id: str
) -> tuple[dict[str, Any], dict[str, Any]]:
diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py
index 8ec15dfdb..51f40f2c8 100644
--- a/plugins/codex-security/scripts/workbench_saved_results.py
+++ b/plugins/codex-security/scripts/workbench_saved_results.py
@@ -95,15 +95,21 @@ from workbench.saved_sources import (
_admit_parent_checkpoint,
_capture_saved_source,
_checkpoint_paths,
+ _CheckpointSnapshotError,
_encode_retained_sources,
_encoded,
_freeze_source_times,
+ _frozen_checkpoint_heads,
_frozen_source_times,
_is_source_order_snapshot,
+ _legacy_checkpoint_observed_at,
_parent_scan_draft,
_read_saved_parent_result,
_read_saved_result,
_require_current_deep_publication,
+ _restore_checkpoint_head,
+ _restore_frozen_model_capture,
+ _retained_checkpoint_state,
_retained_source_state,
_saved_result_paths,
_source_digests,
@@ -241,6 +247,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
manifest_scan.get("preservedSources", {}),
"Published scan source digests are malformed.",
)
+ _, pending_recovery = _retained_checkpoint_state(scan)
+ if pending_recovery:
+ return True
+ published_heads = manifest_scan.get("preservedCheckpointHeads")
+ if published_heads is None:
+ published_heads = _frozen_checkpoint_heads(
+ scan_dir, scan["id"], published_sources, skip_unreadable=True
+ )
+ elif "." not in published_heads:
+ # Older publications recorded root observations without selecting a root head.
+ legacy_heads = _frozen_checkpoint_heads(
+ scan_dir, scan["id"], published_sources, skip_unreadable=True
+ )
+ if "." in legacy_heads:
+ published_heads = {**published_heads, ".": legacy_heads["."]}
+ checkpoint_heads = dict(published_heads)
current_sources = dict(published_sources)
paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
for path in paths:
@@ -252,23 +274,53 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
kind=paths[path],
snapshot_head=path not in published_sources,
write=False,
+ checkpoint_heads=checkpoint_heads,
)
current_sources.update({path: value[0] for path, value in captured.items()})
+ directory = Path(path).parent.as_posix()
+ previous_head = published_heads.get(directory)
+ if (
+ Path(path).name == "checkpoint-head.json"
+ and previous_head in captured
+ and Path(previous_head).parent.name == "checkpoints"
+ ):
+ observed = captured[checkpoint_heads[directory]][1]
+ if observed == _legacy_checkpoint_observed_at(
+ scan_dir, scan["id"], published_sources, previous_head
+ ):
+ checkpoint_heads[directory] = previous_head
except (ContractError, OSError, ValueError):
- continue
- return current_sources != published_sources
+ _restore_checkpoint_head(checkpoint_heads, published_heads, published_sources, path)
+ return current_sources != published_sources or checkpoint_heads != published_heads
except (ContractError, OSError, SystemExit, ValueError):
return False
-def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]:
+def _recovery_source_digests(
+ db: Any, connection: Any, scan: Any
+) -> tuple[dict[str, str], bool, dict[str, str | None]]:
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
frozen_sources: dict[str, str] | None = None
include_parent = True
raw_frozen_sources = scan["retained_source_digests_json"]
if raw_frozen_sources is not None:
- frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
+ retained = json.loads(raw_frozen_sources)
+ frozen_sources, model_source = _retained_source_state(retained)
+ if isinstance(retained.get("sources"), dict) and retained.get("sourceTimes") is not None:
+ _restore_frozen_model_capture(
+ scan_dir, scan["id"], frozen_sources, retained["sourceTimes"], model_source
+ )
+ _freeze_source_times(
+ scan_dir,
+ scan["id"],
+ frozen_sources,
+ retained["sourceTimes"],
+ retained.get("parentContext"),
+ )
include_parent = False
+ checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
+ if pending_recovery:
+ return frozen_sources, False, checkpoint_heads
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
if manifest_path is not None:
@@ -303,6 +355,10 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
"FROM deep_scan_workers WHERE scan_id = ?",
(scan["id"],),
).fetchall()
+ previous_heads = checkpoint_heads
+ if previous_heads is None:
+ previous_heads = _frozen_checkpoint_heads(scan_dir, scan["id"], frozen_sources or {})
+ checkpoint_heads = dict(previous_heads)
paths = dict(_saved_result_paths(scan_dir, workers))
recovery_sources = dict(frozen_sources or {})
parent_context: dict[str, str] = {}
@@ -319,10 +375,25 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
if not _is_source_order_snapshot(relative):
source_times.setdefault(relative, observed)
+ snapshot_error = None
+ captured_records = {}
for relative in paths.keys() - recovery_sources.keys():
try:
- captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
+ captured = _capture_saved_source(
+ scan_dir,
+ relative,
+ scan["id"],
+ kind=paths[relative],
+ checkpoint_heads=checkpoint_heads,
+ )
+ except _CheckpointSnapshotError as exc:
+ snapshot_error = snapshot_error or exc
+ captured = exc.captured
+ captured_records.update(exc.records)
except (ContractError, OSError, ValueError):
+ _restore_checkpoint_head(
+ checkpoint_heads, previous_heads, frozen_sources or {}, relative
+ )
continue
if relative == "checkpoint-head.json":
parent_context["head"] = next(iter(captured))
@@ -331,8 +402,25 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
raise ContractError("checkpoint changed after the scan stopped")
recovery_sources[path] = digest
source_times.setdefault(path, observed)
- _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times, parent_context)
- return recovery_sources, include_parent
+ try:
+ if snapshot_error is not None:
+ raise snapshot_error
+ _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times, parent_context)
+ except (ContractError, OSError) as exc:
+ failure = _CheckpointSnapshotError(
+ str(exc),
+ {
+ path: (digest, source_times[path])
+ for path, digest in recovery_sources.items()
+ if not _is_source_order_snapshot(path)
+ },
+ )
+ failure.records = captured_records
+ failure.heads = checkpoint_heads
+ failure.include_parent = include_parent
+ failure.parent_context = parent_context
+ raise failure from exc
+ return recovery_sources, include_parent, checkpoint_heads
def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
@@ -549,25 +637,44 @@ def preserve_scan_results_locked(
scan_id: str,
*,
recovery_source_digests: dict[str, str] | None = None,
+ recovery_checkpoint_heads: dict[str, str | None] | None = None,
include_parent_with_recovery: bool = False,
+ recovery_snapshot_error: _CheckpointSnapshotError | None = None,
) -> bool:
"""Publish or verify retained terminal results through the workbench host."""
scan = db.require_scan(connection, scan_id)
if scan["status"] != "failed":
return False
frozen_source_digests: dict[str, str] | None = None
+ checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
model_source: list[str] = []
saved_model_source: str | None = None
raw_frozen_sources = scan["retained_source_digests_json"]
+ frozen_source_times = None
if raw_frozen_sources is not None:
- frozen_source_digests, saved_model_source = _retained_source_state(
- json.loads(raw_frozen_sources)
- )
+ retained = json.loads(raw_frozen_sources)
+ frozen_source_digests, saved_model_source = _retained_source_state(retained)
+ if isinstance(retained.get("sources"), dict):
+ frozen_source_times = retained.get("sourceTimes")
if saved_model_source is not None:
model_source.append(saved_model_source)
+ canonical_sources = frozen_source_digests
if recovery_source_digests is not None:
frozen_source_digests = recovery_source_digests
+ checkpoint_heads = recovery_checkpoint_heads
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
+ if frozen_source_digests is not None:
+ _restore_frozen_model_capture(
+ scan_dir, scan_id, frozen_source_digests, frozen_source_times, saved_model_source
+ )
+ if frozen_source_times is not None and recovery_source_digests is None:
+ _freeze_source_times(
+ scan_dir,
+ scan_id,
+ frozen_source_digests,
+ frozen_source_times,
+ retained.get("parentContext"),
+ )
deep_run = connection.execute(
"SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
).fetchone()
@@ -618,11 +725,15 @@ def preserve_scan_results_locked(
db.index_findings(connection, scan_id, findings, scan["completed_at"])
connection.execute(
"UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, "
+ "retained_checkpoint_heads_json = ?, "
"completion_warnings_json = ?, "
"updated_at = ? WHERE id = ? AND status = 'failed'",
(
digest,
_encode_retained_sources(retained_sources, model_source),
+ json.dumps(
+ manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True
+ ),
json.dumps(list(dict.fromkeys(warnings))),
timestamp,
scan_id,
@@ -649,11 +760,23 @@ def preserve_scan_results_locked(
db.verify_manifest_binding(scan, existing)
if existing_scan.get("status") == outcome:
existing_sources = existing_scan.get("preservedSources")
+ existing_heads = existing_scan.get("preservedCheckpointHeads")
if frozen_source_digests is None:
frozen_source_digests = _source_digests(
existing_sources, "Stopped scan source digests could not be frozen."
)
- if existing_sources == frozen_source_digests:
+ checkpoint_heads = existing_heads
+ unchanged = existing_sources == frozen_source_digests and (
+ existing_heads == checkpoint_heads
+ or (
+ "preservedCheckpointHeads" not in existing_scan
+ and recovery_source_digests is None
+ and not pending_recovery
+ )
+ )
+ if unchanged and not any(
+ Path(path).name == "checkpoint-head.json" for path in frozen_source_digests
+ ):
if (
raw_frozen_sources is not None
and scan["seal_manifest_digest"] is not None
@@ -663,45 +786,116 @@ def preserve_scan_results_locked(
return True
record_publication(existing, existing_findings)
return True
- if recovery_source_digests is None:
+ if not unchanged and recovery_source_digests is None and not pending_recovery:
raise ContractError("Stopped scan sources changed after terminal publication.")
binding = {
**db.workbench_completion_binding(scan, scan["completed_at"], existing),
"status": outcome,
}
- if recovery_source_digests is not None:
+ canonical_model_source = saved_model_source
+ canonical_model = existing_scan.get("threatModel") if isinstance(existing_scan, dict) else None
+ if existing is None and frozen_source_digests is not None:
+ # An unsealed live draft is a fallback only when its complete identity
+ # belongs to the frozen set. Otherwise recover from the captured sources.
+ canonical_model = None
+ try:
+ _, canonical_draft = _read_saved_parent_result(scan_dir, scan_id)
+ digest = _digest(canonical_draft)
+ path = f"checkpoints/{digest}.json"
+ if frozen_source_digests.get(path) == digest:
+ canonical_model_source = path
+ canonical_model = canonical_draft.get("threatModel")
+ except (ContractError, OSError, ValueError):
+ # This live fallback is optional.
+ pass
+ if recovery_source_digests is not None and not pending_recovery:
model_source.clear()
- documents = merge_saved_results(
- scan_dir,
- scan_id,
- binding,
- connection.execute(
- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
- (scan_id,),
- ).fetchall(),
- warnings,
- stopped=True,
- scan_mode=scan["mode"],
- finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
- reason=(
- f"Scan {outcome}; saved findings and pending review were preserved. "
- f"{scan['failure_message'] or ''}"
- ).strip(),
- frozen_source_digests=frozen_source_digests,
- frozen_model_source=model_source[0] if model_source else None,
- selected_model_source=model_source,
- project_worker_coverage=scan["mode"] == "deep",
- allow_frozen_legacy_parent=(
- include_parent_with_recovery
- or (
- recovery_source_digests is None
- and frozen_source_digests == {}
- and isinstance(existing_scan, dict)
- and existing_scan.get("sealedAt") is not None
- and "preservedSources" not in existing_scan
- )
- ),
+ # Older pending recoveries retained the parent union without recording the mode.
+ legacy_parent = (
+ include_parent_with_recovery
+ or (pending_recovery is not None and pending_recovery.get("legacyParent", True) is True)
+ or (
+ recovery_source_digests is None
+ and frozen_source_digests == {}
+ and isinstance(existing_scan, dict)
+ and existing_scan.get("sealedAt") is not None
+ and "preservedSources" not in existing_scan
+ )
)
+
+ def pending_selection(heads: dict[str, str | None]) -> dict[str, Any]:
+ return {
+ "heads": heads,
+ "recoveryBaseDigest": scan["seal_manifest_digest"],
+ "legacyParent": legacy_parent,
+ }
+
+ try:
+ documents = merge_saved_results(
+ scan_dir,
+ scan_id,
+ binding,
+ connection.execute(
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
+ (scan_id,),
+ ).fetchall(),
+ warnings,
+ stopped=True,
+ scan_mode=scan["mode"],
+ finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
+ reason=(
+ f"Scan {outcome}; saved findings and pending review were preserved. "
+ f"{scan['failure_message'] or ''}"
+ ).strip(),
+ frozen_source_digests=frozen_source_digests,
+ checkpoint_heads=checkpoint_heads,
+ frozen_model_source=model_source[0] if model_source else None,
+ selected_model_source=model_source,
+ project_worker_coverage=scan["mode"] == "deep",
+ canonical_model=canonical_model,
+ canonical_model_source=canonical_model_source,
+ canonical_sources=canonical_sources,
+ recovery_snapshot_error=recovery_snapshot_error,
+ allow_frozen_legacy_parent=legacy_parent,
+ parent_already_captured=pending_recovery is not None,
+ )
+ except _CheckpointSnapshotError as exc:
+ if frozen_source_digests is None or recovery_snapshot_error is not None:
+ # The stop has already committed. Keep the source set and times in
+ # the database even when auxiliary snapshot files cannot be written.
+ # Changed raw heads remain rejected on retry.
+ captured = exc.captured
+ retained_sources = {path: value[0] for path, value in captured.items()}
+ source_times = {path: value[1] for path, value in captured.items()}
+ retained_heads = exc.heads
+ if recovery_snapshot_error is not None:
+ _prepare_scan_finalization(
+ scan_dir,
+ expected_coverage_mode=db.expected_coverage_mode(scan),
+ completion_binding=binding,
+ recover_unsealed=True,
+ draft_documents=exc.documents,
+ finding_scope=db.requested_finding_scope(scan)
+ if scan["mode"] == "deep"
+ else None,
+ )
+ retained_heads = pending_selection(retained_heads)
+ with connection:
+ connection.execute(
+ "UPDATE scans SET retained_source_digests_json = ?, "
+ "retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
+ "WHERE id = ? AND retained_source_digests_json IS ?",
+ (
+ _encode_retained_sources(
+ retained_sources, exc.model_source, source_times, exc.parent_context
+ ),
+ json.dumps(retained_heads, sort_keys=True),
+ json.dumps(list(dict.fromkeys(warnings))),
+ scan_id,
+ raw_frozen_sources,
+ ),
+ )
+ raise
if documents is None:
unpublished_warnings = list(dict.fromkeys([*warnings, *publication_follow_up_warnings]))
if unpublished_warnings != stored_warnings:
@@ -712,24 +906,46 @@ def preserve_scan_results_locked(
(json.dumps(unpublished_warnings), db.now(), scan_id),
)
return False
- if frozen_source_digests is None or (
- recovery_source_digests is None
- and model_source != ([saved_model_source] if saved_model_source is not None else [])
+ retained_state = None
+ if (
+ frozen_source_digests is None
+ or scan["retained_checkpoint_heads_json"] is None
+ or recovery_source_digests is not None
+ or documents[0]["scan"]["preservedSources"] != frozen_source_digests
+ or model_source != ([saved_model_source] if saved_model_source is not None else [])
):
retained_sources = _source_digests(
documents[0].get("scan", {}).get("preservedSources"),
"Stopped scan source digests could not be frozen.",
)
- with connection:
- connection.execute(
- "UPDATE scans SET retained_source_digests_json = ? "
- "WHERE id = ? AND retained_source_digests_json IS ?",
- (
- _encode_retained_sources(retained_sources, model_source),
- scan_id,
- raw_frozen_sources,
- ),
- )
+ retained_heads = documents[0]["scan"]["preservedCheckpointHeads"]
+ if (
+ recovery_source_digests is not None
+ or pending_recovery
+ or (existing is not None and existing_scan.get("status") == outcome)
+ ):
+ retained_heads = pending_selection(retained_heads)
+ retained_state = (
+ _encode_retained_sources(retained_sources, model_source),
+ json.dumps(retained_heads, sort_keys=True),
+ json.dumps(list(dict.fromkeys(warnings))),
+ scan_id,
+ raw_frozen_sources,
+ )
+
+ def retain_sources() -> None:
+ if retained_state is not None:
+ with connection:
+ connection.execute(
+ "UPDATE scans SET retained_source_digests_json = ?, "
+ "retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
+ "WHERE id = ? AND retained_source_digests_json IS ?",
+ retained_state,
+ )
+
+ first_preservation = recovery_source_digests is None and not pending_recovery
+ if first_preservation:
+ retain_sources()
prepared = _prepare_scan_finalization(
scan_dir,
expected_coverage_mode=db.expected_coverage_mode(scan),
@@ -738,6 +954,8 @@ def preserve_scan_results_locked(
draft_documents=documents,
finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
)
+ if not first_preservation:
+ retain_sources()
warnings.extend(prepared.warnings)
snapshots = _snapshot_published_outputs(scan_dir)
try:
@@ -760,13 +978,24 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
raise SystemExit("Only a stopped scan can recover terminal results.")
if scan["canceled_at"] is not None:
raise SystemExit("Canceled scans cannot recover terminal results.")
- recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan)
+ recovery_snapshot_error = None
+ try:
+ recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests(
+ db, connection, scan
+ )
+ except _CheckpointSnapshotError as exc:
+ recovery_snapshot_error = exc
+ recovery_source_digests = {path: value[0] for path, value in exc.captured.items()}
+ include_parent = exc.include_parent
+ checkpoint_heads = exc.heads
if not preserve_scan_results_locked(
db,
connection,
scan_id,
recovery_source_digests=recovery_source_digests,
+ recovery_checkpoint_heads=checkpoint_heads,
include_parent_with_recovery=include_parent,
+ recovery_snapshot_error=recovery_snapshot_error,
):
raise SystemExit("No saved stopped-scan results were available to recover.")
db.deep_scan.clear_deep_scan_publication_failure(connection, scan_id)
diff --git a/plugins/codex-security/shared/workbench-history-repairs.json b/plugins/codex-security/shared/workbench-history-repairs.json
index 16feecea4..1e9ca60b6 100644
--- a/plugins/codex-security/shared/workbench-history-repairs.json
+++ b/plugins/codex-security/shared/workbench-history-repairs.json
@@ -1,4 +1,13 @@
[
+ [
+ "-- Move checkpoint previews before source-scope history checks.",
+ "WITH input AS (SELECT :history AS history)",
+ "SELECT 'sql' AS operation,",
+ " 'UPDATE schema_migrations SET version = 57 WHERE version = ' || version AS value",
+ "FROM schema_migrations",
+ "WHERE version IN (47, 48, 50, 52, 53, 54)",
+ " AND name = 'freeze stopped scan checkpoint selections';"
+ ],
[
"-- Move source-scope previews before the shared history repairs.",
"WITH input AS (SELECT :history AS history)",
@@ -57,6 +66,18 @@
" FROM moves JOIN schema_migrations USING (version, name)",
"ORDER BY position;"
],
+ [
+ "-- A main-branch reader may have added discovery context under the old checkpoint number.",
+ "SELECT 'timestamp' AS operation,",
+ " 'INSERT INTO schema_migrations (version, name, applied_at) VALUES (47, ' ||",
+ " quote(json_extract(value, '$.name')) || ', ?)' AS value",
+ "FROM json_each(:history) WHERE json_extract(value, '$.version') = 47",
+ " AND EXISTS (SELECT 1 FROM schema_migrations WHERE version = 57",
+ " AND name = 'freeze stopped scan checkpoint selections')",
+ " AND NOT EXISTS (SELECT 1 FROM schema_migrations WHERE version = 47)",
+ " AND EXISTS (SELECT 1 FROM pragma_table_info('deep_scan_runs')",
+ " WHERE name = 'discovery_user_context_json');"
+ ],
[
"-- profiles: preserve legacy columns before the later repairs.",
"WITH",
diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json
index ba1cc5d9e..ac7241638 100644
--- a/plugins/codex-security/shared/workbench-migrations.json
+++ b/plugins/codex-security/shared/workbench-migrations.json
@@ -462,5 +462,12 @@
"statements": [
"UPDATE deep_scan_runs\nSET pending_target_project_path = ''\nWHERE pending_target_project_path IS NULL\n AND status IN ('running', 'succeeded') AND cancel_requested = 0\n AND discovery_user_context_json IS NOT NULL\n AND EXISTS (\n SELECT 1 FROM scans JOIN workspaces ON workspaces.id = scans.workspace_id\n WHERE scans.id = deep_scan_runs.scan_id AND scans.mode = 'deep'\n AND scans.status = 'running' AND scans.canceled_at IS NULL\n AND scans.recipe_json IS NULL AND scans.handoff_status = 'delivered'\n AND scans.handoff_claim_token IS NULL AND scans.continuation_thread_id IS NULL\n AND workspaces.active_scan_id = scans.id AND workspaces.deep_scan_config_json IS NULL\n AND workspaces.created_at = scans.created_at AND scans.created_at = deep_scan_runs.created_at\n );"
]
+ },
+ {
+ "version": 57,
+ "name": "freeze stopped scan checkpoint selections",
+ "statements": [
+ "ALTER TABLE scans ADD COLUMN retained_checkpoint_heads_json TEXT;"
+ ]
}
]
diff --git a/plugins/codex-security/tests/test_checkpoint_publication_authority.py b/plugins/codex-security/tests/test_checkpoint_publication_authority.py
new file mode 100644
index 000000000..5bfc7e332
--- /dev/null
+++ b/plugins/codex-security/tests/test_checkpoint_publication_authority.py
@@ -0,0 +1,3233 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+import sqlite3
+import subprocess
+import sys
+import uuid
+from argparse import Namespace
+from pathlib import Path
+
+import pytest
+from test_deep_scan_successful_publication import add_worker, complete
+from test_deep_scan_successful_publication import publication_scan as publication_scan
+from test_workbench_standard_deep_results import deep_scan_fixture, worker_paths
+from workbench_test_support import (
+ interrupt_publication,
+ run_workbench,
+ saved_draft,
+ sort_checkpoint_entries,
+ write_checkpoint,
+ write_completed_contract,
+)
+
+
+def captured_head(scan_dir, directory):
+ head = scan_dir / directory / "checkpoint-head.json"
+ observation = {
+ **json.loads(head.read_text()),
+ "observedAtNs": str(head.stat().st_mtime_ns),
+ }
+ digest = hashlib.sha256(
+ json.dumps(observation, sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+ relative = (Path(directory) / "checkpoint-heads" / f"{digest}.json").as_posix()
+ assert json.loads((scan_dir / relative).read_text()) == observation
+ return relative
+
+
[email protected]("source_name", ["ordinary-source.json", "sourceTimes"])
[email protected]("replay", ["preserve", "recover"])
+def test_flat_retained_source_name_replays(
+ workbench_api, workbench_db, publication_scan, source_name, replay
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ add_worker(workbench_db, scan, status="canceled")
+ source = scan.scan_dir / source_name
+ source.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": scan.findings,
+ "coverage": scan.coverage,
+ }
+ )
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE scan_id = ?",
+ (str(source), scan.scan_id),
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ stopped = saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped scan."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert isinstance(retained[source_name], str)
+ if replay == "preserve":
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id,
+ claim_token=None,
+ thread_id=None,
+ coordinator_generation=None,
+ ),
+ )["scan"]
+ else:
+ replayed = saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert replayed["findingCount"] == 1
+ assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == findings
+
+
+def test_public_stop_retains_accepted_partial_evidence_and_newer_rejection(tmp_path):
+ state, home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path, workers=2)
+ environment = {"CODEX_HOME": str(home)}
+ contract = tmp_path / "contract"
+ contract.mkdir()
+ write_completed_contract(contract, scan_id, target, relative_path="app.py")
+ finding = json.loads((contract / "findings.json").read_text())["findings"][0]
+ deferred = {
+ "candidateId": "pending-query",
+ "reason": "Validation is pending.",
+ "paths": ["app.py"],
+ }
+ coverage = {
+ "completeness": "partial",
+ "surfaces": [],
+ "explicitExclusions": [],
+ "deferred": [deferred],
+ }
+ workers = []
+ for accepted in (True, False):
+ name = "accepted" if accepted else "interrupted"
+ prompt, output, result = worker_paths(scan_dir, name)
+ worker_id = str(uuid.uuid4())
+ worker_args = (
+ "upsert-deep-scan-worker",
+ "--scan-id",
+ scan_id,
+ "--worker-id",
+ worker_id,
+ "--kind",
+ "discovery",
+ "--prompt-path",
+ str(prompt),
+ "--artifact-dir",
+ str(output),
+ "--attempt",
+ "1",
+ )
+ run_workbench(state, *worker_args, "--status", "running", environment=environment)
+ current = copy.deepcopy(finding)
+ current["identity"]["anchor"] = name
+ current["extensions"] = {"candidateId": name}
+ draft = {"scanId": scan_id, "complete": True, "findings": [current], "coverage": coverage}
+ result.write_text(json.dumps(draft))
+ write_checkpoint(output / "checkpoints", draft)
+ if accepted:
+ run_workbench(
+ state,
+ *worker_args,
+ "--status",
+ "succeeded",
+ "--result-manifest-path",
+ str(result),
+ environment=environment,
+ )
+ else:
+ rejected = {
+ **draft,
+ "complete": False,
+ "findings": [],
+ "coverage": {
+ **coverage,
+ "surfaces": [
+ {
+ "candidateId": name,
+ "label": "Reviewed candidate",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ },
+ }
+ head = write_checkpoint(output / "checkpoints", rejected)
+ pointer = output / "checkpoint-head.json"
+ pointer.write_text(json.dumps({"checkpoint": head.name}))
+ observed = result.stat().st_mtime_ns
+ os.utime(pointer, ns=(observed, observed))
+ workers.append(worker_id)
+
+ stopped = run_workbench(
+ state,
+ "fail-deep-scan",
+ "--scan-id",
+ scan_id,
+ "--message",
+ "Original worker failure.",
+ "--deep-status",
+ "interrupted",
+ environment=environment,
+ )["deepScan"]
+ assert stopped["status"] == "interrupted"
+ scan = run_workbench(state, "get-scan", "--scan-id", scan_id)["scan"]
+ assert scan["findingCount"] == 1
+ findings = json.loads((scan_dir / "findings.json").read_text())["findings"]
+ assert [item["identity"]["anchor"] for item in findings] == ["accepted"]
+ retained_coverage = json.loads((scan_dir / "coverage.json").read_text())
+ assert "completeness" not in retained_coverage
+ assert any(item.get("candidateId") == "pending-query" for item in retained_coverage["deferred"])
+ assert any(
+ item.get("candidateId") == "interrupted" and item["disposition"] == "rejected"
+ for item in retained_coverage["surfaces"]
+ )
+ assert scan["failureMessage"] == "Original worker failure."
+ assert {worker["id"]: worker["status"] for worker in stopped["workers"]} == {
+ workers[0]: "succeeded",
+ workers[1]: "canceled",
+ }
+
+
[email protected]("archived", [False, True], ids=["current", "archived"])
[email protected]("has_head", [True, False], ids=["committed-head", "legacy"])
[email protected]("complete", [False, True], ids=["checkpoint", "complete"])
+def test_recovery_honors_rejection_committed_before_result_replacement(
+ workbench_api, workbench_db, publication_scan, archived, has_head, complete
+):
+ scan = publication_scan()
+ provisional = copy.deepcopy(scan.findings[0])
+ provisional["extensions"] = {"candidateId": "candidate-rejected"}
+ retained = copy.deepcopy(scan.findings[0])
+ retained["identity"]["anchor"] = "independent-finding"
+ retained["extensions"] = {"candidateId": "candidate-retained"}
+ retained["locations"][0]["startLine"] = 20
+ retained["locations"][0]["endLine"] = 21
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result_path = add_worker(workbench_db, scan, status="canceled")
+ if archived:
+ result_path = result_path.parent / "attempts" / "attempt-1" / "result.json"
+ result_path.parent.mkdir(parents=True)
+ previous = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [provisional, retained],
+ "coverage": scan.coverage,
+ }
+ result_path.write_text(json.dumps(previous))
+ old_checkpoint = write_checkpoint(result_path.parent / "checkpoints", previous)
+ rejected = {
+ **previous,
+ "complete": complete,
+ "findings": [retained],
+ "coverage": {
+ **scan.coverage,
+ "surfaces": [
+ {
+ "candidateId": "candidate-rejected",
+ "label": "Validated candidate disposition",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ },
+ }
+ checkpoint = write_checkpoint(result_path.parent / "checkpoints", rejected)
+ if has_head:
+ head = result_path.parent / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ # Model a committed replacement after the older result, even when the
+ # filesystem assigns the same timestamp to these consecutive writes.
+ observed = max(result_path.stat().st_mtime_ns, checkpoint.stat().st_mtime_ns) + 1
+ os.utime(head, ns=(observed, observed))
+ saved_bytes = {path: path.read_bytes() for path in (result_path, old_checkpoint, checkpoint)}
+
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ assert stopped["findingCount"] == len(findings) == (1 if has_head else 2)
+ assert any(finding["identity"]["anchor"] == "independent-finding" for finding in findings)
+ assert all(path.read_bytes() == contents for path, contents in saved_bytes.items())
+ if has_head:
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert any(
+ surface.get("candidateId") == "candidate-rejected"
+ and surface.get("disposition") == "rejected"
+ for surface in coverage["surfaces"]
+ )
+
+
+def save_disposition(scan, directory, disposition):
+ directory.mkdir(parents=True, exist_ok=True)
+ finding = copy.deepcopy(scan.findings[0])
+ finding["extensions"] = {"candidateId": "candidate-disposition"}
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [finding] if disposition == "reported" else [],
+ "coverage": {
+ **scan.coverage,
+ "surfaces": [
+ {
+ "candidateId": "candidate-disposition",
+ "label": "Validated candidate disposition",
+ "disposition": disposition,
+ "receiptRefs": [],
+ }
+ ],
+ },
+ }
+ checkpoint = write_checkpoint(directory / "checkpoints", draft)
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ result = directory / "result.json"
+ observed = (
+ max(
+ checkpoint.stat().st_mtime_ns,
+ result.stat().st_mtime_ns if result.exists() else 0,
+ )
+ + 1
+ )
+ os.utime(head, ns=(observed, observed))
+ return draft
+
+
[email protected]("legacy_publication", [False, True], ids=["observation", "payload"])
[email protected]("selected_time", [100, 300], ids=["restored-older", "unchanged-latest"])
+def test_recovery_needed_compares_checkpoint_observation(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ legacy_publication,
+ selected_time,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ os.utime(result, ns=(200, 200))
+ save_disposition(scan, result.parent, "rejected")
+ head = result.parent / "checkpoint-head.json"
+ os.utime(head, ns=(100, 100))
+ directory = result.parent.relative_to(scan.scan_dir).as_posix()
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ capture = saved._capture_saved_source
+
+ def capture_legacy_head(scan_dir, relative, scan_id, **kwargs):
+ captured = capture(scan_dir, relative, scan_id, **kwargs)
+ heads = kwargs.get("checkpoint_heads")
+ if heads is not None and relative == f"{directory}/checkpoint-head.json":
+ # Legacy writers selected the payload while retaining every observation.
+ heads[directory] = next(
+ path for path in captured if Path(path).parent.name == "checkpoints"
+ )
+ return captured
+
+ with monkeypatch.context() as patch:
+ if legacy_publication:
+ patch.setattr("workbench.retained_coverage._capture_saved_source", capture_legacy_head)
+ patch.setattr(saved, "_capture_saved_source", capture_legacy_head)
+ stopped = saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ os.utime(head, ns=(300, 300))
+ published = saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert published["findingCount"] == 0
+
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ sources = manifest["preservedSources"]
+ assert sorted(
+ int(json.loads((scan.scan_dir / path).read_text())["observedAtNs"])
+ for path in sources
+ if Path(path).parent.as_posix() == f"{directory}/checkpoint-heads"
+ ) == [100, 300]
+ assert Path(manifest["preservedCheckpointHeads"][directory]).parent.name == (
+ "checkpoints" if legacy_publication else "checkpoint-heads"
+ )
+ assert not workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ]
+ os.utime(head, ns=(selected_time, selected_time))
+ assert workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ] is (selected_time == 100)
+ recovered = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert recovered["findingCount"] == (1 if selected_time == 100 else 0)
+ assert not recovered["resultsRecoveryNeeded"]
+ final = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ assert final["preservedSources"] == sources
+ assert final["preservedCheckpointHeads"][directory] == captured_head(scan.scan_dir, directory)
+
+
[email protected]("archived", [False, True], ids=["current-head", "newer-archive"])
[email protected]("disposition", ["reported", "rejected"])
+def test_newer_checkpoint_disposition_precedes_older_archived_head(
+ workbench_api, workbench_db, publication_scan, archived, disposition
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET attempt = 3 WHERE scan_id = ?", (scan.scan_id,)
+ )
+ old = result.parent / "attempts" / "attempt-2"
+ save_disposition(scan, old, "rejected" if disposition == "reported" else "reported")
+ current = result.parent / "attempts" / "attempt-10" if archived else result.parent
+ draft = save_disposition(scan, current, disposition)
+ (current / "result.json").write_text(json.dumps(draft))
+
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+
+ assert stopped["findingCount"] == (1 if disposition == "reported" else 0)
+
+
[email protected]("head_change", ["replaced", "removed", "missing-checkpoint"])
[email protected]("root_head", [False, True], ids=["worker", "root"])
+def test_frozen_stopped_replay_ignores_later_worker_head_changes(
+ workbench_api, workbench_db, publication_scan, monkeypatch, head_change, root_head
+):
+ import finalize_scan_contract
+
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = (
+ scan.scan_dir / "result.json"
+ if root_head
+ else add_worker(workbench_db, scan, status="canceled")
+ )
+ previous = save_disposition(scan, result.parent, "reported")
+ result.write_text(json.dumps(previous))
+ save_disposition(scan, result.parent, "rejected")
+ directory = result.parent.relative_to(scan.scan_dir).as_posix()
+ head = result.parent / "checkpoint-head.json"
+ original_head = (head.read_bytes(), head.stat().st_mtime_ns)
+ original_outputs = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("findings.json", "coverage.json", "scan-manifest.json")
+ }
+ write_bytes = finalize_scan_contract.write_scan_local_bytes
+ failed_writes = []
+
+ def fail_coverage_write(directory, relative, payload, **kwargs):
+ if relative != "coverage.json" or failed_writes:
+ return write_bytes(directory, relative, payload, **kwargs)
+ # Exercise the real writer after findings have reached disk. Remove the
+ # temporary obstruction before the publisher restores its old outputs.
+ failed_writes.append(json.loads((directory / "findings.json").read_text()))
+ path = directory / relative
+ previous_bytes = path.read_bytes()
+ path.unlink()
+ path.mkdir()
+ try:
+ return write_bytes(directory, relative, payload, **kwargs)
+ finally:
+ path.rmdir()
+ path.write_bytes(previous_bytes)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(finalize_scan_contract, "write_scan_local_bytes", fail_coverage_write)
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."
+ ),
+ )
+ row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()
+ if not root_head:
+ assert (head.read_bytes(), head.stat().st_mtime_ns) == original_head
+ expected_heads = {directory: captured_head(scan.scan_dir, directory)}
+ assert len(failed_writes) == 1
+ assert "scanId" in failed_writes[0]
+ assert row["status"] == "failed"
+ assert row["failure_message"] == "Audit stopped."
+ assert row["retained_source_digests_json"]
+ assert row["seal_manifest_digest"] is None
+ assert all(
+ (scan.scan_dir / name).read_bytes() == contents
+ for name, contents in original_outputs.items()
+ )
+ original_sources = row["retained_source_digests_json"]
+ original_run = dict(
+ workbench_db.execute(
+ "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
+ ).fetchone()
+ )
+ if head_change == "replaced":
+ save_disposition(scan, result.parent, "reported")
+ elif head_change == "removed":
+ head.unlink()
+ else:
+ head.write_text(json.dumps({"checkpoint": "a" * 64 + ".json"}))
+
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+
+ assert replayed["findingCount"] == 0
+ assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == []
+ assert json.loads(result.read_text()) == previous
+ row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()
+ assert row["failure_message"] == "Audit stopped."
+ assert row["retained_source_digests_json"] == original_sources
+ assert json.loads(row["retained_checkpoint_heads_json"]) == expected_heads
+ assert row["seal_manifest_digest"]
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert manifest["scan"]["preservedCheckpointHeads"] == expected_heads
+ assert (
+ dict(
+ workbench_db.execute(
+ "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
+ ).fetchone()
+ )
+ == original_run
+ )
+
+
+def test_explicit_recovery_observes_head_change_between_existing_checkpoints(
+ workbench_api, workbench_db, publication_scan
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ previous = save_disposition(scan, result.parent, "reported")
+ result.write_text(json.dumps(previous))
+ save_disposition(scan, result.parent, "rejected")
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 0
+
+ save_disposition(scan, result.parent, "reported")
+
+ context = workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]
+ assert context["resultsRecoveryNeeded"] is True
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == 1
+ assert recovered["resultsRecoveryNeeded"] is False
+
+
+def test_legacy_frozen_publication_keeps_result_fallback_without_saved_heads(
+ workbench_api, workbench_db, publication_scan, monkeypatch
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ previous = save_disposition(scan, result.parent, "reported")
+ result.write_text(json.dumps(previous))
+ save_disposition(scan, result.parent, "rejected")
+ (result.parent / "checkpoint-head.json").unlink()
+
+ with monkeypatch.context() as patch:
+ patch.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ interrupt_publication,
+ )
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."
+ ),
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,)
+ )
+ save_disposition(scan, result.parent, "rejected")
+
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+
+ assert replayed["findingCount"] == 1
+
+
[email protected]("head_time", [0, 1, -1], ids=["tie", "newer", "older"])
[email protected]("root_head", [False, True], ids=["worker", "root"])
+def test_recovery_uses_live_selection_regardless_of_head_timestamp(
+ workbench_api, workbench_db, publication_scan, head_time, root_head
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = (
+ scan.scan_dir / "result.json"
+ if root_head
+ else add_worker(workbench_db, scan, status="canceled")
+ )
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ save_disposition(scan, result.parent, "rejected")
+ head = result.parent / "checkpoint-head.json"
+ observed = head.stat().st_mtime_ns
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ save_disposition(scan, result.parent, "reported")
+ os.utime(head, ns=(observed + head_time, observed + head_time))
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == 1
+ assert recovered["resultsRecoveryNeeded"] is False
+
+
[email protected]("pending_offset", [-100, 0, 100], ids=["older", "tied", "newer"])
[email protected]("head_time", [100, 300], ids=["older-selection", "tied-selection"])
[email protected]("generic", [False, True], ids=["candidate", "generic-task"])
[email protected]("reverse_order", [False, True])
[email protected]("root_head", [False, True], ids=["worker", "root"])
+def test_head_reselection_preserves_equal_or_newer_pending_evidence(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ pending_offset,
+ head_time,
+ generic,
+ reverse_order,
+ root_head,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = (
+ scan.scan_dir / "result.json"
+ if root_head
+ else add_worker(workbench_db, scan, status="canceled")
+ )
+ if root_head:
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ os.utime(result, ns=(50, 50))
+ head = result.parent / "checkpoint-head.json"
+ os.utime(head, ns=(300, 300))
+ saved = workbench_api["saved_results"]
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ selected = save_disposition(scan, result.parent, "rejected")
+ if generic:
+ selected["coverage"]["resolvedDeferred"] = [
+ {"id": "review-task", "reason": "Review completed."}
+ ]
+ checkpoint = write_checkpoint(result.parent / "checkpoints", selected)
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(head_time, head_time))
+ identity = {"id": "review-task"} if generic else {"candidateId": "candidate-disposition"}
+ pending = write_checkpoint(
+ result.parent / "checkpoints",
+ {
+ "scanId": scan.scan_id,
+ "complete": False,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "deferred": [
+ {**identity, "reason": "Validation remains pending.", "paths": ["app.py"]}
+ ],
+ },
+ },
+ )
+ pending_time = head_time + pending_offset
+ os.utime(pending, ns=(pending_time, pending_time))
+ sort_checkpoint_entries(monkeypatch, reverse=reverse_order)
+
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ key, value = next(iter(identity.items()))
+ if generic and not root_head:
+ value = f"{result.parent.name}/{value}"
+ pending_rows = [row for row in coverage["deferred"] if row.get(key) == value]
+ assert bool(pending_rows) is (pending_offset >= 0)
+
+
[email protected]("head_time", [100, 300], ids=["older-selection", "tied-selection"])
[email protected]("result_time", [50, 400], ids=["older-result", "newer-result"])
[email protected]("reverse_order", [False, True])
+def test_reselection_keeps_the_selected_model_and_newer_result_precedence(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ head_time,
+ result_time,
+ reverse_order,
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ saved = workbench_api["saved_results"]
+ head = result.parent / "checkpoint-head.json"
+ draft = save_disposition(scan, result.parent, "reported")
+ draft["threatModel"] = {"summary": "The current result model."}
+ result.write_text(json.dumps(draft))
+ os.utime(result, ns=(result_time, result_time))
+ for summary, observed in (("The earlier model.", 300), ("The selected model.", head_time)):
+ draft["threatModel"] = {"summary": summary}
+ checkpoint = write_checkpoint(result.parent / "checkpoints", draft)
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ if summary == "The earlier model.":
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ sort_checkpoint_entries(monkeypatch, reverse=reverse_order)
+
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ expected = "The current result model." if result_time > head_time else "The selected model."
+ assert manifest["threatModel"]["summary"] == expected
+
+
[email protected]("head_time", [100, 300], ids=["older-selection", "tied-selection"])
[email protected]("reverse_order", [False, True])
+def test_partial_reselection_keeps_independent_accepted_dispositions(
+ workbench_api, workbench_db, publication_scan, monkeypatch, head_time, reverse_order
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ os.utime(result, ns=(50, 50))
+ selected = save_disposition(scan, result.parent, "rejected")
+ head = result.parent / "checkpoint-head.json"
+ os.utime(head, ns=(300, 300))
+ saved = workbench_api["saved_results"]
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ selected["complete"] = False
+ selected["coverage"]["surfaces"][0]["candidateId"] = "independent-candidate"
+ checkpoint = write_checkpoint(result.parent / "checkpoints", selected)
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(head_time, head_time))
+ sort_checkpoint_entries(monkeypatch, reverse=reverse_order)
+
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+
+ assert stopped["findingCount"] == 0
+
+
[email protected]("root_head", [False, True], ids=["worker", "root"])
+def test_publication_metadata_uses_the_captured_head(
+ workbench_api, workbench_db, publication_scan, monkeypatch, root_head
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = (
+ scan.scan_dir / "result.json"
+ if root_head
+ else add_worker(workbench_db, scan, status="canceled")
+ )
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ saved = workbench_api["saved_results"]
+ capture = saved._capture_saved_source
+ relative_head = (result.parent / "checkpoint-head.json").relative_to(scan.scan_dir).as_posix()
+
+ changed = False
+
+ def replace_before_capture(directory, relative, *args, **kwargs):
+ nonlocal changed
+ if relative == relative_head and not changed:
+ changed = True
+ save_disposition(scan, result.parent, "rejected")
+ return capture(directory, relative, *args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.retained_coverage._capture_saved_source", replace_before_capture)
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."
+ ),
+ )["scan"]
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ directory = result.parent.relative_to(scan.scan_dir).as_posix()
+ assert manifest["preservedCheckpointHeads"] == {
+ directory: captured_head(scan.scan_dir, directory)
+ }
+ assert stopped["findingCount"] == 0
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is False
+ )
+
+
[email protected]("legacy_state", ["published", "failed-publication"])
+def test_legacy_checkpoint_selection_is_reconstructed_from_frozen_sources(
+ workbench_api, workbench_db, publication_scan, monkeypatch, legacy_state
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ save_disposition(scan, result.parent, "rejected")
+ saved = workbench_api["saved_results"]
+ prepare = saved._prepare_scan_finalization
+
+ def legacy_documents(*args, **kwargs):
+ kwargs["draft_documents"][0]["scan"].pop("preservedCheckpointHeads")
+ return prepare(*args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ if legacy_state == "published":
+ patch.setattr(saved, "_prepare_scan_finalization", legacy_documents)
+ else:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."
+ ),
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,)
+ )
+ if legacy_state == "failed-publication":
+ saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ context = workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]
+ assert context["findingCount"] == 0
+ assert context["resultsRecoveryNeeded"] is False
+
+
+def test_legacy_omitted_root_head_does_not_require_recovery(
+ workbench_api, workbench_db, publication_scan, monkeypatch
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ saved = workbench_api["saved_results"]
+ _, parent = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", parent)
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ prepare = saved._prepare_scan_finalization
+
+ def legacy_documents(*args, **kwargs):
+ heads = kwargs["draft_documents"][0]["scan"]["preservedCheckpointHeads"]
+ heads.pop(".")
+ for directory, snapshot in heads.items():
+ checkpoint = json.loads((scan.scan_dir / snapshot).read_text())["checkpoint"]
+ heads[directory] = (Path(directory) / "checkpoints" / checkpoint).as_posix()
+ return prepare(*args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_prepare_scan_finalization", legacy_documents)
+ saved.fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ original = manifest_path.read_bytes()
+ manifest = json.loads(original)["scan"]
+ assert "." not in manifest["preservedCheckpointHeads"]
+ assert captured_head(scan.scan_dir, ".") in manifest["preservedSources"]
+ assert not workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ]
+ preserved = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert not preserved["resultsRecoveryNeeded"]
+ assert manifest_path.read_bytes() == original
+ observed = head.stat().st_mtime_ns + 1_000_000
+ os.utime(head, ns=(observed, observed))
+ assert workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ]
+ assert manifest_path.read_bytes() == original
+
+
[email protected]("change", ["new-checkpoint", "identical-result-rewrite"])
+def test_explicit_recovery_preserves_database_only_source_times(
+ workbench_api, workbench_db, publication_scan, monkeypatch, change
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ os.utime(result, ns=(100, 100))
+ saved = workbench_api["saved_results"]
+ write = saved.write_scan_local_bytes
+
+ def fail_source_order(root, relative, contents):
+ if relative.startswith("source-order/"):
+ raise OSError("Synthetic ordering snapshot failure")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_source_order)
+ saved.fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ relative_result = result.relative_to(scan.scan_dir).as_posix()
+ assert retained["sourceTimes"][relative_result] == 100
+ assert not any(path.startswith("source-order/") for path in retained["sources"])
+ if change == "new-checkpoint":
+ save_disposition(scan, result.parent, "rejected")
+ else:
+ result.write_bytes(result.read_bytes())
+ os.utime(result, ns=(300, 300))
+ recovered = saved.recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == (0 if change == "new-checkpoint" else 1)
+ assert not recovered["resultsRecoveryNeeded"]
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ sources = manifest["preservedSources"]
+ times = saved._frozen_source_times(scan.scan_dir, scan.scan_id, sources)
+ for relative, digest in retained["sources"].items():
+ assert sources[relative] == digest
+ assert times[relative] == retained["sourceTimes"][relative]
+ assert len(sources) > len(retained["sources"])
+
+
[email protected]("retry", ["preserve", "recover"])
[email protected]("root_head", [False, True])
[email protected]("failure", ["publication", "source-order", "checkpoint-heads"])
+def test_failed_explicit_recovery_replays_its_frozen_selection(
+ workbench_api, workbench_db, publication_scan, monkeypatch, retry, failure, root_head
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = (
+ scan.scan_dir / "result.json"
+ if root_head
+ else add_worker(workbench_db, scan, status="canceled")
+ )
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ save_disposition(scan, result.parent, "rejected")
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ original = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("findings.json", "coverage.json", "scan-manifest.json")
+ }
+ original_digest = workbench_db.execute(
+ "SELECT seal_manifest_digest FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ save_disposition(scan, result.parent, "reported")
+ saved = workbench_api["saved_results"]
+
+ write = saved.write_scan_local_bytes
+
+ def fail_snapshot(root, relative, contents):
+ if Path(relative).parent.name == failure:
+ raise OSError("Synthetic publication interruption")
+ return write(root, relative, contents)
+
+ selected_head = (result.parent / "checkpoint-head.json").read_bytes()
+ selected_time = (result.parent / "checkpoint-head.json").stat().st_mtime_ns
+ with monkeypatch.context() as patch:
+ if failure == "publication":
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ else:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_snapshot)
+ with pytest.raises(
+ (OSError, saved.ContractError), match="Synthetic publication interruption"
+ ):
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id),
+ )
+ assert all((scan.scan_dir / name).read_bytes() == data for name, data in original.items())
+ assert (
+ workbench_db.execute(
+ "SELECT seal_manifest_digest FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ == original_digest
+ )
+ # The live head returns to the prior published disposition; pending recovery
+ # still needs to publish the selection made before the failed write.
+ save_disposition(scan, result.parent, "rejected")
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is True
+ )
+ if failure == "checkpoint-heads":
+ # Without an immutable head snapshot, replay must reject a changed raw
+ # head until the selected bytes are restored, just like initial stopping.
+ replay = saved.recover_scan_results if retry == "recover" else saved.preserve_scan_results
+ with pytest.raises(
+ saved.ContractError,
+ match="changed after the scan stopped|Frozen stopped-scan checkpoint set is incomplete",
+ ):
+ replay(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id,
+ claim_token=None,
+ thread_id=None,
+ coordinator_generation=None,
+ ),
+ )
+ head = result.parent / "checkpoint-head.json"
+ head.write_bytes(selected_head)
+ os.utime(head, ns=(selected_time, selected_time))
+ if retry == "recover":
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ else:
+ recovered = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert recovered["findingCount"] == 1
+ assert recovered["failureMessage"] == "Audit stopped."
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_checkpoint_heads_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert "recoveryBaseDigest" not in retained
+
+
[email protected]("failure", ["publication", "source-order", "checkpoint-heads"])
+def test_failed_recovery_keeps_model_when_worker_registers_an_unselected_checkpoint(
+ workbench_api, workbench_db, publication_scan, monkeypatch, failure
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ original = save_disposition(scan, result.parent, "reported")
+ result.write_text(json.dumps(original))
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ saved = workbench_api["saved_results"]
+ head = result.parent / "checkpoint-head.json"
+ unselected = write_checkpoint(
+ result.parent / "checkpoints",
+ {**original, "threatModel": {"summary": "Unselected worker model."}},
+ )
+ os.utime(unselected, ns=(400, 400))
+ accepted = write_checkpoint(
+ result.parent / "checkpoints",
+ {**original, "threatModel": {"summary": "Latest accepted model."}},
+ )
+ os.utime(accepted, ns=(200, 200))
+ head.write_text(json.dumps({"checkpoint": accepted.name}))
+ os.utime(head, ns=(200, 200))
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ selected = write_checkpoint(result.parent / "checkpoints", original)
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(300, 300))
+ args = Namespace(scan_id=scan.scan_id)
+
+ write = saved.write_scan_local_bytes
+
+ def fail_snapshot(root, relative, contents):
+ if Path(relative).parent.name == failure:
+ raise OSError("Synthetic publication interruption")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ if failure == "publication":
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ else:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_snapshot)
+ with pytest.raises(
+ (OSError, saved.ContractError), match="Synthetic publication interruption"
+ ):
+ saved.recover_scan_results(workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args)
+ retained = workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ assert (
+ json.loads(retained)["threatModelSource"] == accepted.relative_to(scan.scan_dir).as_posix()
+ )
+
+ # A late worker update must not replace the model already selected for publication.
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE artifact_dir = ?",
+ (str(unselected), str(result.parent)),
+ )
+ saved.recover_scan_results(workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args)
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert manifest["scan"]["threatModel"]["summary"] == "Latest accepted model."
+ published_sources = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert published_sources["threatModelSource"] == json.loads(retained)["threatModelSource"]
+
+
[email protected]("bad_head", ["malformed", "missing-checkpoint", "missing-head"])
[email protected]("prior_disposition", ["reported", "rejected"])
+def test_unreadable_worker_head_does_not_hide_other_recoverable_evidence(
+ workbench_api, workbench_db, publication_scan, bad_head, prior_disposition
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ bad = add_worker(workbench_db, scan, status="canceled")
+ bad.write_text(json.dumps(save_disposition(scan, bad.parent, "reported")))
+ save_disposition(scan, bad.parent, prior_disposition)
+ head = bad.parent / "checkpoint-head.json"
+ original_head = (head.read_bytes(), head.stat().st_mtime_ns)
+ scan.findings[0]["identity"]["anchor"] = "independent-worker"
+ scan.findings[0]["locations"][0]["startLine"] = 20
+ scan.findings[0]["locations"][0]["endLine"] = 21
+ good = add_worker(workbench_db, scan, status="canceled")
+ save_disposition(scan, good.parent, "rejected")
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ assert (head.read_bytes(), head.stat().st_mtime_ns) == original_head
+ directory = bad.parent.relative_to(scan.scan_dir).as_posix()
+ prior_head = captured_head(scan.scan_dir, directory)
+ if bad_head == "missing-head":
+ head.unlink()
+ else:
+ head.write_text(
+ "{" if bad_head == "malformed" else json.dumps({"checkpoint": "a" * 64 + ".json"})
+ )
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is False
+ )
+ save_disposition(scan, good.parent, "reported")
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is True
+ )
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == (2 if prior_disposition == "reported" else 1)
+ assert recovered["resultsRecoveryNeeded"] is False
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ assert manifest["preservedCheckpointHeads"][directory] == prior_head
+
+
+def test_failed_head_snapshot_does_not_claim_uncaptured_authority(
+ workbench_api, workbench_db, publication_scan, monkeypatch
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ save_disposition(scan, result.parent, "rejected")
+ saved = workbench_api["saved_results"]
+ write = saved.write_scan_local_bytes
+ directory = result.parent.relative_to(scan.scan_dir).as_posix()
+
+ def fail_snapshot(root, relative, *args, **kwargs):
+ if relative.startswith(f"{directory}/checkpoint-heads/"):
+ raise OSError("Synthetic snapshot write failure")
+ return write(root, relative, *args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_snapshot)
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."
+ ),
+ )["scan"]
+ row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()
+ assert row["seal_manifest_digest"] is None
+ heads = json.loads(row["retained_checkpoint_heads_json"])
+ relative_head = f"{directory}/checkpoint-head.json"
+ assert heads[directory] == relative_head
+ retained = json.loads(row["retained_source_digests_json"])
+ _, digest, observed = saved._read_saved_result(scan.scan_dir, relative_head, scan.scan_id)
+ assert retained["sources"][relative_head] == digest
+ assert retained["sourceTimes"][relative_head] == observed
+ assert not (result.parent / "checkpoint-heads").exists()
+ assert stopped["resultsRecoveryNeeded"] is True
+ warnings = json.loads(
+ workbench_db.execute(
+ "SELECT completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert any("Synthetic snapshot write failure" in warning for warning in warnings)
+ replayed = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == 0
+
+
[email protected]("root_head", [False, True], ids=["worker", "root"])
+def test_legacy_recovery_can_reselect_an_already_frozen_head(
+ workbench_api, workbench_db, publication_scan, monkeypatch, root_head
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = (
+ scan.scan_dir / "result.json"
+ if root_head
+ else add_worker(workbench_db, scan, status="canceled")
+ )
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ head = result.parent / "checkpoint-head.json"
+ original_head, original_time = head.read_bytes(), head.stat().st_mtime_ns
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ save_disposition(scan, result.parent, "rejected")
+ saved = workbench_api["saved_results"]
+ prepare = saved._prepare_scan_finalization
+
+ def legacy_documents(*args, **kwargs):
+ kwargs["draft_documents"][0]["scan"].pop("preservedCheckpointHeads")
+ return prepare(*args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_prepare_scan_finalization", legacy_documents)
+ rejected = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert rejected["findingCount"] == 0
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,)
+ )
+ head.write_bytes(original_head)
+ os.utime(head, ns=(original_time, original_time))
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is True
+ )
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == 1
+ assert recovered["resultsRecoveryNeeded"] is False
+
+
+def test_recovery_keeps_newer_result_with_legacy_frozen_live_head(
+ workbench_api, workbench_db, publication_scan
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ reported = save_disposition(scan, result.parent, "reported")
+ save_disposition(scan, result.parent, "rejected")
+ head = result.parent / "checkpoint-head.json"
+ checkpoint = result.parent / "checkpoints" / json.loads(head.read_text())["checkpoint"]
+ os.utime(checkpoint, ns=(100, 100))
+ os.utime(head, ns=(200, 200))
+ result.write_text(json.dumps(reported))
+ os.utime(result, ns=(300, 300))
+ saved = workbench_api["saved_results"]
+ sources = {
+ path.relative_to(scan.scan_dir).as_posix(): saved._read_saved_result(
+ scan.scan_dir, path.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )[1]
+ for path in (checkpoint, head, result)
+ }
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET status = 'failed', completed_at = ?, "
+ "retained_source_digests_json = ?, retained_checkpoint_heads_json = NULL "
+ "WHERE id = ?",
+ (scan.timestamp, json.dumps(sources), scan.scan_id),
+ )
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == 1
+ assert recovered["resultsRecoveryNeeded"] is False
+
+
[email protected]("directory_name", ["recoveryBaseDigest", "heads"])
+def test_worker_directory_does_not_collide_with_recovery_metadata(
+ workbench_api, workbench_db, publication_scan, directory_name
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ original = add_worker(workbench_db, scan, status="canceled")
+ directory = scan.scan_dir / directory_name
+ result = directory / "result.json"
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? "
+ "WHERE scan_id = ? AND artifact_dir = ?",
+ (str(directory), str(result), scan.scan_id, str(original.parent)),
+ )
+ result.write_text(json.dumps(save_disposition(scan, directory, "reported")))
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ saved = workbench_api["saved_results"]
+ preserved = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert preserved["findingCount"] == 1
+ save_disposition(scan, directory, "rejected")
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == 0
+ assert recovered["resultsRecoveryNeeded"] is False
+
+
[email protected]("legacy_heads", ["ambiguous", "headless"])
[email protected]("reverse_order", [False, True])
[email protected]("bad_head", ["malformed", "missing-checkpoint"])
+def test_unreadable_heads_preserve_legacy_accepted_results(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ reverse_order,
+ bad_head,
+ legacy_heads,
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ saved = workbench_api["saved_results"]
+ sources = {}
+ for disposition in ("reported", "rejected"):
+ draft = save_disposition(scan, result.parent, disposition)
+ if disposition == "reported":
+ result.write_text(json.dumps(draft))
+ os.utime(result, ns=(300, 300))
+ head = result.parent / "checkpoint-head.json"
+ os.utime(head, ns=(300, 300))
+ sources.update(
+ {
+ path: digest
+ for path, (digest, _) in saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ ).items()
+ }
+ )
+ if legacy_heads == "headless":
+ for relative in list(sources):
+ if "/checkpoint-heads/" in relative:
+ (scan.scan_dir / relative).unlink()
+ del sources[relative]
+ head.unlink()
+ # Content-addressed filenames can place either disposition first.
+ sort_checkpoint_entries(monkeypatch, reverse=reverse_order)
+ relative = result.relative_to(scan.scan_dir).as_posix()
+ sources[relative] = saved._read_saved_result(scan.scan_dir, relative, scan.scan_id)[1]
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET status = 'failed', completed_at = ?, "
+ "retained_source_digests_json = ?, retained_checkpoint_heads_json = NULL "
+ "WHERE id = ?",
+ (scan.timestamp, json.dumps(sources), scan.scan_id),
+ )
+ replayed = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == 1
+ assert replayed["resultsRecoveryNeeded"] is False
+
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,)
+ )
+ head.write_text(
+ "{" if bad_head == "malformed" else json.dumps({"checkpoint": "a" * 64 + ".json"})
+ )
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is False
+ )
+ scan.findings[0]["identity"]["anchor"] = "independent-worker"
+ scan.findings[0]["locations"][0]["startLine"] = 20
+ scan.findings[0]["locations"][0]["endLine"] = 21
+ good = add_worker(workbench_db, scan, status="canceled")
+ save_disposition(scan, good.parent, "reported")
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["findingCount"] == 2
+ assert recovered["resultsRecoveryNeeded"] is False
+
+
+ ("invalid_binding", "snapshot_failure"),
+ [(False, None), (True, None), (True, "source-order"), (True, "checkpoint-heads")],
+)
+def test_invalid_explicit_recovery_does_not_freeze_unpublishable_selection(
+ workbench_api, workbench_db, publication_scan, invalid_binding, snapshot_failure, monkeypatch
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ original = save_disposition(scan, result.parent, "reported")
+ result.write_text(json.dumps(original))
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ prior = tuple(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json, retained_checkpoint_heads_json FROM scans WHERE id = ?",
+ (scan.scan_id,),
+ ).fetchone()
+ )
+ candidate = {
+ **original,
+ "threatModel": {"summary": "Valid first model."},
+ }
+ checkpoint = write_checkpoint(result.parent / "checkpoints", candidate)
+ head = result.parent / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ args = Namespace(scan_id=scan.scan_id)
+ if invalid_binding:
+ saved = workbench_api["saved_results"]
+ write = saved.write_scan_local_bytes
+ prepare = saved._prepare_scan_finalization
+
+ def prepare_mismatched_binding(directory, **kwargs):
+ kwargs["expected_coverage_mode"] = "scoped_path"
+ return prepare(directory, **kwargs)
+
+ def fail_snapshot(root, relative, contents):
+ if Path(relative).parent.name == snapshot_failure:
+ raise OSError("Synthetic snapshot failure")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_snapshot)
+ patch.setattr(saved, "_prepare_scan_finalization", prepare_mismatched_binding)
+ with pytest.raises(saved.ContractError, match="completion binding coverage mode"):
+ saved.recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args
+ )
+ retained = tuple(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json, retained_checkpoint_heads_json FROM scans WHERE id = ?",
+ (scan.scan_id,),
+ ).fetchone()
+ )
+ assert retained == prior
+ else:
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args
+ )
+ corrected = write_checkpoint(
+ result.parent / "checkpoints",
+ {**original, "threatModel": {"summary": "Corrected selected model."}},
+ )
+ head.write_text(json.dumps({"checkpoint": corrected.name}))
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, args
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert manifest["scan"]["threatModel"]["summary"] == "Corrected selected model."
+
+
[email protected]("selected_model", [False, True], ids=["omitted", "replacement"])
+ "result_model", [False, True], ids=["unmodeled-result", "older-result-model"]
+)
+def test_model_less_selected_head_keeps_latest_accepted_model(
+ workbench_api, workbench_db, publication_scan, selected_model, result_model
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan, status="canceled")
+ original = save_disposition(scan, result.parent, "reported")
+ modeled = {}
+ for summary in ("First synthetic accepted model.", "Second synthetic accepted model."):
+ draft = {**original, "threatModel": {"summary": summary}}
+ checkpoint = write_checkpoint(result.parent / "checkpoints", draft)
+ modeled[checkpoint] = summary
+ older, newer = sorted(modeled)
+ head = result.parent / "checkpoint-head.json"
+ saved = workbench_api["saved_results"]
+ for checkpoint, observed in ((older, 100), (newer, 200)):
+ os.utime(checkpoint, ns=(observed, observed))
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ selected = {
+ **original,
+ **({"threatModel": {"summary": "Selected replacement model."}} if selected_model else {}),
+ }
+ checkpoint = write_checkpoint(result.parent / "checkpoints", selected)
+ os.utime(checkpoint, ns=(300, 300))
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(300, 300))
+ result.write_text(
+ json.dumps(
+ {
+ **original,
+ **({"threatModel": {"summary": "Older result model."}} if result_model else {}),
+ }
+ )
+ )
+ os.utime(result, ns=(150, 150) if result_model else (400, 400))
+ originals = {path: path.read_bytes() for path in [result, older, newer, checkpoint, head]}
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )
+ expected = "Selected replacement model." if selected_model else modeled[newer]
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert manifest["scan"]["threatModel"]["summary"] == expected
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert (
+ json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["threatModel"][
+ "summary"
+ ]
+ == expected
+ )
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
+ ("mode", "root_head"), [("standard", True), ("deep", True), ("deep", False)]
+)
[email protected]("newer_sorts_first", [False, True])
[email protected]("restore_model", [False, True], ids=["model-less", "modeled"])
+def test_restored_head_keeps_latest_model_fallback(
+ workbench_api, workbench_db, publication_scan, mode, root_head, newer_sorts_first, restore_model
+):
+ scan = publication_scan(mode=mode)
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ draft = saved_draft(scan.scan_id, complete=True)
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ if root_head:
+ directory = scan.scan_dir
+ else:
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(draft))
+ os.utime(result, ns=(0, 0))
+ directory = result.parent
+ head = directory / "checkpoint-head.json"
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+
+ def select(checkpoint, observed):
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ return head.read_bytes(), observed
+
+ def model():
+ return json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"][
+ "threatModel"
+ ]["summary"]
+
+ empty = write_checkpoint(directory / "checkpoints", draft)
+ os.utime(empty, ns=(50, 50))
+ unmodeled_head = select(empty, 100)
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ models = {
+ write_checkpoint(
+ directory / "checkpoints", {**draft, "threatModel": {"summary": text}}
+ ): text
+ for text in ("Synthetic model A.", "Synthetic model B.")
+ }
+ for checkpoint in models:
+ os.utime(checkpoint, ns=(50, 50))
+ older, newer = sorted(models, reverse=newer_sorts_first)
+ modeled_head = select(older, 200)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Synthetic stop."
+ ),
+ )
+ assert model() == models[older]
+ select(newer, 300)
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert model() == models[newer]
+ originals = {
+ filename: (filename.read_bytes(), filename.stat().st_mtime_ns)
+ for subdirectory in ("checkpoints", "checkpoint-heads")
+ for filename in (directory / subdirectory).glob("*.json")
+ }
+ restored_contents, restored_time = modeled_head if restore_model else unmodeled_head
+ head.write_bytes(restored_contents)
+ os.utime(head, ns=(restored_time, restored_time))
+ recovered = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert model() == models[older if restore_model else newer]
+ assert not recovered["resultsRecoveryNeeded"]
+ published = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json", "report.md")
+ }
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert not replayed["resultsRecoveryNeeded"]
+ assert all(
+ (scan.scan_dir / name).read_bytes() == contents for name, contents in published.items()
+ )
+ assert (head.read_bytes(), head.stat().st_mtime_ns) == (restored_contents, restored_time)
+ assert all(
+ (filename.read_bytes(), filename.stat().st_mtime_ns) == contents
+ for filename, contents in originals.items()
+ )
+
+
+ "checkpoint_result", [False, True], ids=["ordinary-result", "registered-checkpoint"]
+)
+def test_registered_checkpoint_result_keeps_newer_accepted_outcome(
+ workbench_api, workbench_db, publication_scan, checkpoint_result
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan)
+ reported = {
+ **save_disposition(scan, result.parent, "reported"),
+ "threatModel": {"summary": "Newer accepted model."},
+ }
+ head = result.parent / "checkpoint-head.json"
+ accepted = write_checkpoint(result.parent / "checkpoints", reported)
+ head.write_text(json.dumps({"checkpoint": accepted.name}))
+ os.utime(accepted, ns=(300, 300))
+ os.utime(head, ns=(300, 300))
+ workbench_api["saved_results"]._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ if checkpoint_result:
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE id = ?",
+ (str(accepted), result.parent.name),
+ )
+ else:
+ result.write_text(json.dumps(reported))
+ os.utime(result, ns=(300, 300))
+ rejected = {
+ **save_disposition(scan, result.parent, "rejected"),
+ "threatModel": {"summary": "Older selected model."},
+ }
+ older = write_checkpoint(result.parent / "checkpoints", rejected)
+ os.utime(older, ns=(200, 200))
+ head.write_text(json.dumps({"checkpoint": older.name}))
+ os.utime(head, ns=(200, 200))
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ assert stopped["resultsRecoveryNeeded"] is False
+
+ assert (
+ json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["threatModel"][
+ "summary"
+ ]
+ == "Newer accepted model."
+ )
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == 1
+ assert (
+ json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["threatModel"][
+ "summary"
+ ]
+ == "Newer accepted model."
+ )
+
+
[email protected]("registered", [False, True], ids=["head-only", "registered-result"])
[email protected]("interrupted", [False, True], ids=["published", "publication-retry"])
+def test_restored_observation_keeps_independent_registered_result(
+ workbench_api, workbench_db, publication_scan, monkeypatch, registered, interrupted
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="succeeded")
+ reported = {
+ **save_disposition(scan, result.parent, "reported"),
+ "threatModel": {"summary": "Registered reporting model."},
+ }
+ surface = {
+ "id": "registered-surface",
+ "label": "Synthetic reviewed surface",
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ reported["coverage"]["surfaces"].append(surface)
+ accepted = write_checkpoint(result.parent / "checkpoints", reported)
+ os.utime(accepted, ns=(50, 50))
+ stale = {
+ **save_disposition(scan, result.parent, "rejected"),
+ "threatModel": {"summary": "Stale result model."},
+ }
+ stale["coverage"]["surfaces"].append({**surface, "disposition": "needs_follow_up"})
+ stale["coverage"]["openQuestions"] = [{"question": "Obsolete synthetic question."}]
+ stale["coverage"]["explicitExclusions"] = [
+ {"pattern": "old-scope/**", "reason": "Obsolete synthetic exclusion."}
+ ]
+ result.write_text(json.dumps(stale))
+ os.utime(result, ns=(200, 200))
+ head = result.parent / "checkpoint-head.json"
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+
+ def select(observed):
+ head.write_text(json.dumps({"checkpoint": accepted.name}))
+ os.utime(head, ns=(observed, observed))
+
+ select(100)
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ select(300)
+ if registered:
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE id = ?",
+ (str(accepted), result.parent.name),
+ )
+ stopped = saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ sources = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"][
+ "preservedSources"
+ ]
+ prior_coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert not prior_coverage.get("openQuestions")
+ assert not prior_coverage["explicitExclusions"]
+ original = {path: (scan.scan_dir / path).read_bytes() for path in sources}
+ select(100)
+ assert workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ]
+ prepared_counts = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_counts.append(len(prepared[3]["findings"]))
+ if interrupted:
+ raise OSError("Synthetic publication interruption")
+ return write(prepared, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ if interrupted:
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ else:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert prepared_counts == [int(registered)]
+ recovered = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert recovered["findingCount"] == int(registered)
+ assert not recovered["resultsRecoveryNeeded"]
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ assert manifest["preservedSources"] == sources
+ assert manifest["threatModel"]["summary"] == (
+ "Registered reporting model." if registered else "Stale result model."
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ if registered:
+ for field in ("surfaces", "openQuestions", "explicitExclusions"):
+ assert coverage.get(field) == prior_coverage.get(field)
+ else:
+ assert coverage["openQuestions"] == [
+ {**row, "provenance": {"workerId": result.parent.name, "attempt": 1}}
+ for row in stale["coverage"]["openQuestions"]
+ ]
+ assert any(row["pattern"] == "old-scope/**" for row in coverage["explicitExclusions"])
+ assert all((scan.scan_dir / path).read_bytes() == content for path, content in original.items())
+
+
[email protected]("mode", ["standard", "deep"])
[email protected]("interrupted", [False, True], ids=["published", "publication-retry"])
+ "model_choice", ["latest-accepted", "selected-head", "canonical-fallback", "older-accepted"]
+)
[email protected]("legacy_format", [False, True], ids=["current-format", "legacy-format"])
+def test_inherited_model_uses_same_selection_on_initial_and_frozen_publication(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ interrupted,
+ model_choice,
+ legacy_format,
+):
+ scan = publication_scan(mode=mode)
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"]["threatModel"] = {"summary": "Old canonical model."}
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ draft = saved_draft(scan.scan_id, complete=True)
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ if model_choice == "selected-head":
+ draft["threatModel"] = {"summary": "Selected root model."}
+ selected = write_checkpoint(scan.scan_dir / "checkpoints", draft)
+ os.utime(selected, ns=(50, 50))
+ head = scan.scan_dir / "checkpoint-head.json"
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+
+ if model_choice in {"canonical-fallback", "older-accepted"}:
+ unselected = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {**draft, "threatModel": {"summary": "Unaccepted model."}},
+ )
+ os.utime(unselected, ns=(50, 50))
+ sort_checkpoint_entries(monkeypatch, key=lambda name: name != unselected.name)
+
+ def select(checkpoint, observed):
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+
+ select(selected, 200)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ if model_choice != "canonical-fallback":
+ other_model = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {**draft, "threatModel": {"summary": "Latest accepted model."}},
+ )
+ os.utime(other_model, ns=(50, 50))
+ select(other_model, 100 if model_choice == "older-accepted" else 300)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ select(selected, 200)
+ prepared_models = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_models.append(prepared[2]["scan"]["threatModel"]["summary"])
+ if interrupted:
+ raise OSError("Synthetic publication interruption")
+ return write(prepared, **kwargs)
+
+ encode = saved._encode_retained_sources
+ prepare = saved._prepare_scan_finalization
+
+ def legacy_documents(*args, **kwargs):
+ kwargs["draft_documents"][0]["scan"]["preservedCheckpointHeads"].pop(".", None)
+ return prepare(*args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ if legacy_format:
+ patch.setattr(saved, "_prepare_scan_finalization", legacy_documents)
+ patch.setattr(
+ saved,
+ "_encode_retained_sources",
+ lambda sources, model_source, source_times=None: encode(sources, [], source_times),
+ )
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ retained = workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ if legacy_format:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ preserved = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ expected = {
+ "latest-accepted": "Latest accepted model.",
+ "selected-head": "Selected root model.",
+ "canonical-fallback": "Old canonical model.",
+ "older-accepted": "Old canonical model.",
+ }[model_choice]
+ assert prepared_models == [expected]
+ assert json.loads(manifest_path.read_text())["scan"]["threatModel"]["summary"] == expected
+ assert not preserved["resultsRecoveryNeeded"]
+ replayed = workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ assert (
+ saved._retained_source_state(json.loads(replayed))[0]
+ == saved._retained_source_state(json.loads(retained))[0]
+ )
+ if legacy_format:
+ model_source = saved._retained_source_state(json.loads(replayed))[1]
+ if model_source is not None:
+ assert (
+ json.loads((scan.scan_dir / model_source).read_text())["threatModel"]["summary"]
+ == expected
+ )
+ elif not legacy_format:
+ assert replayed == retained
+
+
[email protected]("prepare_only", [False, True], ids=["complete", "prepare"])
+ "model_choice", ["canonical-fallback", "latest-accepted", "selected-head", "older-accepted"]
+)
+def test_successful_completion_keeps_accepted_root_model_fallback(
+ workbench_api, workbench_db, publication_scan, monkeypatch, prepare_only, model_choice
+):
+ scan = publication_scan(mode="standard")
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"]["threatModel"] = {"summary": "Old canonical model."}
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ draft = saved_draft(scan.scan_id, complete=True)
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ if model_choice == "selected-head":
+ draft["threatModel"] = {"summary": "Selected root model."}
+ selected = write_checkpoint(scan.scan_dir / "checkpoints", draft)
+ os.utime(selected, ns=(50, 50))
+ head = scan.scan_dir / "checkpoint-head.json"
+ saved = workbench_api["saved_results"]
+
+ def select(checkpoint, observed):
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+
+ select(selected, 200)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ unaccepted = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {**draft, "threatModel": {"summary": "Unaccepted model."}},
+ )
+ os.utime(unaccepted, ns=(50, 50))
+ sort_checkpoint_entries(monkeypatch, key=lambda name: name != unaccepted.name)
+ if model_choice != "canonical-fallback":
+ modeled = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {**draft, "threatModel": {"summary": "Latest accepted model."}},
+ )
+ os.utime(modeled, ns=(50, 50))
+ select(modeled, 100 if model_choice == "older-accepted" else 300)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ select(selected, 200)
+ original = {
+ path: (path.read_bytes(), path.stat().st_mtime_ns)
+ for directory in ("checkpoints", "checkpoint-heads")
+ for path in (scan.scan_dir / directory).glob("*.json")
+ }
+ completed = complete(workbench_api, workbench_db, scan, prepare_only=prepare_only)
+ expected = {
+ "canonical-fallback": "Old canonical model.",
+ "latest-accepted": "Latest accepted model.",
+ "selected-head": "Selected root model.",
+ "older-accepted": "Old canonical model.",
+ }[model_choice]
+ published = json.loads(manifest_path.read_text())["scan"]
+ assert published["threatModel"]["summary"] == expected
+ assert published["sealedAt"]
+ assert completed["threatModelAvailable"]
+ assert completed["progress"]["status"] == ("running" if prepare_only else "complete")
+ assert all(
+ (path.read_bytes(), path.stat().st_mtime_ns) == value for path, value in original.items()
+ )
+
+
[email protected]("duplicate_order", [None, "first", "last"])
[email protected]("new_model", [False, True])
[email protected]("interrupted", [False, True])
[email protected]("legacy_model_source", [False, True])
+def test_recovery_keeps_known_model_source_with_equal_model_duplicates(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ duplicate_order,
+ new_model,
+ interrupted,
+ legacy_model_source,
+):
+ scan = publication_scan(mode="standard")
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"].update(complete=True, threatModel={"summary": "Canonical model."})
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ _, canonical = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ older = write_checkpoint(
+ scan.scan_dir / "checkpoints", {**canonical, "threatModel": {"summary": "Older model."}}
+ )
+ os.utime(older, ns=(50, 50))
+ if duplicate_order is not None:
+ duplicate = copy.deepcopy(canonical)
+ duplicate["coverage"]["openQuestions"] = [{"question": "Earlier synthetic review."}]
+ duplicate_path = write_checkpoint(scan.scan_dir / "checkpoints", duplicate)
+ os.utime(duplicate_path, ns=(50, 50))
+ sort_checkpoint_entries(
+ monkeypatch,
+ key=lambda name: (
+ name != duplicate_path.name
+ if duplicate_order == "first"
+ else name == duplicate_path.name
+ ),
+ )
+ head = scan.scan_dir / "checkpoint-head.json"
+
+ def select(checkpoint, observed):
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+
+ select(older, 100)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ empty = {key: value for key, value in canonical.items() if key != "threatModel"}
+ selected = write_checkpoint(scan.scan_dir / "checkpoints", empty)
+ os.utime(selected, ns=(50, 50))
+ select(selected, 200)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert retained["threatModelSource"] == f"checkpoints/{saved._digest(canonical)}.json"
+ assert (
+ json.loads(manifest_path.read_text())["scan"]["threatModel"]["summary"]
+ == "Canonical model."
+ )
+ if legacy_model_source:
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_source_digests_json = ? WHERE id = ?",
+ (json.dumps(retained["sources"]), scan.scan_id),
+ )
+ if new_model:
+ newer = write_checkpoint(
+ scan.scan_dir / "checkpoints", {**canonical, "threatModel": {"summary": "New model."}}
+ )
+ os.utime(newer, ns=(50, 50))
+ select(newer, 250)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ select(selected, 300)
+ prepared_models = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_models.append(prepared[2]["scan"]["threatModel"]["summary"])
+ if interrupted:
+ raise OSError("Synthetic publication interruption")
+ return write(prepared, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ if interrupted:
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ else:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ expected = "New model." if new_model else "Canonical model."
+ assert prepared_models == [expected]
+ saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ assert json.loads(manifest_path.read_text())["scan"]["threatModel"]["summary"] == expected
+
+
[email protected]("corrupt", [False, True])
[email protected]("flow", ["stop", "interrupted-stop", "complete", "prepare"])
+def test_skipped_canonical_checkpoint_keeps_valid_model_and_findings(
+ workbench_api, workbench_db, publication_scan, monkeypatch, corrupt, flow
+):
+ scan = publication_scan(mode="standard")
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"].update(complete=True, threatModel={"summary": "Canonical model."})
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ _, canonical = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ checkpoint_dir = scan.scan_dir / "checkpoints"
+ checkpoint_dir.mkdir()
+ canonical_path = checkpoint_dir / f"{saved._digest(canonical)}.json"
+ canonical_path.write_bytes(saved._encoded(canonical))
+ os.utime(canonical_path, ns=(150, 150))
+ if corrupt:
+ canonical_path.write_text("{")
+ original = canonical_path.read_bytes()
+ selected = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {key: value for key, value in canonical.items() if key != "threatModel"},
+ )
+ os.utime(selected, ns=(50, 50))
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(200, 200))
+ if flow in {"complete", "prepare"}:
+ published = complete(workbench_api, workbench_db, scan, prepare_only=flow == "prepare")
+ else:
+ with monkeypatch.context() as patch:
+ if flow == "interrupted-stop":
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."
+ ),
+ )
+ published = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ os.utime(head, ns=(300, 300))
+ published = saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1
+ if flow != "prepare":
+ assert published["findingCount"] == 1
+ final = json.loads(manifest_path.read_text())["scan"]
+ assert final["sealedAt"]
+ assert final["threatModel"]["summary"] == "Canonical model."
+ assert canonical_path.read_bytes() == original
+ if corrupt:
+ assert canonical_path.relative_to(scan.scan_dir).as_posix() not in final["preservedSources"]
+ if flow in {"stop", "interrupted-stop"}:
+ warnings = json.loads(
+ workbench_db.execute(
+ "SELECT completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert any("Preserved unreadable checkpoint" in warning for warning in warnings)
+
+
[email protected]("registered", [None, "A", "B"])
[email protected]("restored_time", [100, 300])
[email protected]("interrupted", [False, True])
+def test_selected_terminal_coverage_excludes_unselected_history(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ registered,
+ restored_time,
+ interrupted,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ checkpoints = {}
+ for name in ("A", "B"):
+ draft = saved_draft(
+ scan.scan_id,
+ complete=True,
+ surfaces=[
+ {
+ "id": "surface",
+ "label": f"{name} surface",
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ draft["threatModel"] = {"summary": f"{name} model"}
+ draft["coverage"]["openQuestions"] = [{"question": f"{name} question"}]
+ draft["coverage"]["explicitExclusions"] = [
+ {"pattern": f"{name}/**", "reason": f"{name} exclusion"}
+ ]
+ checkpoints[name] = write_checkpoint(result.parent / "checkpoints", draft)
+ os.utime(checkpoints[name], ns=(50, 50))
+ head = result.parent / "checkpoint-head.json"
+
+ def select(name, observed):
+ head.write_text(json.dumps({"checkpoint": checkpoints[name].name}))
+ os.utime(head, ns=(observed, observed))
+
+ for name, observed in (("A", 100), ("B", 200), ("A", 300)):
+ select(name, observed)
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ if registered is not None:
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE id = ?",
+ (str(checkpoints[registered]), result.parent.name),
+ )
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ select("A", restored_time)
+ expected = "B" if registered == "B" and restored_time == 100 else "A"
+
+ with monkeypatch.context() as patch:
+ if interrupted:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ if interrupted and restored_time == 100:
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ else:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert [row["label"] for row in coverage["surfaces"]] == [f"{expected} surface"]
+ assert [row["question"] for row in coverage["openQuestions"]] == [f"{expected} question"]
+ assert [row["pattern"] for row in coverage["explicitExclusions"]] == [f"{expected}/**"]
+ assert (
+ json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["threatModel"][
+ "summary"
+ ]
+ == f"{expected} model"
+ )
+ assert not replayed["resultsRecoveryNeeded"]
+
+
+ "mode,layout", [("standard", "root"), ("deep", "root"), ("deep", "worker")]
+)
[email protected]("failure_at", [None, "restore", "modelless"])
[email protected]("modelless_time", [200, 400])
+def test_selected_model_identity_survives_restore_then_modelless_head(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ layout,
+ failure_at,
+ modelless_time,
+):
+ scan = publication_scan(mode=mode)
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if layout == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ draft = saved_draft(scan.scan_id, complete=True)
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ checkpoints = {
+ name: write_checkpoint(
+ directory / "checkpoints", {**draft, "threatModel": {"summary": name}}
+ )
+ for name in ("A", "B")
+ }
+ checkpoints["empty"] = write_checkpoint(directory / "checkpoints", draft)
+ for checkpoint in checkpoints.values():
+ os.utime(checkpoint, ns=(50, 50))
+ head = directory / "checkpoint-head.json"
+ models = []
+
+ for index, (name, observed, phase) in enumerate(
+ (
+ ("A", 100, "initial"),
+ ("B", 300, "newer"),
+ ("A", 100, "restore"),
+ ("empty", modelless_time, "modelless"),
+ )
+ ):
+ head.write_text(json.dumps({"checkpoint": checkpoints[name].name}))
+ os.utime(head, ns=(observed, observed))
+ if index == 0:
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."
+ ),
+ )
+ else:
+ with monkeypatch.context() as patch:
+ if phase == failure_at:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ else:
+ saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ models.append(
+ json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["threatModel"][
+ "summary"
+ ]
+ )
+ assert not replayed["resultsRecoveryNeeded"]
+ assert models == ["A", "B", "A", "B"]
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ assert retained["threatModelSource"] == checkpoints["B"].relative_to(scan.scan_dir).as_posix()
+
+
+ "mode,layout", [("standard", "root"), ("deep", "root"), ("deep", "worker")]
+)
[email protected]("terminal", [False, True])
[email protected]("interrupted", [False, True])
+def test_selected_terminal_coverage_has_root_worker_parity(
+ workbench_api, workbench_db, publication_scan, monkeypatch, mode, layout, terminal, interrupted
+):
+ scan = publication_scan(mode=mode)
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ coverage_path = scan.scan_dir / "coverage.json"
+ initial_coverage = json.loads(coverage_path.read_text())
+ for field in ("surfaces", "explicitExclusions", "openQuestions"):
+ initial_coverage[field] = []
+ coverage_path.write_text(json.dumps(initial_coverage))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if layout == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ checkpoints = {}
+ for name in ("A", "B"):
+ draft = saved_draft(scan.scan_id, complete=terminal if name == "A" else True)
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ draft["coverage"].update(
+ surfaces=[
+ {
+ "id": f"surface-{name}",
+ "label": name,
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ ],
+ explicitExclusions=[{"pattern": f"{name}/**", "reason": f"{name} exclusion"}],
+ openQuestions=[{"question": f"{name} question"}],
+ )
+ draft["threatModel"] = {"summary": f"{name} model"}
+ checkpoints[name] = write_checkpoint(directory / "checkpoints", draft)
+ os.utime(checkpoints[name], ns=(50, 50))
+ head = directory / "checkpoint-head.json"
+ for name, observed in (("A", 100), ("B", 200), ("A", 300)):
+ head.write_text(json.dumps({"checkpoint": checkpoints[name].name}))
+ os.utime(head, ns=(observed, observed))
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ head.write_text(json.dumps({"checkpoint": checkpoints["A"].name}))
+ os.utime(head, ns=(100, 100))
+
+ with monkeypatch.context() as patch:
+ if interrupted:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ else:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ expected = {"A"} if terminal else {"A", "B"}
+ assert {row["label"] for row in coverage["surfaces"]} == expected
+ assert {row["pattern"] for row in coverage["explicitExclusions"]} == {
+ f"{name}/**" for name in expected
+ }
+ assert {row["question"] for row in coverage["openQuestions"]} == {
+ f"{name} question" for name in expected
+ }
+ assert (
+ json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["threatModel"][
+ "summary"
+ ]
+ == "A model"
+ )
+ assert not replayed["resultsRecoveryNeeded"]
+
+
[email protected]("archived_terminal", [False, True])
[email protected]("current_terminal", [False, True])
[email protected]("registered", [False, True])
[email protected]("interrupted", [False, True])
+def test_terminal_coverage_replacement_does_not_cross_into_a_partial_attempt(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ archived_terminal,
+ current_terminal,
+ registered,
+ interrupted,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ directory = result.parent
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET attempt = 2 WHERE scan_id = ?", (scan.scan_id,)
+ )
+ archive = directory / "attempts" / "attempt-1"
+ old = write_checkpoint(
+ archive / "checkpoints", saved_draft(scan.scan_id, complete=archived_terminal)
+ )
+ os.utime(old, ns=(50, 50))
+ (archive / "checkpoint-head.json").write_text(json.dumps({"checkpoint": old.name}))
+ os.utime(archive / "checkpoint-head.json", ns=(100, 100))
+ accepted = saved_draft(scan.scan_id, complete=True)
+ accepted["coverage"].update(
+ surfaces=[
+ {
+ "id": "accepted-surface",
+ "label": "Accepted",
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ ],
+ explicitExclusions=[{"pattern": "accepted/**", "reason": "Accepted exclusion"}],
+ openQuestions=[{"question": "Accepted question"}],
+ )
+ accepted_path = write_checkpoint(directory / "checkpoints", accepted)
+ os.utime(accepted_path, ns=(200, 200))
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": accepted_path.name}))
+ os.utime(head, ns=(200, 200))
+ if registered:
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE id = ?",
+ (str(accepted_path), directory.name),
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ current = write_checkpoint(
+ directory / "checkpoints", saved_draft(scan.scan_id, complete=current_terminal)
+ )
+ os.utime(current, ns=(300, 300))
+ head.write_text(json.dumps({"checkpoint": current.name}))
+ os.utime(head, ns=(300, 300))
+
+ with monkeypatch.context() as patch:
+ if interrupted:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ else:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ expected = [] if current_terminal else ["Accepted"]
+ assert [row["label"] for row in coverage["surfaces"]] == expected
+ assert [row["pattern"] for row in coverage["explicitExclusions"]] == (
+ [] if current_terminal else ["accepted/**"]
+ )
+ assert [row["question"] for row in coverage["openQuestions"]] == (
+ [] if current_terminal else ["Accepted question"]
+ )
+ if not current_terminal:
+ assert "completeness" not in coverage
+ assert not replayed["resultsRecoveryNeeded"]
+
+
[email protected]("valid_rejection", [False, True], ids=["malformed", "valid"])
+def test_tied_rejection_requires_valid_surface_before_removing_finding(
+ workbench_api, workbench_db, publication_scan, valid_rejection
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ os.utime(result, ns=(300, 300))
+ rejected = save_disposition(scan, result.parent, "rejected")
+ if not valid_rejection:
+ rejected["coverage"]["surfaces"][0].pop("label")
+ checkpoint = write_checkpoint(result.parent / "checkpoints", rejected)
+ (result.parent / "checkpoint-head.json").write_text(
+ json.dumps({"checkpoint": checkpoint.name})
+ )
+ os.utime(result.parent / "checkpoint-head.json", ns=(300, 300))
+ original = result.read_bytes()
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == (0 if valid_rejection else 1)
+ assert result.read_bytes() == original
+
+
+ "receipt_state",
+ ["missing", "present", "present-relative", "optional", "shared", "shared-collision"],
+)
[email protected]("head_time", [300, 400], ids=["tied", "newer"])
+def test_rejection_receipt_is_verified_before_suppressing_accepted_finding(
+ workbench_api, workbench_db, publication_scan, receipt_state, head_time
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ # Worker receipts are published beneath the parent artifact directory.
+ output = scan.scan_dir / "artifacts" / "workers" / result.parent.name
+ output.parent.mkdir(parents=True, exist_ok=True)
+ result.parent.rename(output)
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(output), str(output / result.name), output.name),
+ )
+ result = output / result.name
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ os.utime(result, ns=(300, 300))
+ rejected = save_disposition(scan, result.parent, "rejected")
+ if receipt_state != "optional":
+ rejected["coverage"]["surfaces"][0]["receiptRefs"] = [
+ "./artifacts/review.txt"
+ if receipt_state == "present-relative"
+ else "artifacts/review.txt"
+ ]
+ if receipt_state.startswith("shared"):
+ prior_ref = (
+ "artifacts/Review.txt"
+ if receipt_state == "shared-collision"
+ else "artifacts/review.txt"
+ )
+ parent_surface = {
+ "id": "earlier-review",
+ "label": "Earlier review",
+ "disposition": "no_issue_found",
+ "receiptRefs": [prior_ref],
+ }
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps({**scan.coverage, "surfaces": [parent_surface]})
+ )
+ (scan.scan_dir / prior_ref).write_text("Synthetic parent review.")
+ if receipt_state.startswith(("present", "shared")):
+ receipt_dir = scan.scan_dir if receipt_state.startswith("shared") else result.parent
+ receipt = receipt_dir / "artifacts" / "review.txt"
+ receipt.parent.mkdir(exist_ok=True)
+ receipt.write_text("Synthetic rejected candidate review.")
+ checkpoint = write_checkpoint(result.parent / "checkpoints", rejected)
+ (result.parent / "checkpoint-head.json").write_text(
+ json.dumps({"checkpoint": checkpoint.name})
+ )
+ os.utime(result.parent / "checkpoint-head.json", ns=(head_time, head_time))
+ originals = {
+ path: path.read_bytes() for path in (result, result.parent / "checkpoint-head.json")
+ }
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+ unverified = receipt_state in {"missing", "shared-collision"}
+ assert stopped["findingCount"] == int(unverified)
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert ("needs_follow_up" if unverified else "rejected") in {
+ surface["disposition"] for surface in coverage["surfaces"]
+ }
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == stopped["findingCount"]
+ assert all(path.read_bytes() == value for path, value in originals.items())
+
+
[email protected]("rejection", [False, True], ids=["finding", "finding-and-rejection"])
+def test_checkpoint_registered_as_result_keeps_its_own_finding(
+ workbench_api, workbench_db, publication_scan, rejection
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan)
+ draft = save_disposition(scan, result.parent, "reported")
+ if rejection:
+ draft["coverage"]["surfaces"][0]["disposition"] = "rejected"
+ checkpoint = write_checkpoint(result.parent / "checkpoints", draft)
+ head = result.parent / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE id = ?",
+ (str(checkpoint), result.parent.name),
+ )
+ original = checkpoint.read_bytes()
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ assert checkpoint.read_bytes() == original
+
+
[email protected](os.name == "nt", reason="POSIX directory write permission fixture")
[email protected]("writable", [False, True], ids=["blocked-snapshot", "writable-snapshot"])
[email protected]("replace_head", [False, True], ids=["late-file", "late-selection"])
+def test_cancellation_snapshot_failure_keeps_the_original_source_boundary(
+ workbench_api, workbench_db, publication_scan, writable, replace_head
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="running")
+ draft = save_disposition(scan, result.parent, "reported")
+ result.write_text(json.dumps(draft))
+ head = result.parent / "checkpoint-head.json"
+ original_head = head.read_bytes()
+ original_head_time = head.stat().st_mtime_ns
+ snapshots = result.parent / "checkpoint-heads"
+ snapshots.mkdir()
+ try:
+ if not writable:
+ snapshots.chmod(0o500)
+ if os.access(snapshots, os.W_OK):
+ pytest.skip("Current user can write through POSIX directory permissions")
+ workbench_api["saved_results"].cancel_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, thread_id=None),
+ )
+ finally:
+ snapshots.chmod(0o700)
+ late = copy.deepcopy(draft)
+ late["findings"][0]["identity"]["anchor"] = "written-after-cancellation"
+ late["findings"][0]["extensions"]["candidateId"] = "post-cancellation-candidate"
+ late["coverage"]["surfaces"][0]["candidateId"] = "post-cancellation-candidate"
+ checkpoint = write_checkpoint(result.parent / "checkpoints", late)
+ if replace_head:
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ saved = workbench_api["saved_results"]
+
+ def replay():
+ return saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id,
+ claim_token=None,
+ thread_id=None,
+ coordinator_generation=None,
+ ),
+ )["scan"]
+
+ if not writable and replace_head:
+ with pytest.raises(
+ saved.ContractError, match="Frozen stopped-scan checkpoint set is incomplete"
+ ):
+ replay()
+ head.write_bytes(original_head)
+ os.utime(head, ns=(original_head_time, original_head_time))
+ retained = replay()
+ assert retained["findingCount"] == 1
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ assert all(item["identity"]["anchor"] != "written-after-cancellation" for item in findings)
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert (
+ checkpoint.relative_to(scan.scan_dir).as_posix() not in manifest["scan"]["preservedSources"]
+ )
+
+
[email protected](os.name == "nt", reason="POSIX directory write permission fixture")
[email protected]("writable", [False, True], ids=["blocked-snapshot", "writable-snapshot"])
[email protected]("disposition", ["reported", "rejected"])
+def test_snapshot_write_failure_keeps_stopped_publication_pending(
+ workbench_api, workbench_db, publication_scan, writable, disposition
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ if disposition == "rejected":
+ save_disposition(scan, result.parent, disposition)
+ snapshots = result.parent / "checkpoint-heads"
+ snapshots.mkdir()
+ original_outputs = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("findings.json", "coverage.json", "scan-manifest.json")
+ }
+ originals = {
+ path: path.read_bytes() for path in (result, result.parent / "checkpoint-head.json")
+ }
+ try:
+ if not writable:
+ snapshots.chmod(0o500)
+ if os.access(snapshots, os.W_OK):
+ pytest.skip("Current user can write through POSIX directory permissions")
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."
+ ),
+ )
+ row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()
+ assert row["status"] == "failed"
+ if not writable:
+ assert row["seal_manifest_digest"] is None
+ retained = json.loads(row["retained_source_digests_json"])
+ heads = json.loads(row["retained_checkpoint_heads_json"])
+ directory = result.parent.relative_to(scan.scan_dir).as_posix()
+ relative_head = f"{directory}/checkpoint-head.json"
+ assert heads[directory] == relative_head
+ assert set(retained["sourceTimes"]) == set(retained["sources"])
+ for relative, digest in retained["sources"].items():
+ _, expected_digest, observed = workbench_api["saved_results"]._read_saved_result(
+ scan.scan_dir, relative, scan.scan_id
+ )
+ assert digest == expected_digest
+ assert retained["sourceTimes"][relative] == observed
+ assert not list(snapshots.iterdir())
+ assert any(
+ "result publication needs follow-up" in warning
+ for warning in json.loads(row["completion_warnings_json"])
+ )
+ assert all(
+ (scan.scan_dir / name).read_bytes() == value
+ for name, value in original_outputs.items()
+ )
+ else:
+ assert row["seal_manifest_digest"]
+ finally:
+ snapshots.chmod(0o700)
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == (1 if disposition == "reported" else 0)
+ assert all(path.read_bytes() == value for path, value in originals.items())
+
+
[email protected]("snapshot_failure", [False, True])
[email protected]("advance_after_cancel", [False, True])
+def test_cancellation_captures_each_remaining_workers_current_head(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ snapshot_failure,
+ advance_after_cancel,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ first = add_worker(workbench_db, scan, status="running")
+ first.write_text(json.dumps(save_disposition(scan, first.parent, "reported")))
+ second = add_worker(workbench_db, scan, status="running")
+ scan.findings[0]["identity"]["anchor"] = "second-original"
+ second.write_text(json.dumps(save_disposition(scan, second.parent, "reported")))
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET created_at = ? WHERE artifact_dir = ?",
+ ("2026-01-01T00:00:00Z", str(first.parent)),
+ )
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET created_at = ? WHERE artifact_dir = ?",
+ ("2026-01-02T00:00:00Z", str(second.parent)),
+ )
+ saved = workbench_api["saved_results"]
+ write = saved.write_scan_local_bytes
+ first_directory = first.parent.relative_to(scan.scan_dir).as_posix()
+ raced = False
+
+ def publish_second_during_first_snapshot(root, relative, contents):
+ nonlocal raced
+ if relative.startswith(first_directory + "/checkpoint-heads/") and not raced:
+ raced = True
+ scan.findings[0]["identity"]["anchor"] = "second-selected-before-cancel"
+ save_disposition(scan, second.parent, "reported")
+ if snapshot_failure:
+ raise OSError("Synthetic first worker snapshot failure")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(
+ "workbench.saved_sources.write_scan_local_bytes", publish_second_during_first_snapshot
+ )
+ saved.cancel_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, thread_id=None),
+ )
+ assert raced
+ if advance_after_cancel:
+ scan.findings[0]["identity"]["anchor"] = "second-selected-after-cancel"
+ save_disposition(scan, second.parent, "reported")
+ retained = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert retained["findingCount"] >= 2
+ anchors = {
+ finding["identity"]["anchor"]
+ for finding in json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ }
+ assert "second-selected-before-cancel" in anchors
+ assert "second-selected-after-cancel" not in anchors
+
+
[email protected]("blocked_report", [False, True])
+def test_cancellation_freezes_sources_before_report_preparation(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ blocked_report,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="running")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ report = scan.scan_dir / "report.md"
+ if blocked_report:
+ report.unlink()
+ report.mkdir()
+ saved = workbench_api["saved_results"]
+ saved.cancel_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, thread_id=None),
+ )
+ row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()
+ assert row["canceled_at"] is not None
+ assert row["retained_source_digests_json"] is not None
+ if blocked_report:
+ report.rmdir()
+ scan.findings[0]["identity"]["anchor"] = "published-after-cancellation"
+ save_disposition(scan, result.parent, "reported")
+ retained = saved.preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert retained["findingCount"] == 1
+ assert all(
+ finding["identity"]["anchor"] != "published-after-cancellation"
+ for finding in json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ )
+
+
[email protected]("missing_old_checkpoint", [False, True])
+def test_legacy_unreadable_head_does_not_hide_another_workers_new_evidence(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ missing_old_checkpoint,
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ checkpoint = (
+ result.parent
+ / "checkpoints"
+ / json.loads((result.parent / "checkpoint-head.json").read_text())["checkpoint"]
+ )
+ saved = workbench_api["saved_results"]
+ prepare = saved._prepare_scan_finalization
+
+ def legacy_publication(*args, **kwargs):
+ kwargs["draft_documents"][0]["scan"].pop("preservedCheckpointHeads")
+ return prepare(*args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_prepare_scan_finalization", legacy_publication)
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,)
+ )
+ before = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json", "report.md")
+ }
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is False
+ )
+ if missing_old_checkpoint:
+ checkpoint.unlink()
+ scan.findings[0]["identity"]["anchor"] = "independent-new-worker"
+ other = add_worker(workbench_db, scan, status="canceled")
+ save_disposition(scan, other.parent, "reported")
+ assert (
+ workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"]["resultsRecoveryNeeded"]
+ is True
+ )
+ assert all((scan.scan_dir / name).read_bytes() == content for name, content in before.items())
+
+
[email protected]("head_state", ["malformed", "missing-checkpoint", "missing-head", "valid"])
+def test_first_stop_keeps_registered_result_when_head_is_unreadable(
+ workbench_api, workbench_db, publication_scan, head_state
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan)
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ head = result.parent / "checkpoint-head.json"
+ if head_state == "missing-head":
+ head.unlink()
+ elif head_state != "valid":
+ head.write_text(
+ "{" if head_state == "malformed" else json.dumps({"checkpoint": "a" * 64 + ".json"})
+ )
+ original = result.read_bytes()
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ assert not stopped["resultsRecoveryNeeded"]
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == 1
+ assert result.read_bytes() == original
+
+
[email protected]("ordering", ["missing", "malformed", "valid"])
+def test_legacy_ordering_failure_does_not_hide_new_worker_evidence(
+ workbench_api, workbench_db, publication_scan, monkeypatch, ordering
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ saved = workbench_api["saved_results"]
+ prepare = saved._prepare_scan_finalization
+
+ def legacy_publication(*args, **kwargs):
+ kwargs["draft_documents"][0]["scan"].pop("preservedCheckpointHeads")
+ return prepare(*args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_prepare_scan_finalization", legacy_publication)
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,)
+ )
+ assert not workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ]
+ ordering_path = next((scan.scan_dir / "source-order").glob("*.json"))
+ if ordering == "missing":
+ ordering_path.unlink()
+ elif ordering == "malformed":
+ ordering_path.write_text("{")
+ original = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json", "report.md")
+ }
+ scan.findings[0]["identity"]["anchor"] = "independent-new-worker"
+ other = add_worker(workbench_db, scan, status="canceled")
+ save_disposition(scan, other.parent, "reported")
+ assert workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"][
+ "resultsRecoveryNeeded"
+ ]
+ assert all(
+ (scan.scan_dir / name).read_bytes() == contents for name, contents in original.items()
+ )
+ if ordering != "valid":
+ with pytest.raises(saved.ContractError):
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id),
+ )
+ else:
+ assert (
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id),
+ )["scan"]["findingCount"]
+ == 2
+ )
+
+
[email protected]("canceled", [False, True], ids=["failed", "canceled"])
+def test_legacy_retained_heads_survive_loss_after_terminal_output(
+ workbench_api, workbench_db, publication_scan, tmp_path, canceled
+):
+ scan = publication_scan()
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ result = add_worker(workbench_db, scan, status="canceled")
+ result.write_text(json.dumps(save_disposition(scan, result.parent, "reported")))
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = NULL, seal_manifest_digest = NULL, "
+ "canceled_at = ? WHERE id = ?",
+ (scan.timestamp if canceled else None, scan.scan_id),
+ )
+ for name in ("scan-manifest.json", "findings.json", "coverage.json", "report.md"):
+ (scan.scan_dir / name).unlink()
+ database = tmp_path / "legacy-stop.sqlite3"
+ with sqlite3.connect(database) as connection:
+ workbench_db.backup(connection)
+ child_program = """
+import json, os, runpy, sqlite3, sys
+from argparse import Namespace
+from types import SimpleNamespace
+api = runpy.run_path(sys.argv[1], run_name="legacy_stopped_publication_loss")
+api["deep_scan"].configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]}))
+saved = api["saved_results"]
+write = saved._write_prepared_scan_finalization
+def lose_process(*args, **kwargs):
+ write(*args, **kwargs)
+ os._exit(73)
+saved._write_prepared_scan_finalization = lose_process
+connection = sqlite3.connect(sys.argv[2])
+connection.row_factory = sqlite3.Row
+connection.execute("PRAGMA foreign_keys = ON")
+saved.preserve_scan_results(api["_WORKBENCH_DB_CONTEXT"], connection, Namespace(scan_id=sys.argv[3], claim_token=None, thread_id=None, coordinator_generation=None))
+raise AssertionError("terminal write was not reached")
+"""
+ child = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ child_program,
+ workbench_api["__file__"],
+ str(database),
+ scan.scan_id,
+ ],
+ capture_output=True,
+ text=True,
+ )
+ assert child.returncode == 73, child.stderr
+ assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1
+ with sqlite3.connect(database) as connection:
+ connection.row_factory = sqlite3.Row
+ connection.execute("PRAGMA foreign_keys = ON")
+ replayed = workbench_api["saved_results"].preserve_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ connection,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert replayed["findingCount"] == 1
+ assert not replayed["resultsRecoveryNeeded"]
+ assert (
+ connection.execute(
+ "SELECT retained_checkpoint_heads_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ is not None
+ )
diff --git a/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py b/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py
new file mode 100644
index 000000000..27cb7a998
--- /dev/null
+++ b/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py
@@ -0,0 +1,1199 @@
+from __future__ import annotations
+
+import copy
+import json
+import os
+from argparse import Namespace
+from pathlib import Path
+
+import pytest
+from test_deep_scan_successful_publication import add_worker
+from test_deep_scan_successful_publication import publication_scan as publication_scan
+from workbench_test_support import interrupt_publication, saved_draft, write_checkpoint
+
+
+ ("mode", "owner"), [("standard", "root"), ("deep", "root"), ("deep", "worker")]
+)
[email protected]("interrupted", [False, True])
+def test_published_raw_head_compatibility_converts_before_later_selection(
+ workbench_api, workbench_db, publication_scan, monkeypatch, mode, owner, interrupted
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ for name in ("scan-manifest.json", "coverage.json", "findings.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if owner == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ initial = saved_draft(scan.scan_id, complete=True)
+ initial["coverage"] = {**scan.coverage, **initial["coverage"]}
+ checkpoint = write_checkpoint(directory / "checkpoints", initial)
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(100, 100))
+ capture = saved._capture_saved_source
+
+ def legacy_capture(root, relative, scan_id, *, checkpoint_heads=None, **kwargs):
+ if Path(relative).name == "checkpoint-head.json" and checkpoint_heads is not None:
+ checkpoint_heads[Path(relative).parent.as_posix()] = relative
+ return capture(root, relative, scan_id, **{**kwargs, "snapshot_head": False})
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.retained_coverage._capture_saved_source", legacy_capture)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ original = (scan.scan_dir / "scan-manifest.json").read_bytes()
+ assert (
+ head.relative_to(scan.scan_dir).as_posix()
+ in json.loads(original)["scan"]["preservedSources"]
+ )
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+
+ with monkeypatch.context() as patch:
+ if interrupted:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.preserve_scan_results(context, workbench_db, args)
+ assert (scan.scan_dir / "scan-manifest.json").read_bytes() == original
+ result = saved.preserve_scan_results(context, workbench_db, args)["scan"]
+ assert not result["resultsRecoveryNeeded"]
+ normalized = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ assert not any(
+ Path(path).name == "checkpoint-head.json" for path in normalized["preservedSources"]
+ )
+ later = write_checkpoint(directory / "checkpoints", {**initial, "findings": scan.findings})
+ head.write_text(json.dumps({"checkpoint": later.name}))
+ os.utime(head, ns=(200, 200))
+ recovered = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert recovered["findingCount"] == 1
+ assert not recovered["resultsRecoveryNeeded"]
+
+
+def test_early_head_error_keeps_captured_records_during_sibling_capture(
+ workbench_api, workbench_db, publication_scan, monkeypatch
+):
+ scan = publication_scan()
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ first = add_worker(workbench_db, scan, status="canceled").parent
+ second = add_worker(workbench_db, scan, status="canceled").parent
+ with workbench_db:
+ for directory, created_at in [
+ (first, "2026-01-01T00:00:00Z"),
+ (second, "2026-01-02T00:00:00Z"),
+ ]:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET created_at = ? WHERE artifact_dir = ?",
+ (created_at, str(directory)),
+ )
+ original = None
+ for directory, anchor in [(first, "before"), (second, "second")]:
+ finding = copy.deepcopy(scan.findings[0])
+ finding["identity"]["anchor"] = anchor
+ checkpoint = write_checkpoint(
+ directory / "checkpoints", saved_draft(scan.scan_id, complete=True, findings=[finding])
+ )
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(100, 100))
+ if directory == first:
+ original = head.read_bytes()
+ write = saved.write_scan_local_bytes
+ changed = False
+
+ def capture_sibling(root, relative, contents):
+ nonlocal changed
+ path = Path(relative)
+ if path.parent == first.relative_to(scan.scan_dir) / "checkpoint-heads":
+ raise OSError("Synthetic first head failure")
+ if path.parent == second.relative_to(scan.scan_dir) / "checkpoint-heads" and not changed:
+ changed = True
+ finding = copy.deepcopy(scan.findings[0])
+ finding["identity"]["anchor"] = "after"
+ checkpoint = write_checkpoint(
+ first / "checkpoints", saved_draft(scan.scan_id, complete=True, findings=[finding])
+ )
+ (first / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(first / "checkpoint-head.json", ns=(200, 200))
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", capture_sibling)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ assert changed
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ with pytest.raises(
+ saved.ContractError, match="Frozen stopped-scan checkpoint set is incomplete"
+ ):
+ saved.preserve_scan_results(context, workbench_db, args)
+ head = first / "checkpoint-head.json"
+ later = head.read_bytes()
+ head.write_bytes(original)
+ os.utime(head, ns=(100, 100))
+ saved.preserve_scan_results(context, workbench_db, args)
+ _, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert {row["identity"]["anchor"] for row in findings["findings"]} == {"before", "second"}
+ head.write_bytes(later)
+ os.utime(head, ns=(200, 200))
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ _, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert "after" in {row["identity"]["anchor"] for row in findings["findings"]}
+
+
[email protected]("restore_capture", [False, True])
+def test_frozen_model_capture_keeps_earlier_worker_finding(
+ workbench_api, workbench_db, publication_scan, monkeypatch, restore_capture
+):
+ scan = publication_scan()
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ model = {"summary": "Retained scan model."}
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"]["threatModel"] = model
+ manifest_path.write_text(json.dumps(manifest))
+ (scan.scan_dir / "findings.json").unlink()
+ result = add_worker(workbench_db, scan, status="succeeded")
+ worker_draft = saved_draft(scan.scan_id, findings=scan.findings, complete=True)
+ result.write_text(json.dumps(worker_draft))
+ partial = saved_draft(scan.scan_id, complete=False)
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", partial)
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ for path, timestamp in ((result, 100_000), (manifest_path, 150_000), (head, 200_000)):
+ os.utime(path, ns=(timestamp, timestamp))
+ model_draft = saved._parent_scan_draft(
+ scan.scan_id, {"threatModel": model}, {"findings": []}, {}
+ )
+ model_path = f"checkpoints/{saved._digest(model_draft)}.json"
+ initial = []
+ write = saved.write_scan_local_bytes
+
+ def interrupt_capture(root, relative, content):
+ if relative == model_path:
+ raise OSError("Synthetic model capture failure.")
+ return write(root, relative, content)
+
+ def interrupt(prepared, **kwargs):
+ initial.append(prepared.findings["findings"])
+ interrupt_publication()
+
+ with monkeypatch.context() as patch:
+ if restore_capture:
+ patch.setattr("workbench.retained_coverage.write_scan_local_bytes", interrupt_capture)
+ else:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ if not restore_capture:
+ assert len(initial) == len(initial[0]) == 1
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ capture = scan.scan_dir / retained["threatModelSource"]
+ assert retained["threatModelSource"] == model_path
+ assert capture.exists() is not restore_capture
+ saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ final_manifest, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert final_manifest["scan"]["threatModel"] == model
+ assert len(findings["findings"]) == 1
+ assert findings["findings"][0]["identity"] == scan.findings[0]["identity"]
+ if not restore_capture:
+ assert findings["findings"] == initial[0]
+ assert capture.read_bytes() == saved._encoded(model_draft)
+
+
[email protected]("mode", ["standard", "deep"])
[email protected]("late_model", [False, True])
[email protected]("failure", ["source-order", "publication"])
+def test_frozen_model_ignores_unselected_live_manifest_changes(
+ workbench_api, workbench_db, publication_scan, monkeypatch, mode, late_model, failure
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ model = {"summary": "Original canonical model"}
+ manifest["scan"].update(complete=True, threatModel=model)
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ _, canonical = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ selected = write_checkpoint(
+ scan.scan_dir / "checkpoints", {k: v for k, v in canonical.items() if k != "threatModel"}
+ )
+ os.utime(selected, ns=(50, 50))
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(200, 200))
+ write = saved.write_scan_local_bytes
+
+ def fail_order(root, relative, contents):
+ if relative.startswith("source-order/"):
+ raise OSError("Synthetic ordering snapshot failure")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ if failure == "source-order":
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_order)
+ else:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ saved.cancel_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, thread_id=None, claim_token=None),
+ )
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ digest = saved._digest(canonical)
+ assert retained["sources"][f"checkpoints/{digest}.json"] == digest
+ if late_model:
+ manifest["scan"]["threatModel"] = {"summary": "Late unselected model"}
+ manifest_path.write_text(json.dumps(manifest))
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ published, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert published["scan"]["threatModel"] == model
+ assert len(findings["findings"]) == replayed["findingCount"] == 1
+ assert not replayed["resultsRecoveryNeeded"]
+ assert "Late unselected model" not in (scan.scan_dir / "threatmodel.md").read_text()
+
+
+ ("mode", "owner"), [("standard", "root"), ("deep", "root"), ("deep", "worker")]
+)
[email protected]("snapshot_failure", [False, True])
[email protected]("resume", ["preserve", "recover"])
[email protected]("interrupted", [False, True])
+def test_resumed_head_snapshot_allows_later_recovery(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ owner,
+ snapshot_failure,
+ resume,
+ interrupted,
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ for name in ("scan-manifest.json", "coverage.json", "findings.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if owner == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ relative = directory.relative_to(scan.scan_dir)
+ initial = saved_draft(scan.scan_id, complete=True)
+ initial["coverage"] = {**scan.coverage, **initial["coverage"]}
+ checkpoint = write_checkpoint(directory / "checkpoints", initial)
+ os.utime(checkpoint, ns=(50, 50))
+ head = directory / "checkpoint-head.json"
+ original_head = json.dumps({"checkpoint": checkpoint.name})
+ head.write_text(original_head)
+ os.utime(head, ns=(100, 100))
+ write = saved.write_scan_local_bytes
+
+ def fail_head(root, path, contents):
+ if Path(path).parent == relative / "checkpoint-heads":
+ raise OSError("Synthetic head snapshot failure")
+ return write(root, path, contents)
+
+ with monkeypatch.context() as patch:
+ if snapshot_failure:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_head)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ retry = saved.preserve_scan_results if resume == "preserve" else saved.recover_scan_results
+
+ with monkeypatch.context() as patch:
+ if interrupted and snapshot_failure:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ retry(context, workbench_db, args)
+ else:
+ retry(context, workbench_db, args)
+ saved.preserve_scan_results(context, workbench_db, args)
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ assert not any(
+ Path(path).name == "checkpoint-head.json" for path in manifest["preservedSources"]
+ )
+ observation = manifest["preservedCheckpointHeads"][relative.as_posix()]
+ assert json.loads((scan.scan_dir / observation).read_text()) == {
+ "checkpoint": checkpoint.name,
+ "observedAtNs": "100",
+ }
+ assert head.read_text() == original_head
+ later = write_checkpoint(directory / "checkpoints", {**initial, "findings": scan.findings})
+ os.utime(later, ns=(50, 50))
+ head.write_text(json.dumps({"checkpoint": later.name}))
+ os.utime(head, ns=(200, 200))
+ result = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert result["findingCount"] == 1
+ assert not result["resultsRecoveryNeeded"]
+ published, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert len(findings["findings"]) == 1
+ assert observation in published["scan"]["preservedSources"]
+
+
+ ("mode", "failure"),
+ [
+ ("standard", "none"),
+ ("standard", "root-head"),
+ ("deep", "root-head"),
+ ("deep", "worker-head"),
+ ("deep", "source-order"),
+ ],
+)
[email protected]("model_choice", ["canonical", "accepted", "selected"])
[email protected]("live_change", ["coverage", "model"])
+def test_early_head_failure_keeps_frozen_model_choice(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ failure,
+ model_choice,
+ live_change,
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"]["threatModel"] = {"summary": "Canonical model"}
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "coverage.json", "findings.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ draft = saved_draft(scan.scan_id, complete=True, findings=scan.findings)
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ head = scan.scan_dir / "checkpoint-head.json"
+ for label, observed in [
+ ("Accepted model", 180 if model_choice == "accepted" else 100),
+ ("Selected model" if model_choice == "selected" else None, 200),
+ ]:
+ payload = {**draft, **({"threatModel": {"summary": label}} if label else {})}
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", payload)
+ os.utime(checkpoint, ns=(50, 50))
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ if label == "Accepted model":
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ if failure == "worker-head":
+ directory = add_worker(workbench_db, scan, status="canceled").parent
+ worker_checkpoint = write_checkpoint(
+ directory / "checkpoints", saved_draft(scan.scan_id, complete=True)
+ )
+ worker_head = directory / "checkpoint-head.json"
+ worker_head.write_text(json.dumps({"checkpoint": worker_checkpoint.name}))
+ os.utime(worker_head, ns=(200, 200))
+ write = saved.write_scan_local_bytes
+
+ def fail_snapshot(root, relative, contents):
+ parent = Path(relative).parent
+ if (
+ (failure == "root-head" and parent == Path("checkpoint-heads"))
+ or (
+ failure == "worker-head"
+ and parent.name == "checkpoint-heads"
+ and parent != Path("checkpoint-heads")
+ )
+ or (failure == "source-order" and parent == Path("source-order"))
+ ):
+ raise OSError("Synthetic snapshot failure")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_snapshot)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ if failure != "none":
+ if live_change == "coverage":
+ coverage_path = scan.scan_dir / "coverage.json"
+ coverage = json.loads(coverage_path.read_text())
+ coverage["openQuestions"] = [{"question": "Late coverage update"}]
+ coverage_path.write_text(json.dumps(coverage))
+ else:
+ manifest["scan"]["threatModel"] = {"summary": "Late unselected model"}
+ manifest_path.write_text(json.dumps(manifest))
+ result = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ published, findings, _ = saved.finalize_scan(scan.scan_dir)
+ expected = {
+ "canonical": "Canonical model",
+ "accepted": "Accepted model",
+ "selected": "Selected model",
+ }[model_choice]
+ assert published["scan"]["threatModel"] == {"summary": expected}
+ assert result["findingCount"] == len(findings["findings"]) == 1
+ assert not result["resultsRecoveryNeeded"]
+
+
+ ("mode", "owner"), [("standard", "root"), ("deep", "root"), ("deep", "worker")]
+)
[email protected]("change", ["head", "timestamp", "payload"])
[email protected]("resume", ["preserve", "recover"])
+def test_failed_head_fallback_rejects_changes_before_resume(
+ workbench_api, workbench_db, publication_scan, monkeypatch, mode, owner, change, resume
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ for name in ("scan-manifest.json", "coverage.json", "findings.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if owner == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ relative = directory.relative_to(scan.scan_dir)
+ initial = saved_draft(scan.scan_id, complete=True)
+ initial["coverage"] = {**scan.coverage, **initial["coverage"]}
+ checkpoint = write_checkpoint(directory / "checkpoints", initial)
+ original_checkpoint = checkpoint.read_bytes()
+ os.utime(checkpoint, ns=(50, 50))
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(100, 100))
+ original_head = head.read_bytes()
+ original_outputs = {
+ name: (scan.scan_dir / name).read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json")
+ }
+ write = saved.write_scan_local_bytes
+
+ def fail_head(root, path, contents):
+ if Path(path).parent == relative / "checkpoint-heads":
+ raise OSError("Synthetic head snapshot failure")
+ return write(root, path, contents)
+
+ with monkeypatch.context() as patch:
+ patch.setattr("workbench.saved_sources.write_scan_local_bytes", fail_head)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ if change == "head":
+ later = write_checkpoint(directory / "checkpoints", {**initial, "findings": scan.findings})
+ head.write_text(json.dumps({"checkpoint": later.name}))
+ os.utime(head, ns=(100, 100))
+ elif change == "timestamp":
+ os.utime(head, ns=(200, 200))
+ else:
+ checkpoint.write_text(json.dumps({**initial, "findings": scan.findings}))
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ retry = saved.preserve_scan_results if resume == "preserve" else saved.recover_scan_results
+ with pytest.raises(
+ saved.ContractError,
+ match="changed after the scan stopped|Frozen stopped-scan checkpoint set is incomplete",
+ ):
+ retry(context, workbench_db, args)
+ assert all(
+ (scan.scan_dir / name).read_bytes() == value for name, value in original_outputs.items()
+ )
+ head.write_bytes(original_head)
+ os.utime(head, ns=(100, 100))
+ checkpoint.write_bytes(original_checkpoint)
+ os.utime(checkpoint, ns=(50, 50))
+ if change == "head":
+ later.unlink()
+ result = saved.preserve_scan_results(context, workbench_db, args)["scan"]
+ assert result["findingCount"] == 0
+ assert not result["resultsRecoveryNeeded"]
+
+
[email protected]("mode", ["standard", "deep"])
[email protected]("model_format", ["structured", "markdown"])
[email protected]("replacement", ["omitted", "valid", "invalid"])
[email protected]("interrupted", [False, True])
+def test_rejected_historical_model_does_not_poison_recovery(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ model_format,
+ replacement,
+ interrupted,
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ model = {"summary": "Valid canonical model"}
+ invalid = (
+ {"summary": 42} if model_format == "structured" else {"format": "markdown", "content": " "}
+ )
+ manifest["scan"]["threatModel"] = model
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ draft = saved_draft(scan.scan_id, complete=True)
+ draft.update(findings=scan.findings, coverage={**scan.coverage, **draft["coverage"]})
+
+ def select(value, observed):
+ path = write_checkpoint(scan.scan_dir / "checkpoints", value)
+ os.utime(path, ns=(50, 50))
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": path.name}))
+ os.utime(head, ns=(observed, observed))
+ return path.relative_to(scan.scan_dir).as_posix()
+
+ _, canonical = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ canonical_source = f"checkpoints/{saved._digest(canonical)}.json"
+ select({**draft, "threatModel": {"summary": "Older accepted model"}}, 100)
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ invalid_source = select({**draft, "threatModel": invalid}, 200)
+ stopped = saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["findingCount"] == 1
+ assert any("Skipped malformed saved threat model" in warning for warning in stopped["warnings"])
+ assert json.loads(manifest_path.read_text())["scan"]["threatModel"] == model
+ args = Namespace(scan_id=scan.scan_id)
+ chosen = (
+ draft
+ if replacement == "omitted"
+ else {
+ **draft,
+ "threatModel": invalid
+ if replacement == "invalid"
+ else {"summary": "Replacement model"},
+ }
+ )
+ selected_source = select(chosen, 300)
+
+ with monkeypatch.context() as patch:
+ if interrupted:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, args)
+ else:
+ saved.recover_scan_results(context, workbench_db, args)
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ published, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert published["scan"]["threatModel"] == (
+ {"summary": "Replacement model"} if replacement == "valid" else model
+ )
+ retained = workbench_db.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ assert json.loads(retained)["threatModelSource"] == (
+ selected_source if replacement == "valid" else canonical_source
+ )
+ assert invalid_source in published["scan"]["preservedSources"]
+ assert len(findings["findings"]) == replayed["findingCount"] == 1
+ assert not replayed["resultsRecoveryNeeded"]
+
+
+ ("mode", "owner", "registered"),
+ [
+ ("standard", "root", False),
+ ("deep", "root", False),
+ ("deep", "worker", False),
+ ("deep", "worker", True),
+ ],
+)
[email protected]("partial_time", [150, 300])
[email protected]("interrupted", [False, True])
[email protected]("repeated_terminal", [False, True])
+def test_partial_head_retains_previous_terminal_coverage_boundary(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ owner,
+ registered,
+ partial_time,
+ interrupted,
+ repeated_terminal,
+):
+ scan = publication_scan(mode=mode)
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ coverage = {
+ **scan.coverage,
+ "surfaces": [],
+ "explicitExclusions": [],
+ "deferred": [],
+ "openQuestions": [],
+ }
+ (scan.scan_dir / "coverage.json").write_text(json.dumps(coverage))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if owner == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ for label, observed in [("Obsolete", 100), ("Replacement", 200), ("Partial", partial_time)]:
+ draft = saved_draft(scan.scan_id, complete=label != "Partial")
+ draft["coverage"] = {**coverage, **draft["coverage"]}
+ if label != "Partial":
+ draft["coverage"].update(
+ surfaces=[
+ {
+ "id": label,
+ "label": label,
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ ],
+ explicitExclusions=[{"pattern": label + "/**", "reason": label}],
+ openQuestions=[{"question": label}],
+ )
+ checkpoint = write_checkpoint(directory / "checkpoints", draft)
+ os.utime(checkpoint, ns=(50, 50))
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ if label == "Replacement" and registered:
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET result_manifest_path = ? WHERE id = ?",
+ (str(checkpoint), directory.name),
+ )
+ if label == "Obsolete":
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."
+ ),
+ )
+ elif label == "Replacement":
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ if repeated_terminal:
+ os.utime(head, ns=(400, 400))
+ saved._capture_saved_source(
+ scan.scan_dir, head.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ else:
+ with monkeypatch.context() as patch:
+ if interrupted:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ else:
+ saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ _, _, coverage = saved.finalize_scan(scan.scan_dir)
+ expected = {"Replacement"} if partial_time == 300 or registered else {"Obsolete", "Replacement"}
+ assert {row["label"] for row in coverage["surfaces"]} == expected
+ assert {row["pattern"] for row in coverage["explicitExclusions"]} == {
+ label + "/**" for label in expected
+ }
+ assert {row["question"] for row in coverage["openQuestions"]} == expected
+ assert not replayed["resultsRecoveryNeeded"]
+
+
[email protected]("head_time", [100_000, 200_000], ids=["older", "tied"])
+def test_retry_keeps_canonical_finding_when_its_checkpoint_copy_is_unreadable(
+ workbench_api, workbench_db, publication_scan, monkeypatch, head_time
+):
+ scan = publication_scan(mode="standard")
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ model = {"summary": "Canonical model"}
+ manifest["scan"].update(complete=True, threatModel=model)
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(200_000, 200_000))
+ _, canonical = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ empty = {key: value for key, value in canonical.items() if key != "threatModel"}
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", {**empty, "findings": []})
+ broken = checkpoint.parent / f"{saved._digest(canonical)}.json"
+ broken.write_bytes(b"{")
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(head_time, head_time))
+
+ attempted = []
+
+ def interrupt(prepared, **kwargs):
+ attempted.append(
+ (prepared.findings["findings"][0]["title"], prepared.manifest["scan"]["threatModel"])
+ )
+ raise OSError("Synthetic publication interruption")
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ assert attempted == [(scan.findings[0]["title"], model)]
+ with pytest.raises(
+ saved.ContractError, match="Frozen stopped-scan checkpoint set is incomplete"
+ ):
+ saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ assert broken.read_bytes() == b"{"
+ assert "sealedAt" not in json.loads(manifest_path.read_text())["scan"]
+
+
[email protected]("mode", ["standard", "deep"])
[email protected]("failed_capture", [False, True])
[email protected]("late_model", [False, True])
[email protected]("replay", ["preserve", "recover"])
+def test_inherited_model_capture_is_bound_before_publication_retry(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ failed_capture,
+ late_model,
+ replay,
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ model = {"summary": "Original canonical model"}
+ manifest["scan"].update(complete=True, threatModel=model)
+ manifest_path.write_text(json.dumps(manifest))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(150, 150))
+ _, canonical = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ selected = write_checkpoint(
+ scan.scan_dir / "checkpoints", {k: v for k, v in canonical.items() if k != "threatModel"}
+ )
+ os.utime(selected, ns=(50, 50))
+ broken = scan.scan_dir / f"checkpoints/{saved._digest(canonical)}.json"
+ broken.write_text("{")
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(200, 200))
+ model_draft = saved._parent_scan_draft(
+ scan.scan_id, {"threatModel": model}, {"findings": []}, {}
+ )
+ model_source = f"checkpoints/{saved._digest(model_draft)}.json"
+ write = saved.write_scan_local_bytes
+
+ def fail_capture(root, relative, contents):
+ if relative == model_source:
+ raise OSError("Synthetic model capture failure")
+ return write(root, relative, contents)
+
+ with monkeypatch.context() as patch:
+ if failed_capture:
+ patch.setattr("workbench.retained_coverage.write_scan_local_bytes", fail_capture)
+ else:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ row = workbench_db.execute(
+ "SELECT retained_source_digests_json, completion_warnings_json FROM scans WHERE id = ?",
+ (scan.scan_id,),
+ ).fetchone()
+ retained = json.loads(row[0])
+ assert retained["sources"][model_source] == saved._digest(model_draft)
+ assert (scan.scan_dir / model_source).exists() is not failed_capture
+ assert any("Preserved unreadable checkpoint" in warning for warning in json.loads(row[1]))
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ retry = saved.preserve_scan_results if replay == "preserve" else saved.recover_scan_results
+ if late_model:
+ late = copy.deepcopy(manifest)
+ late["scan"]["threatModel"] = {"summary": "Late unselected model"}
+ manifest_path.write_text(json.dumps(late))
+ if failed_capture:
+ with pytest.raises(
+ saved.ContractError, match="Frozen stopped-scan checkpoint set is incomplete"
+ ):
+ retry(context, workbench_db, args)
+ assert "sealedAt" not in json.loads(manifest_path.read_text())["scan"]
+ manifest_path.write_text(json.dumps(manifest))
+ result = retry(context, workbench_db, args)["scan"]
+ published, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert published["scan"]["threatModel"] == model
+ assert len(findings["findings"]) == result["findingCount"] == 1
+ assert not result["resultsRecoveryNeeded"]
+ assert broken.read_text() == "{"
+ assert (
+ saved._frozen_source_times(
+ scan.scan_dir, scan.scan_id, published["scan"]["preservedSources"]
+ )[model_source]
+ == 150
+ )
+
+
+ ("mode", "owner"), [("standard", "root"), ("deep", "root"), ("deep", "worker")]
+)
[email protected]("pending", ["linked", "unlinked", "closed"])
[email protected]("interrupted", [False, True])
+def test_restored_terminal_keeps_surfaces_for_surviving_deferred_work(
+ workbench_api, workbench_db, publication_scan, monkeypatch, mode, owner, pending, interrupted
+):
+ scan = publication_scan(mode=mode)
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ coverage = {
+ **scan.coverage,
+ "surfaces": [],
+ "explicitExclusions": [],
+ "deferred": [],
+ "openQuestions": [],
+ }
+ (scan.scan_dir / "coverage.json").write_text(json.dumps(coverage))
+ for name in ("findings.json", "coverage.json", "scan-manifest.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ directory = (
+ scan.scan_dir
+ if owner == "root"
+ else add_worker(workbench_db, scan, status="canceled").parent
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ terminal = saved_draft(scan.scan_id, complete=True)
+ terminal["coverage"] = {**coverage, **terminal["coverage"]}
+ task = {"id": "review-task", "reason": "Review remains pending."}
+ if pending != "unlinked":
+ task["surfaceIds"] = ["pending-surface"]
+ surface = {
+ "id": "pending-surface",
+ "label": "Unfinished authorization review",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ partial = saved_draft(scan.scan_id, complete=False, deferred=[task], surfaces=[surface])
+ partial["coverage"] = {**coverage, **partial["coverage"]}
+ sequence = [(terminal, 100), (partial, 200)]
+ if pending == "closed":
+ closed = copy.deepcopy(terminal)
+ closed["coverage"]["resolvedDeferred"] = [{"id": task["id"], "reason": "Review completed."}]
+ sequence.append((closed, 300))
+ sequence.append((terminal, 100))
+ for index, (draft, observed) in enumerate(sequence):
+ checkpoint = write_checkpoint(directory / "checkpoints", draft)
+ os.utime(checkpoint, ns=(50, 50))
+ head = directory / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ if index == 0:
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."
+ ),
+ )
+ else:
+ with monkeypatch.context() as patch:
+ if interrupted and index == len(sequence) - 1:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt_publication)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ else:
+ saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ replayed = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ _, _, coverage = saved.finalize_scan(scan.scan_dir)
+ if owner == "worker":
+ surface = {**surface, "id": f"{directory.name}/pending-surface", "workerId": directory.name}
+ task = {**task, "id": f"{directory.name}/review-task", "workerId": directory.name}
+ if "surfaceIds" in task:
+ task["surfaceIds"] = [surface["id"]]
+ assert coverage["surfaces"] == ([surface] if pending == "linked" else [])
+ assert (task in coverage["deferred"]) is (pending != "closed")
+ assert (surface["label"] in (scan.scan_dir / "report.md").read_text()) is (pending == "linked")
+ assert not replayed["resultsRecoveryNeeded"]
+
+
+def test_source_order_failure_retains_accepted_parent_input(
+ workbench_api, workbench_db, publication_scan, monkeypatch
+):
+ from workbench import saved_sources
+
+ scan = publication_scan()
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ scan_dir = scan.scan_dir
+ manifest = scan_dir / "scan-manifest.json"
+ contents = json.loads(manifest.read_text())
+ contents["scan"]["complete"] = True
+ manifest.write_text(json.dumps(contents))
+ _, projected = saved._read_saved_parent_result(scan_dir, scan.scan_id)
+ checkpoints = scan_dir / "checkpoints"
+ publication = {"previousCheckpoint": None, "previousInputCheckpoint": None}
+ accepted = write_checkpoint(
+ checkpoints,
+ {
+ **saved_draft(scan.scan_id, findings=scan.findings, complete=True),
+ "publication": publication,
+ },
+ )
+ selected = write_checkpoint(checkpoints, projected)
+ abandoned = write_checkpoint(
+ checkpoints, {**saved_draft(scan.scan_id, complete=True), "publication": publication}
+ )
+ head = scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name, "inputCheckpoint": accepted.name}))
+ for path in (
+ accepted,
+ selected,
+ head,
+ manifest,
+ scan_dir / "findings.json",
+ scan_dir / "coverage.json",
+ ):
+ os.utime(path, ns=(200, 200))
+ os.utime(abandoned, ns=(300, 300))
+
+ write = saved_sources.write_scan_local_bytes
+
+ def fail_order(root, path, data):
+ if Path(path).parent == Path("source-order"):
+ raise OSError("Synthetic source-order failure")
+ return write(root, path, data)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved_sources, "write_scan_local_bytes", fail_order)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ assert "sealedAt" not in json.loads(manifest.read_text())["scan"]
+ head.write_text(json.dumps({"checkpoint": abandoned.name, "inputCheckpoint": abandoned.name}))
+ os.utime(head, ns=(400, 400))
+ saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ _, findings, _ = saved.finalize_scan(scan_dir)
+ assert [finding["title"] for finding in findings["findings"]] == [scan.findings[0]["title"]]
+
+
+def test_headless_canonical_snapshot_failure(
+ workbench_api, workbench_db, publication_scan, monkeypatch
+):
+ from workbench import saved_sources
+
+ scan = publication_scan()
+ saved = workbench_api["saved_results"]
+ db = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ write = saved_sources.write_scan_local_bytes
+ original = scan.findings[0]["title"]
+ head = scan.scan_dir / "checkpoint-head.json"
+ assert not head.exists()
+
+ def fail_snapshot(root, path, data):
+ if Path(path).parent == Path("checkpoint-heads"):
+ raise OSError("Synthetic canonical observation failure")
+ return write(root, path, data)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved_sources, "write_scan_local_bytes", fail_snapshot)
+ saved.fail_scan(
+ db,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ assert not head.exists()
+ assert "sealedAt" not in json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]
+ raw = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {
+ **saved_draft(scan.scan_id, complete=True),
+ "publication": {"previousCheckpoint": None, "previousInputCheckpoint": None},
+ },
+ )
+ head.write_text(json.dumps({"checkpoint": raw.name, "inputCheckpoint": raw.name}))
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ saved.preserve_scan_results(
+ db,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ manifest, findings, _ = saved.finalize_scan(scan.scan_dir)
+ assert [f["title"] for f in findings["findings"]] == [original]
+ assert "." not in manifest["scan"]["preservedCheckpointHeads"]
+
+
+ ("retry", "selection"),
+ [
+ ("preserve", "first"),
+ ("recover", "first"),
+ ("recover", "old-journal"),
+ ("recover", "frozen"),
+ ],
+)
+def test_interrupted_recovery_keeps_frozen_parent_reconciliation(
+ workbench_api, workbench_db, publication_scan, monkeypatch, retry, selection
+):
+ scan = publication_scan(mode="deep")
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET status = 'failed', completed_at = started_at WHERE id = ?",
+ (scan.scan_id,),
+ )
+ worker = add_worker(workbench_db, scan, status="canceled")
+ draft = saved_draft(scan.scan_id, complete=True)
+ worker.write_text(json.dumps({**draft, "findings": scan.findings}))
+ os.utime(worker, ns=(100_000, 100_000))
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", draft)
+ os.utime(checkpoint, ns=(250_000, 250_000))
+ manifest = scan.scan_dir / "scan-manifest.json"
+ contents = json.loads(manifest.read_text())
+ contents["scan"]["complete"] = True
+ manifest.write_text(json.dumps(contents))
+ findings_file = scan.scan_dir / "findings.json"
+ findings_file.write_text(json.dumps({"findings": []}))
+ for path in (manifest, findings_file, scan.scan_dir / "coverage.json"):
+ os.utime(path, ns=(300_000, 300_000))
+
+ expected = [] if selection == "frozen" else [scan.findings[0]["title"]]
+
+ def interrupt(prepared, **kwargs):
+ assert [row["title"] for row in prepared.findings["findings"]] == expected
+ interrupt_publication()
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", interrupt)
+ if selection == "frozen":
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.preserve_scan_results(context, workbench_db, args)
+ with pytest.raises(OSError, match="Synthetic publication interruption"):
+ saved.recover_scan_results(context, workbench_db, args)
+ if selection == "old-journal":
+ retained = json.loads(
+ workbench_db.execute(
+ "SELECT retained_checkpoint_heads_json FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ )
+ del retained["legacyParent"]
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = ?",
+ (json.dumps(retained), scan.scan_id),
+ )
+ findings_file.write_text(
+ json.dumps({"findings": [{**scan.findings[0], "title": "Late unselected report"}]})
+ )
+ resume = saved.preserve_scan_results if retry == "preserve" else saved.recover_scan_results
+ result = resume(context, workbench_db, args)["scan"]
+ assert result["findingCount"] == len(expected)
+ assert [row["title"] for row in json.loads(findings_file.read_text())["findings"]] == expected
diff --git a/plugins/codex-security/tests/test_checkpoint_schema.py b/plugins/codex-security/tests/test_checkpoint_schema.py
new file mode 100644
index 000000000..459ddefad
--- /dev/null
+++ b/plugins/codex-security/tests/test_checkpoint_schema.py
@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from contextlib import closing
+
+import pytest
+
+
+ "omitted_versions, legacy_checkpoint, main_reader",
+ [
+ ((), False, False),
+ ((57,), False, False),
+ ((), 47, False),
+ ((), 47, True),
+ ((), 48, False),
+ ((), 50, False),
+ ((), 52, False),
+ ((), 53, False),
+ ((), 54, False),
+ ((43,), 47, False),
+ ((44, 45, 46), 47, False),
+ ],
+ ids=[
+ "fresh",
+ "upgrade",
+ "legacy-checkpoint",
+ "legacy-main-reader",
+ "prior-checkpoint-number",
+ "previous-head-checkpoint-number",
+ "refresh-checkpoint-number",
+ "previous-main-checkpoint-number",
+ "checkpoint-before-headless-recovery",
+ "legacy-severity-gap",
+ "legacy-dedupe-gap",
+ ],
+)
+def test_frozen_checkpoint_head_migration_preserves_scan_state(
+ workbench_api, omitted_versions, legacy_checkpoint, main_reader
+):
+ migrations = workbench_api["MIGRATIONS"]
+ timestamp = "2026-07-01T00:00:00Z"
+
+ def migrate(connection, selected):
+ workbench_api["apply_schema_migrations"](
+ connection, selected, lambda: timestamp, workbench_api["backfill_security_targets"]
+ )
+
+ with closing(sqlite3.connect(":memory:")) as connection:
+ connection.row_factory = sqlite3.Row
+ connection.execute("PRAGMA foreign_keys = ON")
+ migrate(
+ connection,
+ tuple(
+ (legacy_checkpoint, name, sql)
+ if legacy_checkpoint and version == 57
+ else (version, name, sql)
+ for version, name, sql in migrations
+ if version not in omitted_versions
+ and not (legacy_checkpoint and version in (54, 55, 56))
+ and not (legacy_checkpoint in (47, 48, 50, 52, 53) and version in (52, 53))
+ and not (legacy_checkpoint in (47, 48, 50) and version in (50, 51))
+ and not (legacy_checkpoint in (47, 48) and version in (48, 49))
+ and not (legacy_checkpoint == 47 and version == 47)
+ ),
+ )
+ if main_reader:
+ connection.execute(next(sql for version, _, sql in migrations if version == 47))
+ connection.execute(
+ "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)",
+ ("workspace", timestamp, timestamp),
+ )
+ for status in ("running", "failed"):
+ connection.execute(
+ "INSERT INTO scans (id, workspace_id, target_path, target_revision, scope, "
+ "mode, scan_dir, status, phase, started_at, created_at, updated_at, "
+ "failure_message, retained_source_digests_json) "
+ "VALUES (?, 'workspace', 'target', 'revision', '.', 'deep', ?, ?, "
+ "'discovery', ?, ?, ?, ?, ?)",
+ (
+ status,
+ f"scans/{status}",
+ status,
+ timestamp,
+ timestamp,
+ timestamp,
+ "Original stop reason." if status == "failed" else None,
+ json.dumps({"workers/review/result.json": "b" * 64})
+ if status == "failed"
+ else None,
+ ),
+ )
+ heads = json.dumps({"workers/review": "workers/review/checkpoints/" + "a" * 64 + ".json"})
+ if 57 not in omitted_versions:
+ connection.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = 'failed'", (heads,)
+ )
+ original_checkpoint_migration = connection.execute(
+ "SELECT * FROM schema_migrations WHERE version = ?",
+ (legacy_checkpoint or 57,),
+ ).fetchone()
+ connection.execute(
+ "INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, status, phase, "
+ "workers, subagents, stop_after_no_new, max_discovery_runs, created_at, updated_at) "
+ "VALUES ('failed', 1, 'v1', 'failed', 'terminal', 1, 0, 7, 10, ?, ?)",
+ (timestamp, timestamp),
+ )
+ context = json.dumps("Original discovery context")
+ if legacy_checkpoint != 47 or main_reader:
+ connection.execute(
+ "UPDATE deep_scan_runs SET discovery_user_context_json = ?", (context,)
+ )
+ original_discovery_migration = connection.execute(
+ "SELECT name, applied_at FROM schema_migrations WHERE version = 47"
+ ).fetchone()
+ before = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")]
+ migrate(connection, migrations)
+ after = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")]
+ for original, updated in zip(before, after, strict=True):
+ retained_heads = updated.pop("retained_checkpoint_heads_json")
+ assert retained_heads == original.get("retained_checkpoint_heads_json")
+ original.pop("retained_checkpoint_heads_json", None)
+ if "source_scopes_json" not in original:
+ source_scopes = updated.pop("source_scopes_json")
+ assert source_scopes is None
+ assert updated == original
+
+ connection.execute(
+ "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = 'failed'", (heads,)
+ )
+ migrate(connection, migrations)
+ assert (
+ connection.execute(
+ "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'failed'"
+ ).fetchone()[0]
+ == heads
+ )
+ assert (
+ connection.execute(
+ "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'running'"
+ ).fetchone()[0]
+ is None
+ )
+ assert [
+ tuple(row)
+ for row in connection.execute(
+ "SELECT version, name FROM schema_migrations WHERE version = 57"
+ )
+ ] == [(57, "freeze stopped scan checkpoint selections")]
+ if original_checkpoint_migration is not None:
+ assert (
+ connection.execute(
+ "SELECT name, applied_at FROM schema_migrations WHERE version = 57"
+ ).fetchone()[:]
+ == original_checkpoint_migration[1:]
+ )
+ assert connection.execute(
+ "SELECT COUNT(*), MAX(version) FROM schema_migrations"
+ ).fetchone()[:] == (len(migrations), 57)
+ for version in (43, 44, 45, 46, 47):
+ assert (
+ connection.execute(
+ "SELECT COUNT(*) FROM schema_migrations WHERE version = ?", (version,)
+ ).fetchone()[0]
+ == 1
+ )
+ assert connection.execute(
+ "SELECT discovery_user_context_json FROM deep_scan_runs"
+ ).fetchone()[0] == (context if legacy_checkpoint != 47 or main_reader else None)
+ if legacy_checkpoint != 47:
+ assert (
+ connection.execute(
+ "SELECT name, applied_at FROM schema_migrations WHERE version = 47"
+ ).fetchone()
+ == original_discovery_migration
+ )
+ assert connection.execute("PRAGMA foreign_key_check").fetchall() == []
+
+
+def test_current_checkpoint_schema_reader_does_not_wait_for_writer(workbench_api, tmp_path):
+ database_path = tmp_path / "workbench.sqlite3"
+
+ def migrate(connection):
+ connection.row_factory = sqlite3.Row
+ workbench_api["apply_schema_migrations"](
+ connection,
+ workbench_api["MIGRATIONS"],
+ lambda: "2026-07-01T00:00:00Z",
+ workbench_api["backfill_security_targets"],
+ )
+
+ with closing(sqlite3.connect(database_path)) as writer:
+ writer.execute("PRAGMA journal_mode = WAL")
+ migrate(writer)
+ writer.execute("BEGIN IMMEDIATE")
+ with closing(sqlite3.connect(database_path, timeout=0)) as reader:
+ migrate(reader)
+ writer.rollback()
diff --git a/plugins/codex-security/tests/test_deep_coverage_recovery.py b/plugins/codex-security/tests/test_deep_coverage_recovery.py
index eda32ba35..b974bd737 100644
--- a/plugins/codex-security/tests/test_deep_coverage_recovery.py
+++ b/plugins/codex-security/tests/test_deep_coverage_recovery.py
@@ -91,6 +91,43 @@ def test_worker_projection_retains_independent_links_receipts_and_frozen_replay(
assert all(p.read_bytes() == contents for p, contents in originals.items())
+def test_partial_root_rejections_close_worker_local_candidates_independently(tmp_path):
+ workers = []
+ for name in ("first", "second"):
+ task = {"id": "review", "candidateId": "local", "reason": f"Review {name}."}
+ source = worker(
+ tmp_path, name, saved_draft("coverage-scan", complete=True, deferred=[task])
+ )
+ source.update(
+ status="succeeded",
+ result_manifest_path=str(Path(source["artifact_dir"]) / "result.json"),
+ )
+ workers.append(source)
+
+ head = tmp_path / "checkpoint-head.json"
+ for name, observed, pending in [("first", 200, ["Review second."]), ("second", 300, [])]:
+ rejection = {
+ "id": name,
+ "label": f"Reviewed {name}.",
+ "candidateId": "local",
+ "disposition": "rejected",
+ "provenance": {"workerId": name, "attempt": 1, "candidateId": "local"},
+ }
+ checkpoint = write_checkpoint(
+ tmp_path / "checkpoints", saved_draft("coverage-scan", surfaces=[rejection])
+ )
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ documents = recover(tmp_path, workers, project=True)
+ assert [
+ row["reason"] for row in documents[2]["deferred"] if row["id"] != "scan-stopped"
+ ] == pending
+ replay = recover(
+ tmp_path, workers, project=True, frozen=documents[0]["scan"]["preservedSources"]
+ )
+ assert replay[2] == documents[2]
+
+
def test_accepted_attempt_reconciles_legacy_parent_worker_projection(tmp_path):
surface = {
"id": "api",
@@ -190,6 +227,40 @@ def test_partial_parent_without_arrays_retains_closed_worker_surface_and_frozen_
assert all(path.read_bytes() == contents for path, contents in originals.items())
+def test_finding_only_reducer_preserves_earlier_partial_root_coverage(tmp_path):
+ scan_id = "coverage-scan"
+ surface = {"id": "route", "label": "Route", "disposition": "no_issue_found", "receiptRefs": []}
+ question = {"question": "Which routes are public?"}
+ earlier = saved_draft(scan_id, surfaces=[surface])
+ earlier["coverage"]["openQuestions"] = [question]
+ head = tmp_path / "checkpoint-head.json"
+ for observed, draft in [(100_000, earlier), (200_000, saved_draft(scan_id))]:
+ checkpoint = write_checkpoint(tmp_path / "checkpoints", draft)
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ saved._capture_saved_source(tmp_path, "checkpoint-head.json", scan_id)
+
+ finding_path = (
+ Path(saved.__file__).resolve().parents[1] / "examples/completed-scan/findings.json"
+ )
+ finding = json.loads(finding_path.read_text())["findings"][0]
+ result = {"scanId": scan_id, "findings": [finding]}
+ reducer = worker(tmp_path, "reducer", result)
+ output = Path(reducer["artifact_dir"])
+ write_checkpoint(output / "checkpoints", result)
+ os.utime(output / "result.json", ns=(300_000, 300_000))
+ reducer.update(
+ kind="dedup",
+ status="succeeded",
+ completed_at="2026-01-01T00:00:00Z",
+ result_manifest_path=str(output / "result.json"),
+ )
+
+ _, findings, coverage = recover(tmp_path, [reducer])
+ assert {row["title"] for row in findings["findings"]} == {finding["title"]}
+ assert (coverage["surfaces"], coverage.get("openQuestions", [])) == ([surface], [question])
+
+
@pytest.mark.parametrize("collision", [False, True])
@pytest.mark.parametrize("parent_has_worker_metadata", [False, True])
def test_worker_links_follow_final_ids_without_redirecting_parent_links(
@@ -1028,6 +1099,42 @@ def test_projection_normalizes_surface_receipts_without_rewriting_extension_meta
assert json.loads((Path(source["artifact_dir"]) / "result.json").read_text()) == draft
[email protected]("prior", ["reported", "pending"])
[email protected]("earlier_name", ["Review.txt", "review.txt"])
+def test_anonymous_worker_receipt_precedes_another_workers_rejection(tmp_path, earlier_name, prior):
+ example = Path(__file__).resolve().parents[1] / "examples/completed-scan/findings.json"
+ finding = json.loads(example.read_text())["findings"][0]
+ finding["extensions"] = {"candidateId": "review"}
+ task = {"candidateId": "review", "reason": "Review is pending."}
+ records = {"findings": [finding]} if prior == "reported" else {"deferred": [task]}
+ earlier_ref = f"artifacts/{earlier_name}"
+ ref = "artifacts/review.txt"
+ early = {"label": "Review", "disposition": "no_issue_found", "receiptRefs": [earlier_ref]}
+ rejection = {**early, "candidateId": "review", "disposition": "rejected", "receiptRefs": [ref]}
+ first = worker(tmp_path, "first", saved_draft("coverage-scan", complete=True, surfaces=[early]))
+ second = worker(tmp_path, "second", saved_draft("coverage-scan", complete=True, **records))
+ for source in (first, second):
+ result = Path(source["artifact_dir"]) / "result.json"
+ source.update(status="succeeded", result_manifest_path=str(result))
+ for receipt in (earlier_ref, ref):
+ (tmp_path / receipt).write_text("Synthetic review.")
+ output = Path(second["artifact_dir"])
+ checkpoint = write_checkpoint(
+ output / "checkpoints", saved_draft("coverage-scan", complete=True, surfaces=[rejection])
+ )
+ head = output / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(200, 200))
+
+ _, findings, coverage = recover(tmp_path, [first, second], project=True, warnings=[])
+ collision = earlier_ref != ref
+ assert len(findings["findings"]) == int(collision and prior == "reported")
+ candidates = [row.get("provenance", {}).get("candidateId") for row in coverage["deferred"]]
+ assert ("review" in candidates) == (collision and prior == "pending")
+ dispositions = [row["disposition"] for row in coverage["surfaces"]]
+ assert dispositions == ["no_issue_found", "needs_follow_up" if collision else "rejected"]
+
+
@pytest.mark.parametrize("reverse_sources", [False, True])
@pytest.mark.parametrize("reverse_surfaces", [False, True])
def test_current_receipt_spelling_precedes_historical_receipts(
diff --git a/plugins/codex-security/tests/test_root_candidate_authority.py b/plugins/codex-security/tests/test_root_candidate_authority.py
new file mode 100644
index 000000000..fb01f655a
--- /dev/null
+++ b/plugins/codex-security/tests/test_root_candidate_authority.py
@@ -0,0 +1,648 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+from argparse import Namespace
+
+import pytest
+from test_deep_scan_successful_publication import add_worker, complete
+from test_deep_scan_successful_publication import publication_scan as publication_scan
+from workbench_test_support import saved_draft, sort_checkpoint_entries, write_checkpoint
+
+
+def canonical_disposition(scan, disposition, complete_marker):
+ findings = copy.deepcopy(scan.findings)
+ if disposition == "rejected":
+ findings = []
+ else:
+ field = "provenance" if disposition == "provenance-reported" else "extensions"
+ findings[0].setdefault(field, {})["candidateId"] = "candidate-review"
+ coverage = {
+ **scan.coverage,
+ "surfaces": [
+ {
+ "id": "reviewed-surface",
+ "candidateId": "candidate-review",
+ "label": "Reviewed surface",
+ "disposition": disposition.removeprefix("provenance-"),
+ "receiptRefs": [],
+ }
+ ],
+ "deferred": [],
+ }
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ if complete_marker:
+ manifest["scan"]["complete"] = True
+ for name, document in (
+ ("scan-manifest.json", manifest),
+ ("findings.json", {"findings": findings}),
+ ("coverage.json", coverage),
+ ):
+ path = scan.scan_dir / name
+ path.write_text(json.dumps(document))
+ os.utime(path, ns=(200, 200))
+
+
[email protected]("disposition", ["reported", "rejected", "provenance-reported"])
[email protected]("complete_marker", [False, True], ids=["legacy-terminal", "terminal"])
[email protected]("pending_time", [100, 200, 300], ids=["older", "tied", "newer"])
[email protected]("flow", ["complete", "prepare", "stop", "interrupted-stop"])
+def test_canonical_candidate_resolution_keeps_legacy_checkpoint_authority(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ disposition,
+ complete_marker,
+ pending_time,
+ flow,
+):
+ scan = publication_scan(mode="standard")
+ pending = saved_draft(
+ scan.scan_id,
+ deferred=[{"candidateId": "candidate-review", "reason": "Validation is pending."}],
+ )
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", pending)
+ os.utime(checkpoint, ns=(pending_time, pending_time))
+ canonical_disposition(scan, disposition, complete_marker)
+ original = checkpoint.read_bytes()
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ prepared_coverage = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_coverage.append(copy.deepcopy(prepared[4]))
+ if flow == "interrupted-stop":
+ raise OSError("Synthetic candidate publication interruption")
+ return write(prepared, **kwargs)
+
+ if flow in {"complete", "prepare"}:
+ completed = complete(workbench_api, workbench_db, scan, prepare_only=flow == "prepare")
+ assert completed["progress"]["status"] == ("running" if flow == "prepare" else "complete")
+ else:
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."
+ ),
+ )
+ if flow == "interrupted-stop":
+ row = workbench_db.execute(
+ "SELECT * FROM scans WHERE id = ?", (scan.scan_id,)
+ ).fetchone()
+ assert row["retained_source_digests_json"]
+ assert row["seal_manifest_digest"] is None
+ # A retry must make the same choice from frozen sources, even after a
+ # later file-authored draft reopens this candidate.
+ late = {**scan.coverage, "deferred": pending["coverage"]["deferred"]}
+ (scan.scan_dir / "coverage.json").write_text(json.dumps(late))
+ saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert "completeness" not in coverage
+ for document in [coverage, *prepared_coverage]:
+ assert any(
+ row.get("candidateId") == "candidate-review" for row in document["deferred"]
+ ) is (pending_time > 200)
+ assert any(
+ row.get("candidateId") == "candidate-review"
+ and row["disposition"] == disposition.removeprefix("provenance-")
+ for row in document["surfaces"]
+ )
+ if flow in {"stop", "interrupted-stop"}:
+ before = (scan.scan_dir / "scan-manifest.json").read_bytes()
+ recovered = saved.recover_scan_results(
+ context, workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert not recovered["resultsRecoveryNeeded"]
+ assert (scan.scan_dir / "scan-manifest.json").read_bytes() == before
+ assert json.loads((scan.scan_dir / "coverage.json").read_text()) == coverage
+ assert checkpoint.read_bytes() == original
+
+
[email protected]("disposition", ["reported", "rejected", "provenance-reported"])
+def test_accepted_pending_root_tie_survives_canonical_disposition(
+ workbench_api, workbench_db, publication_scan, disposition
+):
+ scan = publication_scan(mode="standard")
+ pending = saved_draft(
+ scan.scan_id,
+ deferred=[{"candidateId": "candidate-review", "reason": "Accepted pending review."}],
+ )
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", pending)
+ os.utime(checkpoint, ns=(100, 100))
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(200, 200))
+ saved = workbench_api["saved_results"]
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ canonical_disposition(scan, disposition, True)
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert "completeness" not in coverage
+ assert any(row.get("candidateId") == "candidate-review" for row in coverage["deferred"])
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert json.loads((scan.scan_dir / "coverage.json").read_text()) == coverage
+
+
[email protected]("disposition", ["reported", "rejected", "provenance-reported"])
[email protected]("pending_time", [99, 100, 101], ids=["older", "tied", "newer"])
[email protected]("interrupted", [False, True])
+def test_restored_root_observation_keeps_tied_pending_payload(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ disposition,
+ pending_time,
+ interrupted,
+):
+ scan = publication_scan(mode="standard")
+ canonical_disposition(scan, disposition, True)
+ saved = workbench_api["saved_results"]
+ _, terminal = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", terminal)
+ os.utime(checkpoint, ns=(50, 50))
+ head = scan.scan_dir / "checkpoint-head.json"
+ for observed in (100, 300, 100):
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ pending = saved_draft(
+ scan.scan_id,
+ deferred=[{"candidateId": "candidate-review", "reason": "Review remains pending."}],
+ )
+ pending_path = write_checkpoint(scan.scan_dir / "checkpoints", pending)
+ os.utime(pending_path, ns=(pending_time, pending_time))
+ prepared_coverage = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_coverage.append(copy.deepcopy(prepared[4]))
+ if interrupted:
+ raise OSError("Synthetic restored-observation publication interruption")
+ return write(prepared, **kwargs)
+
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert len(prepared_coverage) == 1
+ for document in [coverage, *prepared_coverage]:
+ assert any(
+ row.get("candidateId") == "candidate-review" for row in document["deferred"]
+ ) is (pending_time >= 100)
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert json.loads((scan.scan_dir / "coverage.json").read_text()) == coverage
+
+
+ "root_complete", [False, True, None], ids=["partial", "terminal", "legacy"]
+)
+ "complete_worker_last", [False, True], ids=["partial-last", "complete-last"]
+)
[email protected]("disposition", ["reported", "rejected"])
[email protected]("flow", ["stop", "recover"])
[email protected]("interrupted", [False, True])
+def test_root_outcome_uses_its_own_completion_marker(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ root_complete,
+ complete_worker_last,
+ disposition,
+ flow,
+ interrupted,
+):
+ scan = publication_scan()
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ canonical_disposition(scan, disposition, True)
+ _, root_draft = saved._read_saved_parent_result(scan.scan_dir, scan.scan_id)
+ if root_complete is None:
+ root_draft.pop("complete")
+ else:
+ root_draft["complete"] = root_complete
+ for name, document in (
+ ("findings.json", {"findings": []}),
+ ("coverage.json", scan.coverage),
+ ):
+ (scan.scan_dir / name).write_text(json.dumps(document))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ os.utime(scan.scan_dir / name, ns=(0, 0))
+
+ results = sorted(
+ [add_worker(workbench_db, scan), add_worker(workbench_db, scan)],
+ key=lambda path: path.parent.name,
+ )
+ # Registration timestamps are equal, so IDs determine both arrival orders.
+ for result, worker_complete in zip(results, [not complete_worker_last, complete_worker_last]):
+ result.write_text(
+ json.dumps(
+ saved_draft(
+ scan.scan_id,
+ complete=worker_complete,
+ )
+ )
+ )
+ os.utime(result, ns=(100, 100))
+ assert [
+ row["result_manifest_path"]
+ for row in workbench_db.execute(
+ "SELECT result_manifest_path FROM deep_scan_workers "
+ "WHERE scan_id = ? ORDER BY created_at, id",
+ (scan.scan_id,),
+ )
+ ] == [str(result) for result in results]
+ stop_args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."
+ )
+ if flow == "recover":
+ saved.fail_scan(context, workbench_db, stop_args)
+ selected = write_checkpoint(scan.scan_dir / "checkpoints", root_draft)
+ os.utime(selected, ns=(100, 100))
+ head = scan.scan_dir / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(200, 200))
+ pending = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ saved_draft(
+ scan.scan_id,
+ deferred=[{"candidateId": "candidate-review", "reason": "Validation is pending."}],
+ ),
+ )
+ os.utime(pending, ns=(200, 200))
+ originals = {path: path.read_bytes() for path in [selected, pending, *results]}
+ prepared_coverage = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_coverage.append(copy.deepcopy(prepared[4]))
+ if interrupted:
+ raise OSError("Synthetic mixed-source publication interruption")
+ return write(prepared, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ if flow == "stop":
+ saved.fail_scan(context, workbench_db, stop_args)
+ elif interrupted:
+ with pytest.raises(OSError, match="Synthetic mixed-source publication interruption"):
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ else:
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ published = saved.preserve_scan_results(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+ assert not published["resultsRecoveryNeeded"]
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert len(prepared_coverage) == 1
+ for document in [coverage, *prepared_coverage]:
+ assert any(
+ row.get("candidateId") == "candidate-review" for row in document["deferred"]
+ ) is (root_complete is False)
+ manifest = (scan.scan_dir / "scan-manifest.json").read_bytes()
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert json.loads((scan.scan_dir / "coverage.json").read_text()) == coverage
+ assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
[email protected]("mode", ["standard", "deep"])
[email protected]("wrapped", [False, True], ids=["flat", "wrapped"])
+ "recorded_heads", [False, True], ids=["absent-selection", "empty-selection"]
+)
[email protected]("reverse_roots", [False, True])
[email protected]("reverse_listing", [False, True])
[email protected]("interrupted", [False, True])
[email protected]("later_selection", ["rejected", "pending"])
+def test_legacy_tied_root_parent_preserves_accepted_pending_evidence(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ mode,
+ wrapped,
+ recorded_heads,
+ reverse_roots,
+ reverse_listing,
+ interrupted,
+ later_selection,
+):
+ scan = publication_scan(mode=mode)
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ finding = copy.deepcopy(scan.findings[0])
+ finding["extensions"] = {"candidateId": "candidate-review"}
+ reported = saved_draft(scan.scan_id, complete=True, findings=[finding])
+ rejected = saved_draft(
+ scan.scan_id,
+ complete=True,
+ surfaces=[
+ {
+ "id": "reviewed-surface",
+ "candidateId": "candidate-review",
+ "label": "Reviewed surface",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ pending = saved_draft(
+ scan.scan_id,
+ complete=True,
+ deferred=[{"candidateId": "candidate-review", "reason": "Validation remains pending."}],
+ )
+ roots = [("rejected", rejected), ("pending", pending)]
+ if reverse_roots:
+ roots.reverse()
+ captures = {}
+ checkpoints = {}
+ head = scan.scan_dir / "checkpoint-head.json"
+ for name, draft in [("reported", reported), *roots]:
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", draft)
+ observed = 100 if name == "reported" else 500
+ os.utime(checkpoint, ns=(observed, observed))
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ captures.update(
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ )
+ checkpoints[name] = checkpoint
+ head.unlink()
+ frozen = {path: digest for path, (digest, _) in captures.items()}
+ retained = (
+ {
+ "sources": frozen,
+ "sourceTimes": {path: observed for path, (_, observed) in captures.items()},
+ }
+ if wrapped
+ else frozen
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET status = 'failed', completed_at = ?, retained_source_digests_json = ?, "
+ "retained_checkpoint_heads_json = ? WHERE id = ?",
+ (scan.timestamp, json.dumps(retained), "{}" if recorded_heads else None, scan.scan_id),
+ )
+ originals = {path: (scan.scan_dir / path).read_bytes() for path in frozen}
+ sort_checkpoint_entries(monkeypatch, reverse=reverse_listing)
+ prepared_documents = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_documents.append(copy.deepcopy((prepared[3], prepared[4])))
+ if interrupted:
+ raise OSError("Synthetic legacy-root publication interruption")
+ return write(prepared, **kwargs)
+
+ preserve_args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ if interrupted:
+ with pytest.raises(OSError, match="Synthetic legacy-root publication interruption"):
+ saved.preserve_scan_results(context, workbench_db, preserve_args)
+ else:
+ saved.preserve_scan_results(context, workbench_db, preserve_args)
+ published = saved.preserve_scan_results(context, workbench_db, preserve_args)["scan"]
+ assert published["findingCount"] == 1
+ assert not published["resultsRecoveryNeeded"]
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = manifest_path.read_bytes()
+ assert json.loads(manifest)["scan"]["preservedCheckpointHeads"] == {}
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert len(prepared_documents) == 1
+ for current_findings, current_coverage in [(findings, coverage), *prepared_documents]:
+ assert len(current_findings["findings"]) == 1
+ assert any(
+ row.get("candidateId") == "candidate-review" for row in current_coverage["deferred"]
+ )
+ replayed = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert replayed["findingCount"] == 1
+ assert not replayed["resultsRecoveryNeeded"]
+ before_sources = json.loads(manifest)["scan"]["preservedSources"]
+ after_sources = json.loads(manifest_path.read_text())["scan"]["preservedSources"]
+ assert before_sources.items() <= after_sources.items()
+ added = after_sources.keys() - before_sources.keys()
+ # Explicit recovery records ordering missing from flat legacy captures.
+ assert len(added) == (0 if wrapped else 1)
+ for relative in added:
+ assert relative.startswith("source-order/")
+ payload = (scan.scan_dir / relative).read_bytes()
+ assert hashlib.sha256(payload).hexdigest() == after_sources[relative]
+ assert json.loads(payload) == {
+ "scanId": scan.scan_id,
+ "sources": {
+ path: {"digest": digest, "observedAtNs": str(observed)}
+ for path, (digest, observed) in captures.items()
+ },
+ }
+ assert json.loads((scan.scan_dir / "findings.json").read_text()) == findings
+ assert json.loads((scan.scan_dir / "coverage.json").read_text()) == coverage
+ if wrapped:
+ assert manifest_path.read_bytes() == manifest
+ manifest = manifest_path.read_bytes()
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert manifest_path.read_bytes() == manifest
+
+ head.write_text(json.dumps({"checkpoint": checkpoints[later_selection].name}))
+ os.utime(head, ns=(600, 600))
+ recovered = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert recovered["findingCount"] == (0 if later_selection == "rejected" else 1)
+ assert not recovered["resultsRecoveryNeeded"]
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert any(row.get("candidateId") == "candidate-review" for row in coverage["deferred"]) is (
+ later_selection == "pending"
+ )
+ assert all(
+ (scan.scan_dir / path).read_bytes() == contents for path, contents in originals.items()
+ )
+
+
[email protected]("wrapped", [False, True], ids=["flat", "wrapped"])
[email protected]("flow", ["preserve", "recover"])
[email protected]("accepted_first", [False, True])
[email protected]("interrupted", [False, True])
[email protected]("accepted_time", [99, 100, 101], ids=["older", "tied", "newer"])
[email protected]("identity", ["candidate-only", "explicit", "alias"])
+def test_legacy_pending_ties_aggregate_accepted_provenance(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ wrapped,
+ flow,
+ accepted_first,
+ interrupted,
+ accepted_time,
+ identity,
+):
+ scan = publication_scan(mode="standard")
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ finding = copy.deepcopy(scan.findings[0])
+ finding["extensions"] = {"candidateId": "candidate-review"}
+ pending = {"candidateId": "candidate-review", "reason": "Accepted pending review."}
+ if identity != "candidate-only":
+ pending["id"] = "pending-review" if identity == "alias" else "candidate-review"
+ unobserved = {**pending, "reason": "Unobserved pending details."}
+ rejected = saved_draft(
+ scan.scan_id,
+ complete=True,
+ surfaces=[
+ {
+ "id": "reviewed-surface",
+ "candidateId": "candidate-review",
+ "label": "Reviewed surface",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ observations = [
+ (saved_draft(scan.scan_id, findings=[finding], complete=True), 50),
+ (saved_draft(scan.scan_id, deferred=[pending]), accepted_time),
+ (rejected, 100),
+ (
+ saved_draft(
+ scan.scan_id,
+ deferred=[{"id": "independent-review", "reason": "Another task remains."}],
+ ),
+ 200,
+ ),
+ ]
+ captures = {}
+ head = scan.scan_dir / "checkpoint-head.json"
+ for draft, observed in observations:
+ draft["coverage"] = {**scan.coverage, **draft["coverage"]}
+ checkpoint = write_checkpoint(scan.scan_dir / "checkpoints", draft)
+ os.utime(checkpoint, ns=(observed, observed))
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(observed, observed))
+ captures.update(
+ saved._capture_saved_source(scan.scan_dir, "checkpoint-head.json", scan.scan_id)
+ )
+ head.unlink()
+ raw = saved_draft(scan.scan_id, deferred=[unobserved])
+ raw["coverage"] = {**scan.coverage, **raw["coverage"]}
+ unobserved_path = write_checkpoint(scan.scan_dir / "checkpoints", raw)
+ os.utime(unobserved_path, ns=(100, 100))
+ captures.update(
+ saved._capture_saved_source(
+ scan.scan_dir, unobserved_path.relative_to(scan.scan_dir).as_posix(), scan.scan_id
+ )
+ )
+ frozen = {path: digest for path, (digest, _) in captures.items()}
+ retained = (
+ {
+ "sources": frozen,
+ "sourceTimes": {path: observed for path, (_, observed) in captures.items()},
+ }
+ if wrapped
+ else frozen
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE scans SET status = 'failed', completed_at = ?, retained_source_digests_json = ?, "
+ "retained_checkpoint_heads_json = '{}' WHERE id = ?",
+ (scan.timestamp, json.dumps(retained), scan.scan_id),
+ )
+ originals = {path: (scan.scan_dir / path).read_bytes() for path in frozen}
+ sort_checkpoint_entries(
+ monkeypatch, key=lambda name: (name == unobserved_path.name) == accepted_first
+ )
+ prepared_documents = []
+ write = saved._write_prepared_scan_finalization
+
+ def publish(prepared, **kwargs):
+ prepared_documents.append(copy.deepcopy((prepared[3], prepared[4])))
+ if interrupted:
+ raise OSError("Synthetic tied-pending publication interruption")
+ return write(prepared, **kwargs)
+
+ args = Namespace(
+ scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ )
+ action = saved.preserve_scan_results if flow == "preserve" else saved.recover_scan_results
+ with monkeypatch.context() as patch:
+ patch.setattr(saved, "_write_prepared_scan_finalization", publish)
+ if interrupted:
+ with pytest.raises(OSError, match="Synthetic tied-pending publication interruption"):
+ action(context, workbench_db, args)
+ else:
+ action(context, workbench_db, args)
+ published = saved.preserve_scan_results(context, workbench_db, args)["scan"]
+ retained_pending = accepted_time >= 100
+ assert published["findingCount"] == int(retained_pending)
+ assert not published["resultsRecoveryNeeded"]
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert len(prepared_documents) == 1
+ for current_findings, current_coverage in [(findings, coverage), *prepared_documents]:
+ assert len(current_findings["findings"]) == int(retained_pending)
+ candidate_rows = [
+ row
+ for row in current_coverage["deferred"]
+ if row.get("candidateId") == "candidate-review"
+ ]
+ assert bool(candidate_rows) is retained_pending
+ if retained_pending:
+ assert pending["reason"] in {row["reason"] for row in candidate_rows}
+ assert any(row["id"] == "independent-review" for row in current_coverage["deferred"])
+ recovered = saved.recover_scan_results(context, workbench_db, args)["scan"]
+ assert recovered["findingCount"] == int(retained_pending)
+ assert not recovered["resultsRecoveryNeeded"]
+ assert json.loads((scan.scan_dir / "findings.json").read_text()) == findings
+ assert json.loads((scan.scan_dir / "coverage.json").read_text()) == coverage
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = manifest_path.read_bytes()
+ saved.recover_scan_results(context, workbench_db, args)
+ assert manifest_path.read_bytes() == manifest
+ assert all(
+ (scan.scan_dir / path).read_bytes() == contents for path, contents in originals.items()
+ )
diff --git a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py
index 1568c5600..f44efc8b5 100644
--- a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py
+++ b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py
@@ -798,7 +798,7 @@ def test_head_snapshot_reads_content_and_time_from_one_descriptor(
}
-def test_legacy_live_head_sources_keep_their_recorded_digest(
+def test_legacy_live_head_sources_convert_their_recorded_observation(
tmp_path: Path, checkpoint_scan
) -> None:
scan_id, pending, closed, binding = checkpoint_scan
@@ -819,7 +819,18 @@ def test_legacy_live_head_sources_keep_their_recorded_digest(
)
assert first is not None
assert pending["coverage"]["deferred"][0] not in first[2]["deferred"]
- assert first[0]["scan"]["preservedSources"] == frozen
+ published = first[0]["scan"]["preservedSources"]
+ assert "checkpoint-head.json" not in published
+ assert published[paths[0]] == frozen[paths[0]]
+ assert (
+ saved._read_saved_result(tmp_path, "checkpoint-head.json", scan_id)[1]
+ == frozen["checkpoint-head.json"]
+ )
+ snapshot = next(path for path in published if path.startswith("checkpoint-heads/"))
+ assert json.loads((tmp_path / snapshot).read_text()) == {
+ "checkpoint": checkpoint.name,
+ "observedAtNs": "200",
+ }
select(tmp_path, checkpoint, 300)
with pytest.raises(
saved.ContractError, match="Frozen stopped-scan checkpoint set is incomplete"
@@ -834,6 +845,8 @@ def test_legacy_live_head_sources_keep_their_recorded_digest(
reason="interrupted",
frozen_source_digests=frozen,
)
+ replayed = replay_saved_results(saved, first, tmp_path, scan_id, binding, [], stopped=True)
+ assert replayed[2] == first[2]
@pytest.mark.parametrize("evidence", ["deferred", "reported"])
@@ -1339,3 +1352,50 @@ def test_accepted_head_survives_superseded_terminal_timestamps(
assert replay[1] == result[1]
assert warnings == []
assert all(path.read_bytes() == content for path, content in original.items())
+
+
[email protected]("scan_id", ["legacy-tied-head", "frozen-worker-heads"])
+def test_legacy_tied_worker_heads_keep_pending_work_on_replay(tmp_path: Path, scan_id: str) -> None:
+ pending, closed = drafts(scan_id)
+ binding = saved_binding("deep_repository", repository="synthetic")
+ output = tmp_path / "worker"
+ completed = write_checkpoint(output / "checkpoints", closed)
+ reopened = write_checkpoint(output / "checkpoints", pending)
+ frozen = {}
+ for checkpoint in (completed, reopened):
+ os.utime(checkpoint, ns=(100, 100))
+ select(output, checkpoint, 500)
+ frozen.update(
+ {
+ path: digest
+ for path, (digest, _) in saved._capture_saved_source(
+ tmp_path, "worker/checkpoint-head.json", scan_id
+ ).items()
+ }
+ )
+ workers = [saved_discovery_worker(output, "worker", 1)]
+ first = saved.merge_saved_results(
+ tmp_path,
+ scan_id,
+ binding,
+ workers,
+ [],
+ stopped=True,
+ reason="interrupted",
+ frozen_source_digests=frozen,
+ )
+ replay = replay_saved_results(saved, first, tmp_path, scan_id, binding, workers, stopped=True)
+ repeated = saved.merge_saved_results(
+ tmp_path,
+ scan_id,
+ binding,
+ workers,
+ [],
+ stopped=True,
+ reason="interrupted",
+ frozen_source_digests=replay[0]["scan"]["preservedSources"],
+ checkpoint_heads=replay[0]["scan"]["preservedCheckpointHeads"],
+ )
+ for documents in (first, replay, repeated):
+ assert pending["coverage"]["deferred"][0] in documents[2]["deferred"]
+ assert not documents[2].get("resolvedDeferred")
diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py
index 69b0cd909..6a1f843bf 100644
--- a/plugins/codex-security/tests/test_workbench_completion_binding.py
+++ b/plugins/codex-security/tests/test_workbench_completion_binding.py
@@ -1062,6 +1062,7 @@ def test_deep_completion_rejects_invalid_target_even_with_recoverable_inventory(
recovered = scan_command(state_dir, "recover-scan-results", scan_id)["scan"]
assert recovered["resultsRecoveryNeeded"] is False
+ assert json.loads(manifest_path.read_text())["scan"]["target"]["kind"] == "directory_snapshot"
preserved_coverage = json.loads(coverage_path.read_text())
assert preserved_coverage["inventoryStrategy"] == "repository"
diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py
index 75146583c..83d4695c0 100644
--- a/plugins/codex-security/tests/test_workbench_deep_scan.py
+++ b/plugins/codex-security/tests/test_workbench_deep_scan.py
@@ -81,6 +81,7 @@ def begin_target_scan(
def restore_v53_schema(state_dir: Path) -> None:
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
+ connection.execute("ALTER TABLE scans DROP COLUMN retained_checkpoint_heads_json")
connection.execute("ALTER TABLE workspaces DROP COLUMN deep_scan_config_json")
connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN pending_target_project_path")
connection.execute("DELETE FROM schema_migrations WHERE version > 53")
diff --git a/plugins/codex-security/tests/test_workbench_saved_source_order.py b/plugins/codex-security/tests/test_workbench_saved_source_order.py
index 7a4fcfece..dd017fc2d 100644
--- a/plugins/codex-security/tests/test_workbench_saved_source_order.py
+++ b/plugins/codex-security/tests/test_workbench_saved_source_order.py
@@ -8,12 +8,19 @@ from pathlib import Path
import pytest
from test_workbench_checkpoint_heads import drafts, select
-from test_workbench_standard_deep_results import accepted_standard_worker, deep_scan_fixture
+from test_workbench_standard_deep_results import (
+ accepted_standard_worker,
+ deep_scan_fixture,
+ standard_finding,
+)
from workbench_test_support import (
+ cancel_scan,
fail_deep_scan,
get_scan,
+ preserve_scan_results,
saved_binding,
saved_discovery_worker,
+ saved_draft,
scan_command,
write_checkpoint,
)
@@ -33,9 +40,15 @@ def call_workbench(monkeypatch, state, codex_home, *args):
workbench_db.main()
[email protected]("termination", ["canceled", "failed", "interrupted"])
@pytest.mark.parametrize("latest_pending", [True, False])
[email protected]("failure_stage", ["publication", "source-order", "head-and-source-order"])
def test_failed_publication_keeps_order_after_identical_result_rewrite(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch, latest_pending: bool
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ latest_pending: bool,
+ failure_stage: str,
+ termination: str,
) -> None:
state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path)
worker_id, result = accepted_standard_worker(state, codex_home, scan_dir, scan_id)
@@ -50,16 +63,65 @@ def test_failed_publication_keeps_order_after_identical_result_rewrite(
def fail_publication(*args, **kwargs):
raise OSError("injected publication failure")
+ write = saved_sources.write_scan_local_bytes
+
+ def fail_snapshot(root, relative, contents):
+ if relative.startswith("source-order/") or (
+ failure_stage == "head-and-source-order" and "/checkpoint-heads/" in relative
+ ):
+ raise OSError("injected snapshot failure")
+ return write(root, relative, contents)
+
with monkeypatch.context() as patch:
- patch.setattr(saved, "_write_prepared_scan_finalization", fail_publication)
- call_workbench(patch, state, codex_home, "cancel-scan", "--scan-id", scan_id)
+ if failure_stage == "publication":
+ patch.setattr(saved, "_write_prepared_scan_finalization", fail_publication)
+ else:
+ patch.setattr(saved_sources, "write_scan_local_bytes", fail_snapshot)
+ call_workbench(
+ patch,
+ state,
+ codex_home,
+ "cancel-scan" if termination == "canceled" else "fail-deep-scan",
+ "--scan-id",
+ scan_id,
+ *(
+ ["--deep-status", termination, "--message", "Worker stopped."]
+ if termination != "canceled"
+ else []
+ ),
+ )
with sqlite3.connect(state / "workbench.sqlite3") as connection:
frozen = json.loads(
connection.execute(
"SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan_id,)
).fetchone()[0]
)
- assert result.relative_to(scan_dir).as_posix() in frozen
+ sources = frozen.get("sources", frozen)
+ assert result.relative_to(scan_dir).as_posix() in sources
+
+ if failure_stage != "publication":
+ with monkeypatch.context() as patch:
+ patch.setattr(saved_sources, "write_scan_local_bytes", fail_snapshot)
+ with pytest.raises((saved.ContractError, OSError), match="snapshot failure"):
+ call_workbench(
+ patch,
+ state,
+ codex_home,
+ "preserve-scan-results",
+ "--scan-id",
+ scan_id,
+ "--thread-id",
+ "standard-worker-thread",
+ )
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ assert (
+ json.loads(
+ connection.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan_id,)
+ ).fetchone()[0]
+ )
+ == frozen
+ )
result.write_bytes(result.read_bytes())
os.utime(result, ns=(300, 300))
@@ -86,10 +148,21 @@ def test_failed_publication_keeps_order_after_identical_result_rewrite(
if latest_pending
else []
)
- assert (
- json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["preservedSources"]
- == frozen
- )
+ published_sources = json.loads((scan_dir / "scan-manifest.json").read_text())["scan"][
+ "preservedSources"
+ ]
+ times = saved._frozen_source_times(scan_dir, scan_id, published_sources)
+ for path, digest in sources.items():
+ if Path(path).name == "checkpoint-head.json":
+ assert saved._read_saved_result(scan_dir, path, scan_id)[1] == digest
+ assert path not in published_sources
+ captured = saved._capture_saved_source(scan_dir, path, scan_id, write=False)
+ for snapshot, (expected, observed) in captured.items():
+ assert published_sources[snapshot] == expected
+ if Path(snapshot).parent.name == "checkpoint-heads":
+ assert times[snapshot] == observed == frozen["sourceTimes"][path]
+ else:
+ assert published_sources[path] == digest
@pytest.mark.parametrize("explicit_recovery", [False, True])
@@ -217,3 +290,176 @@ def test_order_metadata_is_not_new_evidence(tmp_path: Path) -> None:
state, "recover-scan-results", scan_id, environment={"CODEX_HOME": str(codex_home)}
)
assert (scan_dir / "scan-manifest.json").read_bytes() == manifest
+
+
[email protected]("head_time", [100, 200, 500], ids=["older", "tied", "newer"])
+def test_restored_head_uses_its_captured_observation(tmp_path: Path, head_time: int) -> None:
+ scan_id = "restored-observation"
+ finding = standard_finding(tmp_path, scan_id, tmp_path)
+ finding.setdefault("extensions", {})["candidateId"] = "candidate-a"
+ reported = saved_draft(
+ scan_id,
+ findings=[finding],
+ complete=True,
+ surfaces=[
+ {
+ "id": "candidate-a",
+ "candidateId": "candidate-a",
+ "label": "Candidate",
+ "disposition": "reported",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ rejected = saved_draft(
+ scan_id,
+ surfaces=[
+ {
+ "id": "candidate-a",
+ "candidateId": "candidate-a",
+ "label": "Candidate",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ complete=True,
+ )
+ output = tmp_path / "worker"
+ output.mkdir()
+ checkpoints = [
+ write_checkpoint(output / "checkpoints", draft) for draft in (rejected, reported)
+ ]
+ for checkpoint, observed in zip(checkpoints, (300, 400)):
+ os.utime(checkpoint, ns=(50, 50))
+ select(output, checkpoint, observed)
+ saved._capture_saved_source(tmp_path, "worker/checkpoint-head.json", scan_id)
+ result = output / "result.json"
+ result.write_text(json.dumps(reported))
+ os.utime(result, ns=(200, 200))
+ select(output, checkpoints[0], head_time)
+ original = {path: path.read_bytes() for path in (result, *checkpoints)}
+ workers = [saved_discovery_worker(output)]
+ binding = saved_binding("deep_repository", repository="synthetic")
+ binding["target"]["targetId"] = "synthetic-target"
+ first = saved.merge_saved_results(
+ tmp_path, scan_id, binding, workers, [], stopped=True, reason="interrupted"
+ )
+ for _ in range(2):
+ assert len(first[1]["findings"]) == int(head_time < 200)
+ first = saved.merge_saved_results(
+ tmp_path,
+ scan_id,
+ binding,
+ workers,
+ [],
+ stopped=True,
+ reason="interrupted",
+ frozen_source_digests=first[0]["scan"]["preservedSources"],
+ checkpoint_heads=first[0]["scan"]["preservedCheckpointHeads"],
+ )
+ assert all(path.read_bytes() == contents for path, contents in original.items())
+
+
[email protected]("termination", ["failed", "interrupted", "canceled"])
+def test_first_stop_freezes_sources_before_report_preparation(
+ tmp_path: Path, termination: str
+) -> None:
+ state, codex_home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path)
+ _, result = accepted_standard_worker(state, codex_home, scan_dir, scan_id)
+ finding = standard_finding(tmp_path, scan_id, target)
+ finding.setdefault("extensions", {})["candidateId"] = "candidate-a"
+ reported = saved_draft(
+ scan_id,
+ findings=[finding],
+ complete=True,
+ surfaces=[
+ {
+ "id": "candidate-a",
+ "candidateId": "candidate-a",
+ "label": "Candidate",
+ "disposition": "reported",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ result.write_text(json.dumps(reported))
+ os.utime(result, ns=(100, 100))
+ checkpoint = write_checkpoint(result.parent / "checkpoints", reported)
+ select(result.parent, checkpoint, 100)
+ report = scan_dir / "report.md"
+ report.mkdir()
+ environment = {"CODEX_HOME": str(codex_home)}
+ if termination == "canceled":
+ cancel_scan(state, scan_id, "standard-worker-thread", environment=environment)
+ else:
+ fail_deep_scan(state, codex_home, scan_id, deep_status=termination)
+ stopped = get_scan(state, scan_id)["scan"]
+ assert any("report.md: expected a regular file" in warning for warning in stopped["warnings"])
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ frozen = connection.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?", (scan_id,)
+ ).fetchone()[0]
+ assert frozen is not None
+ report.rmdir()
+ rejection = saved_draft(
+ scan_id,
+ surfaces=[
+ {
+ "id": "candidate-a",
+ "candidateId": "candidate-a",
+ "label": "Candidate",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ ],
+ complete=True,
+ )
+ latest = write_checkpoint(result.parent / "checkpoints", rejection)
+ select(result.parent, latest, 300)
+ preserved = preserve_scan_results(
+ state, scan_id, "standard-worker-thread", environment=environment
+ )["scan"]
+ assert preserved["findingCount"] == 1
+ assert preserved["resultsRecoveryNeeded"] is (termination != "canceled")
+ published = (scan_dir / "scan-manifest.json").read_bytes()
+ preserve_scan_results(state, scan_id, "standard-worker-thread", environment=environment)
+ assert (scan_dir / "scan-manifest.json").read_bytes() == published
+ assert result.read_text() == json.dumps(reported)
+
+
+def test_invalid_explicit_recovery_does_not_replace_first_selection(tmp_path: Path) -> None:
+ state, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path)
+ _, result = accepted_standard_worker(state, codex_home, scan_dir, scan_id)
+ initial = json.loads(result.read_text())
+ os.utime(result, ns=(100, 100))
+ checkpoint = write_checkpoint(result.parent / "checkpoints", initial)
+ select(result.parent, checkpoint, 100)
+ fail_deep_scan(state, codex_home, scan_id, deep_status="failed")
+ artifacts = {
+ name: (scan_dir / name).read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json", "report.md")
+ }
+ query = (
+ "SELECT seal_manifest_digest, retained_source_digests_json, retained_checkpoint_heads_json "
+ "FROM scans WHERE id = ?"
+ )
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ original = connection.execute(query, (scan_id,)).fetchone()
+ original_checkpoint = checkpoint.read_bytes()
+ checkpoint.write_text(json.dumps({**initial, "scanId": "other-scan"}))
+ valid = {**initial, "complete": True, "threatModel": {"summary": "Completed review."}}
+ later = write_checkpoint(result.parent / "checkpoints", valid)
+ select(result.parent, later, 200)
+ environment = {"CODEX_HOME": str(codex_home)}
+ failed = scan_command(
+ state, "recover-scan-results", scan_id, environment=environment, check=False
+ )
+ assert failed["returncode"] != 0
+ assert "Frozen stopped-scan checkpoint set is incomplete" in failed["stderr"]
+ with sqlite3.connect(state / "workbench.sqlite3") as connection:
+ assert connection.execute(query, (scan_id,)).fetchone() == original
+ assert all((scan_dir / name).read_bytes() == contents for name, contents in artifacts.items())
+ checkpoint.write_bytes(original_checkpoint)
+ scan_command(state, "recover-scan-results", scan_id, environment=environment)
+ manifest = json.loads((scan_dir / "scan-manifest.json").read_text())
+ assert manifest["scan"]["threatModel"]["summary"] == "Completed review."
diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
index a8580249a..4113e8df5 100644
--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
@@ -98,6 +98,7 @@ EXPECTED_MIGRATIONS = [
(54, "snapshot deep scan setup configuration"),
(55, "retain undelivered headless deep scan results"),
(56, "recover unfinished legacy headless deep scans"),
+ (57, "freeze stopped scan checkpoint selections"),
]
diff --git a/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py b/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py
index b7cbcb791..a61128b7f 100644
--- a/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py
+++ b/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import os
import uuid
from pathlib import Path
@@ -18,6 +19,7 @@ from workbench_test_support import (
saved_draft,
upsert_deep_worker,
worker_paths,
+ write_checkpoint,
)
@@ -225,3 +227,88 @@ def test_stopped_worker_links_follow_explicit_surface_identity_through_collision
)
assert coverage_path.read_bytes() == sealed
assert result.read_bytes() == original
+
+
+def test_pending_worker_surface_receipt_precedes_historical_receipt_repair(tmp_path: Path) -> None:
+ state_dir, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path)
+ worker_id, result = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id)
+ output = result.parent
+ artifacts = output / "artifacts"
+ artifacts.mkdir()
+ current_receipt = artifacts / "Review.md"
+ current_receipt.write_text("Current API review.\n")
+ historical_receipt = artifacts / "review.md"
+ if historical_receipt.exists():
+ pytest.skip("This regression requires case-sensitive receipt paths.")
+ historical_receipt.write_text("Historical writer review.\n")
+ other_receipt = artifacts / "other.md"
+ other_receipt.write_text("Current writer review.\n")
+ surface = {
+ "id": "api",
+ "label": "API",
+ "disposition": "no_issue_found",
+ "receiptRefs": ["artifacts/Review.md"],
+ }
+ writer = {
+ **surface,
+ "id": "writer",
+ "label": "Writer",
+ "receiptRefs": ["artifacts/other.md"],
+ }
+ task = {"id": "review-api", "reason": "Review remains.", "surfaceIds": ["api"]}
+ result.write_text(
+ json.dumps(saved_draft(scan_id, complete=True, surfaces=[surface, writer], deferred=[task]))
+ )
+ os.utime(result, ns=(100, 100))
+ historical = write_checkpoint(
+ output / "checkpoints",
+ saved_draft(
+ scan_id,
+ complete=True,
+ surfaces=[
+ {
+ **writer,
+ "receiptRefs": ["artifacts/review.md", "artifacts/missing.md"],
+ }
+ ],
+ ),
+ )
+ os.utime(historical, ns=(50, 50))
+ selected = write_checkpoint(
+ output / "checkpoints", saved_draft(scan_id, complete=True, surfaces=[writer])
+ )
+ os.utime(selected, ns=(300, 300))
+ head = output / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ os.utime(head, ns=(400, 400))
+ originals = {
+ path: path.read_bytes()
+ for path in (result, historical, selected, head, current_receipt, historical_receipt)
+ }
+ assert not (scan_dir / "coverage.json").exists()
+
+ fail_deep_scan(state_dir, codex_home, scan_id)
+
+ coverage_path = scan_dir / "coverage.json"
+ coverage = json.loads(coverage_path.read_text())
+ surfaces = {row["label"]: row for row in coverage["surfaces"]}
+ retained = surfaces["API"]
+ assert retained["disposition"] == "no_issue_found"
+ assert retained["receiptRefs"] == [current_receipt.relative_to(scan_dir).as_posix()]
+ assert surfaces["Writer"]["receiptRefs"] == [other_receipt.relative_to(scan_dir).as_posix()]
+ retained_task = next(
+ row
+ for row in coverage["deferred"]
+ if row.get("provenance") == {"workerId": worker_id, "attempt": 1, "sourceId": "review-api"}
+ )
+ assert retained_task["surfaceIds"] == [retained["id"]]
+ assert not any(
+ "Skipped malformed coverage receipt" in warning
+ for warning in get_scan(state_dir, scan_id)["scan"]["warnings"]
+ )
+ sealed = coverage_path.read_bytes()
+ preserve_scan_results(
+ state_dir, scan_id, "standard-worker-thread", environment={"CODEX_HOME": str(codex_home)}
+ )
+ assert coverage_path.read_bytes() == sealed
+ assert all(path.read_bytes() == content for path, content in originals.items())
diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py
index 210e92460..6cfdf6882 100644
--- a/plugins/codex-security/tests/workbench_test_support.py
+++ b/plugins/codex-security/tests/workbench_test_support.py
@@ -56,6 +56,21 @@ def write_checkpoint(checkpoint_dir: Path, payload: Any) -> Path:
return checkpoint_path
+def sort_checkpoint_entries(monkeypatch, *, key=None, reverse=False):
+ from workbench import saved_sources
+
+ children = saved_sources._children
+ monkeypatch.setattr(
+ saved_sources,
+ "_children",
+ lambda root, relative: sorted(children(root, relative), key=key, reverse=reverse),
+ )
+
+
+def interrupt_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption")
+
+
def saved_coverage(*, deferred=(), surfaces=()):
return {
"surfaces": list(surfaces),
diff --git a/sdk/typescript/tests-ts/stopped-scan-results.test.ts b/sdk/typescript/tests-ts/stopped-scan-results.test.ts
index a1ae586b4..65aefca9f 100644
--- a/sdk/typescript/tests-ts/stopped-scan-results.test.ts
+++ b/sdk/typescript/tests-ts/stopped-scan-results.test.ts
@@ -120,14 +120,15 @@ const stoppedScanProbe = [
" final_manifest = json.loads(manifest_path.read_text(encoding='utf-8'))",
" final_findings = json.loads((scan_dir / 'findings.json').read_text(encoding='utf-8'))['findings']",
" connection.close()",
- " frozen_sources = json.loads(frozen_after_success)",
+ " frozen_state = json.loads(frozen_after_success)",
+ " frozen_sources = frozen_state['sources'] if isinstance(frozen_state.get('sources'), dict) else frozen_state",
" head_relative = next(path for path in frozen_sources if path.startswith('checkpoint-heads/'))",
" head_path = scan_dir / head_relative",
" head = json.loads(head_path.read_text(encoding='utf-8'))",
" snapshot_bytes = (scan_dir / 'checkpoints' / head['checkpoint']).read_bytes()",
" snapshot = json.loads(snapshot_bytes)",
" head_evidence = head_path.read_bytes()",
- " print(json.dumps({'firstFailed': first_failed, 'frozenAfterFailure': frozen_after_failure, 'retryPublished': retry_published, 'frozenAfterSuccess': json.loads(frozen_after_success) if frozen_after_success else None, 'status': final_manifest['scan']['status'], 'findingCount': len(final_findings), 'scanId': scan_id, 'head': head, 'headPath': head_relative, 'headDigest': hashlib.sha256(head_evidence).hexdigest(), 'snapshot': snapshot, 'snapshotDigest': hashlib.sha256(snapshot_bytes).hexdigest()}))",
+ " print(json.dumps({'firstFailed': first_failed, 'frozenAfterFailure': frozen_after_failure, 'retryPublished': retry_published, 'frozenAfterSuccess': frozen_state, 'frozenSources': frozen_sources, 'status': final_manifest['scan']['status'], 'findingCount': len(final_findings), 'scanId': scan_id, 'head': head, 'headPath': head_relative, 'headDigest': hashlib.sha256(head_evidence).hexdigest(), 'snapshot': snapshot, 'snapshotDigest': hashlib.sha256(snapshot_bytes).hexdigest()}))",
" raise SystemExit(0)",
"if source == 'late-checkpoint':",
" manifest_before = (scan_dir / 'scan-manifest.json').read_bytes()",
@@ -244,7 +245,7 @@ test("retries a legacy stopped seal after transient publication failure", () =>
expect(recovered.headPath).toBe(
`checkpoint-heads/${recovered.headDigest}.json`,
);
- expect(recovered.frozenAfterSuccess).toMatchObject({
+ expect(recovered.frozenSources).toMatchObject({
[`checkpoints/${recovered.head.checkpoint}`]: recovered.snapshotDigest,
[recovered.headPath]: recovered.headDigest,
});