OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(deep-scan): retain unfinished deadline checkpoints (#1710)
Michael D'Angelo3e0bf51bba5f
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.jsonindex b34ac93cf..3b8c90052 100644--- a/plugins/codex-security/plugin-files.json+++ b/plugins/codex-security/plugin-files.json@@ -82,6 +82,7 @@"scripts/workbench/runtime_toml.py","scripts/workbench/saved_coverage.py","scripts/workbench/saved_outputs.py",+ "scripts/workbench/saved_sources.py","scripts/workbench/storage.py","scripts/workbench_cli.py","scripts/workbench_constants.py",@@ -95,6 +96,7 @@"scripts/workbench_publication.py","scripts/workbench_remediation.py","scripts/workbench_saved_results.py",+ "scripts/workbench/retained_coverage.py","scripts/workbench_scan_history.py","scripts/workbench_scan_start.py","scripts/workbench_scan_usage.py",diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.pyindex 8ae81df50..ed71e008d 100644--- a/plugins/codex-security/scripts/deep_scan_workbench.py+++ b/plugins/codex-security/scripts/deep_scan_workbench.py@@ -1747,9 +1747,9 @@ def finish_deep_scan_locked(""",(scan_id,),).fetchone()+ discovery_deadline = args.terminal_reason == "capped" and deep_scan_deadline_reached(run)zero_discovery_deadline = (- args.terminal_reason == "capped"- and deep_scan_deadline_reached(run)+ discovery_deadlineand run["completion_sequence"] == 0and (standard_scan_manifest@@ -1781,9 +1781,13 @@ def finish_deep_scan_locked(kind="file",)promotion = promote_staged_file(staged_manifest_path, manifest_path)- if zero_discovery_deadline:+ if zero_discovery_deadline or (discovery_deadline and standard_scan_manifest):deadline_publication = saved_results.preserve_deep_deadline_draft(- dependencies(), connection, scan, allow_empty=not standard_scan_manifest+ dependencies(),+ connection,+ scan,+ preserve_reduction=not zero_discovery_deadline,+ allow_empty=not standard_scan_manifest,)timestamp = dependencies().now()connection.execute(diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.pyindex 84c50abf9..949e58aa3 100644--- a/plugins/codex-security/scripts/finalize_scan_contract.py+++ b/plugins/codex-security/scripts/finalize_scan_contract.py@@ -1211,6 +1211,7 @@ def _recover_unsealed_findings(f"Skipped malformed {auxiliary} for finding {index + 1}: {exc}.")_validate_schema_node(finding, finding_schema, context, schema)+ _require_safe_json_value(finding, context)if current_contract:_validate_current_findings({"findings": [finding]}, schema_dir)except ContractError as exc:@@ -1268,9 +1269,12 @@ def _recover_unsealed_coverage(scan_dir: Path,warnings: list[str],discarded_findings: list[str],-) -> None:+ *,+ worker_draft: bool = False,+) -> set[str]:schema = _read_json(schema_dir / "coverage.schema.json")properties = _require_dict(schema, "properties", "coverage.schema")+ discarded_fields: set[str] = set()if (coverage.get("mode") == "deep_repository"and coverage.get("inventoryStrategy") != "repository"@@ -1280,16 +1284,24 @@ def _recover_unsealed_coverage(surface_ids: set[str] = set()receipt_paths: dict[str, str] = {}- for field, label in (+ fields = [("surfaces", "coverage surface"),("explicitExclusions", "coverage exclusion"),("deferred", "deferred coverage item"),- ):+ ]+ if worker_draft:+ if "openQuestions" in coverage:+ _normalize_unsealed_open_questions(coverage)+ fields.append(("openQuestions", "coverage open question"))+ if "resolvedDeferred" in coverage:+ fields.append(("resolvedDeferred", "resolved deferred item"))+ for field, label in fields:array_schema = _require_dict(properties, field, "coverage.schema.properties")item_schema = _require_dict(array_schema, "items", f"coverage.schema.properties.{field}")items = coverage.get(field)if not isinstance(items, list):warnings.append(f"Skipped malformed {label} records: expected an array.")+ discarded_fields.add(field)coverage[field] = []continue@@ -1302,7 +1314,9 @@ def _recover_unsealed_coverage(if field == "surfaces":surface_receipt_paths = dict(receipt_paths)surface_id = _require_str(item, "id", context)- if surface_id in surface_ids:+ if worker_draft:+ _require_str(item, "label", context)+ if surface_id in surface_ids and not worker_draft:raise ContractError(f"{context}.id: duplicate surface id")disposition = item.get("disposition")surface_recovered = False@@ -1352,12 +1366,18 @@ def _recover_unsealed_coverage(item["receiptRefs"] = recovered_receiptsif surface_recovered:item["disposition"] = "needs_follow_up"+ discarded_fields.add(field)if field == "deferred":_validate_deferred_paths(item, context)_validate_schema_node(item, item_schema, context)+ if field == "resolvedDeferred":+ _validate_resolved_deferred({"resolvedDeferred": [item]})+ if worker_draft:+ _require_safe_json_value(item, context)except ContractError as exc:warnings.append(f"Skipped malformed {label} {index + 1}: {exc}.")+ discarded_fields.add(field)continueif field == "surfaces":@@ -1369,6 +1389,13 @@ def _recover_unsealed_coverage(if discarded_findings:for surface in coverage["surfaces"]:+ if (+ worker_draft+ and surface.get("candidateId")+ and surface["disposition"] in {"rejected", "not_applicable"}+ ):+ # A valid explicit outcome can still resolve that worker's history.+ continuesurface["disposition"] = "needs_follow_up"coverage["deferred"].extend({"id": f"discarded-finding-{index}", "reason": warning}@@ -1383,9 +1410,13 @@ def _recover_unsealed_coverage("coverage.resolvedDeferred",)_validate_resolved_deferred(coverage)+ if worker_draft:+ _require_safe_json_value(coverage["resolvedDeferred"], "coverage.resolvedDeferred")except ContractError as exc:coverage.pop("resolvedDeferred")warnings.append(f"Skipped malformed resolved deferred work: {exc}.")+ discarded_fields.add("resolvedDeferred")+ return discarded_fieldsdef _recover_unsealed_hardening(diff --git a/plugins/codex-security/scripts/workbench/retained_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.pynew file mode 100644index 000000000..fdf1a95f8--- /dev/null+++ b/plugins/codex-security/scripts/workbench/retained_coverage.py@@ -0,0 +1,511 @@+"""Reconcile retained coverage observations during saved-result recovery."""++from __future__ import annotations++import copy+import hashlib+import json+import os+import re+from collections.abc import Callable, Iterator+from pathlib import Path+from typing import Any++from finalize_scan_contract import (+ ContractError,+ _read_json,+ _require_portable_relative_path,+ _require_str,+ _validate_resolved_deferred,+ _validate_schema_node,+ finding_candidate_id,+ open_scan_local_file_descriptor,+)+from workbench.saved_coverage import _project_worker_coverage_row, _saved_coverage_id+++def _encoded(value: Any) -> bytes:+ return json.dumps(+ value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")+ ).encode()+++def _digest(value: Any) -> str:+ return hashlib.sha256(_encoded(value)).hexdigest()+++def _finding_key(finding: dict[str, Any]) -> str:+ # Wording and evidence may improve between checkpoints; distinct source locations+ # must not collide merely because two workers chose the same semantic identity.+ provenance = finding.get("provenance")+ identity = (+ provenance.get("preservedIdentity", finding.get("identity"))+ if isinstance(provenance, dict)+ else finding.get("identity")+ )+ if not isinstance(identity, dict):+ normalized = dict(finding)+ normalized.pop("identity", None)+ _ensure_finding_identity(normalized)+ identity = normalized["identity"]+ locations = finding.get("locations", [])+ if not isinstance(locations, list):+ locations = []+ return _digest(+ [+ finding.get("ruleId"),+ identity,+ sorted(+ (+ (+ location.get("path"),+ location.get("startLine"),+ location.get("endLine", location.get("startLine")),+ )+ for location in locations+ if isinstance(location, dict)+ ),+ key=_encoded,+ ),+ ]+ )+++def _worker_candidate_key(+ worker_id: str, candidate_id: str, finding: dict[str, Any]+) -> tuple[str, str, Any, Any, Any]:+ """Identify one worker-local candidate without merging unrelated locations."""+ provenance = finding.get("provenance")+ identity = (+ provenance.get("preservedIdentity", finding.get("identity"))+ if isinstance(provenance, dict)+ else finding.get("identity")+ )+ if not isinstance(identity, dict):+ normalized = dict(finding)+ _ensure_finding_identity(normalized)+ identity = normalized.get("identity")+ anchor = identity.get("anchor") if isinstance(identity, dict) else None+ instance = identity.get("instance") if isinstance(identity, dict) else None+ return worker_id, candidate_id, finding.get("ruleId"), anchor, instance+++def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:+ """Return substantive finding content without generated identity or provenance."""+ return {+ key: value+ for key, value in finding.items()+ if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}+ }+++def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None:+ if not isinstance(finding, dict) or "identity" in finding:+ return+ if candidate_only and not finding_candidate_id(finding):+ return+ extensions = finding.get("extensions")+ source = str(+ (extensions.get("candidateId") if isinstance(extensions, dict) else None)+ or finding.get("title")+ or "finding"+ )+ anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"+ finding["identity"] = {"anchor": anchor}+++def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:+ """Yield canonical and historical findings without trusting candidate IDs."""+ pending = [finding]+ seen: set[int] = set()+ while pending:+ current = pending.pop()+ marker = id(current)+ if marker in seen:+ continue+ seen.add(marker)+ yield current+ provenance = current.get("provenance")+ if not isinstance(provenance, dict):+ continue+ previous = provenance.get("previousFindings")+ if isinstance(previous, list):+ pending.extend(item for item in reversed(previous) if isinstance(item, dict))+ sources = provenance.get("sourceFindings")+ if isinstance(sources, list):+ pending.extend(+ source["finding"]+ for source in reversed(sources)+ if isinstance(source, dict) and isinstance(source.get("finding"), dict)+ )+++def _append_previous_finding(+ item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False+) -> None:+ if not isinstance(item.get("previousFindings"), list):+ item["previousFindings"] = []+ history = item["previousFindings"]+ if not any(+ isinstance(previous, dict)+ and _finding_key(previous) == _finding_key(finding)+ and _finding_content(previous) == _finding_content(finding)+ for previous in history+ ):+ history.append(copy.deepcopy(finding) if clone else finding)+++def _has_candidate_metadata(row: dict[str, Any]) -> bool:+ return any(key in row for key in ("candidateId", "candidate", "finding"))+++def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]:+ rows = coverage.get(field, [])+ return rows if isinstance(rows, list) else []+++def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:+ if draft.get("complete") is False:+ return []+ coverage = draft["coverage"]+ rows = coverage.get("resolvedDeferred", [])+ try:+ # Invalid closure metadata cannot discard the evidence it names.+ _validate_schema_node(rows, schema, "coverage.resolvedDeferred")+ _validate_resolved_deferred({**coverage, "deferred": _coverage_rows(coverage, "deferred")})+ except ContractError:+ return []+ return rows+++def _merge_tied_parent_observations(+ current: dict[str, Any], previous: dict[str, Any]+) -> dict[str, Any]:+ merged = copy.deepcopy(current)+ for finding in previous["findings"]:+ if finding not in merged["findings"]:+ merged["findings"].append(copy.deepcopy(finding))+ coverage = merged["coverage"]+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):+ output = coverage.setdefault(field, [])+ if isinstance(output, list):+ for row in _coverage_rows(previous["coverage"], field):+ if row not in output:+ output.append(copy.deepcopy(row))+ pending_ids = {+ identity+ for row in _coverage_rows(coverage, "deferred")+ if isinstance(row, dict)+ for identity in (row.get("id"), row.get("candidateId"))+ if isinstance(identity, str)+ }+ closure_schema = _read_json(+ Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json"+ )["properties"]["resolvedDeferred"]+ closures = {}+ for observed in (current, previous):+ for row in _resolved_deferred_rows(observed, closure_schema):+ if row["id"] not in pending_ids:+ closures.setdefault(row["id"], copy.deepcopy(row))+ coverage.pop("resolvedDeferred", None)+ if closures:+ coverage["resolvedDeferred"] = list(closures.values())+ if current.get("complete") is False or previous.get("complete") is False:+ merged["complete"] = False+ coverage.pop("completeness", None)+ return merged+++def _inherit_coverage_sources(+ row: Any,+ originals: list[Any],+ sources: dict[int, tuple[dict[str, Any], list[str]]],+) -> None:+ if not isinstance(row, dict):+ return+ for original in originals:+ saved = sources.get(id(original))+ if saved is None:+ continue+ # Retain the object as well as its ID while copies are reconciled.+ paths = sources.setdefault(id(row), (row, []))[1]+ for relative in saved[1]:+ if relative not in paths:+ paths.append(relative)+++def _generic_surface_updates(+ sources: list[tuple[str, dict[str, Any], str | None]],+ source_order: dict[str, tuple[int, int]],+ closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],+ active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],+ resolved_candidates: dict[tuple[str | None, str], str],+ reopened_generic: set[tuple[str | None, str]],+ surface_schema: dict[str, Any],+ deferred_rows: dict[str, list[Any]],+ ambiguous_deferred: set[tuple[str | None, str]],+ canonical_rows: set[int],+ coverage_sources: dict[int, tuple[dict[str, Any], list[str]]],+ *,+ surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],+ eligible_receipt_sources: set[str],+ receipt_root: Path | None = None,+ receipt_warnings: list[str] | None = None,+ fallback_surfaces: list[dict[str, Any]] | None = None,+) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:+ if not closed_deferred and not reopened_generic and receipt_root is None:+ return set(), []++ def receipt_error(ref: Any) -> ContractError | None:+ if receipt_root is None or not isinstance(ref, str):+ return ContractError("Saved coverage receipt is unavailable.")+ try:+ descriptor = open_scan_local_file_descriptor(+ receipt_root, ref, "Saved coverage receipt"+ )+ os.close(descriptor)+ except ContractError as exc:+ return exc+ return None++ def linked(row: dict[str, Any], identity: str | None) -> bool:+ surface_ids = row.get("surfaceIds", [])+ return identity is not None and (+ row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)+ )++ saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}+ for relative, draft, owner in sources:+ for row in _coverage_rows(draft["coverage"], "surfaces"):+ if isinstance(row, dict) and isinstance(row.get("id"), str):+ saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))+ replaced: set[int] = set()+ updates: list[tuple[str | None, dict[str, Any], set[int]]] = []+ # Current selections across all surfaces take precedence over historical refs.+ receipt_paths: dict[str, str] = {}+ retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}+ if receipt_root is not None:+ receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []+ for key, matches in saved_surfaces.items():+ retained = [+ match for match in matches if surface_is_retained(match[0], key[0], match[1])+ ]+ if not retained:+ continue+ latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])+ retained_surfaces[key] = latest_surface+ receipt_surfaces.extend(+ (key[0], row)+ for relative, row in retained+ if source_order[relative] == source_order[latest_source]+ )+ for relative, draft, owner in sources:+ for row in _coverage_rows(draft["coverage"], "surfaces"):+ if (+ isinstance(row, dict)+ and "id" not in row+ and surface_is_retained(relative, owner, row)+ ):+ receipt_surfaces.append((owner, row))+ receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])+ for owner, latest_surface in receipt_surfaces:+ # Recovery repairs disposition and receipts before validating row+ # metadata. A row discarded there cannot reserve a receipt spelling.+ metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")+ metadata.setdefault("id", _saved_coverage_id(metadata))+ metadata.update(disposition="needs_follow_up", receiptRefs=[])+ try:+ _require_str(metadata, "id", "coverage.surfaces")+ _validate_schema_node(metadata, surface_schema, "coverage.surfaces")+ except ContractError:+ continue+ refs = latest_surface.get("receiptRefs", [])+ for ref in refs if isinstance(refs, list) else []:+ if receipt_error(ref) is None:+ normalized = _require_portable_relative_path(ref, "Saved coverage receipt")+ receipt_paths.setdefault(normalized.lower(), normalized)+ for relative, draft, owner in sources:+ closed_ids = {+ identity+ for (saved_owner, identity), (_, _, source) in closed_deferred.items()+ if saved_owner == owner and source == relative+ }+ reopened_ids = {+ identity+ for (saved_owner, identity), (_, _, source) in active_deferred.items()+ if saved_owner == owner and source == relative and (owner, identity) in reopened_generic+ }+ if not closed_ids and not reopened_ids and receipt_root is None:+ continue+ for surface in _coverage_rows(draft["coverage"], "surfaces"):+ if not isinstance(surface, dict):+ continue+ reopening = surface.get("disposition") == "needs_follow_up"+ work_ids = reopened_ids if reopening else closed_ids+ repair_only = not work_ids+ if repair_only and receipt_root is None:+ continue+ identity = surface.get("id")+ if not isinstance(identity, str):+ continue+ matches = saved_surfaces[(owner, identity)]+ # Receipt repair alone cannot close or reopen a saved surface.+ if repair_only and any(+ row.get("disposition") != surface.get("disposition") for _, row in matches+ ):+ continue+ # Older writers could assign one ID to distinct surfaces in a draft.+ # A closure cannot identify which of those observations it replaces.+ by_source = dict(matches)+ if any(row != by_source[saved_path] for saved_path, row in matches):+ continue+ if any(_has_candidate_metadata(row) for _, row in matches):+ continue+ if any(+ row is not surface+ and source_order[saved_path] >= source_order[relative]+ and row.get("disposition") != surface.get("disposition")+ for saved_path, row in matches+ ):+ continue++ if not reopening and any(+ saved_owner == owner+ and (+ not isinstance(row.get("id") or row.get("candidateId"), str)+ or (+ isinstance(row.get("id"), str)+ and (owner, row["id"]) in ambiguous_deferred+ and not _has_candidate_metadata(row)+ )+ )+ and linked(row, identity)+ for saved_path, _, saved_owner in sources+ for row in deferred_rows[saved_path]+ if isinstance(row, dict)+ ):+ continue+ if not reopening and any(+ saved_owner == owner+ and (owner, deferred_id) not in closed_deferred+ and (+ (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None+ or (owner, candidate_id) not in resolved_candidates+ )+ and linked(row, identity)+ for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()+ ):+ continue+ # An accepted checkpoint can update a saved surface by ID without+ # optional surfaceIds links on its generic task.+ receipt_matches = [+ (saved_path, row)+ for saved_path, row in matches+ if receipt_root is None+ or saved_path in eligible_receipt_sources+ or surface_is_retained(saved_path, owner, row)+ ]+ if not receipt_matches:+ continue+ latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]+ target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface+ if target is None:+ continue+ update = copy.deepcopy(target)+ # Historical evidence can repair a retained row without promoting+ # an unselected observation's label, notes, or other metadata.+ refs = copy.deepcopy(latest_surface.get("receiptRefs", []))+ update["receiptRefs"] = refs+ replaced_receipt = False+ replacement_paths = dict(receipt_paths)+ if isinstance(refs, list):+ repaired = (+ not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)+ )+ if repaired:+ for ref in refs:+ normalized = _require_portable_relative_path(ref, "Saved coverage receipt")+ key = normalized.lower()+ if key in replacement_paths and replacement_paths[key] != normalized:+ repaired = False+ break+ replacement_paths[key] = normalized+ for _, row in receipt_matches:+ previous_refs = row.get("receiptRefs", [])+ for ref in previous_refs if isinstance(previous_refs, list) else []:+ if ref not in refs:+ failure = receipt_error(ref) if repaired else None+ replaceable = failure is not None and isinstance(+ failure.__cause__, FileNotFoundError+ )+ if repaired and isinstance(ref, str):+ try:+ normalized = _require_portable_relative_path(+ ref, "Saved coverage receipt"+ )+ except ContractError as error:+ failure = error+ replaceable = True+ else:+ key = normalized.lower()+ previous = replacement_paths.get(key)+ if previous is not None and previous != normalized:+ failure = ContractError(+ f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"+ )+ replaceable = True+ elif not replaceable:+ replacement_paths[key] = normalized+ if replaceable:+ replaced_receipt = True+ if receipt_warnings is not None:+ receipt_warnings.append(+ f"Replaced invalid saved coverage receipt {ref}: {failure}"+ )+ continue+ refs.append(ref)+ if repair_only and not replaced_receipt:+ continue+ try:+ _validate_schema_node(update, surface_schema, "coverage.surfaces")+ except ContractError:+ continue+ replaced_sources = {+ id(row)+ for _, row in matches+ if reopening+ or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}+ }+ replaced.update(replaced_sources)+ existing = next(+ (entry for entry in updates if entry[0] == owner and entry[1] == update), None+ )+ if existing is None:+ receipt_paths.update(replacement_paths)+ updates.append((owner, update, replaced_sources))+ else:+ existing[2].update(replaced_sources)+ retained = next(+ entry[1] for entry in updates if entry[0] == owner and entry[1] == update+ )+ _inherit_coverage_sources(+ retained,+ [latest_surface, *(row for _, row in matches if id(row) in replaced_sources)],+ coverage_sources,+ )+ if id(latest_surface) in canonical_rows:+ canonical_rows.add(id(retained))+ return replaced, updates+++def _deferred_candidate_id(+ row: dict[str, Any],+ owner: str | None,+ ambiguous_deferred: set[tuple[str | None, str]],+) -> str | None:+ identity = row.get("candidateId") or row.get("id")+ if not isinstance(identity, str):+ return None+ if (owner, identity) in ambiguous_deferred and not _has_candidate_metadata(row):+ return None+ return identitydiff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.pynew file mode 100644index 000000000..585ebdf60--- /dev/null+++ b/plugins/codex-security/scripts/workbench/saved_sources.py@@ -0,0 +1,169 @@+"""Read immutable saved-result observations and their frozen ordering."""++from __future__ import annotations++import json+import os+import re+from pathlib import Path+from typing import Any++from finalize_scan_contract import (+ ContractError,+ _read_scan_local_json_with_metadata,+ write_scan_local_bytes,+)+from workbench.retained_coverage import _digest, _encoded+++def _checkpoint_head_directory(relative: str) -> Path | None:+ path = Path(relative)+ if path.name == "checkpoint-head.json":+ return path.parent+ if path.parent.name == "checkpoint-heads":+ return path.parent.parent+ return None+++def _capture_saved_source(+ scan_dir: Path,+ relative: str,+ scan_id: str,+ *,+ kind: str | None = None,+ snapshot_head: bool = True,+ write: bool = True,+ created_paths: set[str] | None = None,+) -> dict[str, tuple[str, int]]:+ if not snapshot_head or Path(relative).name != "checkpoint-head.json":+ _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)+ return {relative: (digest, observed)}+ head, _, observed = _read_saved_result(scan_dir, relative, scan_id)+ observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}+ directory = Path(relative).parent+ selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()+ _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)+ digest = _digest(observation)+ snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()+ # Capture the selected file even if the worker created it after directory enumeration.+ if write and not (scan_dir / snapshot).exists():+ if created_paths is not None and not os.path.lexists(scan_dir / snapshot):+ created_paths.add(snapshot)+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))+ return {+ snapshot: (digest, int(observation["observedAtNs"])),+ selected: (selected_digest, selected_time),+ }+++def _is_source_order_snapshot(relative: str) -> bool:+ path = Path(relative)+ return path.parent == Path("source-order") and bool(+ re.fullmatch(r"[0-9a-f]{64}\.json", path.name)+ )+++def _read_saved_result(+ scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None+) -> tuple[dict[str, Any], str, int]:+ draft, _, metadata = _read_scan_local_json_with_metadata(+ scan_dir, relative, "Saved scan checkpoint"+ )+ directory = _checkpoint_head_directory(relative)+ if directory is not None:+ checkpoint = draft.get("checkpoint")+ if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):+ raise ContractError("checkpoint head does not name a saved checkpoint")+ _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)+ if Path(relative).name == "checkpoint-head.json":+ return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns+ observed = draft.get("observedAtNs")+ if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):+ raise ContractError("checkpoint head has no observation time")+ return draft, _digest(draft), int(observed)+ if draft.get("scanId") != scan_id:+ raise ContractError("checkpoint belongs to a different scan")+ if not _is_source_order_snapshot(relative) and (+ not isinstance(draft.get("findings"), list)+ or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)+ ):+ raise ContractError("checkpoint has no semantic findings or coverage")+ return draft, _digest(draft), metadata.st_mtime_ns+++def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:+ times: dict[str, int] = {}+ snapshots = [path for path in sources if _is_source_order_snapshot(path)]+ for path in snapshots:+ record, digest, _ = _read_saved_result(scan_dir, path, scan_id)+ if digest != sources[path] or not isinstance(record.get("sources"), dict):+ raise ContractError("saved source ordering changed after the scan stopped")+ for relative, observation in record["sources"].items():+ if (+ not isinstance(observation, dict)+ or relative not in sources+ or observation.get("digest") != sources[relative]+ or not isinstance(observation.get("observedAtNs"), str)+ or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])+ ):+ raise ContractError("saved source ordering does not match its frozen sources")+ observed = int(observation["observedAtNs"])+ if relative in times and times[relative] != observed:+ raise ContractError("saved source ordering has conflicting observations")+ times[relative] = observed+ if snapshots and sources.keys() - set(snapshots) - times.keys():+ raise ContractError("saved source ordering is incomplete")+ return times+++def _freeze_source_times(+ scan_dir: Path,+ scan_id: str,+ sources: dict[str, str],+ times: dict[str, int],+ *,+ created_paths: set[str] | None = None,+) -> None:+ # Identical result rewrites must not change the order of frozen review evidence.+ observations = {+ path: {"digest": digest, "observedAtNs": str(times[path])}+ for path, digest in sources.items()+ if not _is_source_order_snapshot(path)+ }+ if not observations:+ return+ record = {"scanId": scan_id, "sources": observations}+ digest = _digest(record)+ path = f"source-order/{digest}.json"+ if not (scan_dir / path).exists():+ if created_paths is not None and not os.path.lexists(scan_dir / path):+ created_paths.add(path)+ write_scan_local_bytes(scan_dir, path, _encoded(record))+ if _read_saved_result(scan_dir, path, scan_id)[1] != digest:+ raise ContractError("saved source ordering does not match its digest")+ sources[path] = digest+++def _source_digests(value: Any, error: str) -> dict[str, str]:+ if not isinstance(value, dict) or not all(+ isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()+ ):+ raise ContractError(error)+ return value+++def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:+ if isinstance(value, dict) and isinstance(value.get("sources"), dict):+ sources = _source_digests(+ value["sources"], "Saved stopped-scan source digests are malformed."+ )+ model_source = value.get("threatModelSource")+ if not isinstance(model_source, str) or model_source not in sources:+ raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")+ return sources, model_source+ return _source_digests(value, "Saved stopped-scan source digests are malformed."), None+++def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:+ state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources+ return json.dumps(state, sort_keys=True)diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.pyindex cd709d050..d9a94ff31 100644--- a/plugins/codex-security/scripts/workbench_saved_results.py+++ b/plugins/codex-security/scripts/workbench_saved_results.py@@ -12,7 +12,7 @@ import reimport sqlite3import statimport sys-from collections.abc import Callable, Iterator+from collections.abc import Iteratorfrom contextlib import contextmanagerfrom pathlib import Pathfrom typing import Any@@ -29,15 +29,13 @@ from finalize_scan_contract import (_read_saved_threat_model,_read_scan_local_json,_read_scan_local_json_bytes,- _read_scan_local_json_with_metadata,_recover_unsealed_coverage,_recover_unsealed_findings,_remove_scan_local_file_if_exists,_require_portable_relative_path,_require_safe_json_value,- _require_str,_validate_completion_binding,- _validate_resolved_deferred,+ _validate_current_findings,_validate_schema_node,_write_prepared_scan_finalization,finalize_scan,@@ -64,6 +62,80 @@ _RESERVED_ARTIFACT_PATHS = json.loads()+from workbench.retained_coverage import (+ _append_previous_finding as _append_previous_finding,+)+from workbench.retained_coverage import (+ _coverage_rows as _coverage_rows,+)+from workbench.retained_coverage import (+ _deferred_candidate_id as _deferred_candidate_id,+)+from workbench.retained_coverage import (+ _digest as _digest,+)+from workbench.retained_coverage import (+ _encoded as _encoded,+)+from workbench.retained_coverage import (+ _ensure_finding_identity as _ensure_finding_identity,+)+from workbench.retained_coverage import (+ _finding_content as _finding_content,+)+from workbench.retained_coverage import (+ _finding_key as _finding_key,+)+from workbench.retained_coverage import (+ _generic_surface_updates as _generic_surface_updates,+)+from workbench.retained_coverage import (+ _has_candidate_metadata as _has_candidate_metadata,+)+from workbench.retained_coverage import (+ _inherit_coverage_sources as _inherit_coverage_sources,+)+from workbench.retained_coverage import (+ _merge_tied_parent_observations as _merge_tied_parent_observations,+)+from workbench.retained_coverage import (+ _resolved_deferred_rows as _resolved_deferred_rows,+)+from workbench.retained_coverage import (+ _retained_findings as _retained_findings,+)+from workbench.retained_coverage import (+ _worker_candidate_key as _worker_candidate_key,+)+from workbench.saved_sources import (+ _capture_saved_source as _capture_saved_source,+)+from workbench.saved_sources import (+ _checkpoint_head_directory as _checkpoint_head_directory,+)+from workbench.saved_sources import (+ _encode_retained_sources as _encode_retained_sources,+)+from workbench.saved_sources import (+ _freeze_source_times as _freeze_source_times,+)+from workbench.saved_sources import (+ _frozen_source_times as _frozen_source_times,+)+from workbench.saved_sources import (+ _is_source_order_snapshot as _is_source_order_snapshot,+)+from workbench.saved_sources import (+ _read_saved_result as _read_saved_result,+)+from workbench.saved_sources import (+ _retained_source_state as _retained_source_state,+)+from workbench.saved_sources import (+ _source_digests as _source_digests,+)++def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]:scan_dir = Path(scan["scan_dir"])fields: dict[str, Any] = {"threatModelAvailable": False}@@ -119,16 +191,6 @@ def refresh_completed_scan(return db.scan_context(connection, scan["id"])-def _encoded(value: Any) -> bytes:- return json.dumps(- value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")- ).encode()---def _digest(value: Any) -> str:- return hashlib.sha256(_encoded(value)).hexdigest()--def _children(scan_dir: Path, relative: str) -> list[str]:cursor = scan_dirfor part in Path(relative).parts:@@ -215,134 +277,6 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[stcontinue-def _checkpoint_head_directory(relative: str) -> Path | None:- path = Path(relative)- if path.name == "checkpoint-head.json":- return path.parent- if path.parent.name == "checkpoint-heads":- return path.parent.parent- return None---def _capture_saved_source(- scan_dir: Path,- relative: str,- scan_id: str,- *,- kind: str | None = None,- snapshot_head: bool = True,- write: bool = True,- created_paths: set[str] | None = None,-) -> dict[str, tuple[str, int]]:- if not snapshot_head or Path(relative).name != "checkpoint-head.json":- _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)- return {relative: (digest, observed)}- head, _, observed = _read_saved_result(scan_dir, relative, scan_id)- observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}- directory = Path(relative).parent- selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()- _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)- digest = _digest(observation)- snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()- # Capture the selected file even if the worker created it after directory enumeration.- if write and not (scan_dir / snapshot).exists():- if created_paths is not None and not os.path.lexists(scan_dir / snapshot):- created_paths.add(snapshot)- write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))- return {- snapshot: (digest, int(observation["observedAtNs"])),- selected: (selected_digest, selected_time),- }---def _is_source_order_snapshot(relative: str) -> bool:- path = Path(relative)- return path.parent == Path("source-order") and bool(- re.fullmatch(r"[0-9a-f]{64}\.json", path.name)- )---def _read_saved_result(- scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None-) -> tuple[dict[str, Any], str, int]:- draft, _, metadata = _read_scan_local_json_with_metadata(- scan_dir, relative, "Saved scan checkpoint"- )- directory = _checkpoint_head_directory(relative)- if directory is not None:- checkpoint = draft.get("checkpoint")- if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):- raise ContractError("checkpoint head does not name a saved checkpoint")- _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)- if Path(relative).name == "checkpoint-head.json":- return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns- observed = draft.get("observedAtNs")- if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):- raise ContractError("checkpoint head has no observation time")- return draft, _digest(draft), int(observed)- if draft.get("scanId") != scan_id:- raise ContractError("checkpoint belongs to a different scan")- if not _is_source_order_snapshot(relative) and (- not isinstance(draft.get("findings"), list)- or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)- ):- raise ContractError("checkpoint has no semantic findings or coverage")- return draft, _digest(draft), metadata.st_mtime_ns---def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:- times: dict[str, int] = {}- snapshots = [path for path in sources if _is_source_order_snapshot(path)]- for path in snapshots:- record, digest, _ = _read_saved_result(scan_dir, path, scan_id)- if digest != sources[path] or not isinstance(record.get("sources"), dict):- raise ContractError("saved source ordering changed after the scan stopped")- for relative, observation in record["sources"].items():- if (- not isinstance(observation, dict)- or relative not in sources- or observation.get("digest") != sources[relative]- or not isinstance(observation.get("observedAtNs"), str)- or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])- ):- raise ContractError("saved source ordering does not match its frozen sources")- observed = int(observation["observedAtNs"])- if relative in times and times[relative] != observed:- raise ContractError("saved source ordering has conflicting observations")- times[relative] = observed- if snapshots and sources.keys() - set(snapshots) - times.keys():- raise ContractError("saved source ordering is incomplete")- return times---def _freeze_source_times(- scan_dir: Path,- scan_id: str,- sources: dict[str, str],- times: dict[str, int],- *,- created_paths: set[str] | None = None,-) -> None:- # Identical result rewrites must not change the order of frozen review evidence.- observations = {- path: {"digest": digest, "observedAtNs": str(times[path])}- for path, digest in sources.items()- if not _is_source_order_snapshot(path)- }- if not observations:- return- record = {"scanId": scan_id, "sources": observations}- digest = _digest(record)- path = f"source-order/{digest}.json"- if not (scan_dir / path).exists():- if created_paths is not None and not os.path.lexists(scan_dir / path):- created_paths.add(path)- write_scan_local_bytes(scan_dir, path, _encoded(record))- if _read_saved_result(scan_dir, path, scan_id)[1] != digest:- raise ContractError("saved source ordering does not match its digest")- sources[path] = digest--def _parent_scan_draft(scan_id: str,parent_scan: dict[str, Any],@@ -382,31 +316,6 @@ def _read_saved_parent_result(return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)-def _source_digests(value: Any, error: str) -> dict[str, str]:- if not isinstance(value, dict) or not all(- isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()- ):- raise ContractError(error)- return value---def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:- if isinstance(value, dict) and isinstance(value.get("sources"), dict):- sources = _source_digests(- value["sources"], "Saved stopped-scan source digests are malformed."- )- model_source = value.get("threatModelSource")- if not isinstance(model_source, str) or model_source not in sources:- raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")- return sources, model_source- return _source_digests(value, "Saved stopped-scan source digests are malformed."), None---def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:- state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources- return json.dumps(state, sort_keys=True)--def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:try:scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))@@ -562,453 +471,6 @@ def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:return _saved_results_changed(db, connection, scan)-def _finding_key(finding: dict[str, Any]) -> str:- # Wording and evidence may improve between checkpoints; distinct source locations- # must not collide merely because two workers chose the same semantic identity.- provenance = finding.get("provenance")- identity = (- provenance.get("preservedIdentity", finding.get("identity"))- if isinstance(provenance, dict)- else finding.get("identity")- )- if not isinstance(identity, dict):- normalized = dict(finding)- normalized.pop("identity", None)- _ensure_finding_identity(normalized)- identity = normalized["identity"]- locations = finding.get("locations", [])- if not isinstance(locations, list):- locations = []- return _digest(- [- finding.get("ruleId"),- identity,- sorted(- (- (- location.get("path"),- location.get("startLine"),- location.get("endLine", location.get("startLine")),- )- for location in locations- if isinstance(location, dict)- ),- key=_encoded,- ),- ]- )---def _worker_candidate_key(- worker_id: str, candidate_id: str, finding: dict[str, Any]-) -> tuple[str, str, Any, Any, Any]:- """Identify one worker-local candidate without merging unrelated locations."""- provenance = finding.get("provenance")- identity = (- provenance.get("preservedIdentity", finding.get("identity"))- if isinstance(provenance, dict)- else finding.get("identity")- )- if not isinstance(identity, dict):- normalized = dict(finding)- _ensure_finding_identity(normalized)- identity = normalized.get("identity")- anchor = identity.get("anchor") if isinstance(identity, dict) else None- instance = identity.get("instance") if isinstance(identity, dict) else None- return worker_id, candidate_id, finding.get("ruleId"), anchor, instance---def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:- """Return substantive finding content without generated identity or provenance."""- return {- key: value- for key, value in finding.items()- if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}- }---def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None:- if not isinstance(finding, dict) or "identity" in finding:- return- if candidate_only and not finding_candidate_id(finding):- return- extensions = finding.get("extensions")- source = str(- (extensions.get("candidateId") if isinstance(extensions, dict) else None)- or finding.get("title")- or "finding"- )- anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"- finding["identity"] = {"anchor": anchor}---def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:- """Yield canonical and historical findings without trusting candidate IDs."""- pending = [finding]- seen: set[int] = set()- while pending:- current = pending.pop()- marker = id(current)- if marker in seen:- continue- seen.add(marker)- yield current- provenance = current.get("provenance")- if not isinstance(provenance, dict):- continue- previous = provenance.get("previousFindings")- if isinstance(previous, list):- pending.extend(item for item in reversed(previous) if isinstance(item, dict))- sources = provenance.get("sourceFindings")- if isinstance(sources, list):- pending.extend(- source["finding"]- for source in reversed(sources)- if isinstance(source, dict) and isinstance(source.get("finding"), dict)- )---def _append_previous_finding(- item: dict[str, Any], finding: dict[str, Any], *, clone: bool = False-) -> None:- if not isinstance(item.get("previousFindings"), list):- item["previousFindings"] = []- history = item["previousFindings"]- if not any(- isinstance(previous, dict)- and _finding_key(previous) == _finding_key(finding)- and _finding_content(previous) == _finding_content(finding)- for previous in history- ):- history.append(copy.deepcopy(finding) if clone else finding)---def _has_candidate_metadata(row: dict[str, Any]) -> bool:- return any(key in row for key in ("candidateId", "candidate", "finding"))---def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]:- rows = coverage.get(field, [])- return rows if isinstance(rows, list) else []---def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:- if draft.get("complete") is False:- return []- coverage = draft["coverage"]- rows = coverage.get("resolvedDeferred", [])- try:- # Invalid closure metadata cannot discard the evidence it names.- _validate_schema_node(rows, schema, "coverage.resolvedDeferred")- _validate_resolved_deferred({**coverage, "deferred": _coverage_rows(coverage, "deferred")})- except ContractError:- return []- return rows---def _merge_tied_parent_observations(- current: dict[str, Any], previous: dict[str, Any]-) -> dict[str, Any]:- merged = copy.deepcopy(current)- for finding in previous["findings"]:- if finding not in merged["findings"]:- merged["findings"].append(copy.deepcopy(finding))- coverage = merged["coverage"]- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):- output = coverage.setdefault(field, [])- if isinstance(output, list):- for row in _coverage_rows(previous["coverage"], field):- if row not in output:- output.append(copy.deepcopy(row))- pending_ids = {- identity- for row in _coverage_rows(coverage, "deferred")- if isinstance(row, dict)- for identity in (row.get("id"), row.get("candidateId"))- if isinstance(identity, str)- }- closure_schema = _read_json(- Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"- )["properties"]["resolvedDeferred"]- closures = {}- for observed in (current, previous):- for row in _resolved_deferred_rows(observed, closure_schema):- if row["id"] not in pending_ids:- closures.setdefault(row["id"], copy.deepcopy(row))- coverage.pop("resolvedDeferred", None)- if closures:- coverage["resolvedDeferred"] = list(closures.values())- if current.get("complete") is False or previous.get("complete") is False:- merged["complete"] = False- coverage.pop("completeness", None)- return merged---def _deferred_candidate_id(- row: dict[str, Any],- owner: str | None,- ambiguous_deferred: set[tuple[str | None, str]],-) -> str | None:- identity = row.get("candidateId") or row.get("id")- if not isinstance(identity, str):- return None- if (owner, identity) in ambiguous_deferred and not _has_candidate_metadata(row):- return None- return identity---def _generic_surface_updates(- sources: list[tuple[str, dict[str, Any], str | None]],- source_order: dict[str, tuple[int, int]],- closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],- active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],- resolved_candidates: dict[tuple[str | None, str], str],- reopened_generic: set[tuple[str | None, str]],- surface_schema: dict[str, Any],- deferred_rows: dict[str, list[Any]],- ambiguous_deferred: set[tuple[str | None, str]],- *,- surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],- eligible_receipt_sources: set[str],- receipt_root: Path | None = None,- receipt_warnings: list[str] | None = None,- fallback_surfaces: list[dict[str, Any]] | None = None,-) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:- if not closed_deferred and not reopened_generic and receipt_root is None:- return set(), []-- def receipt_error(ref: Any) -> ContractError | None:- if receipt_root is None or not isinstance(ref, str):- return ContractError("Saved coverage receipt is unavailable.")- try:- descriptor = open_scan_local_file_descriptor(- receipt_root, ref, "Saved coverage receipt"- )- os.close(descriptor)- except ContractError as exc:- return exc- return None-- def linked(row: dict[str, Any], identity: str | None) -> bool:- surface_ids = row.get("surfaceIds", [])- return identity is not None and (- row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)- )-- saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}- for relative, draft, owner in sources:- for row in _coverage_rows(draft["coverage"], "surfaces"):- if isinstance(row, dict) and isinstance(row.get("id"), str):- saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))- replaced: set[int] = set()- updates: list[tuple[str | None, dict[str, Any], set[int]]] = []- # Current selections across all surfaces take precedence over historical refs.- receipt_paths: dict[str, str] = {}- retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}- if receipt_root is not None:- receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []- for key, matches in saved_surfaces.items():- retained = [- match for match in matches if surface_is_retained(match[0], key[0], match[1])- ]- if not retained:- continue- latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])- retained_surfaces[key] = latest_surface- receipt_surfaces.extend(- (key[0], row)- for relative, row in retained- if source_order[relative] == source_order[latest_source]- )- for relative, draft, owner in sources:- for row in _coverage_rows(draft["coverage"], "surfaces"):- if (- isinstance(row, dict)- and "id" not in row- and surface_is_retained(relative, owner, row)- ):- receipt_surfaces.append((owner, row))- receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])- for owner, latest_surface in receipt_surfaces:- # Recovery repairs disposition and receipts before validating row- # metadata. A row discarded there cannot reserve a receipt spelling.- metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")- metadata.setdefault("id", _saved_coverage_id(metadata))- metadata.update(disposition="needs_follow_up", receiptRefs=[])- try:- _require_str(metadata, "id", "coverage.surfaces")- _validate_schema_node(metadata, surface_schema, "coverage.surfaces")- except ContractError:- continue- refs = latest_surface.get("receiptRefs", [])- for ref in refs if isinstance(refs, list) else []:- if receipt_error(ref) is None:- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")- receipt_paths.setdefault(normalized.lower(), normalized)- for relative, draft, owner in sources:- closed_ids = {- identity- for (saved_owner, identity), (_, _, source) in closed_deferred.items()- if saved_owner == owner and source == relative- }- reopened_ids = {- identity- for (saved_owner, identity), (_, _, source) in active_deferred.items()- if saved_owner == owner and source == relative and (owner, identity) in reopened_generic- }- if not closed_ids and not reopened_ids and receipt_root is None:- continue- for surface in _coverage_rows(draft["coverage"], "surfaces"):- if not isinstance(surface, dict):- continue- reopening = surface.get("disposition") == "needs_follow_up"- work_ids = reopened_ids if reopening else closed_ids- repair_only = not work_ids- if repair_only and receipt_root is None:- continue- identity = surface.get("id")- if not isinstance(identity, str):- continue- matches = saved_surfaces[(owner, identity)]- # Receipt repair alone cannot close or reopen a saved surface.- if repair_only and any(- row.get("disposition") != surface.get("disposition") for _, row in matches- ):- continue- # Older writers could assign one ID to distinct surfaces in a draft.- # A closure cannot identify which of those observations it replaces.- by_source = dict(matches)- if any(row != by_source[saved_path] for saved_path, row in matches):- continue- if any(_has_candidate_metadata(row) for _, row in matches):- continue- if any(- row is not surface- and source_order[saved_path] >= source_order[relative]- and row.get("disposition") != surface.get("disposition")- for saved_path, row in matches- ):- continue-- if not reopening and any(- saved_owner == owner- and (- not isinstance(row.get("id") or row.get("candidateId"), str)- or (- isinstance(row.get("id"), str)- and (owner, row["id"]) in ambiguous_deferred- and not _has_candidate_metadata(row)- )- )- and linked(row, identity)- for saved_path, _, saved_owner in sources- for row in deferred_rows[saved_path]- if isinstance(row, dict)- ):- continue- if not reopening and any(- saved_owner == owner- and (owner, deferred_id) not in closed_deferred- and (- (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None- or (owner, candidate_id) not in resolved_candidates- )- and linked(row, identity)- for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()- ):- continue- # An accepted checkpoint can update a saved surface by ID without- # optional surfaceIds links on its generic task.- receipt_matches = [- (saved_path, row)- for saved_path, row in matches- if receipt_root is None- or saved_path in eligible_receipt_sources- or surface_is_retained(saved_path, owner, row)- ]- if not receipt_matches:- continue- latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]- target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface- if target is None:- continue- update = copy.deepcopy(target)- # Historical evidence can repair a retained row without promoting- # an unselected observation's label, notes, or other metadata.- refs = copy.deepcopy(latest_surface.get("receiptRefs", []))- update["receiptRefs"] = refs- replaced_receipt = False- replacement_paths = dict(receipt_paths)- if isinstance(refs, list):- repaired = (- not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)- )- if repaired:- for ref in refs:- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")- key = normalized.lower()- if key in replacement_paths and replacement_paths[key] != normalized:- repaired = False- break- replacement_paths[key] = normalized- for _, row in receipt_matches:- previous_refs = row.get("receiptRefs", [])- for ref in previous_refs if isinstance(previous_refs, list) else []:- if ref not in refs:- failure = receipt_error(ref) if repaired else None- replaceable = failure is not None and isinstance(- failure.__cause__, FileNotFoundError- )- if repaired and isinstance(ref, str):- try:- normalized = _require_portable_relative_path(- ref, "Saved coverage receipt"- )- except ContractError as error:- failure = error- replaceable = True- else:- key = normalized.lower()- previous = replacement_paths.get(key)- if previous is not None and previous != normalized:- failure = ContractError(- f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"- )- replaceable = True- elif not replaceable:- replacement_paths[key] = normalized- if replaceable:- replaced_receipt = True- if receipt_warnings is not None:- receipt_warnings.append(- f"Replaced invalid saved coverage receipt {ref}: {failure}"- )- continue- refs.append(ref)- if repair_only and not replaced_receipt:- continue- try:- _validate_schema_node(update, surface_schema, "coverage.surfaces")- except ContractError:- continue- replaced_sources = {- id(row)- for _, row in matches- if reopening- or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}- }- replaced.update(replaced_sources)- existing = next(- (entry for entry in updates if entry[0] == owner and entry[1] == update), None- )- if existing is None:- receipt_paths.update(replacement_paths)- updates.append((owner, update, replaced_sources))- else:- existing[2].update(replaced_sources)- return replaced, updates--@contextmanagerdef preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:"""Keep rejected completion attempts from becoming accepted parent observations."""@@ -1061,6 +523,7 @@ def merge_saved_results(allow_frozen_legacy_parent: bool = False,frozen_model_source: str | None = None,selected_model_source: list[str] | None = None,+ preserve_canonical_parent: bool = False,project_worker_coverage: bool = False,created_paths: set[str] | None = None,current_contract: bool = False,@@ -1088,7 +551,7 @@ def merge_saved_results(warnings.append(f"Could not read the saved parent draft: {exc}")parent_manifest = Noneparent = None- if parent_manifest is not None and parent is not None:+ if parent_manifest is not None and parent is not None and not preserve_canonical_parent:parent_scan = parent_manifest["scan"]try:previous_head, _, head_modified = _read_saved_result(@@ -1177,8 +640,9 @@ def merge_saved_results(for saved_directory in (directory, head_snapshots):paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))- paths["checkpoint-head.json"] = None- checkpoints("checkpoints", None)+ if not preserve_canonical_parent:+ paths["checkpoint-head.json"] = None+ checkpoints("checkpoints", None)for worker in workers:try:outputs = _worker_outputs(scan_dir, worker)@@ -1244,6 +708,7 @@ def merge_saved_results(if latest_reducer not in frozen_source_digests:latest_reducer = None+ discarded_by_source: dict[str, list[str]] = {}for relative, worker_id in paths.items():try:draft, digest, observed = _read_saved_result(@@ -1257,6 +722,91 @@ def merge_saved_results(if _checkpoint_head_directory(relative) is not None:saved_heads[relative] = draft["checkpoint"]continue+ if preserve_canonical_parent and worker_id is not None:+ # Recover imported records without relaxing canonical parent validation.+ if "complete" in draft and not isinstance(draft["complete"], bool):+ draft["complete"] = False+ warnings.append(+ f"{relative}: Invalid complete marker; retained as a progress checkpoint."+ )+ schema_dir = Path(__file__).resolve().parent.parent / "schemas"+ if "threatModel" in draft:+ try:+ model = draft["threatModel"]+ if isinstance(model, dict):+ model = {**model, "origin": "recovered"}+ schema = _read_json(schema_dir / "scan-manifest.schema.json")+ _validate_schema_node(+ model,+ schema["properties"]["scan"]["properties"]["threatModel"],+ "threatModel",+ schema,+ )+ _require_safe_json_value(model, "threatModel")+ except ContractError as exc:+ draft.pop("threatModel")+ warnings.append(f"{relative}: Skipped malformed threat model: {exc}.")+ directory = Path(relative).parent+ if directory.name == "checkpoints":+ directory = directory.parent+ recovered_findings = []+ for index, finding in enumerate(draft["findings"]):+ _ensure_finding_identity(finding)+ recovered = {"scanId": scan_id, "findings": [finding]}+ source_warnings: list[str] = []+ discarded = _recover_unsealed_findings(+ {"scan": {"id": scan_id, "target": binding["target"]}},+ recovered,+ schema_dir,+ scan_dir / directory,+ source_warnings,+ )+ if recovered["findings"]:+ try:+ _validate_current_findings(recovered, schema_dir)+ except ContractError as exc:+ warning = f"Skipped malformed finding: {exc}."+ source_warnings.append(warning)+ discarded.append(warning)+ recovered["findings"] = []+ if discarded:+ discarded_by_source.setdefault(relative, []).extend(discarded)+ for recovered_finding in recovered["findings"]:+ writeup = recovered_finding.pop("writeup", None)+ if writeup is not None:+ source_warnings.append(+ "Retained worker writeup at "+ f"{directory.as_posix()}/{writeup['reportPath']}; "+ "the parent report renders this finding inline."+ )+ recovered_findings.extend(recovered["findings"])+ warnings.extend(+ f"{relative} finding {index + 1}: {warning}" for warning in source_warnings+ )+ draft["findings"] = recovered_findings+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):+ rows = draft["coverage"].get(field)+ if not isinstance(rows, list):+ continue+ for index, row in enumerate(rows):+ if field == "openQuestions" and isinstance(row, str):+ row = rows[index] = {"question": row}+ if not isinstance(row, dict):+ continue+ row["workerId"] = worker_id+ refs = row.get("receiptRefs")+ if not isinstance(refs, list):+ continue+ for ref_index, ref in enumerate(refs):+ if not isinstance(ref, str):+ continue+ try:+ normalized = _require_portable_relative_path(ref, "receiptRef")+ except ContractError:+ # Keep malformed references for the finalizer's diagnostics.+ continue+ if normalized.startswith("artifacts/"):+ refs[ref_index] = f"{directory.as_posix()}/{normalized}"# Recovery expects coverage, but reducer results only contain findings# and context. Add an empty value after hashing the original result.sources.append((relative, {"coverage": {}, **draft}, worker_id))@@ -1335,9 +885,12 @@ def merge_saved_results(if parent is None and latest_reducer is not None:parent = drafts_by_path[latest_reducer]- project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(- sources, parent, source_order, enabled=project_worker_coverage- )+ if preserve_canonical_parent:+ project_worker_coverage, surface_links = False, []+ else:+ project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(+ sources, parent, source_order, enabled=project_worker_coverage+ )# Keep the actual retained row even when recovery adds previous findings.linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}linked_surface_outputs: dict[int, dict[str, Any]] = {}@@ -1360,6 +913,12 @@ def merge_saved_results(output.append(retained)retained = output[output.index(retained)]record_coverage_owner(retained, item, owner, field)+ _inherit_coverage_sources(retained, [item], coverage_sources)+ if preserve_canonical_parent:+ if id(item) in canonical_rows:+ canonical_rows.add(id(retained))+ if field == "surfaces":+ retained_surface_rows.add(id(item))return retaineddef linked_tasks(@@ -1439,6 +998,50 @@ def merge_saved_results(row["id"] = identityreserved.add(identity)+ # A discarded local surface must not fall back to another source's reused ID.+ surface_ids_by_source: dict[tuple[str, str], list[str]] = {+ (relative, row["id"]): []+ for relative, draft, _ in all_sources+ for row in (+ draft["coverage"].get("surfaces", [])+ if isinstance(draft["coverage"].get("surfaces"), list)+ else []+ )+ if isinstance(row, dict) and isinstance(row.get("id"), str)+ }+ discarded_coverage_sources: set[str] = set()+ if preserve_canonical_parent:+ for relative, draft, worker_id in sources:+ if worker_id is None:+ continue+ for field in ("surfaces", "explicitExclusions", "deferred"):+ rows = draft["coverage"].get(field)+ for row in rows if isinstance(rows, list) else []:+ if isinstance(row, dict):+ row.setdefault("id", _saved_coverage_id(row))+ if field == "surfaces":+ row.setdefault("receiptRefs", [])+ source_warnings = []+ discarded_fields = _recover_unsealed_coverage(+ draft["coverage"],+ Path(__file__).resolve().parent.parent / "schemas",+ scan_dir,+ source_warnings,+ discarded_by_source.get(relative, []),+ # Reconcile local IDs and valid outcomes against worker history below.+ worker_draft=True,+ )+ if discarded_fields & {+ "surfaces",+ "explicitExclusions",+ "deferred",+ "resolvedDeferred",+ }:+ discarded_coverage_sources.add(relative)+ for row in draft["coverage"]["deferred"]:+ row["workerId"] = worker_id+ warnings.extend(f"{relative}: {warning}" for warning in source_warnings)+if parent is None and not sources:return Noneif (@@ -1557,17 +1160,43 @@ def merge_saved_results({"question": item.strip()} if isinstance(item, str) else itemfor item in coverage["openQuestions"]]+ # Closure reconciliation can reuse original parent rows instead of their copies.canonical_rows = ({id(item)+ for parent_coverage in (+ (coverage, parent["coverage"]) if preserve_canonical_parent else (coverage,)+ )for field in ("surfaces", "explicitExclusions", "deferred")- for item in _coverage_rows(coverage, field)+ for item in _coverage_rows(parent_coverage, field)}if parent_is_canonicalelse set())+ # Links use source-local IDs until all retained surfaces receive their final IDs.+ coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] = {}+ source_owners = {relative: owner for relative, _, owner in all_sources}+ retained_surface_rows: set[int] = set()+ for relative, draft, _ in all_sources:+ for field in ("surfaces", "deferred"):+ rows = draft["coverage"].get(field)+ for row in rows if isinstance(rows, list) else []:+ if isinstance(row, dict):+ paths_for_row = coverage_sources.setdefault(id(row), (row, []))[1]+ if relative not in paths_for_row:+ paths_for_row.append(relative)+ if parent:+ for field in ("surfaces", "deferred"):+ rows, originals = coverage.get(field), parent["coverage"].get(field)+ if isinstance(rows, list) and isinstance(originals, list):+ for row, original in zip(rows, originals):+ _inherit_coverage_sources(row, [original], coverage_sources)+ if field == "surfaces":+ retained_surface_rows.update(id(original) for original in originals)+findings: list[dict[str, Any]] = []finding_positions: dict[str, int] = {}+ canonical_finding_keys: set[str] = set()represented: dict[str, str | None] = {}represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {}represented_history: dict[str, set[str]] = {}@@ -1598,6 +1227,20 @@ def merge_saved_results(relative: _coverage_rows(draft["coverage"], "deferred")for relative, draft, _ in all_sources}+ # Accepted heads can replace equal tasks before they reach output coalescing.+ equivalent_tasks: dict[str | None, list[dict[str, Any]]] = {}+ for relative, _, owner in all_sources:+ known = equivalent_tasks.setdefault(owner, [])+ for row in deferred_rows[relative]:+ if not isinstance(row, dict):+ continue+ for previous in known:+ if previous == row:+ _inherit_coverage_sources(previous, [row], coverage_sources)+ coverage_sources[id(row)] = (row, coverage_sources[id(previous)][1])+ break+ else:+ known.append(row)# A legacy source can contain independent tasks with the same explicit ID.# Later rewrites cannot make an ID-only closure identify one of those tasks.ambiguous_deferred: set[tuple[str | None, str]] = set()@@ -1808,23 +1451,34 @@ def merge_saved_results(worker_id is not Noneand relative in selected_observationsand draft.get("complete") is not False+ and relative not in discarded_by_source+ and relative not in discarded_coverage_sources):order = source_order[relative]selected_terminal_orders[worker_id] = max(selected_terminal_orders.get(worker_id, order), order)terminal_worker_orders: dict[str | None, tuple[int, int]] = {}+ terminal_coverage_orders: dict[str | None, tuple[int, int]] = {}for relative, draft, worker_id in sources:- if relative in current_results and draft.get("complete") is not False:+ if (+ relative in current_results+ and draft.get("complete") is not False+ and relative not in discarded_by_source+ ):order = source_order[relative]terminal_worker_orders[worker_id] = max(terminal_worker_orders.get(worker_id, order), order)+ if relative not in discarded_coverage_sources:+ terminal_coverage_orders[worker_id] = max(+ terminal_coverage_orders.get(worker_id, order), order+ )def source_coverage_selection(relative: str,worker_id: str | None,- ) -> tuple[bool, set[str], bool, bool]:+ ) -> tuple[bool, set[str], bool, bool, bool]:worker_result_order = terminal_worker_orders.get(worker_id)selected_coverage_superseded = worker_id in selected_terminal_orders and (source_order[relative] < selected_terminal_orders[worker_id]@@ -1853,28 +1507,53 @@ def merge_saved_results(or source_order[relative] < worker_result_order))+ worker_coverage_order = terminal_coverage_orders.get(worker_id)+ coverage_superseded = (+ superseded+ if worker_id is None+ else (+ relative not in current_results+ and worker_coverage_order is not None+ and (+ relative not in selected_observations+ or source_order[relative] < worker_coverage_order+ )+ )+ )# A failed result write can leave pending work outside the accepted result.accepted_order = ((0, parent_modified)if worker_id is None and parent is not None- else worker_result_order+ else worker_coverage_order)retain_pending = (- superseded+ coverage_supersededand (worker_id in headed_workers or (worker_id is None and parent_heads))and accepted_order is not Noneand source_order[relative] >= accepted_orderand any(isinstance(row, dict) for row in deferred_rows[relative]))- return selected_coverage_superseded, selected_candidates, superseded, retain_pending+ return (+ selected_coverage_superseded,+ selected_candidates,+ superseded,+ retain_pending,+ coverage_superseded,+ )def coverage_row_is_retained(field: str,item: Any,worker_id: str | None,- selection: tuple[bool, set[str], bool, bool],+ selection: tuple[bool, set[str], bool, bool, bool],) -> bool:- selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection+ (+ selected_coverage_superseded,+ selected_candidates,+ superseded,+ retain_pending,+ coverage_superseded,+ ) = selection# A selected terminal checkpoint replaces ordinary coverage, while# candidate evidence and pending tasks retain their own reconciliation.if (@@ -1884,7 +1563,7 @@ def merge_saved_results():return False# A selected outcome must retain its evidence even if its result write failed.- if superseded and not (+ if coverage_superseded and not (isinstance(item, dict)and ((field == "deferred" and retain_pending)@@ -2016,6 +1695,8 @@ def merge_saved_results(coverage_schema["surfaces"]["items"],deferred_rows,ambiguous_deferred,+ canonical_rows,+ coverage_sources,receipt_root=scan_dir if project_worker_coverage else None,receipt_warnings=warnings,fallback_surfaces=fallback_surfaces,@@ -2065,21 +1746,34 @@ def merge_saved_results(and ambiguous_pending_surface(surface, owner)and isinstance(coverage.get("surfaces"), list)):- append_coverage_row(+ retained = append_coverage_row(coverage["surfaces"],{**surface, "receiptRefs": surface.get("receiptRefs", [])},owner,"surfaces",)+ _inherit_coverage_sources(retained, [surface], coverage_sources)+ retained_surface_rows.add(id(surface))for relative, draft, worker_id in all_sources:selection = source_coverage_selection(relative, worker_id)- selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection+ (+ selected_coverage_superseded,+ selected_candidates,+ superseded,+ retain_pending,+ coverage_superseded,+ ) = selectionskip_superseded_findings = (supersededand not stoppedand all(valid_finding(finding) for finding in (parent["findings"] if parent else [])))- if skip_superseded_findings and not selected_candidates and not retain_pending:+ if (+ skip_superseded_findings+ and coverage_superseded+ and not selected_candidates+ and not retain_pending+ ):continueif (not skip_superseded_findings@@ -2139,6 +1833,8 @@ def merge_saved_results(continueif valid_finding(finding):finding_positions.setdefault(_finding_key(finding), len(findings))+ if preserve_canonical_parent:+ canonical_finding_keys.add(_finding_key(finding))findings.append(finding)continueif relative != "parent" and parent and value in parent["findings"]:@@ -2152,8 +1848,14 @@ def merge_saved_results("rejected","not_applicable",}:+ if preserve_canonical_parent:+ rejected_history.setdefault((worker_id, candidate_id), []).append(finding)for item in _coverage_rows(coverage, "surfaces"):- if isinstance(item, dict) and item.get("candidateId") == candidate_id:+ if (+ isinstance(item, dict)+ and item.get("candidateId") == candidate_id+ and (not preserve_canonical_parent or item.get("workerId") == worker_id)+ ):_append_previous_finding(item, finding)continuefor key in ("findingId", "occurrenceId", "fingerprints"):@@ -2201,8 +1903,10 @@ def merge_saved_results(if key in finding_positions:retained = findings[finding_positions[key]]if finding != retained:- if not represented_by_parent and _finding_strength(finding) > _finding_strength(- retained+ if (+ key not in canonical_finding_keys+ and not represented_by_parent+ and _finding_strength(finding) > _finding_strength(retained)):previous = copy.deepcopy(retained)previous_history = previous["provenance"].pop("previousFindings", [])@@ -2243,10 +1947,10 @@ def merge_saved_results(continuefinding_positions[key] = len(findings)findings.append(finding)- if superseded and not selected_candidates and not retain_pending:+ if coverage_superseded and not selected_candidates and not retain_pending:continuefor field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):- if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:+ if coverage_superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:continueitems = draft["coverage"].get(field, [])if field == "deferred":@@ -2259,7 +1963,7 @@ def merge_saved_results(# recovery and warnings rather than silently changing its contract.continuefor item in items:- source_item = item+ source_item = original_item = itemif not coverage_row_is_retained(field, item, worker_id, selection):continueif id(item) in replaced_rows.get(field, ()):@@ -2302,8 +2006,14 @@ def merge_saved_results(worker_row_owners[id(existing)] = worker_idif field == "surfaces" and id(source_item) in linked_surface_sources:linked_surface_outputs[id(source_item)] = existing+ _inherit_coverage_sources(existing, [original_item], coverage_sources)+ if field == "surfaces":+ retained_surface_rows.add(id(original_item))continueretained = append_coverage_row(output, item, worker_id, field)+ _inherit_coverage_sources(retained, [original_item], coverage_sources)+ if field == "surfaces":+ retained_surface_rows.add(id(original_item))if field == "surfaces" and id(source_item) in linked_surface_sources:linked_surface_outputs[id(source_item)] = retained@@ -2320,6 +2030,34 @@ def merge_saved_results(finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity)identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}"identities[key] = variant+ recovered_worker_ids = {worker_id for _, _, worker_id in sources if worker_id is not None}+ surface_positions: dict[str, int] = {}+ resolved_surface_targets: dict[tuple[str | None, str], list[str]] = {}++ def remember_surface_id(item: dict[str, Any], local_id: str) -> None:+ final_id = item["id"]+ surface_positions.setdefault(final_id, len(surface_positions))+ shadowed_alias = (+ project_worker_coverage+ and id(item) not in named_surface_outputs+ and (named := worker_surfaces.get((worker_row_owners.get(id(item)), local_id)))+ is not None+ and id(named) in named_surface_outputs+ )+ for relative in coverage_sources.get(id(item), (item, []))[1]:+ # A candidate-derived ID cannot claim a link explicitly naming another surface.+ if not shadowed_alias:+ targets = surface_ids_by_source.setdefault((relative, local_id), [])+ if final_id not in targets:+ targets.append(final_id)+ candidate_id = item.get("candidateId")+ if isinstance(candidate_id, str):+ key = (source_owners[relative], candidate_id)+ if key in resolved and resolved[key] == item.get("disposition"):+ targets = resolved_surface_targets.setdefault(key, [])+ if final_id not in targets:+ targets.append(final_id)+# Bind links to retained output objects before their IDs become unique.# Candidate metadata can give an anonymous row the same preliminary ID as# another surface, so only the final surface ID can be used for its links.@@ -2387,17 +2125,73 @@ def merge_saved_results(reserved = {row["id"] for row in items if isinstance(row, dict) and isinstance(row.get("id"), str)}+ normalized_items = []for item in items:+ normalized_items.append(item)if not isinstance(item, dict):continueif id(item) in canonical_rows:if isinstance(item.get("id"), str):used.add(item["id"])+ if field == "surfaces":+ remember_surface_id(item, item["id"])continueitem.setdefault("id", _saved_coverage_id(item))if not isinstance(item["id"], str):# Preserve malformed rows for per-record recovery, including frozen replay.continue+ local_id = item["id"]+ scoped = (+ preserve_canonical_parent+ and isinstance(worker_id := item.get("workerId"), str)+ and worker_id in recovered_worker_ids+ )+ if scoped:+ # Resolve worker-local candidate/deferred history before scoping IDs.+ item["id"] = f"{worker_id}/{item['id']}"+ if isinstance(item.get("surfaceIds"), list):+ links: list[Any] = []+ for surface_id in item["surfaceIds"]:+ targets: list[str] = []+ declared = False+ if field == "deferred" and isinstance(surface_id, str):+ for relative in coverage_sources.get(id(item), (item, []))[1]:+ source_key = (relative, surface_id)+ history_keys = [source_key]+ if source_key not in surface_ids_by_source:+ # A deferred-only checkpoint can refer to an earlier+ # observation from this logical worker, including retries.+ # Reused IDs can denote several retained observations.+ history_keys = [+ (historical, surface_id)+ for historical, _, owner in all_sources+ if owner == source_owners[relative]+ and source_order[historical] <= source_order[relative]+ ]+ for key in history_keys:+ declared |= key in surface_ids_by_source+ for target in surface_ids_by_source.get(key, []):+ if target not in targets:+ targets.append(target)+ if targets:+ targets.sort(key=surface_positions.__getitem__)+ for target in targets:+ if target not in links:+ links.append(target)+ elif not declared:+ links.append(+ f"{worker_id}/{surface_id}"+ if scoped and isinstance(surface_id, str)+ else surface_id+ )+ item["surfaceIds"] = links+ if scoped:+ if item in normalized_items[:-1]:+ # A previous finish may have published files before its DB commit.+ if field == "surfaces":+ remember_surface_id(item, local_id)+ normalized_items.pop()+ continueif item["id"] in used:if project_worker_coverage:base = item["id"]@@ -2407,9 +2201,36 @@ def merge_saved_results(suffix += 1else:item["id"] = f"{item['id']}-{_digest(item)[:16]}"+ if preserve_canonical_parent and item in normalized_items[:-1]:+ if field == "surfaces":+ remember_surface_id(item, local_id)+ normalized_items.pop()+ continueused.add(item["id"])if field == "surfaces":item.setdefault("receiptRefs", [])+ remember_surface_id(item, local_id)+ if preserve_canonical_parent and isinstance(items, list):+ coverage[field] = normalized_items+ if field == "surfaces":+ # Candidate resolution replaces pending surfaces independently of generic+ # tasks. Preserve their original links, even when the outcome changes ID+ # or a distinct retained surface shares the old source-local ID.+ for relative, draft, owner in all_sources:+ originals = draft["coverage"].get("surfaces", [])+ for original in originals if isinstance(originals, list) else []:+ if (+ not isinstance(original, dict)+ or id(original) in retained_surface_rows+ or original.get("disposition") != "needs_follow_up"+ or not isinstance(local_id := original.get("id"), str)+ or not isinstance(candidate_id := original.get("candidateId"), str)+ ):+ continue+ targets = surface_ids_by_source.setdefault((relative, local_id), [])+ for target in resolved_surface_targets.get((owner, candidate_id), []):+ if target not in targets:+ targets.append(target)for task, links in retained_surface_links.values():task["surfaceIds"] = [links[identity]["id"] if isinstance(identity, str) and identity in links else identity@@ -2440,27 +2261,50 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:def preserve_deep_deadline_draft(- db: Any, connection: Any, scan: Any, *, allow_empty: bool = False+ db: Any,+ connection: Any,+ scan: Any,+ *,+ preserve_reduction: bool = False,+ allow_empty: bool = False,) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None:- """Recover a settled, zero-accepted Deep deadline without sealing the parent."""+ """Recover unfinished reviews at a settled Deep deadline without sealing the parent."""scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))+ warnings = json.loads(scan["completion_warnings_json"])+ binding = db.workbench_completion_binding(scan, db.now())+ workers = connection.execute(+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",+ (scan["id"],),+ ).fetchall()+ if preserve_reduction:+ # Completed reviews already went through reduction. Only add evidence+ # from unfinished reviews; raw accepted drafts must not undo that result.+ workers = [+ worker+ for worker in workers+ if worker["kind"] == "discovery" and worker["status"] == "canceled"+ ]+ if not workers:+ return None+ _prepare_scan_finalization(+ scan_dir,+ expected_coverage_mode=db.expected_coverage_mode(scan),+ completion_binding=binding,+ allow_missing_writeups=True,+ )snapshots = _snapshot_published_outputs(scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json"))- warnings = json.loads(scan["completion_warnings_json"])- binding = db.workbench_completion_binding(scan, db.now())created_paths: set[str] = set()try:documents = merge_saved_results(scan_dir,scan["id"],binding,- connection.execute(- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",- (scan["id"],),- ).fetchall(),+ workers,warnings,- stopped=True,+ stopped=not preserve_reduction,+ preserve_canonical_parent=preserve_reduction,project_worker_coverage=True,created_paths=created_paths,current_contract=True,@@ -2478,28 +2322,46 @@ def preserve_deep_deadline_draft((json.dumps(list(dict.fromkeys(warnings))), scan["id"]),)return None+ if preserve_reduction:+ marker = {+ "id": "scan-deadline",+ "reason": (+ "The configured discovery time limit elapsed. Saved findings and "+ "pending work from unfinished source reviews were preserved."+ ),+ }+ if not any(+ marker["id"] in (row.get("id"), row.get("candidateId"))+ for key in ("deferred", "resolvedDeferred")+ for row in documents[2].get(key, [])+ ):+ documents[2]["deferred"].append(marker)documents[0]["scan"].pop("complete", None)- # A capped deadline can retain interrupted records even though its- # completion binding requires strict validation of completed findings.expected_coverage_mode = db.expected_coverage_mode(scan)- _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)- _populate_unsealed_artifact_envelope(*documents, binding)- if expected_coverage_mode == "deep_repository":- documents[2]["inventoryStrategy"] = "repository"- elif expected_coverage_mode == "scoped_path":- documents[2]["inventoryStrategy"] = "scoped_path"- schema_dir = Path(__file__).resolve().parent.parent / "schemas"- discarded_findings = _recover_unsealed_findings(- documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True- )- _recover_unsealed_coverage(documents[2], schema_dir, scan_dir, warnings, discarded_findings)+ if not preserve_reduction:+ # A capped deadline can retain interrupted records even though its+ # completion binding requires strict validation of completed findings.+ _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)+ _populate_unsealed_artifact_envelope(*documents, binding)+ if expected_coverage_mode == "deep_repository":+ documents[2]["inventoryStrategy"] = "repository"+ elif expected_coverage_mode == "scoped_path":+ documents[2]["inventoryStrategy"] = "scoped_path"+ schema_dir = Path(__file__).resolve().parent.parent / "schemas"+ discarded_findings = _recover_unsealed_findings(+ documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True+ )+ _recover_unsealed_coverage(+ documents[2], schema_dir, scan_dir, warnings, discarded_findings+ )# Reuse stopped-result recovery's per-record validation, scope checks,# source lineage, and receipt validation before admitting this draft.prepared = _prepare_scan_finalization(scan_dir,expected_coverage_mode=expected_coverage_mode,completion_binding=binding,- recover_unsealed=True,+ recover_unsealed=not preserve_reduction,+ allow_missing_writeups=preserve_reduction,draft_documents=documents,finding_scope=db.requested_finding_scope(scan),)diff --git a/plugins/codex-security/tests/test_deep_coverage_recovery.py b/plugins/codex-security/tests/test_deep_coverage_recovery.pyindex 74a91492c..1eb8982bf 100644--- a/plugins/codex-security/tests/test_deep_coverage_recovery.py+++ b/plugins/codex-security/tests/test_deep_coverage_recovery.py@@ -1033,6 +1033,8 @@ def test_current_receipt_spelling_precedes_historical_receipts(schema,{"old": [task], "current": []},set(),+ canonical_rows=set(),+ coverage_sources={},surface_is_retained=lambda _relative, _owner, _row: True,eligible_receipt_sources={"old", "current"},receipt_root=tmp_path,diff --git a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.pyindex 7f494f9b4..08b897f1f 100644--- a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py+++ b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py@@ -431,6 +431,8 @@ def test_deadline_interrupt_preserves_only_committed_publication(def test_mid_merge_failure_removes_only_created_recovery_evidence(tmp_path: Path, monkeypatch, workbench_api, write_kind: str):+ from workbench import saved_sources+scan = deadline_fixture(tmp_path, "valid")canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))before = {@@ -458,6 +460,7 @@ def test_mid_merge_failure_removes_only_created_recovery_evidence(raise RuntimeError("Synthetic merge failure")monkeypatch.setattr(saved, "write_scan_local_bytes", fail_during_merge)+ monkeypatch.setattr(saved_sources, "write_scan_local_bytes", fail_during_merge)with sqlite3.connect(scan.database) as connection:connection.row_factory = sqlite3.Rowrecord = workbench_api["require_scan"](connection, scan.scan_id)@@ -476,6 +479,8 @@ def test_mid_merge_failure_removes_only_created_recovery_evidence(def test_failed_head_snapshot_does_not_interrupt_publication_rollback(tmp_path, monkeypatch, workbench_api, snapshot_failure):+ from workbench import saved_sources+scan = deadline_fixture(tmp_path, "valid")canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))before = {@@ -495,6 +500,7 @@ def test_failed_head_snapshot_does_not_interrupt_publication_rollback(write(directory, relative, contents)monkeypatch.setattr(saved, "write_scan_local_bytes", fail_before_snapshot_directory)+ monkeypatch.setattr(saved_sources, "write_scan_local_bytes", fail_before_snapshot_directory)with sqlite3.connect(scan.database) as connection:connection.row_factory = sqlite3.Rowconnection.execute(diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.pynew file mode 100644index 000000000..789fd62a5--- /dev/null+++ b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py@@ -0,0 +1,598 @@+from __future__ import annotations++import copy+import json+import os++import pytest+from test_deep_scan_deadline_recovery import worker_finding+from test_deep_scan_stop_conditions import canceled_discovery_output, complete, finish+from test_deep_scan_stop_conditions import deadline_scan as deadline_scan+from test_deep_scan_stop_conditions import publication_scan as publication_scan+from test_deep_scan_stop_conditions import saturated_scan as saturated_scan+from workbench_test_support import saved_draft, write_checkpoint++[email protected]("source", ["result", "checkpoint", "archived-checkpoint"])+ "marker",+ [+ pytest.param("false", id="string-false"),+ pytest.param(None, id="null"),+ pytest.param(0, id="zero"),+ pytest.param(1, id="one"),+ pytest.param(False, id="incomplete"),+ pytest.param(True, id="complete"),+ pytest.param("omitted", id="omitted"),+ ],+)+def test_deadline_requires_boolean_completion_before_superseding_worker_history(+ workbench_api, workbench_db, deadline_scan, source, marker+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan, attempt=2)+ if source == "archived-checkpoint":+ output = output.parent / "attempts" / "attempt-1"+ earlier = worker_finding(scan, "earlier-worker-finding")+ rejected = worker_finding(scan, "rejected-worker-finding")+ rejected["provenance"]["candidateId"] = "rejected-worker-finding"+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[earlier, rejected],+ deferred=[{"id": "review", "reason": "Review remains."}],+ ),+ )+ os.utime(checkpoint, ns=(100, 100))+ sibling = worker_finding(scan, "latest-valid-sibling")+ latest = saved_draft(+ scan.scan_id,+ complete=marker,+ findings=[sibling],+ surfaces=[+ {+ "id": "rejected-outcome",+ "candidateId": "rejected-worker-finding",+ "label": "Rejected after review",+ "disposition": "rejected",+ "receiptRefs": [],+ }+ ],+ closures=[{"id": "review", "reason": "Review completed."}],+ )+ if marker == "omitted":+ latest.pop("complete")+ if source == "result":+ result = output / "result.json"+ result.write_text(json.dumps(latest))+ else:+ result = write_checkpoint(output / "checkpoints", latest)+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": result.name}))+ os.utime(head, ns=(300, 300))+ os.utime(result, ns=(200, 200))+ original = {path: path.read_bytes() for path in (checkpoint, result)}++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ terminal = marker is True or marker == "omitted"+ invalid = not isinstance(marker, bool) and marker != "omitted"+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ titles = {finding["title"] for finding in findings}+ assert sibling["title"] in titles+ assert rejected["title"] not in titles+ if not terminal or source == "result":+ assert (earlier["title"] in titles) == (not terminal)+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) == (not terminal)+ assert (+ any(+ "Invalid complete marker" in warning and result.name in warning+ for warning in completed["warnings"]+ )+ == invalid+ )+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ for path, contents in original.items():+ assert path.read_bytes() == contents+ assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == (+ workbench_api["saved_results"]._digest(json.loads(contents))+ )++[email protected]("source", ["result", "checkpoint", "archived-result"])[email protected]("invalid", ["rationale", "code_evidence", "sourceFindingIds"])+def test_deadline_validates_each_recovered_finding_against_the_completed_contract(+ workbench_api, workbench_db, deadline_scan, source, invalid+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)+ if source == "archived-result":+ output = output.parent / "attempts" / "attempt-1"+ earlier = worker_finding(scan, "earlier-worker-finding")+ rejected = worker_finding(scan, "rejected-worker-finding")+ rejected["provenance"]["candidateId"] = "rejected-worker-finding"+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[earlier, rejected],+ deferred=[+ {"id": "review", "reason": "Review remains."},+ {"id": "finished", "reason": "Separate review remains."},+ ],+ ),+ )+ os.utime(checkpoint, ns=(100, 100))+ sibling = worker_finding(scan, "valid-worker-sibling")+ malformed = worker_finding(scan, "malformed-worker-finding")+ malformed["provenance"]["candidateId"] = "malformed-worker-finding"+ if invalid == "rationale":+ malformed["severity"]["rationale"] = " \t "+ elif invalid == "code_evidence":+ malformed["code_evidence"] = []+ else:+ malformed["provenance"]["sourceFindingIds"] = ["source-finding", "source-finding"]+ latest = saved_draft(+ scan.scan_id,+ complete=True,+ findings=[malformed, sibling],+ surfaces=[+ {+ "id": "reported-outcome",+ "candidateId": "malformed-worker-finding",+ "label": "Unverified finding",+ "disposition": "reported",+ "receiptRefs": [],+ },+ {+ "id": "rejected-outcome",+ "candidateId": "rejected-worker-finding",+ "label": "Rejected after review",+ "disposition": "rejected",+ "receiptRefs": [],+ },+ ],+ closures=[{"id": "finished", "reason": "Separate review completed."}],+ )+ if source == "checkpoint":+ result = write_checkpoint(output / "checkpoints", latest)+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": result.name}))+ os.utime(head, ns=(300, 300))+ else:+ result = output / "result.json"+ result.write_text(json.dumps(latest))+ os.utime(result, ns=(200, 200))+ original = {path: path.read_bytes() for path in (checkpoint, result)}+ if source == "archived-result":+ selected = write_checkpoint(output / "checkpoints", latest)+ os.utime(selected, ns=(200, 200))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": selected.name}))+ os.utime(head, ns=(300, 300))+ original[selected] = selected.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert {finding["title"] for finding in findings} == {+ scan.findings[0]["title"],+ earlier["title"],+ sibling["title"],+ }+ retained = next(finding for finding in findings if finding["title"] == sibling["title"])+ assert retained["provenance"]["workerId"] == worker_id+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert any(row["reason"] == "Review remains." for row in coverage["deferred"])+ assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"])+ surfaces = {row["label"]: row for row in coverage["surfaces"]}+ assert surfaces["Unverified finding"]["disposition"] == "needs_follow_up"+ assert surfaces["Rejected after review"]["disposition"] == "rejected"+ assert any(invalid in warning and result.name in warning for warning in completed["warnings"])+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ for path, contents in original.items():+ assert path.read_bytes() == contents+ assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == (+ workbench_api["saved_results"]._digest(json.loads(contents))+ )+++ "model",+ [+ {"assets": []},+ {"summary": "Invalid " + chr(0xD800)},+ {"summary": "Reviewed boundaries."},+ {"format": "markdown", "content": "# Reviewed boundaries"},+ ],+)+def test_deadline_recovers_optional_worker_model_without_losing_findings(+ workbench_api, workbench_db, deadline_scan, model+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ finding = worker_finding(scan, "worker-model-finding")+ draft = {**saved_draft(scan.scan_id, findings=[finding]), "threatModel": model}+ checkpoint = write_checkpoint(output / "checkpoints", draft)+ original = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ invalid = "assets" in model or chr(0xD800) in model.get("summary", "")+ if invalid:+ assert "threatModel" not in manifest+ assert any(+ "threat model" in warning and checkpoint.name in warning+ for warning in completed["warnings"]+ )+ else:+ assert manifest["threatModel"] == {**model, "origin": "recovered"}+ assert checkpoint.read_bytes() == original+++def test_deadline_still_rejects_malformed_canonical_model(+ workbench_api, workbench_db, deadline_scan+):+ from finalize_scan_contract import ContractError++ scan = deadline_scan+ manifest_file = scan.scan_dir / "scan-manifest.json"+ manifest = json.loads(manifest_file.read_text())+ manifest["scan"]["threatModel"] = {"assets": []}+ manifest_file.write_text(json.dumps(manifest))+ _, output = canceled_discovery_output(workbench_db, scan)+ write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id))+ with pytest.raises(ContractError, match="threatModel"):+ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+++def test_deadline_still_rejects_canonical_findings_outside_the_completed_contract(+ workbench_api, workbench_db, deadline_scan+):+ from finalize_scan_contract import ContractError++ scan = deadline_scan+ finding = worker_finding(scan, "valid-worker-finding")+ scan.findings[0]["severity"]["rationale"] = " \t "+ findings_path = scan.scan_dir / "findings.json"+ findings_path.write_text(json.dumps({"findings": scan.findings}))+ original = findings_path.read_bytes()+ _, output = canceled_discovery_output(workbench_db, scan)+ write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, findings=[finding]),+ )++ with pytest.raises(ContractError, match="rationale"):+ finish(workbench_api, workbench_db, scan, terminal_reason="capped")++ assert findings_path.read_bytes() == original++[email protected]("label", [" \t ", " Reviewed Résumé 😀 "])+def test_deadline_applies_existing_nonblank_surface_label_contract(+ workbench_api, workbench_db, deadline_scan, label+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[worker_finding(scan, "worker-label-finding")],+ surfaces=[+ {"id": "entry", "label": label, "disposition": "reported", "receiptRefs": []}+ ],+ ),+ )+ original = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert len(surfaces) == bool(label.strip())+ if surfaces:+ assert surfaces[0]["label"] == label+ else:+ assert any(+ "label: expected a non-empty string" in warning and checkpoint.name in warning+ for warning in completed["warnings"]+ )+ assert checkpoint.read_bytes() == original+++ "field",+ ["finding", "surfaces", "explicitExclusions", "deferred", "openQuestions", "resolvedDeferred"],+)[email protected]("malformed", [False, True], ids=["valid-unicode", "lone-surrogate"])+def test_deadline_applies_existing_unicode_contract_per_worker_record(+ workbench_api, workbench_db, deadline_scan, field, malformed+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ text = "Review " + (chr(0xD800) if malformed else "Résumé 😀")+ finding = worker_finding(scan, "unicode-worker-finding")+ draft = saved_draft(scan.scan_id, findings=[finding])+ if field == "finding":+ finding["summary"] = text+ elif field == "surfaces":+ draft["coverage"][field] = [+ {"id": "entry", "label": text, "disposition": "reported", "receiptRefs": []}+ ]+ elif field == "openQuestions":+ draft["coverage"][field] = [text]+ elif field == "explicitExclusions":+ draft["coverage"][field] = [{"pattern": "vendor/", "reason": text}]+ else:+ draft["coverage"][field] = [{"id": "review", "reason": text}]+ if field == "resolvedDeferred":+ draft["complete"] = True+ previous = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]),+ )+ os.utime(previous, ns=(100, 100))+ checkpoint = write_checkpoint(output / "checkpoints", draft)+ os.utime(checkpoint, ns=(200, 200))+ if field == "resolvedDeferred":+ (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name}))+ original = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert len(findings) == (1 if malformed and field == "finding" else 2)+ assert any("well-formed Unicode" in warning for warning in completed["warnings"]) == malformed+ if field == "resolvedDeferred":+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert any(row["reason"] == "Review pending." for row in coverage["deferred"]) == malformed+ elif not malformed:+ document = (+ findings+ if field == "finding"+ else json.loads((scan.scan_dir / "coverage.json").read_text())+ )+ assert text in json.dumps(document, ensure_ascii=False)+ assert checkpoint.read_bytes() == original+++ "reason",+ [pytest.param("\u0085", id="historical-nel"), pytest.param("Review completed.", id="valid")],+)+def test_deadline_recovers_historical_closure_text_without_losing_a_valid_sibling(+ workbench_api, workbench_db, deadline_scan, reason+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ previous = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ deferred=[+ {"id": "review", "reason": "First review pending."},+ {"id": "sibling", "reason": "Valid sibling review pending."},+ ],+ ),+ )+ os.utime(previous, ns=(100, 100))+ draft = saved_draft(+ scan.scan_id,+ complete=True,+ closures=[+ {"id": "review", "reason": reason},+ {"id": "sibling", "reason": "Valid sibling review completed."},+ ],+ )+ checkpoint = write_checkpoint(output / "checkpoints", draft)+ result = output / "result.json"+ result.write_text(json.dumps(draft))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))+ for path in (checkpoint, result, head):+ os.utime(path, ns=(200, 200))+ original = {path: path.read_bytes() for path in (previous, checkpoint, result)}++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ invalid = not reason.strip()+ assert completed["progress"]["status"] == "complete"+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ pending = {row["reason"] for row in coverage["deferred"]}+ assert ("First review pending." in pending) == invalid+ assert "Valid sibling review pending." not in pending+ report = (scan.scan_dir / "report.md").read_text()+ assert ("First review pending." in report) == invalid+ assert "Valid sibling review pending." not in report+ assert any("expected a non-empty string" in warning for warning in completed["warnings"]) == (+ invalid+ )+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ assert manifest["sealedAt"]+ for path, contents in original.items():+ assert path.read_bytes() == contents+ assert manifest["preservedSources"][path.relative_to(scan.scan_dir).as_posix()] == (+ workbench_api["saved_results"]._digest(json.loads(contents))+ )+++ ("terminal_findings", "malformed", "selected_head"),+ [+ (False, None, False),+ (True, None, False),+ (False, "surface", False),+ (True, "surface", False),+ (False, "closure", False),+ (True, "closure", False),+ (True, "surface", True),+ (False, "missing-receipt", False),+ (True, "missing-receipt", False),+ (False, "disposition", False),+ (True, "disposition", False),+ (False, "receipt-array", False),+ (True, "receipt-array", False),+ ],+ ids=[+ "valid-coverage-empty-findings",+ "valid-coverage-retained-finding",+ "malformed-surface-empty-findings",+ "malformed-surface-retained-finding",+ "malformed-closure-empty-findings",+ "malformed-closure-retained-finding",+ "malformed-selected-head",+ "missing-receipt-empty-findings",+ "missing-receipt-retained-finding",+ "disposition-empty-findings",+ "disposition-retained-finding",+ "receipt-array-empty-findings",+ "receipt-array-retained-finding",+ ],+)+def test_malformed_terminal_coverage_preserves_only_earlier_coverage(+ workbench_api, workbench_db, deadline_scan, terminal_findings, malformed, selected_head+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ finding = worker_finding(scan, "earlier-worker-finding")+ rejected = worker_finding(scan, "rejected-sibling")+ rejected["provenance"]["candidateId"] = "rejected-sibling"+ surface = {+ "id": "entry",+ "label": "Unfinished review",+ "disposition": "needs_follow_up",+ "receiptRefs": [],+ }+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[finding, rejected],+ surfaces=[surface],+ deferred=[+ {"id": "review", "reason": "Review remains.", "surfaceIds": ["entry"]},+ {"id": "finished", "reason": "Separate review remains."},+ ],+ ),+ )+ os.utime(checkpoint, ns=(100, 100))+ invalid_surface = copy.deepcopy(surface)+ if malformed == "surface":+ invalid_surface.pop("label")+ else:+ invalid_surface.update(id="latest", label="Latest review", disposition="no_issue_found")+ if malformed == "missing-receipt":+ invalid_surface["receiptRefs"] = ["artifacts/missing.txt"]+ elif malformed == "disposition":+ invalid_surface["disposition"] = "unknown"+ elif malformed == "receipt-array":+ invalid_surface["receiptRefs"] = "invalid"+ closures = [{"id": "finished", "reason": "Separate review completed."}]+ if malformed == "closure":+ closures.append({"id": "review", "reason": "Invalid " + chr(0xD800)})+ terminal = saved_draft(+ scan.scan_id,+ complete=True,+ findings=[finding] if terminal_findings else [],+ surfaces=[+ {+ "id": "sibling-outcome",+ "candidateId": "rejected-sibling",+ "label": "Sibling reviewed",+ "disposition": "rejected",+ "receiptRefs": [],+ },+ *([invalid_surface] if malformed not in (None, "closure") else []),+ ],+ closures=closures,+ )+ if selected_head:+ result = write_checkpoint(output / "checkpoints", terminal)+ (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": result.name}))+ else:+ result = output / "result.json"+ result.write_text(json.dumps(terminal))+ os.utime(result, ns=(200, 200))+ original = {path: path.read_bytes() for path in (checkpoint, result)}++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert any(row["title"] == finding["title"] for row in findings) == terminal_findings+ assert all(row["title"] != rejected["title"] for row in findings)+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert any(+ row.get("workerId") == worker_id and row["label"] == surface["label"]+ for row in coverage["surfaces"]+ ) == bool(malformed)+ assert any(row["reason"] == "Review remains." for row in coverage["deferred"]) == bool(+ malformed+ )+ assert not any(row["reason"] == "Separate review remains." for row in coverage["deferred"])+ assert (+ next(row for row in coverage["surfaces"] if row["label"] == "Sibling reviewed")[+ "disposition"+ ]+ == "rejected"+ )+ if malformed in {"missing-receipt", "disposition", "receipt-array"}:+ latest = next(row for row in coverage["surfaces"] if row["label"] == "Latest review")+ assert latest["disposition"] == "needs_follow_up"+ assert latest["receiptRefs"] == []+ assert {path: path.read_bytes() for path in original} == original++[email protected]("canceled_worker", [False, True])+def test_optional_export_failure_does_not_block_deadline(+ workbench_api, workbench_db, deadline_scan, tmp_path, capsys, canceled_worker+):+ scan = deadline_scan+ sentinel = tmp_path / "untouched.sarif"+ sentinel.write_text("Preserve this file.\n")+ export = scan.scan_dir / "exports" / "results.sarif"+ export.parent.mkdir()+ export.symlink_to(sentinel)+ if canceled_worker:+ _, output = canceled_discovery_output(workbench_db, scan)+ write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, findings=[worker_finding(scan, "saved-worker-finding")]),+ )++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ assert completed["progress"]["status"] == "complete"+ assert "automatic SARIF export failed" in capsys.readouterr().err+ assert export.is_symlink()+ assert sentinel.read_text() == "Preserve this file.\n"+++def test_standard_parent_closure_preserves_unrelated_task_without_id(+ workbench_api, workbench_db, publication_scan+):+ scan = publication_scan(mode="standard")+ scan.coverage["resolvedDeferred"] = [{"id": "finished", "reason": "Review completed."}]+ scan.coverage["deferred"] = [{"reason": "Unrelated pending review."}]+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))++ complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert len(coverage["deferred"]) == 1+ assert coverage["deferred"][0]["id"]+ assert coverage["deferred"][0]["reason"] == "Unrelated pending review."diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.pynew file mode 100644index 000000000..d86b711de--- /dev/null+++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py@@ -0,0 +1,512 @@+from __future__ import annotations++import copy+import hashlib+import json+import os+import sqlite3+import traceback++import pytest+from test_deep_scan_stop_conditions import (+ canceled_discovery_output,+ complete,+ finish,+)+from test_deep_scan_stop_conditions import (+ deadline_scan as deadline_scan,+)+from test_deep_scan_stop_conditions import (+ publication_scan as publication_scan,+)+from test_deep_scan_stop_conditions import (+ saturated_scan as saturated_scan,+)+from workbench_test_support import saved_draft, write_checkpoint+++def worker_finding(scan, anchor):+ finding = copy.deepcopy(scan.findings[0])+ finding["identity"] = {"anchor": anchor}+ finding["title"] = anchor+ finding["provenance"] = {"source": "local_plugin"}+ return finding+++ ("task_state", "task_id", "marker_count"),+ [+ ("resolvedDeferred", "scan-deadline", 0),+ ("resolvedDeferred", "prior-review", 1),+ ("deferred", "scan-deadline", 0),+ ],+)+def test_deadline_completion_preserves_existing_tasks_and_recovered_findings(+ workbench_api, workbench_db, deadline_scan, task_state, task_id, marker_count+):+ scan = deadline_scan+ task = {"id": task_id, "reason": "Existing review outcome."}+ scan.coverage[task_state] = [task]+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ recovered = worker_finding(scan, "saved-before-deadline")+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[recovered],+ deferred=[{"id": "unfinished-review", "reason": "Review remains unfinished."}],+ ),+ )+ original_checkpoint = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ assert completed["progress"]["status"] == "complete"+ assert (+ workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0]+ == "complete"+ )+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert task in coverage[task_state]+ if task_state == "resolvedDeferred":+ assert coverage[task_state] == [task]+ assert any(+ row.get("workerId") == worker_id and row["reason"] == "Review remains unfinished."+ for row in coverage["deferred"]+ )+ assert (+ sum(row.get("id") == "scan-deadline" and row != task for row in coverage["deferred"])+ == marker_count+ )+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert {row["title"] for row in findings} == {scan.findings[0]["title"], recovered["title"]}+ assert (+ next(row for row in findings if row["title"] == recovered["title"])["provenance"][+ "workerId"+ ]+ == worker_id+ )+ assert checkpoint.read_bytes() == original_checkpoint+++def interrupt_finish_before_commit(workbench_api, database, scan):+ child = os.fork()+ if child == 0:+ connection = sqlite3.connect(database)+ connection.row_factory = sqlite3.Row++ class CrashBeforeCommit:+ def __getattr__(self, name):+ return getattr(connection, name)++ def commit(self):+ os._exit(0)++ try:+ finish(workbench_api, CrashBeforeCommit(), scan, terminal_reason="capped")+ except BaseException:+ traceback.print_exc()+ os._exit(1)+ _, status = os.waitpid(child, 0)+ assert os.waitstatus_to_exitcode(status) == 0+++def test_discarded_worker_finding_leaves_its_review_pending(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ canonical = {+ "id": "accepted",+ "label": "Accepted review",+ "disposition": "reported",+ "receiptRefs": [],+ }+ scan.coverage["surfaces"] = [canonical]+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ invalid = worker_finding(scan, "incomplete-worker-finding")+ invalid.pop("remediation")+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[invalid],+ surfaces=[+ {+ "id": "entry",+ "label": "Worker review",+ "disposition": "reported",+ "receiptRefs": [],+ }+ ],+ ),+ )+ original = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert canonical in coverage["surfaces"]+ surface = next(row for row in coverage["surfaces"] if row.get("workerId") == worker_id)+ assert surface["disposition"] == "needs_follow_up"+ assert any(+ row.get("workerId") == worker_id and "remediation" in row["reason"]+ for row in coverage["deferred"]+ )+ assert completed["progress"]["status"] == "complete"+ assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1+ assert checkpoint.read_bytes() == original+++ "invalid_terminal", [False, True], ids=["valid-terminal", "invalid-terminal"]+)+def test_terminal_worker_authority_requires_valid_findings(+ workbench_api, workbench_db, deadline_scan, invalid_terminal+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ previous = worker_finding(scan, "earlier-valid-finding")+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[previous],+ deferred=[{"id": "review", "reason": "Review pending."}],+ ),+ )+ os.utime(checkpoint, ns=(100, 100))+ sibling = worker_finding(scan, "valid-terminal-sibling")+ terminal_findings = [sibling]+ if invalid_terminal:+ invalid = copy.deepcopy(previous)+ invalid.pop("remediation")+ terminal_findings.append(invalid)+ result = output / "result.json"+ result.write_text(+ json.dumps(+ saved_draft(+ scan.scan_id,+ complete=True,+ findings=terminal_findings,+ closures=[{"id": "review", "reason": "Review completed."}],+ )+ )+ )+ os.utime(result, ns=(200, 200))+ original = {path: path.read_bytes() for path in (checkpoint, result)}++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert any(row["title"] == previous["title"] for row in findings) == invalid_terminal+ assert any(row["title"] == sibling["title"] for row in findings)+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert not any(row["reason"] == "Review pending." for row in coverage["deferred"])+ assert {path: path.read_bytes() for path in original} == original+++def test_malformed_worker_finding_preserves_a_valid_sibling_outcome(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ earlier = worker_finding(scan, "rejected-candidate")+ earlier["provenance"]["candidateId"] = "candidate"+ checkpoint = write_checkpoint(+ output / "checkpoints", saved_draft(scan.scan_id, findings=[earlier])+ )+ os.utime(checkpoint, ns=(100, 100))+ invalid = worker_finding(scan, "malformed-sibling")+ invalid.pop("remediation")+ outcome = {+ "id": "review",+ "label": "Candidate reviewed",+ "candidateId": "candidate",+ "disposition": "rejected",+ "receiptRefs": [],+ }+ result = output / "result.json"+ result.write_text(+ json.dumps(saved_draft(scan.scan_id, complete=True, findings=[invalid], surfaces=[outcome]))+ )+ os.utime(result, ns=(200, 200))++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert all(row["title"] != earlier["title"] for row in findings)+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert (+ next(row for row in coverage["surfaces"] if row.get("workerId") == worker_id)["disposition"]+ == "rejected"+ )+ assert any("remediation" in row["reason"] for row in coverage["deferred"])++[email protected]("invalid", ["surfaces", "missing-receipt"])+def test_deadline_keeps_canonical_coverage_validation_strict(+ workbench_api, workbench_db, deadline_scan, invalid+):+ from finalize_scan_contract import ContractError++ scan = deadline_scan+ scan.coverage["surfaces"] = (+ "invalid"+ if invalid == "surfaces"+ else [+ {+ "id": "accepted",+ "label": "Accepted review",+ "disposition": "no_issue_found",+ "receiptRefs": ["artifacts/missing.txt"],+ }+ ]+ )+ coverage_file = scan.scan_dir / "coverage.json"+ coverage_file.write_text(json.dumps(scan.coverage))+ original = coverage_file.read_bytes()+ _, output = canceled_discovery_output(workbench_db, scan)+ write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id))++ with pytest.raises(ContractError, match="surfaces|receiptRefs"):+ finish(workbench_api, workbench_db, scan, terminal_reason="capped")++ assert coverage_file.read_bytes() == original+ assert (+ workbench_db.execute(+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)+ ).fetchone()[0]+ == "running"+ )++[email protected]("worker_writeup_exists", [False, True], ids=["missing", "present"])+def test_worker_writeups_remain_bound_to_their_source_directory(+ workbench_api, workbench_db, deadline_scan, worker_writeup_exists+):+ scan = deadline_scan+ report_path = "findings/review/review.md"+ scan.findings[0]["writeup"] = {"reportPath": report_path}+ parent_writeup = scan.scan_dir / report_path+ parent_writeup.parent.mkdir(parents=True)+ parent_writeup.write_text("# Accepted parent writeup\n")+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings}))+ originals = {parent_writeup: parent_writeup.read_bytes()}+ sources = []+ for archived in (False, True):+ _, output = canceled_discovery_output(workbench_db, scan, attempt=2 if archived else 1)+ source = output.parent / "attempts" / "attempt-1" if archived else output+ sources.append(source.relative_to(scan.scan_dir).as_posix())+ finding = worker_finding(scan, "archived-worker" if archived else "current-worker")+ if worker_writeup_exists:+ writeup = source / report_path+ writeup.parent.mkdir(parents=True)+ writeup.write_text(f"# Separate worker writeup: {finding['title']}\n")+ originals[writeup] = writeup.read_bytes()+ checkpoint = write_checkpoint(+ source / "checkpoints", saved_draft(scan.scan_id, findings=[finding])+ )+ originals[checkpoint] = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert len(findings) == 3+ assert next(row for row in findings if row["title"] == scan.findings[0]["title"])[+ "writeup"+ ] == {"reportPath": report_path}+ assert all(+ "writeup" not in row for row in findings if row["title"] != scan.findings[0]["title"]+ )+ writeup_warnings = [warning for warning in completed["warnings"] if "writeup" in warning]+ assert len(writeup_warnings) == 2+ expected = (+ "Retained worker writeup at" if worker_writeup_exists else "Skipped malformed writeup"+ )+ assert all(expected in warning for warning in writeup_warnings)+ assert all(any(source in warning for warning in writeup_warnings) for source in sources)+ report = (scan.scan_dir / "report.md").read_text()+ assert "archived-worker" in report and "current-worker" in report+ assert {path: path.read_bytes() for path in originals} == originals+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)+ ("colliding_ids", "parent_closure", "reviewed", "worker_like_id"),+ [+ (False, False, False, False),+ (True, False, False, False),+ (False, True, False, False),+ (True, True, False, False),+ (False, True, True, False),+ (False, True, True, True),+ ],+ ids=[+ "pending",+ "legacy-collision",+ "parent-closure",+ "parent-closure-with-collision",+ "reviewed-surface",+ "reviewed-surface-with-worker-like-id",+ ],+)+def test_deadline_recovery_is_idempotent_after_crash_before_commit(+ workbench_api,+ workbench_db,+ deadline_scan,+ tmp_path,+ colliding_ids,+ parent_closure,+ reviewed,+ worker_like_id,+):+ scan = deadline_scan+ if parent_closure:+ scan.coverage["resolvedDeferred"] = [+ {"id": "prior-review", "reason": "Earlier review completed."}+ ]+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ database = tmp_path / "crash-recovery.db"+ connection = sqlite3.connect(database)+ connection.row_factory = sqlite3.Row+ workbench_db.backup(connection)+ worker_id, output = canceled_discovery_output(connection, scan)+ surface_id = f"{worker_id}/entry" if worker_like_id else "entry"+ surfaces = [+ {+ "id": surface_id,+ "label": "Worker review",+ "disposition": "no_issue_found" if reviewed else "needs_follow_up",+ "receiptRefs": [],+ }+ ]+ if colliding_ids:+ surfaces.append({**surfaces[0], "label": "Separate worker review"})+ write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=surfaces,+ findings=[]+ if reviewed+ else [worker_finding(scan, "saved-before-crash"), copy.deepcopy(scan.findings[0])],+ deferred=[]+ if reviewed+ else [{"id": "review", "reason": "Review pending.", "surfaceIds": [surface_id]}],+ ),+ )+ connection.close()+ interrupt_finish_before_commit(workbench_api, database, scan)+ before = json.loads((scan.scan_dir / "coverage.json").read_text())+ findings_before = json.loads((scan.scan_dir / "findings.json").read_text())+ assert len([row for row in before["surfaces"] if row.get("workerId") == worker_id]) == len(+ surfaces+ )+ if reviewed:+ assert before["surfaces"][0]["id"] == f"{worker_id}/{surface_id}"+ assert len([row for row in before["deferred"] if row["id"] == "scan-deadline"]) == 1+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ assert (+ connection.execute(+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)+ ).fetchone()[0]+ == "running"+ )+ finish(workbench_api, connection, scan, terminal_reason="capped")+ complete(workbench_api, connection, scan)++ after = json.loads((scan.scan_dir / "coverage.json").read_text())+ findings_after = json.loads((scan.scan_dir / "findings.json").read_text())+ for document in (findings_before, findings_after):+ for finding in document["findings"]:+ # Existing recovery can materialize an empty optional history list.+ if finding["provenance"].get("previousFindings") == []:+ finding["provenance"].pop("previousFindings")+ assert findings_after == findings_before+ assert after == before+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)[email protected]("earlier_checkpoint_first", [False, True])[email protected]("colliding_ids", [False, True])+def test_deadline_rejection_history_is_stable_across_repeated_crashes(+ workbench_api, workbench_db, deadline_scan, tmp_path, earlier_checkpoint_first, colliding_ids+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ finding = worker_finding(scan, "reviewed-candidate")+ finding["provenance"]["candidateId"] = "candidate"+ # Check both hash filename orders without adding invalid or unrelated checkpoints.+ for attempt in range(1000):+ finding["title"] = f"Candidate review {attempt}"+ earlier = saved_draft(scan.scan_id, findings=[finding])+ outcomes = [+ {+ "id": "entry",+ "candidateId": "candidate",+ "finding": copy.deepcopy(finding),+ "label": "Candidate reviewed",+ "disposition": "rejected",+ "receiptRefs": [],+ }+ ]+ if colliding_ids:+ outcomes.append({**copy.deepcopy(outcomes[0]), "label": "Separate observation"})+ terminal = saved_draft(scan.scan_id, complete=True, surfaces=outcomes)+ earlier_hash = hashlib.sha256(json.dumps(earlier).encode()).hexdigest()+ terminal_hash = hashlib.sha256(json.dumps(terminal).encode()).hexdigest()+ if (earlier_hash < terminal_hash) == earlier_checkpoint_first:+ break+ else:+ pytest.fail("Could not select checkpoint filename order")+ old_checkpoint = write_checkpoint(output / "checkpoints", earlier)+ checkpoint = write_checkpoint(output / "checkpoints", terminal)+ assert (old_checkpoint.name < checkpoint.name) == earlier_checkpoint_first+ os.utime(old_checkpoint, ns=(100, 100))+ os.utime(checkpoint, ns=(200, 200))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))+ (output / "result.json").unlink(missing_ok=True)+ originals = {path: path.read_bytes() for path in (old_checkpoint, checkpoint, head)}+ database = tmp_path / "repeated-crash.db"+ with sqlite3.connect(database) as connection:+ workbench_db.backup(connection)++ publications = []+ for _ in range(3):+ interrupt_finish_before_commit(workbench_api, database, scan)+ publications.append(json.loads((scan.scan_dir / "coverage.json").read_text()))+ with sqlite3.connect(database) as connection:+ assert (+ connection.execute(+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)+ ).fetchone()[0]+ == "running"+ )+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ finish(workbench_api, connection, scan, terminal_reason="capped")+ complete(workbench_api, connection, scan)+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert all(publication == coverage for publication in publications)+ surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert {row["label"] for row in surfaces} == {row["label"] for row in outcomes}+ assert len(surfaces) == len({row["id"] for row in surfaces}) == len(outcomes)+ assert all(len(row["previousFindings"]) == 1 for row in surfaces)+ assert all(row["previousFindings"][0]["title"] == finding["title"] for row in surfaces)+ assert all(row["disposition"] == "rejected" for row in surfaces)+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert len(findings) == 1 and findings[0]["title"] == scan.findings[0]["title"]+ assert {path: path.read_bytes() for path in originals} == originalsdiff --git a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.pynew file mode 100644index 000000000..13b693904--- /dev/null+++ b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py@@ -0,0 +1,603 @@+from __future__ import annotations++import json+import os+import sqlite3+import subprocess+import sys+from pathlib import Path++import pytest+from test_deep_scan_stop_conditions import canceled_discovery_output, complete, finish+from test_deep_scan_stop_conditions import deadline_scan as deadline_scan+from test_deep_scan_stop_conditions import publication_scan as publication_scan+from test_deep_scan_stop_conditions import saturated_scan as saturated_scan+from workbench_test_support import (+ replay_saved_results,+ saved_binding,+ saved_discovery_worker,+ saved_draft,+ write_checkpoint,+)+++def surface(label, identity="entry", disposition="needs_follow_up"):+ return {"id": identity, "label": label, "disposition": disposition, "receiptRefs": []}+++def pending(identity, reason, surface_id="entry"):+ return {"id": identity, "reason": reason, "surfaceIds": [surface_id]}+++def interrupt_in_fresh_python(database, scan, hash_seed):+ script = Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"+ result = subprocess.run(+ [+ sys.executable,+ "-B",+ "-c",+ """+import runpy+import sys+from pathlib import Path+from types import SimpleNamespace+sys.path.insert(0, str(Path(sys.argv[1]).parents[1] / "tests"))+from test_deep_scan_deadline_recovery import interrupt_finish_before_commit++api = runpy.run_path(sys.argv[1], run_name="surface_link_recovery_test")+api["deep_scan"]._dependencies = SimpleNamespace(+ **api, preserve_stopped_results=api["preserve_stopped_results_after_transition"]+)+scan = SimpleNamespace(scan_id=sys.argv[3], scan_dir=Path(sys.argv[4]))+interrupt_finish_before_commit(api, Path(sys.argv[2]), scan)+""",+ str(script),+ str(database),+ scan.scan_id,+ str(scan.scan_dir),+ ],+ cwd=Path(__file__).parent,+ env={**os.environ, "PYTHONHASHSEED": str(hash_seed)},+ capture_output=True,+ text=True,+ )+ assert result.returncode == 0, result.stdout + result.stderr+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)[email protected]("parent_collision", [False, True])+def test_deadline_surface_links_follow_sources_across_repeated_crashes(+ workbench_api, workbench_db, deadline_scan, tmp_path, parent_collision+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)+ archived = output.parent / "attempts" / "attempt-01"+ shared = pending("shared-review", "Review all retained observations.")+ checkpoints = []+ for index, (source, label) in enumerate(+ [(archived, "download"), (output, "upload"), (output, "export")], start=1+ ):+ # Each is a valid raw input checkpoint saved before the writer merges history.+ checkpoint = write_checkpoint(+ source / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=[surface(label)],+ deferred=[pending(f"{label}-review", f"Review {label}."), shared],+ ),+ )+ os.utime(checkpoint, ns=(index * 100, index * 100))+ checkpoints.append(checkpoint)+ canonical_id = f"{worker_id}/entry" if parent_collision else "parent-entry"+ canonical_surface = surface("Canonical parent", canonical_id)+ canonical_pending = pending("parent-review", "Parent review remains.", canonical_id)+ scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_pending])+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ originals = {path: path.read_bytes() for path in checkpoints}+ database = tmp_path / "surface-links.db"+ with sqlite3.connect(database) as connection:+ workbench_db.backup(connection)++ publications = []+ for hash_seed in (1, 17, 83):+ interrupt_in_fresh_python(database, scan, hash_seed)+ publications.append(json.loads((scan.scan_dir / "coverage.json").read_text()))+ with sqlite3.connect(database) as connection:+ assert (+ connection.execute(+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)+ ).fetchone()[0]+ == "running"+ )+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ finish(workbench_api, connection, scan, terminal_reason="capped")+ completed = complete(workbench_api, connection, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert completed["progress"]["status"] == "complete"+ assert all(publication == coverage for publication in publications)+ assert canonical_surface in coverage["surfaces"]+ assert canonical_pending in coverage["deferred"]+ surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert {row["label"] for row in surfaces} == {"download", "upload", "export"}+ assert len(surfaces) == len({row["id"] for row in surfaces}) == 3+ tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id]+ for row in surfaces:+ task = next(item for item in tasks if item["reason"] == f"Review {row['label']}.")+ assert task["surfaceIds"] == [row["id"]]+ shared_tasks = [row for row in tasks if row["reason"] == shared["reason"]]+ assert len(shared_tasks) == 1+ assert shared_tasks[0]["surfaceIds"] == [row["id"] for row in surfaces]+ assert {path: path.read_bytes() for path in originals} == originals++[email protected]("with_reduction", [False, True], ids=["zero-accepted", "reduced"])+def test_deadline_retains_all_targets_of_one_legacy_surface_id(+ workbench_api, workbench_db, publication_scan, request, with_reduction+):+ scan = request.getfixturevalue("deadline_scan") if with_reduction else publication_scan()+ if not with_reduction:+ with workbench_db:+ workbench_db.execute(+ "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', "+ "created_at = '2000-01-01T00:00:00+00:00', manifest_path = NULL, "+ "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?",+ (scan.scan_id,),+ )+ _, output = canceled_discovery_output(workbench_db, scan)+ checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=[surface("download"), surface("upload")],+ deferred=[pending("review", "Review both legacy observations.")],+ ),+ )+ original = checkpoint.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ surfaces = [row for row in coverage["surfaces"] if row["label"] in {"download", "upload"}]+ assert len(surfaces) == len({row["id"] for row in surfaces}) == 2+ task = next(+ row for row in coverage["deferred"] if row["reason"] == "Review both legacy observations."+ )+ assert task["surfaceIds"] == [row["id"] for row in surfaces]+ assert checkpoint.read_bytes() == original+++def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ earlier = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, surfaces=[surface("Earlier review", "prior")]),+ )+ current = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, deferred=[pending("review", "Review prior surface.", "prior")]),+ )+ os.utime(earlier, ns=(100, 100))+ os.utime(current, ns=(200, 200))+ originals = {path: path.read_bytes() for path in (earlier, current)}++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ retained = next(row for row in coverage["surfaces"] if row["label"] == "Earlier review")+ task = next(row for row in coverage["deferred"] if row["reason"] == "Review prior surface.")+ assert task["surfaceIds"] == [retained["id"]] == [f"{worker_id}/prior"]+ assert {path: path.read_bytes() for path in originals} == originals+++def test_deadline_reopened_generic_surface_copy_retains_its_link(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ canonical = surface("Canonical parent", f"{worker_id}/entry")+ scan.coverage["surfaces"] = [canonical]+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ drafts = [+ saved_draft(+ scan.scan_id,+ surfaces=[surface("Worker review")],+ deferred=[pending("review", "Initial review pending.")],+ ),+ saved_draft(+ scan.scan_id,+ complete=True,+ surfaces=[surface("Worker review", disposition="no_issue_found")],+ closures=[{"id": "review", "reason": "Initial review completed."}],+ ),+ saved_draft(+ scan.scan_id,+ surfaces=[surface("Worker review")],+ deferred=[pending("review", "Review reopened.")],+ ),+ ]+ originals = {}+ for index, draft in enumerate(drafts, start=1):+ checkpoint = write_checkpoint(output / "checkpoints", draft)+ os.utime(checkpoint, ns=(index * 100, index * 100))+ originals[checkpoint] = checkpoint.read_bytes()+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))+ originals[head] = head.read_bytes()++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ retained = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert len(retained) == 1+ assert retained[0]["disposition"] == "needs_follow_up"+ task = next(row for row in coverage["deferred"] if row["reason"] == "Review reopened.")+ assert task["surfaceIds"] == [retained[0]["id"]]+ assert canonical in coverage["surfaces"]+ assert {path: path.read_bytes() for path in originals} == originals+++def test_stopped_recovery_preserves_source_link_union(workbench_api, tmp_path):+ saved = workbench_api["saved_results"]+ scan_id = "surface-links"+ binding = saved_binding("deep_repository")+ output = tmp_path / "worker" / "output"+ archived = output.parent / "attempts" / "attempt-01"+ shared = pending("review", "Review retained observations.")+ originals = {}+ for source, label in ((archived, "download"), (output, "upload")):+ checkpoint = write_checkpoint(+ source / "checkpoints",+ saved_draft(scan_id, surfaces=[surface(label)], deferred=[shared]),+ )+ originals[checkpoint] = checkpoint.read_bytes()+ workers = [saved_discovery_worker(output, "worker", 2)]++ documents = saved.merge_saved_results(+ tmp_path, scan_id, binding, workers, [], stopped=True, reason="interrupted"+ )+ replay = replay_saved_results(saved, documents, tmp_path, scan_id, binding, workers)++ assert replay[2] == documents[2]+ coverage = documents[2]+ assert {row["label"] for row in coverage["surfaces"]} == {"download", "upload"}+ tasks = [row for row in coverage["deferred"] if row["reason"] == shared["reason"]]+ assert len(tasks) == 1+ assert tasks[0]["surfaceIds"] == [row["id"] for row in coverage["surfaces"]]+ assert {path: path.read_bytes() for path in originals} == originals+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)[email protected]("changed_task", [False, True], ids=["copied-task", "updated-task"])+def test_deadline_retains_equal_task_origins_from_selected_worker_result(+ workbench_api, workbench_db, deadline_scan, tmp_path, changed_task+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)+ archived = output.parent / "attempts" / "attempt-01"+ previous_task = pending("download-review", "Download review pending.")+ archived_draft = saved_draft(+ scan.scan_id, surfaces=[surface("download")], deferred=[previous_task]+ )+ checkpoint = write_checkpoint(archived / "checkpoints", archived_draft)+ os.utime(checkpoint, ns=(100, 100))+ archived_head = archived / "checkpoint-head.json"+ archived_head.write_text(json.dumps({"checkpoint": checkpoint.name}))+ os.utime(archived_head, ns=(110, 110))+ archived_result = archived / "result.json"+ archived_result.write_bytes(checkpoint.read_bytes())+ os.utime(archived_result, ns=(120, 120))+ current_task = pending("upload-review", "Upload review pending.")+ retained_task = (+ pending("download-review", "Download review updated.") if changed_task else previous_task+ )+ incoming = saved_draft(+ scan.scan_id,+ surfaces=[surface("upload")],+ deferred=[current_task, retained_task] if changed_task else [current_task],+ )+ raw_checkpoint = write_checkpoint(output / "checkpoints", incoming)+ os.utime(raw_checkpoint, ns=(200, 200))+ # The writer preserves the old pending task while publishing its latest surface.+ merged = saved_draft(+ scan.scan_id, surfaces=[surface("upload")], deferred=[current_task, retained_task]+ )+ selected = write_checkpoint(output / "checkpoints", merged)+ os.utime(selected, ns=(210, 210))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": selected.name}))+ os.utime(head, ns=(220, 220))+ result = output / "result.json"+ result.write_bytes(selected.read_bytes())+ os.utime(result, ns=(230, 230))+ originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()}+ database = tmp_path / "selected-result.db"+ with sqlite3.connect(database) as connection:+ workbench_db.backup(connection)++ publications = []+ for hash_seed in (1, 17, 83):+ interrupt_in_fresh_python(database, scan, hash_seed)+ publications.append(json.loads((scan.scan_dir / "coverage.json").read_text()))+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ finish(workbench_api, connection, scan, terminal_reason="capped")+ complete(workbench_api, connection, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert all(publication == coverage for publication in publications)+ surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert {row["label"] for row in surfaces} == {"download", "upload"}+ upload = next(row for row in surfaces if row["label"] == "upload")+ tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id]+ assert len(tasks) == 2+ current = next(row for row in tasks if row["reason"] == current_task["reason"])+ retained = next(row for row in tasks if row["reason"] == retained_task["reason"])+ assert current["surfaceIds"] == [upload["id"]]+ assert retained["surfaceIds"] == (+ [upload["id"]] if changed_task else [row["id"] for row in surfaces]+ )+ assert {path: path.read_bytes() for path in originals} == originals+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)+ "selected_result", [False, True], ids=["raw-checkpoint", "selected-result"]+)+def test_deadline_dropped_legacy_surface_does_not_rebind_its_pending_task(+ workbench_api, workbench_db, deadline_scan, tmp_path, selected_result+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)+ archived = output.parent / "attempts" / "attempt-01"+ download_task = pending("download-review", "Download review pending.")+ archived_checkpoint = write_checkpoint(+ archived / "checkpoints",+ saved_draft(scan.scan_id, surfaces=[surface("download")], deferred=[download_task]),+ )+ os.utime(archived_checkpoint, ns=(100, 100))+ archived_result = archived / "result.json"+ archived_result.write_bytes(archived_checkpoint.read_bytes())+ os.utime(archived_result, ns=(110, 110))+ upload_task = pending("upload-review", "Upload review pending.")+ # Historical artifacts can contain labels rejected during finalization.+ malformed_surface = surface("\u0085")+ raw_checkpoint = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, surfaces=[malformed_surface], deferred=[upload_task]),+ )+ os.utime(raw_checkpoint, ns=(200, 200))+ if selected_result:+ selected = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=[malformed_surface],+ deferred=[upload_task, download_task],+ ),+ )+ os.utime(selected, ns=(210, 210))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": selected.name}))+ os.utime(head, ns=(220, 220))+ result = output / "result.json"+ result.write_bytes(selected.read_bytes())+ os.utime(result, ns=(230, 230))+ originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()}+ database = tmp_path / "dropped-surface.db"+ with sqlite3.connect(database) as connection:+ workbench_db.backup(connection)++ publications = []+ for hash_seed in (1, 17, 83):+ interrupt_in_fresh_python(database, scan, hash_seed)+ publications.append(json.loads((scan.scan_dir / "coverage.json").read_text()))+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ finish(workbench_api, connection, scan, terminal_reason="capped")+ completed = complete(workbench_api, connection, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert completed["progress"]["status"] == "complete"+ assert all(publication == coverage for publication in publications)+ surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert len(surfaces) == 1 and surfaces[0]["label"] == "download"+ tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id]+ retained_upload = next(row for row in tasks if row["reason"] == upload_task["reason"])+ retained_download = next(row for row in tasks if row["reason"] == download_task["reason"])+ assert retained_upload["surfaceIds"] == []+ assert retained_download["surfaceIds"] == [surfaces[0]["id"]]+ assert any(+ raw_checkpoint.relative_to(scan.scan_dir).as_posix() in warning+ and "coverage.surfaces[0].label: expected a non-empty string" in warning+ for warning in completed["warnings"]+ )+ assert {path: path.read_bytes() for path in originals} == originals+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)+ "history",+ ["parent-collision", "current-checkpoints", "retry-checkpoints", "coalesced-history"],+)+def test_deadline_inherited_surface_links_follow_historical_observations(+ workbench_api, workbench_db, deadline_scan, tmp_path, history+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(+ workbench_db, scan, attempt=2 if history == "retry-checkpoints" else 1+ )+ archived = output.parent / "attempts" / "attempt-01"+ first_output = archived if history == "retry-checkpoints" else output+ earlier = write_checkpoint(+ first_output / "checkpoints",+ saved_draft(scan.scan_id, surfaces=[surface("Earlier observation")]),+ )+ os.utime(earlier, ns=(100, 100))+ expected_labels = {"Earlier observation"}+ if history != "parent-collision":+ newer = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, surfaces=[surface("Newer observation")]),+ )+ os.utime(newer, ns=(200, 200))+ expected_labels.add("Newer observation")+ inherited_task = pending("history-review", "Review the inherited surface.")+ unknown_task = pending("unknown-review", "Review an unresolved reference.", "not-declared")+ deferred_only = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, deferred=[inherited_task, unknown_task]),+ )+ os.utime(deferred_only, ns=(300, 300))+ if history == "coalesced-history":+ # The merged checkpoint copies a task whose earlier raw source inherited history.+ selected = write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=[surface("Newer observation")],+ deferred=[inherited_task, unknown_task],+ ),+ )+ os.utime(selected, ns=(400, 400))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": selected.name}))+ os.utime(head, ns=(410, 410))+ if history == "current-checkpoints":+ # An interrupted later write cannot identify an earlier task's target.+ future = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, surfaces=[surface("Later observation")]),+ )+ os.utime(future, ns=(400, 400))+ canonical_id = f"{worker_id}/entry" if history == "parent-collision" else "parent-entry"+ canonical_surface = surface("Canonical parent", canonical_id)+ canonical_task = pending("parent-review", "Parent review remains.", canonical_id)+ scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_task])+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ originals = {path: path.read_bytes() for path in output.parent.rglob("*") if path.is_file()}+ database = tmp_path / "historical-links.db"+ with sqlite3.connect(database) as connection:+ workbench_db.backup(connection)++ publications = []+ for hash_seed in (1, 17, 83):+ interrupt_in_fresh_python(database, scan, hash_seed)+ publications.append(json.loads((scan.scan_dir / "coverage.json").read_text()))+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ finish(workbench_api, connection, scan, terminal_reason="capped")+ completed = complete(workbench_api, connection, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert completed["progress"]["status"] == "complete"+ assert all(publication == coverage for publication in publications)+ assert canonical_surface in coverage["surfaces"]+ assert canonical_task in coverage["deferred"]+ worker_surfaces = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ historical = [row for row in worker_surfaces if row["label"] in expected_labels]+ assert {row["label"] for row in historical} == expected_labels+ assert len(historical) == len(expected_labels)+ if history == "current-checkpoints":+ assert any(row["label"] == "Later observation" for row in worker_surfaces)+ tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id]+ inherited = next(row for row in tasks if row["reason"] == inherited_task["reason"])+ unknown = next(row for row in tasks if row["reason"] == unknown_task["reason"])+ assert inherited["surfaceIds"] == [row["id"] for row in historical]+ assert unknown["surfaceIds"] == [f"{worker_id}/not-declared"]+ assert {path: path.read_bytes() for path in originals} == originals+++ not hasattr(os, "fork"), reason="Crash boundary uses a forked workbench process."+)+ ("changed_surface_id", "legacy_duplicate_id"),+ [(False, False), (True, False), (True, True)],+ ids=["same-id", "changed-id", "legacy-duplicate-id"],+)+def test_deadline_candidate_supersession_preserves_independent_surface_links(+ workbench_api, workbench_db, deadline_scan, tmp_path, changed_surface_id, legacy_duplicate_id+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan)+ canonical_id = f"{worker_id}/entry"+ canonical_surface = surface("Canonical parent", canonical_id)+ canonical_task = pending("parent-review", "Parent review remains.", canonical_id)+ scan.coverage.update(surfaces=[canonical_surface], deferred=[canonical_task])+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ independent = pending("independent-review", "Review another caller.")+ candidate_surface = {**surface("Candidate review"), "candidateId": "candidate"}+ earlier_surfaces = [candidate_surface]+ if legacy_duplicate_id:+ # Legacy artifacts may reuse an ID for a candidate and an independent observation.+ earlier_surfaces.append(surface("Independent observation"))+ earlier = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, surfaces=earlier_surfaces, deferred=[independent]),+ )+ os.utime(earlier, ns=(100, 100))+ outcome_id = "outcome" if changed_surface_id else "entry"+ outcome = {+ **surface("Candidate reviewed", outcome_id, "rejected"),+ "candidateId": "candidate",+ }+ raw_checkpoint = write_checkpoint(+ output / "checkpoints", saved_draft(scan.scan_id, complete=True, surfaces=[outcome])+ )+ os.utime(raw_checkpoint, ns=(200, 200))+ # The selected merged checkpoint retains independent work before result publication.+ selected = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, complete=True, surfaces=[outcome], deferred=[independent]),+ )+ os.utime(selected, ns=(210, 210))+ head = output / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": selected.name}))+ os.utime(head, ns=(220, 220))+ originals = {path: path.read_bytes() for path in output.rglob("*") if path.is_file()}+ database = tmp_path / "candidate-links.db"+ with sqlite3.connect(database) as connection:+ workbench_db.backup(connection)++ publications = []+ for hash_seed in (1, 17, 83):+ interrupt_in_fresh_python(database, scan, hash_seed)+ publications.append(json.loads((scan.scan_dir / "coverage.json").read_text()))+ with sqlite3.connect(database) as connection:+ connection.row_factory = sqlite3.Row+ finish(workbench_api, connection, scan, terminal_reason="capped")+ completed = complete(workbench_api, connection, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert completed["progress"]["status"] == "complete"+ assert all(publication == coverage for publication in publications)+ assert canonical_surface in coverage["surfaces"]+ assert canonical_task in coverage["deferred"]+ retained = [row for row in coverage["surfaces"] if row.get("workerId") == worker_id]+ assert len(retained) == (2 if legacy_duplicate_id else 1)+ outcome = next(row for row in retained if row["label"] == "Candidate reviewed")+ assert outcome["disposition"] == "rejected"+ if legacy_duplicate_id:+ observation = next(row for row in retained if row["label"] == "Independent observation")+ assert observation["disposition"] == "needs_follow_up"+ assert all(row["id"] != canonical_id for row in retained)+ tasks = [row for row in coverage["deferred"] if row.get("workerId") == worker_id]+ assert len(tasks) == 1 and tasks[0]["reason"] == independent["reason"]+ assert tasks[0]["surfaceIds"] == [row["id"] for row in retained]+ assert {path: path.read_bytes() for path in originals} == originalsdiff --git a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py b/plugins/codex-security/tests/test_deep_scan_stop_conditions.pyindex 1c2389e15..6e67fbf2c 100644--- a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py+++ b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py@@ -1,5 +1,6 @@from __future__ import annotations+import copyimport jsonimport osimport sqlite3@@ -72,6 +73,299 @@ def finish(workbench_api, connection, scan, *, terminal_reason="saturated"):)["deepScan"]+def deadline_scan(saturated_scan, workbench_db):+ with workbench_db:+ workbench_db.execute(+ "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?",+ (saturated_scan.scan_id,),+ )+ return saturated_scan+++def canceled_discovery_output(connection, scan, *, attempt=1):+ result = add_worker(connection, scan, status="canceled")+ worker_id = result.parent.name+ output = scan.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output"+ output.parent.mkdir(parents=True)+ result.parent.rename(output)+ with connection:+ connection.execute(+ "UPDATE deep_scan_workers SET artifact_dir = ?, prompt_path = ?, "+ "result_manifest_path = NULL, attempt = ? WHERE id = ?",+ (str(output), str(output.parent / "prompt.md"), attempt, worker_id),+ )+ return worker_id, output++[email protected]("missing_writeup", [False, True], ids=["present", "missing"])+def test_deadline_preserves_optional_reduced_writeup_behavior(+ workbench_api, workbench_db, deadline_scan, missing_writeup+):+ scan = deadline_scan+ report_path = "findings/reduced/reduced.md"+ scan.findings[0]["writeup"] = {"reportPath": report_path}+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings}))+ if not missing_writeup:+ report = scan.scan_dir / report_path+ report.parent.mkdir(parents=True)+ report.write_text("# Reduced finding\n")+ _, output = canceled_discovery_output(workbench_db, scan)+ write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, deferred=[{"reason": "Review the remaining upload paths."}]),+ )++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ assert completed["progress"]["status"] == "complete"+ finding = json.loads((scan.scan_dir / "findings.json").read_text())["findings"][0]+ assert finding["title"] == scan.findings[0]["title"]+ assert ("writeup" not in finding) == missing_writeup+ assert any("Skipped missing writeup" in warning for warning in completed["warnings"]) == (+ missing_writeup+ )+++def test_deadline_recovers_valid_findings_from_an_invalid_archived_retry(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)+ archived = output.parent / "attempts" / "attempt-1"+ archived.mkdir(parents=True)+ valid = copy.deepcopy(scan.findings[0])+ valid["identity"]["anchor"] = "valid-archived-finding"+ valid["title"] = "Valid finding saved before retry"+ valid["provenance"] = {"source": "local_plugin"}+ invalid = copy.deepcopy(valid)+ invalid["identity"]["anchor"] = "invalid-archived-finding"+ invalid.pop("remediation")+ result = archived / "result.json"+ result.write_text(+ json.dumps(saved_draft(scan.scan_id, complete=True, findings=[valid, invalid]))+ )+ original = result.read_bytes()+ write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, deferred=[{"reason": "Retry review remains unfinished."}]),+ )++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert {item["identity"]["anchor"] for item in findings} == {+ scan.findings[0]["identity"]["anchor"],+ valid["identity"]["anchor"],+ }+ recovered = next(item for item in findings if item["title"] == valid["title"])+ assert recovered["provenance"]["workerId"] == worker_id+ assert any("remediation" in warning for warning in completed["warnings"])+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ assert manifest["preservedSources"][result.relative_to(scan.scan_dir).as_posix()] == (+ workbench_api["saved_results"]._digest(json.loads(original))+ )+ assert result.read_bytes() == original++[email protected]("invalid", ["finding", "unsafe-writeup"])+def test_deadline_does_not_recover_invalid_reduced_findings(+ workbench_api, workbench_db, deadline_scan, invalid+):+ from finalize_scan_contract import ContractError++ scan = deadline_scan+ if invalid == "finding":+ scan.findings[0].pop("remediation")+ else:+ scan.findings[0]["writeup"] = {"reportPath": "../invalid.md"}+ findings_path = scan.scan_dir / "findings.json"+ findings_path.write_text(json.dumps({"findings": scan.findings}))+ original = findings_path.read_bytes()+ _, output = canceled_discovery_output(workbench_db, scan)+ write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id))++ with pytest.raises(ContractError, match="remediation|reportPath"):+ finish(workbench_api, workbench_db, scan, terminal_reason="capped")++ assert findings_path.read_bytes() == original+ assert (+ workbench_db.execute(+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)+ ).fetchone()[0]+ == "running"+ )+++def test_deadline_preserves_worker_findings_with_optional_and_shared_identity(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ no_identity = copy.deepcopy(scan.findings[0])+ no_identity.pop("identity")+ no_identity["title"] = "Worker finding without authored identity"+ no_identity["provenance"] = {"source": "local_plugin"}+ first = copy.deepcopy(no_identity)+ first["title"] = "Shared semantic anchor at distinct locations"+ first["identity"] = {"anchor": "shared-worker-anchor"}+ first["locations"] = [{"path": "src/first.py", "startLine": 1}]+ second = copy.deepcopy(first)+ second["locations"] = [{"path": "src/second.py", "startLine": 1}]+ malformed = copy.deepcopy(first)+ malformed.pop("remediation")+ write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, findings=[no_identity, first, second, malformed]),+ )++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert len(findings) == 4+ assert any(item["title"] == no_identity["title"] for item in findings)+ shared = [item for item in findings if item["identity"]["anchor"] == "shared-worker-anchor"]+ assert {item["locations"][0]["path"] for item in shared} == {"src/first.py", "src/second.py"}+ assert any("remediation" in warning for warning in completed["warnings"])+++def test_deadline_resolves_worker_local_deferred_ids_before_scoping(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ previous = write_checkpoint(+ output / "checkpoints",+ saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]),+ )+ os.utime(previous, ns=(100, 100))+ current = saved_draft(scan.scan_id, complete=True)+ current["coverage"]["resolvedDeferred"] = [{"id": "review", "reason": "Review completed."}]+ latest = write_checkpoint(output / "checkpoints", current)+ os.utime(latest, ns=(200, 200))+ (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": latest.name}))++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert not any(item["reason"] == "Review pending." for item in coverage["deferred"])+++def test_deadline_does_not_turn_invalid_worker_receipts_into_parent_artifacts(+ workbench_api, workbench_db, deadline_scan+):+ scan = deadline_scan+ _, output = canceled_discovery_output(workbench_db, scan)+ (output / "notes.txt").write_text("Not a worker receipt under artifacts/.\n")+ write_checkpoint(+ output / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=[+ {+ "id": "entry",+ "label": "Unverified review",+ "disposition": "no_issue_found",+ "receiptRefs": ["notes.txt", "../review.txt", "/artifacts/review.txt"],+ }+ ],+ ),+ )++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ completed = complete(workbench_api, workbench_db, scan)++ surface = json.loads((scan.scan_dir / "coverage.json").read_text())["surfaces"][0]+ assert surface["receiptRefs"] == []+ assert surface["disposition"] == "needs_follow_up"+ assert (+ len(+ [+ warning+ for warning in completed["warnings"]+ if "malformed coverage receipt" in warning+ ]+ )+ == 3+ )++[email protected]("archived", [False, True], ids=["current", "archived"])+def test_deadline_preserves_worker_receipts_and_surface_links(+ workbench_api, workbench_db, deadline_scan, archived+):+ scan = deadline_scan+ canonical_receipt = scan.scan_dir / "artifacts" / "reduced.txt"+ canonical_receipt.parent.mkdir()+ canonical_receipt.write_text("Reduced review evidence.\n")+ canonical_surface = {+ "id": f"{scan.completed_worker_id}/entry",+ "workerId": scan.completed_worker_id,+ "label": "Reduced review",+ "disposition": "no_issue_found",+ "receiptRefs": ["artifacts/reduced.txt"],+ }+ scan.coverage["surfaces"] = [canonical_surface]+ (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))+ expected = {}+ for label in ("upload", "download"):+ worker_id, output = canceled_discovery_output(+ workbench_db, scan, attempt=2 if archived else 1+ )+ source = output.parent / "attempts" / "attempt-1" if archived else output+ receipt = source / "artifacts" / "review.txt"+ receipt.parent.mkdir(parents=True)+ receipt.write_text(f"Reviewed {label}.\n")+ if archived:+ current_receipt = output / "artifacts" / "review.txt"+ current_receipt.parent.mkdir(parents=True)+ current_receipt.write_text("Different current-attempt evidence.\n")+ checkpoint = write_checkpoint(+ source / "checkpoints",+ saved_draft(+ scan.scan_id,+ surfaces=[+ {+ "id": "entry",+ "label": label,+ "disposition": "no_issue_found",+ "receiptRefs": ["artifacts/review.txt"],+ }+ ],+ deferred=[{"id": "follow-up", "reason": label, "surfaceIds": ["entry"]}],+ ),+ )+ expected[label] = (worker_id, receipt, checkpoint, checkpoint.read_bytes())++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert len(coverage["surfaces"]) == 3+ assert canonical_surface in coverage["surfaces"]+ sources = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"][+ "preservedSources"+ ]+ for label, (worker_id, receipt, checkpoint, original) in expected.items():+ surface = next(item for item in coverage["surfaces"] if item["label"] == label)+ deferred = next(item for item in coverage["deferred"] if item["reason"] == label)+ assert surface["id"] == f"{worker_id}/entry"+ assert deferred["id"] == f"{worker_id}/follow-up"+ assert deferred["surfaceIds"] == [surface["id"]]+ assert surface["workerId"] == deferred["workerId"] == worker_id+ assert surface["disposition"] == "no_issue_found"+ assert surface["receiptRefs"] == [receipt.relative_to(scan.scan_dir).as_posix()]+ assert (scan.scan_dir / surface["receiptRefs"][0]).read_text() == f"Reviewed {label}.\n"+ assert sources[checkpoint.relative_to(scan.scan_dir).as_posix()] == workbench_api[+ "saved_results"+ ]._digest(json.loads(original))+ assert checkpoint.read_bytes() == original++def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts(workbench_api, workbench_db, saturated_scan):@@ -125,6 +419,148 @@ def test_saturated_finish_retains_failed_discovery_diagnostic(assert_published_aggregate(scan)[email protected]("stop", ["deadline", "run-limit", "saturated"])+def test_unfinished_checkpoints_after_a_completed_reduction(+ workbench_api, workbench_db, saturated_scan, stop+):+ scan = saturated_scan+ result = add_worker(workbench_db, scan, status="canceled")+ recovered = copy.deepcopy(scan.findings[0])+ recovered["identity"]["anchor"] = "unfinished-upload-review"+ recovered["title"] = "Separate upload handler bypasses containment"+ recovered["provenance"] = {"source": "local_plugin"}+ duplicate = copy.deepcopy(scan.findings[0])+ duplicate["severity"]["level"] = "critical"+ duplicate["remediation"] = "Unreduced remediation from an unfinished review."+ pending = {+ "id": "pending-upload",+ "reason": "The remaining upload paths need review.",+ "paths": ["subdir/extract.py"],+ "surfaceIds": ["upload"],+ }+ checkpoint = write_checkpoint(+ result.parent / "checkpoints",+ saved_draft(+ scan.scan_id,+ findings=[recovered, duplicate],+ deferred=[pending],+ surfaces=[+ {+ "id": "upload",+ "label": "Upload handler",+ "disposition": "needs_follow_up",+ "receiptRefs": [],+ }+ ],+ ),+ )+ source_bytes = checkpoint.read_bytes()+ # Earlier parent, discovery, and reducer drafts must not undo reduction.+ discarded = copy.deepcopy(recovered)+ discarded["identity"]["anchor"] = "discarded-during-reduction"+ for directory in (+ scan.scan_dir,+ scan.scan_dir / "workers" / scan.completed_worker_id,+ scan.scan_dir / "workers" / scan.reducer_id,+ ):+ discarded_checkpoint = write_checkpoint(+ directory / "checkpoints",+ saved_draft(scan.scan_id, findings=[discarded]),+ )+ if directory == scan.scan_dir:+ head = directory / "checkpoint-head.json"+ head.write_text(json.dumps({"checkpoint": discarded_checkpoint.name}))+ observed = (+ max(+ (directory / name).stat().st_mtime_ns+ for name in ("scan-manifest.json", "findings.json", "coverage.json")+ )+ + 1_000_000_000+ )+ for path in (discarded_checkpoint, head):+ os.utime(path, ns=(observed, observed))+ with workbench_db:+ if stop == "deadline":+ workbench_db.execute(+ "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' "+ "WHERE scan_id = ?",+ (scan.scan_id,),+ )+ elif stop == "run-limit":+ workbench_db.execute(+ "UPDATE deep_scan_runs SET discovery_runs_dispatched = max_discovery_runs "+ "WHERE scan_id = ?",+ (scan.scan_id,),+ )++ reason = "saturated" if stop == "saturated" else "capped"+ finished = finish(workbench_api, workbench_db, scan, terminal_reason=reason)++ assert finished["status"] == "succeeded"+ worker = next(item for item in finished["workers"] if item["id"] == result.parent.name)+ assert worker["status"] == "canceled"+ assert worker["completionSequence"] is None+ row = workbench_db.execute(+ "SELECT status, seal_manifest_digest FROM scans WHERE id = ?", (scan.scan_id,)+ ).fetchone()+ assert tuple(row) == ("running", None)+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())+ assert "sealedAt" not in manifest["scan"]+ completed = complete(workbench_api, workbench_db, scan)+ assert completed["progress"]["status"] == "complete"+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ expected_anchors = {scan.findings[0]["identity"]["anchor"]}+ if stop == "deadline":+ expected_anchors.add(recovered["identity"]["anchor"])+ retained = next(item for item in findings if item["title"] == recovered["title"])+ assert retained["provenance"]["workerId"] == result.parent.name+ assert (+ checkpoint.relative_to(scan.scan_dir).as_posix() in manifest["scan"]["preservedSources"]+ )+ assert {item["identity"]["anchor"] for item in findings} == expected_anchors+ canonical = next(item for item in findings if item["title"] == scan.findings[0]["title"])+ for field in ("severity", "confidence", "remediation"):+ assert canonical[field] == scan.findings[0][field]+ assert (+ canonical["provenance"]["sourceFindings"]+ == scan.findings[0]["provenance"]["sourceFindings"]+ )+ if stop == "deadline":+ assert any(+ item["remediation"] == duplicate["remediation"]+ for item in canonical["provenance"]["previousFindings"]+ )+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())+ assert any(item["reason"] == pending["reason"] for item in coverage["deferred"]) == (+ stop == "deadline"+ )+ assert any(item["disposition"] == "needs_follow_up" for item in coverage["surfaces"]) == (+ stop == "deadline"+ )+ assert checkpoint.read_bytes() == source_bytes+ names = ("scan-manifest.json", "findings.json", "coverage.json", "report.md")+ sealed = {name: (scan.scan_dir / name).read_bytes() for name in names}+ finish(workbench_api, workbench_db, scan, terminal_reason=reason)+ complete(workbench_api, workbench_db, scan)+ assert {name: (scan.scan_dir / name).read_bytes() for name in names} == sealed+++def test_deadline_after_all_reviews_complete_keeps_reduced_coverage(+ workbench_api, workbench_db, saturated_scan+):+ scan = saturated_scan+ with workbench_db:+ workbench_db.execute(+ "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?",+ (scan.scan_id,),+ )++ finish(workbench_api, workbench_db, scan, terminal_reason="capped")+ complete(workbench_api, workbench_db, scan)++ assert_published_aggregate(scan)++@pytest.mark.parametrize(("terminal_reason", "kind"),[("saturated", "setup"), ("saturated", "dedup"), ("capped", "discovery")],@@ -156,14 +592,21 @@ def test_finish_preserves_other_worker_failure_checks(@pytest.mark.parametrize("existing_head", [False, True])[email protected]("with_reduction", [False, True], ids=["zero-accepted", "reduced"])def test_rejected_deadline_publication_does_not_leave_parent_observations(- workbench_api, workbench_db, publication_scan, existing_head+ workbench_api, workbench_db, publication_scan, request, existing_head, with_reduction):- scan = publication_scan()+ scan = request.getfixturevalue("saturated_scan") if with_reduction else publication_scan()canonical_names = ("scan-manifest.json", "findings.json", "coverage.json")for name in canonical_names:os.utime(scan.scan_dir / name, ns=(1_000_000_000, 1_000_000_000))pending = {"id": "authorization-review", "reason": "Review the remaining authorization path."}+ if with_reduction:+ result = add_worker(workbench_db, scan, status="canceled")+ write_checkpoint(+ result.parent / "checkpoints",+ saved_draft(scan.scan_id, deferred=[pending]),+ )if existing_head:checkpoint = write_checkpoint(scan.scan_dir / "checkpoints",diff --git a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py b/plugins/codex-security/tests/test_workbench_checkpoint_heads.pyindex 125dcbd4c..014f294e9 100644--- a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py+++ b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py@@ -425,14 +425,15 @@ def test_recovery_keeps_generic_work_that_collides_with_candidate_aliases(repeated = replay_saved_results(saved, replay, tmp_path, scan_id, binding, workers)assert first[2] == replay[2] == repeated[2]for result in (first, replay, repeated):- assert [row for row in result[2]["deferred"] if row["id"] != "scan-stopped"] == [generic]- assert (- sum(- row["label"] == "Generic review" and row["disposition"] == "needs_follow_up"- for row in result[2]["surfaces"]- )- == 1- )+ pending_surfaces = [+ row+ for row in result[2]["surfaces"]+ if row["label"] == "Generic review" and row["disposition"] == "needs_follow_up"+ ]+ assert len(pending_surfaces) == 1+ assert [row for row in result[2]["deferred"] if row["id"] != "scan-stopped"] == [+ {**generic, "surfaceIds": [pending_surfaces[0]["id"]]}+ ]assert any(row.get("candidateId") == "candidate-a" and row["disposition"] == outcomefor row in result[2]["surfaces"]diff --git a/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.pynew file mode 100644index 000000000..502bf5b9f--- /dev/null+++ b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py@@ -0,0 +1,145 @@+from __future__ import annotations++import copy+import json+import os+from types import SimpleNamespace++import pytest+from test_deep_scan_successful_publication import complete+from test_deep_scan_successful_publication import publication_scan as publication_scan+from workbench_test_support import saved_draft, write_checkpoint+++ ("label", "error"),+ [+ pytest.param(None, None, id="absent-rejection"),+ pytest.param("Reviewed candidate", None, id="valid-rejection"),+ pytest.param("Invalid \ud800", "well-formed Unicode", id="surrogate-label"),+ pytest.param("\u0085", "label: expected a non-empty string", id="nel-label"),+ pytest.param(" ", "label: expected a non-empty string", id="blank-label"),+ ],+)+def test_stopped_scan_requires_valid_rejection_evidence_before_sealing(+ workbench_api, workbench_db, publication_scan, label, error+):+ scan = publication_scan(mode="standard")+ report_path = scan.scan_dir / "report.md"+ report_path.unlink()+ finding = copy.deepcopy(scan.findings[0])+ finding["provenance"] = {"source": "local_plugin", "candidateId": "candidate-a"}+ checkpoint = write_checkpoint(+ scan.scan_dir / "checkpoints", saved_draft(scan.scan_id, findings=[finding])+ )+ os.utime(checkpoint, ns=(100, 100))+ checkpoint_bytes = checkpoint.read_bytes()+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))+ scan.coverage["surfaces"] = (+ []+ if label is None+ else [+ {+ "id": "rejection",+ "candidateId": "candidate-a",+ "label": label,+ "disposition": "rejected",+ "receiptRefs": [],+ }+ ]+ )+ coverage_path = scan.scan_dir / "coverage.json"+ coverage_path.write_text(json.dumps(scan.coverage))+ coverage_bytes = coverage_path.read_bytes()++ workbench_api["saved_results"].fail_scan(+ workbench_api["_WORKBENCH_DB_CONTEXT"],+ workbench_db,+ SimpleNamespace(+ scan_id=scan.scan_id,+ cost_json=None,+ claim_token=None,+ message="Synthetic interruption",+ ),+ )++ row = workbench_db.execute(+ "SELECT status, completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,)+ ).fetchone()+ assert row["status"] == "failed"+ assert checkpoint.read_bytes() == checkpoint_bytes+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ warnings = json.loads(row["completion_warnings_json"])+ if error:+ assert not manifest.get("sealedAt")+ assert not report_path.exists()+ assert coverage_path.read_bytes() == coverage_bytes+ assert any(+ "result publication needs follow-up" in warning and error in warning+ for warning in warnings+ )+ else:+ assert manifest["sealedAt"]+ assert report_path.is_file()+ assert warnings == []+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]+ assert [item["title"] for item in findings] == ([finding["title"]] if label is None else [])+ surfaces = json.loads(coverage_path.read_text())["surfaces"]+ assert [(item.get("candidateId"), item["disposition"]) for item in surfaces] == (+ [] if label is None else [("candidate-a", "rejected")]+ )+++ ("reason", "invalid"),+ [+ pytest.param("Review completed.", False, id="valid-closure"),+ pytest.param("Invalid \ud800", True, id="surrogate-reason"),+ ],+)+def test_standard_completion_requires_valid_closure_evidence_before_sealing(+ workbench_api, workbench_db, publication_scan, reason, invalid+):+ scan = publication_scan(mode="standard")+ report_path = scan.scan_dir / "report.md"+ report_path.unlink()+ checkpoint = write_checkpoint(+ scan.scan_dir / "checkpoints",+ saved_draft(+ scan.scan_id,+ deferred=[{"id": "review", "reason": "Review authorization remains pending."}],+ ),+ )+ os.utime(checkpoint, ns=(100, 100))+ checkpoint_bytes = checkpoint.read_bytes()+ scan.coverage["deferred"] = []+ closure = {"id": "review", "reason": reason}+ scan.coverage["resolvedDeferred"] = [closure]+ coverage_path = scan.scan_dir / "coverage.json"+ coverage_path.write_text(json.dumps(scan.coverage))+ coverage_bytes = coverage_path.read_bytes()++ if invalid:+ with pytest.raises(SystemExit, match="well-formed Unicode"):+ complete(workbench_api, workbench_db, scan)+ else:+ completed = complete(workbench_api, workbench_db, scan)+ assert completed["warnings"] == []++ assert checkpoint.read_bytes() == checkpoint_bytes+ status = workbench_db.execute(+ "SELECT status FROM scans WHERE id = ?", (scan.scan_id,)+ ).fetchone()[0]+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]+ if invalid:+ assert status == "running"+ assert not manifest.get("sealedAt")+ assert not report_path.exists()+ assert coverage_path.read_bytes() == coverage_bytes+ else:+ assert status == "complete"+ assert manifest["sealedAt"]+ assert report_path.is_file()+ coverage = json.loads(coverage_path.read_text())+ assert coverage["deferred"] == []+ assert coverage["resolvedDeferred"] == [closure]