Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(findings): preserve valid evidence during stopped scan recovery (#171)

Hai Duong0ac43dc8c758
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index ded753096..9edf8e3c5 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -70,6 +70,7 @@
"scripts/finding_preview.py",
"scripts/launch_codex_security_mcp",
"scripts/launch_codex_security_mcp.cmd",
+ "scripts/legacy_findings.py",
"scripts/posix_scan_local_files.py",
"scripts/report_projection.py",
"scripts/review_state.py",
diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py
index babe102f5..705238561 100644
--- a/plugins/codex-security/scripts/finalize_scan_contract.py
+++ b/plugins/codex-security/scripts/finalize_scan_contract.py
@@ -29,6 +29,10 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
import report_projection
import servicenow_avit
import threat_model_projection
+from legacy_findings import (
+ _legacy_sealed_findings_for_validation,
+ _normalize_legacy_string_list_fields,
+)
from posix_scan_local_files import (
descriptor_relative_writes_available as _descriptor_relative_writes_available,
)
@@ -1119,6 +1123,48 @@ def _finding_strength(finding: dict[str, Any]) -> tuple[int, int, int]:
)
+def _recover_code_evidence(
+ finding: dict[str, Any],
+ item_schema: dict[str, Any],
+ context: str,
+ warnings: list[str],
+) -> dict[str, Any]:
+ evidence = finding["codeEvidence"]
+ # Validate before the legacy adapter resolves references and deduplicates IDs.
+ by_id: dict[str, dict[str, Any] | None] = {}
+ if isinstance(evidence, list):
+ for index, row in enumerate(evidence):
+ location = f"{context}.codeEvidence[{index}]"
+ try:
+ _validate_schema_node(row, item_schema, location)
+ _require_safe_json_value(row, location)
+ _require_str(row, "code", location)
+ _require_safe_relative_path(row["path"], f"{location}.path")
+ evidence_id = row["id"]
+ if evidence_id in by_id:
+ if not _schema_values_equal(by_id[evidence_id], row):
+ # Conflicting sources cannot both identify one evidence row.
+ by_id[evidence_id] = None
+ raise ContractError(f"{location}.id: duplicate code-evidence id")
+ by_id[evidence_id] = row
+ except ContractError as exc:
+ warnings.append(f"Skipped malformed codeEvidence: {exc}.")
+ rows = [row for row in by_id.values() if row is not None]
+ if len(rows) == len(evidence):
+ return finding
+ else:
+ warnings.append(
+ f"Skipped malformed codeEvidence: {context}.codeEvidence: expected an array."
+ )
+ rows = []
+ repaired = finding.copy()
+ if rows:
+ repaired["codeEvidence"] = rows
+ else:
+ repaired.pop("codeEvidence")
+ return repaired
+
+
def _recover_unsealed_findings(
manifest: dict[str, Any],
findings: dict[str, Any],
@@ -1153,11 +1199,23 @@ def _recover_unsealed_findings(
discarded: list[str] = []
finding_positions: dict[str, int] = {}
writeup_paths: set[str] = set()
+ evidence_schema = None
for index, finding in enumerate(_require_list(findings, "findings", "findings")):
context = f"findings.findings[{index}]"
+ evidence_warnings: list[str] = []
try:
if not isinstance(finding, dict):
raise ContractError(f"{context}: expected an object")
+ if "codeEvidence" in finding:
+ if evidence_schema is None:
+ evidence_schema = (
+ _current_finding_definition(schema_dir, "codeEvidence")
+ if current_contract
+ else finding_properties["codeEvidence"]["items"]
+ )
+ finding = _recover_code_evidence(
+ finding, evidence_schema, context, evidence_warnings
+ )
compatible_findings = _legacy_sealed_findings_for_validation({"findings": [finding]})[
"findings"
]
@@ -1172,6 +1230,10 @@ def _recover_unsealed_findings(
if "instance" in identity:
fields.append((identity, "instance", f"{context}.identity", "instance"))
normalized_fields = ["legacy finding details"] if normalized_legacy_details else []
+ taxonomy = finding.get("taxonomy")
+ if isinstance(taxonomy, dict) and "cwe" not in taxonomy:
+ taxonomy["cwe"] = []
+ normalized_fields.append("missing taxonomy.cwe to an empty list")
for parent, field, field_context, label in fields:
value = _require_str(parent, field, field_context)
if SLUG_RE.fullmatch(value):
@@ -1285,6 +1347,7 @@ def _recover_unsealed_findings(
if "writeup" in finding:
writeup_paths.add(finding["writeup"]["reportPath"])
+ warnings.extend(evidence_warnings)
if normalized_fields:
warnings.append(
f"Recovered finding {index + 1}: normalized {', '.join(normalized_fields)}."
@@ -2153,9 +2216,7 @@ def validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:
_validate_schema_node(payload, schema, schema_path.stem, schema)
-def _validate_current_findings(findings: dict[str, Any], schema_dir: Path) -> None:
- """Use the model's current finding contract for new completed results."""
-
+def _current_finding_definition(schema_dir: Path, name: str) -> dict[str, Any]:
draft = _read_json(schema_dir / "tools" / "scan-draft.schema.json")
common = _read_json(schema_dir / "definitions" / "artifact-common.schema.json")
documents = {document["$id"]: document for document in (draft, common)}
@@ -2174,7 +2235,12 @@ def _validate_current_findings(findings: dict[str, Any], schema_dir: Path) -> No
resolved = {**resolve(referenced), **resolved}
return resolved
- finding_schema = resolve(draft["$defs"]["finding"])
+ return resolve(draft["$defs"][name])
+
+
+def _validate_current_findings(findings: dict[str, Any], schema_dir: Path) -> None:
+ """Use the model's current finding contract for new completed results."""
+ finding_schema = _current_finding_definition(schema_dir, "finding")
for index, finding in enumerate(findings["findings"]):
# These fields belong to the host, not the authored finding contract.
authored = {
@@ -2185,146 +2251,6 @@ def _validate_current_findings(findings: dict[str, Any], schema_dir: Path) -> No
_validate_schema_node(authored, finding_schema, f"findings.findings[{index}]")
-def _filter_unknown_legacy_evidence_refs(section: dict[str, Any], evidence_ids: set[str]) -> None:
- for refs_field in ("evidenceRefs", "evidence_refs"):
- refs = section.get(refs_field)
- if isinstance(refs, list):
- section[refs_field] = [
- ref for ref in refs if isinstance(ref, str) and ref.strip() and ref in evidence_ids
- ]
-
-
-def _normalize_legacy_string_list_fields(section: dict[str, Any], fields: tuple[str, ...]) -> None:
- for field in fields:
- if field not in section:
- continue
- value = section[field]
- if isinstance(value, str):
- normalized = [value] if value.strip() else []
- elif isinstance(value, list):
- normalized = [item for item in value if isinstance(item, str) and item.strip()]
- else:
- normalized = []
- if normalized:
- section[field] = normalized
- else:
- section.pop(field)
-
-
-def _remove_unsupported_legacy_scalar_fields(
- section: dict[str, Any], fields: tuple[str, ...]
-) -> None:
- for field in fields:
- if field in section and (not isinstance(section[field], str) or section[field] == ""):
- section.pop(field)
-
-
-def _legacy_sealed_findings_for_validation(findings: dict[str, Any]) -> dict[str, Any]:
- compatible = copy.deepcopy(findings)
- finding_items = compatible.get("findings")
- if not isinstance(finding_items, list):
- return compatible
- for finding in finding_items:
- if not isinstance(finding, dict):
- continue
- canonical_evidence = finding.get("codeEvidence")
- canonical_evidence = canonical_evidence if isinstance(canonical_evidence, list) else []
- evidence_ids = {
- evidence["id"]
- for evidence in canonical_evidence
- if isinstance(evidence, dict) and isinstance(evidence.get("id"), str) and evidence["id"]
- }
- legacy_evidence = finding.get("code_evidence")
- if isinstance(legacy_evidence, list):
- compatible_legacy_evidence = []
- for evidence in report_projection.code_evidence_rows(legacy_evidence):
- evidence_id = evidence.get("id")
- if not isinstance(evidence_id, str) or not evidence_id.strip():
- continue
- if evidence_id in evidence_ids:
- continue
- evidence_ids.add(evidence_id)
- compatible_legacy_evidence.append(evidence)
- finding["code_evidence"] = compatible_legacy_evidence
- elif "code_evidence" in finding:
- finding.pop("code_evidence")
- for section_name, list_fields in (
- ("rootCause", ("evidenceRefs", "evidence_refs")),
- ("root_cause", ("evidenceRefs", "evidence_refs")),
- (
- "validation",
- (
- "assertions",
- "counterEvidence",
- "evidence",
- "evidenceRefs",
- "evidence_refs",
- "limitations",
- ),
- ),
- (
- "attackPath",
- (
- "assumptions",
- "blindspots",
- "controls",
- "evidenceRefs",
- "evidence_refs",
- "limitations",
- "preconditions",
- "steps",
- ),
- ),
- ):
- section = finding.get(section_name)
- if not isinstance(section, dict):
- continue
- _normalize_legacy_string_list_fields(section, list_fields)
- _filter_unknown_legacy_evidence_refs(section, evidence_ids)
- legacy_root_cause = finding.get("root_cause")
- if isinstance(legacy_root_cause, dict):
- _remove_unsupported_legacy_scalar_fields(
- legacy_root_cause, ("summary", "code", "language")
- )
- elif (
- "root_cause" in finding
- and legacy_root_cause is not None
- and (not isinstance(legacy_root_cause, str) or legacy_root_cause == "")
- ):
- finding.pop("root_cause")
- validation = finding.get("validation")
- if isinstance(validation, dict):
- _remove_unsupported_legacy_scalar_fields(
- validation, ("method", "status", "summary", "disposition", "result")
- )
- attack_path = finding.get("attackPath")
- if not isinstance(attack_path, dict):
- continue
- _remove_unsupported_legacy_scalar_fields(attack_path, ("summary",))
- for field in ("dataFlow", "data_flow", "dataflow", "reachability"):
- detail = attack_path.get(field)
- if isinstance(detail, dict):
- detail_scalar_fields = ("summary", "source", "sink", "outcome")
- if field == "reachability":
- detail_scalar_fields += ("attacker", "entrypoint")
- _remove_unsupported_legacy_scalar_fields(detail, detail_scalar_fields)
- _normalize_legacy_string_list_fields(
- detail, ("evidenceRefs", "evidence_refs", "transformations")
- )
- _filter_unknown_legacy_evidence_refs(detail, evidence_ids)
- if field == "reachability":
- _normalize_legacy_string_list_fields(detail, ("preconditions",))
- elif not isinstance(detail, str) or detail == "":
- attack_path.pop(field, None)
- for field in ("impact", "likelihood"):
- detail = attack_path.get(field)
- if isinstance(detail, dict):
- _remove_unsupported_legacy_scalar_fields(detail, ("level", "rationale", "why"))
- elif detail is not None and (not isinstance(detail, str) or detail == ""):
- attack_path.pop(field)
- return compatible
-
-
def _validate_canonical_schemas_before_projection(
manifest: dict[str, Any],
findings: dict[str, Any],
diff --git a/plugins/codex-security/scripts/legacy_findings.py b/plugins/codex-security/scripts/legacy_findings.py
new file mode 100644
index 000000000..71e97f457
--- /dev/null
+++ b/plugins/codex-security/scripts/legacy_findings.py
@@ -0,0 +1,148 @@
+"""Normalize legacy finding details for scan contract validation."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+import report_projection
+
+
+def _filter_unknown_legacy_evidence_refs(section: dict[str, Any], evidence_ids: set[str]) -> None:
+ for refs_field in ("evidenceRefs", "evidence_refs"):
+ refs = section.get(refs_field)
+ if isinstance(refs, list):
+ section[refs_field] = [
+ ref for ref in refs if isinstance(ref, str) and ref.strip() and ref in evidence_ids
+ ]
+
+
+def _normalize_legacy_string_list_fields(section: dict[str, Any], fields: tuple[str, ...]) -> None:
+ for field in fields:
+ if field not in section:
+ continue
+ value = section[field]
+ if isinstance(value, str):
+ normalized = [value] if value.strip() else []
+ elif isinstance(value, list):
+ normalized = [item for item in value if isinstance(item, str) and item.strip()]
+ else:
+ normalized = []
+ if normalized:
+ section[field] = normalized
+ else:
+ section.pop(field)
+
+
+def _remove_unsupported_legacy_scalar_fields(
+ section: dict[str, Any], fields: tuple[str, ...]
+) -> None:
+ for field in fields:
+ if field in section and (not isinstance(section[field], str) or section[field] == ""):
+ section.pop(field)
+
+
+def _legacy_sealed_findings_for_validation(findings: dict[str, Any]) -> dict[str, Any]:
+ compatible = copy.deepcopy(findings)
+ finding_items = compatible.get("findings")
+ if not isinstance(finding_items, list):
+ return compatible
+ for finding in finding_items:
+ if not isinstance(finding, dict):
+ continue
+ canonical_evidence = finding.get("codeEvidence")
+ canonical_evidence = canonical_evidence if isinstance(canonical_evidence, list) else []
+ evidence_ids = {
+ evidence["id"]
+ for evidence in canonical_evidence
+ if isinstance(evidence, dict) and isinstance(evidence.get("id"), str) and evidence["id"]
+ }
+ legacy_evidence = finding.get("code_evidence")
+ if isinstance(legacy_evidence, list):
+ compatible_legacy_evidence = []
+ for evidence in report_projection.code_evidence_rows(legacy_evidence):
+ evidence_id = evidence.get("id")
+ if not isinstance(evidence_id, str) or not evidence_id.strip():
+ continue
+ if evidence_id in evidence_ids:
+ continue
+ evidence_ids.add(evidence_id)
+ compatible_legacy_evidence.append(evidence)
+ finding["code_evidence"] = compatible_legacy_evidence
+ elif "code_evidence" in finding:
+ finding.pop("code_evidence")
+ for section_name, list_fields in (
+ ("rootCause", ("evidenceRefs", "evidence_refs")),
+ ("root_cause", ("evidenceRefs", "evidence_refs")),
+ (
+ "validation",
+ (
+ "assertions",
+ "counterEvidence",
+ "evidence",
+ "evidenceRefs",
+ "evidence_refs",
+ "limitations",
+ ),
+ ),
+ (
+ "attackPath",
+ (
+ "assumptions",
+ "blindspots",
+ "controls",
+ "evidenceRefs",
+ "evidence_refs",
+ "limitations",
+ "preconditions",
+ "steps",
+ ),
+ ),
+ ):
+ section = finding.get(section_name)
+ if not isinstance(section, dict):
+ continue
+ _normalize_legacy_string_list_fields(section, list_fields)
+ _filter_unknown_legacy_evidence_refs(section, evidence_ids)
+ legacy_root_cause = finding.get("root_cause")
+ if isinstance(legacy_root_cause, dict):
+ _remove_unsupported_legacy_scalar_fields(
+ legacy_root_cause, ("summary", "code", "language")
+ )
+ elif (
+ "root_cause" in finding
+ and legacy_root_cause is not None
+ and (not isinstance(legacy_root_cause, str) or legacy_root_cause == "")
+ ):
+ finding.pop("root_cause")
+ validation = finding.get("validation")
+ if isinstance(validation, dict):
+ _remove_unsupported_legacy_scalar_fields(
+ validation, ("method", "status", "summary", "disposition", "result")
+ )
+ attack_path = finding.get("attackPath")
+ if not isinstance(attack_path, dict):
+ continue
+ _remove_unsupported_legacy_scalar_fields(attack_path, ("summary",))
+ for field in ("dataFlow", "data_flow", "dataflow", "reachability"):
+ detail = attack_path.get(field)
+ if isinstance(detail, dict):
+ detail_scalar_fields = ("summary", "source", "sink", "outcome")
+ if field == "reachability":
+ detail_scalar_fields += ("attacker", "entrypoint")
+ _remove_unsupported_legacy_scalar_fields(detail, detail_scalar_fields)
+ _normalize_legacy_string_list_fields(
+ detail, ("evidenceRefs", "evidence_refs", "transformations")
+ )
+ _filter_unknown_legacy_evidence_refs(detail, evidence_ids)
+ if field == "reachability":
+ _normalize_legacy_string_list_fields(detail, ("preconditions",))
+ elif not isinstance(detail, str) or detail == "":
+ attack_path.pop(field, None)
+ for field in ("impact", "likelihood"):
+ detail = attack_path.get(field)
+ if isinstance(detail, dict):
+ _remove_unsupported_legacy_scalar_fields(detail, ("level", "rationale", "why"))
+ elif detail is not None and (not isinstance(detail, str) or detail == ""):
+ attack_path.pop(field)
+ return compatible
diff --git a/plugins/codex-security/scripts/workbench/retained_records.py b/plugins/codex-security/scripts/workbench/retained_records.py
index 5b12556b4..4e84a755b 100644
--- a/plugins/codex-security/scripts/workbench/retained_records.py
+++ b/plugins/codex-security/scripts/workbench/retained_records.py
@@ -17,7 +17,6 @@ from finalize_scan_contract import (
_recover_unsealed_coverage,
_recover_unsealed_findings,
_require_safe_json_value,
- _validate_current_findings,
_validate_schema_node,
finding_candidate_id,
)
@@ -280,15 +279,8 @@ def _recover_preserved_worker_source(
schema_dir,
scan_dir / directory,
source_warnings,
+ current_contract=True,
)
- if recovered["findings"]:
- try:
- _validate_current_findings(recovered, schema_dir)
- except ContractError as exc:
- warning = f"Skipped malformed finding: {exc}."
- source_warnings.append(warning)
- discarded.append(warning)
- recovered["findings"] = []
if discarded:
discarded_findings.extend(discarded)
for recovered_finding in recovered["findings"]:
diff --git a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
index d6a1e1249..80df432dd 100644
--- a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
+++ b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
@@ -148,7 +148,7 @@ def test_deadline_recovers_partial_worker_findings_before_completion(tmp_path, c
if canonical:
canonical_parent(scan, saved_draft(scan.scan_id))
- defects = {"severity": None, "codeEvidence": [None], "rootCause": {"summary": 7}}
+ defects = {"severity": None, "confidence": {"level": "high"}, "rootCause": {"summary": 7}}
partials = []
for same_identity in (False, True):
for field, value in defects.items():
@@ -195,6 +195,41 @@ def test_deadline_recovers_partial_worker_findings_before_completion(tmp_path, c
assert_retained_finding(scan)
+def test_deadline_preserves_worker_evidence_when_one_excerpt_is_malformed(tmp_path):
+ scan = deadline_fixture(tmp_path, "valid")
+ revised = copy.deepcopy(scan.finding)
+ revised["severity"]["level"] = "critical"
+ excerpt = {**revised["codeEvidence"][0], "id": "worker-excerpt"}
+ revised["codeEvidence"] = [excerpt, None]
+ revised["rootCause"]["evidenceRefs"] = [excerpt["id"], "missing-excerpt"]
+ revised["attackPath"]["evidenceRefs"] = [excerpt["id"]]
+ distinct = copy.deepcopy(revised)
+ distinct["identity"]["anchor"] = "partial-code-evidence"
+ distinct["provenance"]["candidateId"] = "partial-code-evidence"
+ sources = {
+ path: path.read_bytes() for path in append_worker_findings(scan, [revised, distinct])
+ }
+
+ assert finish(scan)["deepScan"]["terminalReason"] == "capped"
+ run_workbench(
+ scan.state, "complete-scan", "--scan-id", scan.scan_id, environment=scan.environment
+ )
+
+ retained = json.loads((scan.directory / "findings.json").read_text())["findings"]
+ by_anchor = {row["identity"]["anchor"]: row for row in retained}
+ assert set(by_anchor) == {scan.finding["identity"]["anchor"], distinct["identity"]["anchor"]}
+ for row in retained:
+ assert row["severity"] == revised["severity"]
+ assert row["codeEvidence"] == [excerpt]
+ assert row["rootCause"] == {**revised["rootCause"], "evidenceRefs": [excerpt["id"]]}
+ (previous,) = by_anchor[scan.finding["identity"]["anchor"]]["provenance"]["previousFindings"]
+ assert previous["severity"] == scan.finding["severity"]
+ assert previous["codeEvidence"] == scan.finding["codeEvidence"]
+ warnings = get_scan(scan.state, scan.scan_id)["scan"]["warnings"]
+ assert sum("Skipped malformed codeEvidence" in warning for warning in warnings) == 2
+ assert all(path.read_bytes() == contents for path, contents in sources.items())
+
+
@pytest.mark.parametrize("checkpoint_kind", ["empty", "valid", "invalid"])
def test_coordinator_deadline_preserves_only_usable_worker_checkpoints(
tmp_path: Path, checkpoint_kind: str
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index 7beeee7eb..309ad78f4 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -20,6 +20,14 @@ from workbench_test_support import ScanFixtureTestCase, windows_file_backend
EXAMPLE_DIR = Path(__file__).resolve().parent.parent / "examples" / "legacy-completed-scan"
CANONICAL_FILES = ("scan-manifest.json", "findings.json", "coverage.json")
+SOURCE_EVIDENCE = {
+ "id": "source",
+ "label": "Source",
+ "path": "src/extract.py",
+ "startLine": 41,
+ "code": "read_input()",
+ "explanation": "Reads the input.",
+}
class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@@ -1563,7 +1571,9 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_new_findings_use_current_fields_without_coercion(self) -> None:
invalid_details = [
+ {"codeEvidence": [{"id": "incomplete"}]},
{"code_evidence": []},
+ {"taxonomy": {"category": "path-traversal"}},
{"root_cause": "A legacy cause."},
{"rootCause": {"summary": "Cause.", "code": "legacy()"}},
{"rootCause": {"summary": "Cause.", "language": "python"}},
@@ -2214,16 +2224,107 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
["Recovered finding 2: retained stronger duplicate logical finding."],
)
- def test_stopped_recovery_preserves_finding_with_malformed_optional_remediation_fields(
- self,
- ) -> None:
+ def test_stopped_recovery_preserves_supported_evidence_and_its_references(self) -> None:
+ finding = self.findings["findings"][0]
+ legacy = {"id": "legacy", "code": "inspect_input()"}
+ duplicate = {**SOURCE_EVIDENCE, "id": "ambiguous"}
+ finding["codeEvidence"] = [
+ SOURCE_EVIDENCE,
+ {"id": "legacy"},
+ {**SOURCE_EVIDENCE, "id": "outside", "path": "../other.py"},
+ duplicate,
+ {**duplicate, "startLine": 99},
+ ]
+ finding["code_evidence"] = [legacy]
+ refs = ["outside", "legacy", {}, "ambiguous", "source"]
+ finding["rootCause"] = {"summary": "Source review.", "evidenceRefs": refs}
+ finding["validation"] = {"evidence_refs": refs}
+ finding["attackPath"] = {"dataFlow": {"evidenceRefs": refs}}
+ self.write_scan()
+
+ prepared = FINALIZER._prepare_scan_finalization(
+ self.scan_dir,
+ completion_binding={**self.completion_binding(), "status": "interrupted"},
+ recover_unsealed=True,
+ )
+ FINALIZER._write_prepared_scan_finalization(prepared)
+ (saved,) = self.read_json("findings.json")["findings"]
+ self.assertEqual(saved["codeEvidence"], [SOURCE_EVIDENCE])
+ self.assertEqual(saved["code_evidence"], [legacy])
+ surviving_refs = ["legacy", "source"]
+ self.assertEqual(saved["rootCause"]["evidenceRefs"], surviving_refs)
+ self.assertEqual(saved["validation"]["evidence_refs"], surviving_refs)
+ self.assertEqual(saved["attackPath"]["dataFlow"]["evidenceRefs"], surviving_refs)
+ self.assertEqual(saved["taxonomy"], finding["taxonomy"])
+ self.assertTrue(any("codeEvidence" in warning for warning in prepared.warnings))
+
+ def test_recovery_uses_the_selected_evidence_schema(self) -> None:
+ evidence = [{**SOURCE_EVIDENCE, "explanation": " "}]
+ for current in (False, True):
+ with self.subTest(current_contract=current):
+ draft = copy.deepcopy(self.findings)
+ draft["findings"][0]["codeEvidence"] = evidence
+ discarded = FINALIZER._recover_unsealed_findings(
+ self.manifest,
+ draft,
+ EXAMPLE_DIR.parents[1] / "schemas",
+ self.scan_dir,
+ [],
+ current_contract=current,
+ )
+ self.assertEqual(discarded, [])
+ (saved,) = draft["findings"]
+ self.assertEqual(saved.get("codeEvidence"), None if current else evidence)
+
+ def test_stopped_recovery_backfills_missing_cwe_and_rejects_invalid_findings(self) -> None:
+ missing_cwe = copy.deepcopy(self.finding)
+ del missing_cwe["taxonomy"]["cwe"]
+ invalid_cwe = copy.deepcopy(self.finding)
+ invalid_cwe["taxonomy"]["cwe"] = "CWE-22"
+ missing_identity = copy.deepcopy(missing_cwe)
+ del missing_identity["identity"]
+ missing_rationale = copy.deepcopy(missing_cwe)
+ del missing_rationale["confidence"]["rationale"]
+ missing_rationale["codeEvidence"] = "A source excerpt."
+ self.findings["findings"] = [missing_cwe, invalid_cwe, missing_identity, missing_rationale]
+ self.write_scan()
+
+ prepared = FINALIZER._prepare_scan_finalization(
+ self.scan_dir,
+ completion_binding={**self.completion_binding(), "status": "canceled"},
+ recover_unsealed=True,
+ )
+ FINALIZER._write_prepared_scan_finalization(prepared)
+ (saved,) = self.read_json("findings.json")["findings"]
+ self.assertEqual(saved["title"], missing_cwe["title"])
+ self.assertEqual(saved["taxonomy"], {**missing_cwe["taxonomy"], "cwe": []})
+ self.assertFalse(
+ any(w.startswith("Skipped malformed codeEvidence") for w in prepared.warnings)
+ )
+ self.assertTrue(
+ any(
+ "finding 1" in warning and "cwe" in warning.lower() for warning in prepared.warnings
+ )
+ )
+ for number, field in ((2, "cwe"), (3, "identity"), (4, "rationale")):
+ self.assertFalse(any(f"Recovered finding {number}:" in w for w in prepared.warnings))
+ self.assertTrue(
+ any(
+ f"Skipped malformed finding {number}:" in warning and field in warning
+ for warning in prepared.warnings
+ ),
+ prepared.warnings,
+ )
+
+ def test_stopped_recovery_preserves_finding_with_malformed_optional_fields(self) -> None:
for field, value in (
+ ("codeEvidence", "A source excerpt."),
("remediationTests", "Reject unsafe archive paths."),
("remediationTests", [{"test": "Reject unsafe archive paths."}]),
("preventiveControls", "Validate every archive path."),
):
with self.subTest(field=field, value=value):
- self.findings["findings"][0][field] = value
+ self.findings["findings"] = [{**self.finding, field: value}]
self.write_scan()
prepared = FINALIZER._prepare_scan_finalization(
@@ -2241,7 +2342,6 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
for warning in prepared.warnings
)
)
- self.findings["findings"][0].pop(field)
def test_recovery_publishes_findings_without_unsafe_deferred_paths(self) -> None:
valid = {"id": "review", "reason": "Repository review is incomplete.", "paths": ["."]}
diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py
index b6775ebdb..7646cfd9a 100644
--- a/plugins/codex-security/tests/test_workbench_completion_binding.py
+++ b/plugins/codex-security/tests/test_workbench_completion_binding.py
@@ -1353,17 +1353,24 @@ def test_completion_rejects_bad_findings_and_failed_scan_preserves_valid_rows(
stopped = get_scan(state_dir, scan_id)["scan"]
assert stopped["progress"]["status"] == "failed"
- assert stopped["findingCount"] == 1
+ assert stopped["findingCount"] == 2
retained = json.loads((scan_dir / "findings.json").read_text())["findings"]
- assert len(retained) == 1
- assert retained[0]["summary"] == valid["summary"]
+ by_anchor = {row["identity"]["anchor"]: row for row in retained}
+ assert set(by_anchor) == {valid["identity"]["anchor"], "invalid-evidence-id"}
+ clean = by_anchor[valid["identity"]["anchor"]]
+ assert clean["summary"] == valid["summary"]
+ assert clean["codeEvidence"] == valid["codeEvidence"]
+ repaired = by_anchor["invalid-evidence-id"]
+ assert "codeEvidence" not in repaired
+ assert repaired["locations"] == valid["locations"]
+ assert repaired["rootCause"] == {**invalid_evidence_id["rootCause"], "evidenceRefs": []}
assert json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["status"] == "failed"
assert (scan_dir / "report.md").is_file()
sarif = json.loads((scan_dir / "exports/results.sarif").read_text())
assert sarif["runs"][0]["invocations"][0]["executionSuccessful"] is False
warnings = json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["warnings"]
- assert warnings
+ assert any("Skipped malformed codeEvidence" in warning for warning in warnings)
notifications = sarif["runs"][0]["invocations"][0]["toolExecutionNotifications"]
assert all(
{"level": "warning", "message": {"text": warning}} in notifications for warning in warnings