Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(reports): share projection and validation logic (#2183)

Michael D'Angelo2344b21828ca
diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py
index 3ae1c062c..8f2d3d722 100644
--- a/plugins/codex-security/scripts/finalize_scan_contract.py
+++ b/plugins/codex-security/scripts/finalize_scan_contract.py
@@ -2205,17 +2205,9 @@ def _legacy_sealed_findings_for_validation(findings: dict[str, Any]) -> dict[str
legacy_evidence = finding.get("code_evidence")
if isinstance(legacy_evidence, list):
compatible_legacy_evidence = []
- for evidence in legacy_evidence:
- if not isinstance(evidence, dict):
- continue
+ for evidence in report_projection.code_evidence_rows(legacy_evidence):
evidence_id = evidence.get("id")
- evidence_code = evidence.get("code")
- if (
- not isinstance(evidence_id, str)
- or not evidence_id.strip()
- or not isinstance(evidence_code, str)
- or not evidence_code.strip()
- ):
+ if not isinstance(evidence_id, str) or not evidence_id.strip():
continue
if evidence_id in evidence_ids:
continue
@@ -2524,23 +2516,17 @@ def _finding_evidence_strength(finding: dict[str, Any]) -> int:
section = finding.get(section_name)
if not isinstance(section, dict):
continue
- for evidence_name in ("codeEvidence", "code_evidence"):
- embedded = section.get(evidence_name)
- if not isinstance(embedded, list):
+ for item in report_projection.embedded_code_evidence(section):
+ code = item["code"]
+ if code in seen_codes:
continue
- for item in embedded:
- if not isinstance(item, dict):
+ evidence_id = item.get("id")
+ if isinstance(evidence_id, str) and evidence_id.strip():
+ if evidence_id in seen_ids:
continue
- code = item.get("code")
- if not isinstance(code, str) or not code.strip() or code in seen_codes:
- continue
- evidence_id = item.get("id")
- if isinstance(evidence_id, str) and evidence_id.strip():
- if evidence_id in seen_ids:
- continue
- seen_ids.add(evidence_id)
- seen_codes.add(code)
- strength += 1
+ seen_ids.add(evidence_id)
+ seen_codes.add(code)
+ strength += 1
code = section.get("code")
if isinstance(code, str) and code.strip() and code not in seen_codes:
seen_codes.add(code)
diff --git a/plugins/codex-security/scripts/finding_preview.py b/plugins/codex-security/scripts/finding_preview.py
index cab0dce67..d943a9cbe 100644
--- a/plugins/codex-security/scripts/finding_preview.py
+++ b/plugins/codex-security/scripts/finding_preview.py
@@ -7,12 +7,12 @@ import json
import re
import sys
from bisect import bisect_right
-from itertools import chain, islice
+from itertools import islice
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
-from report_projection import merged_root_cause
+from report_projection import code_evidence_rows, merged_root_cause
from workbench_constants import (
FINDING_ATTACK_PATH_PREVIEW_BYTES,
FINDING_CODE_EVIDENCE_LIMIT,
@@ -35,14 +35,12 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
("rootCause", "root_cause"),
FINDING_ROOT_CAUSE_PREVIEW_BYTES,
(
- "summary",
"description",
"detail",
"cause",
"rationale",
"why",
"explanation",
- "evidenceRefs",
"evidence_refs",
),
(
@@ -54,21 +52,13 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
("validation",),
FINDING_VALIDATION_PREVIEW_BYTES,
(
- "summary",
"conclusion",
- "method",
- "status",
"disposition",
"result",
"rationale",
"evidenceRef",
"evidence_ref",
- "evidenceRefs",
"evidence_refs",
- "assertions",
- "evidence",
- "counterEvidence",
- "limitations",
),
(
(("summary", "conclusion", "rationale", "detail", "disposition"), 800),
@@ -85,23 +75,14 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
("attackPath",),
FINDING_ATTACK_PATH_PREVIEW_BYTES,
(
- "narrative",
"summary",
"description",
- "dataFlow",
"data_flow",
"dataflow",
"path",
- "reachability",
- "steps",
- "authScope",
"auth_scope",
- "vector",
- "preconditions",
- "assumptions",
"impact",
"likelihood",
- "evidenceRefs",
"evidence_refs",
),
(
@@ -277,6 +258,7 @@ def bounded_finding_section(
key = next((alias for alias in aliases if alias in value), None)
if key is not None:
ordered[key] = bounded_json_value(value[key], [field_bytes])
+ # Reserved fields already come first. Order the remaining aliases and fields next.
for key in (*priority_keys, *value):
if key in value and key not in ordered:
ordered[key] = value[key]
@@ -295,14 +277,8 @@ def merged_bounded_code_evidence(value: dict[str, Any]) -> tuple[str | None, Any
if not catalogs:
return evidence_keys[0], value[evidence_keys[0]]
bounded = {}
- for item in chain.from_iterable(catalogs):
- if not (
- isinstance(item, dict)
- and isinstance(item.get("id"), str)
- and bool(item["id"].strip())
- and isinstance(item.get("code"), str)
- and bool(item["code"].strip())
- ):
+ for item in code_evidence_rows(*catalogs):
+ if not isinstance(item.get("id"), str) or not item["id"].strip():
continue
if item["id"] in bounded:
continue
diff --git a/plugins/codex-security/scripts/report_projection.py b/plugins/codex-security/scripts/report_projection.py
index 0e8692545..c554c248f 100644
--- a/plugins/codex-security/scripts/report_projection.py
+++ b/plugins/codex-security/scripts/report_projection.py
@@ -7,6 +7,8 @@ import argparse
import json
import re
from collections import Counter
+from collections.abc import Iterator
+from itertools import chain
from typing import Any
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "informational": 4}
@@ -60,12 +62,15 @@ def _text(value: Any, fallback: str) -> str:
return _escape_markdown(normalized)
-def _strings(value: Any) -> list[str]:
+def _string_values(value: Any) -> Iterator[str]:
if isinstance(value, str):
- value = [value]
- if not isinstance(value, list):
- return []
- return [text for item in value if (text := _text(item, ""))]
+ yield value
+ elif isinstance(value, list):
+ yield from (item for item in value if isinstance(item, str))
+
+
+def _strings(value: Any) -> list[str]:
+ return [text for item in _string_values(value) if (text := _text(item, ""))]
def _path_label(value: str, *, ensure_ascii: bool = False) -> str:
@@ -139,7 +144,7 @@ def uses_deep_presentation(coverage: dict[str, Any], findings: list[dict[str, An
return any(_has_deep_child_metadata(finding) for finding in findings)
-def _deep_title_parts(finding: dict[str, Any]) -> tuple[str, str | None]:
+def _deep_title_parts(finding: dict[str, Any], report_id: str) -> tuple[str, str | None]:
title = finding.get("title")
if not isinstance(title, str):
return "Untitled finding", None
@@ -149,7 +154,7 @@ def _deep_title_parts(finding: dict[str, Any]) -> tuple[str, str | None]:
return normalized, None
annotation = match.group(2)
extensions = finding.get("extensions")
- recognized_ids = [_deep_report_id(finding)]
+ recognized_ids = [report_id]
if isinstance(extensions, dict):
ledger_row_id = extensions.get("ledgerRowId")
if isinstance(ledger_row_id, str) and ledger_row_id.strip():
@@ -171,12 +176,6 @@ def _deep_finding_groups(
return list(groups.values())
-def _deep_group_titles(group: list[tuple[int, dict[str, Any], str | None]]) -> str:
- return "<br>".join(
- dict.fromkeys(_cell(_deep_title_parts(finding)[0]) for _, finding, _ in group)
- )
-
-
def _deep_group_levels(
group: list[tuple[int, dict[str, Any], str | None]],
field: str,
@@ -186,47 +185,34 @@ def _deep_group_levels(
return "<br>".join(sorted(levels, key=lambda level: order.get(level, len(order))))
-def _deep_group_report_labels(
- group: list[tuple[int, dict[str, Any], str | None]],
-) -> list[str]:
+def _deep_group_row(group: list[tuple[int, dict[str, Any], str | None]]) -> str:
report_ids = [_deep_report_id(finding) for _, finding, _ in group]
- report_id_counts = Counter(report_ids)
- labels = [
- (_deep_title_parts(finding)[1] if report_id_counts[report_id] > 1 else report_id)
- or report_id
- for report_id, (_, finding, _) in zip(report_ids, group, strict=True)
- ]
- label_counts = Counter(labels)
- return [
- (
- finding.get("identity", {}).get("instance")
- if label_counts[label] > 1 and isinstance(finding.get("identity"), dict)
- else label
- )
- or _deep_report_id(finding)
- for label, (_, finding, _) in zip(labels, group, strict=True)
- ]
-
+ report_counts = Counter(report_ids)
+ titles = []
+ labels = []
+ for report_id, (_, finding, _) in zip(report_ids, group, strict=True):
+ title, annotation = _deep_title_parts(finding, report_id)
+ titles.append(_cell(title))
+ labels.append(annotation if report_counts[report_id] > 1 and annotation else report_id)
-def _deep_group_report_links(group: list[tuple[int, dict[str, Any], str | None]]) -> str:
- labels = _deep_group_report_labels(group)
- return "<br>".join(
- f"[{_cell(label)}](#finding-{number})"
- for label, (number, _, _) in zip(labels, group, strict=True)
+ label_counts = Counter(labels)
+ report_links = []
+ writeup_links = []
+ for report_id, label, (number, finding, path) in zip(report_ids, labels, group, strict=True):
+ identity = finding.get("identity")
+ if label_counts[label] > 1 and isinstance(identity, dict):
+ label = identity.get("instance") or report_id
+ label = _cell(label)
+ report_links.append(f"[{label}](#finding-{number})")
+ writeup_links.append(f"[Open {label}]({path})" if path else f"{label}: inline below")
+ return (
+ f"| {'<br>'.join(dict.fromkeys(titles))} | {'<br>'.join(report_links)} "
+ f"| {_deep_group_levels(group, 'severity', SEVERITY_ORDER)} "
+ f"| {_deep_group_levels(group, 'confidence', CONFIDENCE_ORDER)} "
+ f"| {'<br>'.join(writeup_links)} |"
)
-def _deep_group_writeup_links(group: list[tuple[int, dict[str, Any], str | None]]) -> str:
- labels = _deep_group_report_labels(group)
- links: list[str] = []
- for label, (_, _, report_path) in zip(labels, group, strict=True):
- report_id = _cell(label)
- links.append(
- f"[Open {report_id}]({report_path})" if report_path else f"{report_id}: inline below"
- )
- return "<br>".join(links)
-
-
def _writeup_report_path(finding: dict[str, Any]) -> str | None:
writeup = finding.get("writeup")
if writeup is None:
@@ -255,51 +241,43 @@ def _bullet_section(heading: str, items: list[str]) -> list[str]:
return ["", heading, *(f"- {item}" for item in items)] if items else []
-def _code_evidence_catalog(finding: dict[str, Any]) -> dict[str, dict[str, Any]]:
- catalog: dict[str, dict[str, Any]] = {}
- for key in ("codeEvidence", "code_evidence"):
- raw = finding.get(key, [])
- if not isinstance(raw, list):
- continue
- for item in raw:
- if (
- isinstance(item, dict)
- and isinstance(item.get("id"), str)
+def code_evidence_rows(*catalogs: Any) -> Iterator[dict[str, Any]]:
+ for items in catalogs:
+ if isinstance(items, list):
+ yield from (
+ item
+ for item in items
+ if isinstance(item, dict)
and isinstance(item.get("code"), str)
and item["code"].strip()
- ):
- catalog.setdefault(item["id"], item)
+ )
+
+
+def embedded_code_evidence(section: dict[str, Any]) -> Iterator[dict[str, Any]]:
+ return code_evidence_rows(section.get("codeEvidence"), section.get("code_evidence"))
+
+
+def _code_evidence_catalog(finding: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ catalog = {}
+ for item in embedded_code_evidence(finding):
+ if isinstance(item.get("id"), str):
+ catalog.setdefault(item["id"], item)
return catalog
def _section_code_evidence(
- finding: dict[str, Any], *sections: dict[str, Any]
+ catalog: dict[str, dict[str, Any]], *sections: dict[str, Any]
) -> list[dict[str, Any]]:
- catalog = _code_evidence_catalog(finding)
- resolved: list[dict[str, Any]] = []
- for section in sections:
- for key in ("evidenceRefs", "evidence_refs"):
- refs = section.get(key, [])
- if isinstance(refs, str):
- refs = [refs]
- if isinstance(refs, list):
- resolved.extend(
- catalog[ref] for ref in refs if isinstance(ref, str) and ref in catalog
- )
- for key in ("codeEvidence", "code_evidence"):
- embedded = section.get(key, [])
- if isinstance(embedded, list):
- resolved.extend(
- item
- for item in embedded
- if isinstance(item, dict)
- and isinstance(item.get("code"), str)
- and item["code"].strip()
- )
unique: dict[tuple[str, str], dict[str, Any]] = {}
- for item in resolved:
- key = (str(item.get("id", "")), item["code"])
- unique.setdefault(key, item)
+ for section in sections:
+ referenced = (
+ catalog[ref]
+ for key in ("evidenceRefs", "evidence_refs")
+ for ref in _string_values(section.get(key))
+ if ref in catalog
+ )
+ for item in chain(referenced, embedded_code_evidence(section)):
+ unique.setdefault((str(item.get("id", "")), item["code"]), item)
return list(unique.values())
@@ -359,12 +337,7 @@ def merged_root_cause(value: dict[str, Any]) -> tuple[str | None, Any]:
]
if evidence_values:
merged["evidenceRefs"] = list(
- dict.fromkeys(
- item
- for evidence in evidence_values
- for item in (evidence if isinstance(evidence, list) else [evidence])
- if isinstance(item, str)
- )
+ dict.fromkeys(item for evidence in evidence_values for item in _string_values(evidence))
)
embedded_evidence = [
@@ -396,9 +369,9 @@ def merged_root_cause(value: dict[str, Any]) -> tuple[str | None, Any]:
def _root_cause_code_evidence(
- finding: dict[str, Any], root_cause: dict[str, Any]
+ finding: dict[str, Any], root_cause: dict[str, Any], catalog: dict[str, dict[str, Any]]
) -> list[dict[str, Any]]:
- evidence = _section_code_evidence(finding, root_cause)
+ evidence = _section_code_evidence(catalog, root_cause)
legacy_code = root_cause.get("code")
if not isinstance(legacy_code, str) or not legacy_code.strip():
return evidence
@@ -599,33 +572,17 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]:
if isinstance((value := attack_path.get(key)), (str, dict))
]
dataflow: dict[str, Any] = {}
- for key in ("summary", "source", "sink", "outcome"):
- value = next(
- (
- section[key]
- for section in dataflow_sections
- if key in section and isinstance(section[key], str) and section[key].strip()
- ),
- None,
- )
- if value is not None:
- dataflow[key] = value
- transformations = list(
- dict.fromkeys(
- transformation
- for section in dataflow_sections
- for transformation in (
- [section.get("transformations")]
- if isinstance(section.get("transformations"), str)
- else section.get("transformations")
- if isinstance(section.get("transformations"), list)
- else []
- )
- if isinstance(transformation, str) and transformation.strip()
- )
+ for section in dataflow_sections:
+ for key in ("summary", "source", "sink", "outcome"):
+ value = section.get(key)
+ if isinstance(value, str) and value.strip():
+ dataflow.setdefault(key, value)
+ transformations = dict.fromkeys(
+ value
+ for section in dataflow_sections
+ for value in _string_values(section.get("transformations"))
+ if value.strip()
)
- if transformations:
- dataflow["transformations"] = transformations
raw_reachability = attack_path.get("reachability")
reachability = (
{"summary": raw_reachability}
@@ -654,10 +611,11 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]:
raw_root_cause if isinstance(raw_root_cause, str) else root_cause.get("summary"),
"",
)
- root_cause_code_evidence = _root_cause_code_evidence(finding, root_cause)
- validation_code_evidence = _section_code_evidence(finding, validation)
- dataflow_code_evidence = _section_code_evidence(finding, attack_path, *dataflow_sections)
- reachability_code_evidence = _section_code_evidence(finding, reachability)
+ catalog = _code_evidence_catalog(finding)
+ root_cause_code_evidence = _root_cause_code_evidence(finding, root_cause, catalog)
+ validation_code_evidence = _section_code_evidence(catalog, validation)
+ dataflow_code_evidence = _section_code_evidence(catalog, attack_path, *dataflow_sections)
+ reachability_code_evidence = _section_code_evidence(catalog, reachability)
dataflow_summary = _text(dataflow.get("summary"), "") or (
f"The canonical finding records the affected path at {_locations(finding)}, "
"but no expanded source-to-sink narrative was recorded."
@@ -704,7 +662,9 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]:
for label, key in (("Source", "source"), ("Sink", "sink"), ("Outcome", "outcome")):
if dataflow.get(key):
lines.extend(["", f"- **{label}:** {_text(dataflow[key], 'not recorded')}"])
- lines.extend(_bullet_section("Transformations:", _strings(dataflow.get("transformations"))))
+ lines.extend(
+ _bullet_section("Transformations:", [_text(value, "") for value in transformations])
+ )
lines.extend(_code_evidence_lines(dataflow_code_evidence))
lines.extend(["", "#### Reachability", "", reachability_summary])
for label, key in (
@@ -718,10 +678,7 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]:
lines.extend(["", f"- **{label}:** {_text(reachability[key], 'not recorded')}"])
preconditions = list(
dict.fromkeys(
- [
- *_strings(attack_path.get("preconditions")),
- *_strings(reachability.get("preconditions")),
- ]
+ _strings(attack_path.get("preconditions")) + _strings(reachability.get("preconditions"))
)
)
lines.extend(_bullet_section("Preconditions:", preconditions))
@@ -912,13 +869,7 @@ def build_report_markdown(
"| --- | --- | --- | --- | --- |",
]
)
- for group in deep_finding_groups:
- lines.append(
- f"| {_deep_group_titles(group)} | {_deep_group_report_links(group)} "
- f"| {_deep_group_levels(group, 'severity', SEVERITY_ORDER)} "
- f"| {_deep_group_levels(group, 'confidence', CONFIDENCE_ORDER)} "
- f"| {_deep_group_writeup_links(group)} |"
- )
+ lines.extend(_deep_group_row(group) for group in deep_finding_groups)
else:
lines.extend(
[
diff --git a/plugins/codex-security/scripts/validate_scan_contract.py b/plugins/codex-security/scripts/validate_scan_contract.py
index bd92747df..3e1849386 100644
--- a/plugins/codex-security/scripts/validate_scan_contract.py
+++ b/plugins/codex-security/scripts/validate_scan_contract.py
@@ -57,14 +57,10 @@ def receipt(validated: dict[str, Any]) -> dict[str, Any]:
}
-def _parse_args(argv: list[str]) -> argparse.Namespace:
+def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--scan-dir", required=True, type=Path)
- return parser.parse_args(argv)
-
-
-def main(argv: list[str] | None = None) -> int:
- args = _parse_args(sys.argv[1:] if argv is None else argv)
+ args = parser.parse_args(argv)
try:
validated = validate_contract(args.scan_dir)
except (OSError, ValueError, RecursionError) as exc:
diff --git a/plugins/codex-security/scripts/validate_tracking_source.py b/plugins/codex-security/scripts/validate_tracking_source.py
index 34e18aee0..abe3e24ab 100644
--- a/plugins/codex-security/scripts/validate_tracking_source.py
+++ b/plugins/codex-security/scripts/validate_tracking_source.py
@@ -53,17 +53,13 @@ def validate_source(
)
-def _parse_args(argv: list[str]) -> argparse.Namespace:
+def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("scan_dir", type=Path)
selector = parser.add_mutually_exclusive_group()
selector.add_argument("--finding-id")
selector.add_argument("--fingerprint")
- return parser.parse_args(argv)
-
-
-def main(argv: list[str] | None = None) -> int:
- args = _parse_args(sys.argv[1:] if argv is None else argv)
+ args = parser.parse_args(argv)
try:
findings = validate_source(
args.scan_dir,
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index 98fccb592..7beeee7eb 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -54,6 +54,10 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.write_scan()
return FINALIZER.finalize_scan(self.scan_dir)
+ def seal_and_read_json(self, name: str) -> dict:
+ self.write_sealed_scan()
+ return self.read_json(name)
+
def compatible_finding(self, findings: dict) -> dict:
original = copy.deepcopy(findings)
compatible = FINALIZER._legacy_sealed_findings_for_validation(findings)
@@ -114,6 +118,10 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
cwd=cwd,
)
+ def assert_finalization_fails(self, message: str) -> None:
+ with self.assertRaisesRegex(FINALIZER.ContractError, message):
+ FINALIZER.finalize_scan(self.scan_dir)
+
def write_scan(self) -> None:
self.write_json("scan-manifest.json", self.manifest)
self.write_json("findings.json", self.findings)
@@ -235,9 +243,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
return {name: (self.scan_dir / name).read_bytes() for name in names}
def test_finalize_seals_manifest_deterministically(self) -> None:
- self.write_scan()
-
- manifest_result, findings_result, coverage_result = FINALIZER.finalize_scan(self.scan_dir)
+ manifest_result, findings_result, coverage_result = self.write_sealed_scan()
first_manifest = (self.scan_dir / "scan-manifest.json").read_bytes()
first_findings = (self.scan_dir / "findings.json").read_bytes()
first_sarif = (self.scan_dir / "exports" / "results.sarif").read_bytes()
@@ -284,9 +290,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_finalizes_canonical_document_beyond_previous_size_limit(self) -> None:
self.manifest["metadata"] = "x" * (16 * 1024 * 1024)
- self.write_scan()
-
- manifest, _, _ = FINALIZER.finalize_scan(self.scan_dir)
+ manifest, _, _ = self.write_sealed_scan()
self.assertEqual(len(manifest["metadata"]), 16 * 1024 * 1024)
@@ -301,9 +305,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
for _ in range(258):
nested = [nested]
self.findings["findings"][0]["extensions"]["nested"] = nested
- self.write_scan()
-
- _, findings, _ = FINALIZER.finalize_scan(self.scan_dir)
+ _, findings, _ = self.write_sealed_scan()
self.assertEqual(findings["findings"][0]["extensions"]["nested"], nested)
@@ -372,23 +374,20 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
validate.assert_not_called()
def test_sealed_rerun_rejects_malformed_unicode_extensions_without_mutating_files(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["extensions"]["note"] = "bad-" + chr(0xD800)
self.rewrite_sealed_artifact("findings.json", findings)
before = {
path.name: path.read_bytes() for path in self.scan_dir.iterdir() if path.is_file()
}
- with self.assertRaisesRegex(FINALIZER.ContractError, "well-formed Unicode"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("well-formed Unicode")
after = {path.name: path.read_bytes() for path in self.scan_dir.iterdir() if path.is_file()}
self.assertEqual(before, after)
def test_sealed_rerun_preserves_existing_document_bytes(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["summary"] = "é" * 128
compact = (json.dumps(findings, ensure_ascii=False, separators=(",", ":")) + "\n").encode(
"utf-8"
@@ -422,8 +421,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_accepts_legacy_unknown_evidence_references(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["codeEvidence"] = [
{
"id": "shared-source",
@@ -463,8 +461,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_malformed_legacy_evidence_references(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["code_evidence"] = [
{"id": "legacy-source", "code": "legacy_source()"}
]
@@ -493,16 +490,11 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.findings["findings"][0]["validation"] = {"evidenceRefs": [None]}
self.write_scan()
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- r"validation\.evidenceRefs: expected strings",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(r"validation\.evidenceRefs: expected strings")
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_non_string_legacy_validation_scalars(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["validation"] = {
"method": [],
"summary": {"legacy": True},
@@ -512,8 +504,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_accepts_formerly_free_form_finding_details(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["validation"] = {
"evidence": {"kind": "trace"},
"counterEvidence": [None, "The mitigation was checked."],
@@ -530,8 +521,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_blank_legacy_attack_path_details(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["attackPath"] = {
field: ""
for field in (
@@ -548,17 +538,12 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_rejects_nullable_canonical_evidence_catalog(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["codeEvidence"] = None
self.rewrite_sealed_artifact("findings.json", findings)
self.assertIsNone(self.compatible_finding(findings)["codeEvidence"])
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- r"codeEvidence: expected (?:an |schema type )?array",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(r"codeEvidence: expected (?:an |schema type )?array")
def test_accepts_custom_schema_beyond_previous_complexity_limits(self) -> None:
schema = {
@@ -625,11 +610,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.coverage["inventoryStrategy"] = "deep_repository_repeated_discovery_v2"
self.write_scan()
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- "coverage.schema.inventoryStrategy: unsupported value",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("coverage.schema.inventoryStrategy: unsupported value")
def test_finalize_rejects_sealed_deep_inventory_strategy_alias(self) -> None:
self.coverage["mode"] = "deep_repository"
@@ -676,8 +657,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.findings["findings"][0]["writeup"] = {"reportPath": report_path}
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "expected a file inside"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected a file inside")
with self.assertRaisesRegex(FINALIZER.ContractError, "expected a file inside"):
FINALIZER._prepare_scan_finalization(
@@ -687,15 +667,13 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
writeup_path = self.scan_dir / report_path
writeup_path.parent.mkdir(parents=True)
writeup_path.mkdir()
- with self.assertRaisesRegex(FINALIZER.ContractError, "regular non-symlink file"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("regular non-symlink file")
writeup_path.rmdir()
writeup_path.write_text("# Archive extraction\n", encoding="utf-8")
FINALIZER.finalize_scan(self.scan_dir)
writeup_path.unlink()
- with self.assertRaisesRegex(FINALIZER.ContractError, "expected a file inside"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected a file inside")
sealed_files = self.scan_file_bytes(*CANONICAL_FILES)
with self.assertRaises(FINALIZER.ContractError):
FINALIZER._prepare_scan_finalization(self.scan_dir, allow_missing_writeups=True)
@@ -706,21 +684,18 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.manifest["scan"]["hardening"] = {"portfolioPath": portfolio_path}
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "expected a file inside"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected a file inside")
portfolio = self.scan_dir / portfolio_path
portfolio.parent.mkdir(parents=True)
portfolio.mkdir()
- with self.assertRaisesRegex(FINALIZER.ContractError, "regular non-symlink file"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("regular non-symlink file")
portfolio.rmdir()
portfolio.write_text("# Structural hardening\n", encoding="utf-8")
FINALIZER.finalize_scan(self.scan_dir)
portfolio.unlink()
- with self.assertRaisesRegex(FINALIZER.ContractError, "expected a file inside"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected a file inside")
def test_rerun_regenerates_sarif_projection_without_mutating_seal(self) -> None:
self.write_sealed_scan()
@@ -1210,8 +1185,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.assertEqual(payload, {"source": "original"})
def test_sarif_projection_revalidates_sealed_findings(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["locations"][0]["path"] = "../../outside.py"
self.rewrite_sealed_artifact("findings.json", findings)
@@ -1296,12 +1270,10 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
"_artifact_record",
side_effect=record_then_replace,
):
- with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("sealed artifact changed")
def test_sarif_projection_requires_canonical_sealed_artifacts(self) -> None:
- self.write_sealed_scan()
- sealed_manifest = self.read_json("scan-manifest.json")
+ sealed_manifest = self.seal_and_read_json("scan-manifest.json")
for required_path in ("findings.json", "coverage.json"):
with self.subTest(required_path=required_path):
@@ -1331,8 +1303,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
FINALIZER.write_sarif_projection(self.scan_dir)
def test_sarif_projection_revalidates_manifest_schema(self) -> None:
- self.write_sealed_scan()
- manifest = self.read_json("scan-manifest.json")
+ manifest = self.seal_and_read_json("scan-manifest.json")
manifest["scan"]["startedAt"] = "not-a-time"
self.write_json("scan-manifest.json", manifest)
@@ -1343,8 +1314,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.findings["findings"][0]["locations"][0]["path"] = "src/bad" + chr(0xDCFF) + ".py"
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "safe repository-relative"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("safe repository-relative")
self.assertNotIn("sealedAt", self.read_json("scan-manifest.json")["scan"])
@@ -1453,12 +1423,10 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_rejects_changed_canonical_artifact_after_sealing(self) -> None:
self.write_sealed_scan()
(self.scan_dir / "findings.json").write_text("{}\n", encoding="utf-8")
- with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("sealed artifact changed")
def test_rejects_sealed_findings_missing_derived_identity(self) -> None:
- self.write_sealed_scan()
- sealed_findings = self.read_json("findings.json")
+ sealed_findings = self.seal_and_read_json("findings.json")
for field in ("findingId", "occurrenceId", "fingerprints"):
with self.subTest(field=field):
@@ -1466,8 +1434,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
findings["findings"][0].pop(field)
self.rewrite_sealed_artifact("findings.json", findings)
- with self.assertRaisesRegex(FINALIZER.ContractError, field):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(field)
def test_finalize_generates_unsealed_reports_from_canonical_json(self) -> None:
self.manifest["scan"]["scope"].update(
@@ -1584,9 +1551,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_finalize_does_not_project_model_authored_runtime_status(self) -> None:
stale_status = "SDK finalization and sealing intentionally pending."
self.manifest["scan"]["scope"]["runtimeStatus"] = stale_status
- self.write_scan()
-
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
manifest = self.read_json("scan-manifest.json")
report = (self.scan_dir / "report.md").read_text(encoding="utf-8")
@@ -1630,9 +1595,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
finding["rootCause"] = "The extraction destination is not contained."
finding["validation"] = {"evidence": "A traversal input escapes the destination."}
finding["attackPath"] = {"dataflow": "archive entry -> filesystem write"}
- self.write_scan()
-
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
report = (self.scan_dir / "report.md").read_text(encoding="utf-8")
self.assertIn("The extraction destination is not contained.", report)
@@ -1652,9 +1615,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
}
]
}
- self.write_scan()
-
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
report = (self.scan_dir / "report.md").read_text(encoding="utf-8")
self.assertIn("**Source control** — `src/evidence.py:37`\n", report)
@@ -1706,11 +1667,9 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
}
self.write_scan()
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- "rootCause.evidenceRefs: unknown code-evidence ids: missing-evidence",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(
+ "rootCause.evidenceRefs: unknown code-evidence ids: missing-evidence"
+ )
@pytest.mark.cross_platform
def test_finalize_rejects_unknown_nested_code_evidence_reference(self) -> None:
@@ -1721,17 +1680,13 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
}
self.write_scan()
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- "attackPath.dataflow.evidenceRefs: unknown code-evidence ids: "
- "missing-dataflow-evidence",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(
+ "attackPath.dataflow.evidenceRefs: unknown code-evidence ids: missing-dataflow-evidence"
+ )
@pytest.mark.cross_platform
def test_sealed_rerun_accepts_legacy_scalar_finding_details(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["validation"] = {
"assertions": "The destination escapes the extraction root.",
"counterEvidence": None,
@@ -1768,31 +1723,24 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_rejects_malformed_canonical_root_cause(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["rootCause"] = {"summary": []}
self.rewrite_sealed_artifact("findings.json", findings)
sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- r"rootCause\.summary",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(r"rootCause\.summary")
self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
@pytest.mark.cross_platform
def test_sealed_rerun_accepts_nullable_legacy_evidence_catalog(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["code_evidence"] = None
with self.preserving_sealed_findings(findings):
self.assertNotIn("code_evidence", self.compatible_finding(findings))
def test_sealed_rerun_and_export_preserve_nullable_legacy_evidence_path(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["code_evidence"] = [
{"id": "legacy-source", "code": "legacy_source()", "path": None}
]
@@ -1802,16 +1750,14 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.assertEqual(json.loads(result.stdout), findings)
def test_sealed_rerun_accepts_empty_legacy_root_cause(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["root_cause"] = ""
with self.preserving_sealed_findings(findings):
pass
@pytest.mark.cross_platform
def test_sealed_rerun_renders_legacy_evidence_with_reversed_bounds(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
finding = findings["findings"][0]
finding["code_evidence"] = [
{
@@ -1830,8 +1776,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_malformed_legacy_evidence_rows(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["code_evidence"] = [
None,
"legacy source",
@@ -1849,8 +1794,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
@pytest.mark.cross_platform
def test_sealed_rerun_accepts_legacy_sequence_attack_path_details(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["attackPath"] = {
"dataflow": ["archive entry path", "filesystem write"],
"reachability": ["authenticated uploader"],
@@ -1862,10 +1806,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.findings["findings"][0]["validation"] = {"score": float("nan")}
self.write_scan()
- with self.assertRaisesRegex(
- FINALIZER.ContractError, "invalid JSON.*non-finite JSON number.*NaN"
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("invalid JSON.*non-finite JSON number.*NaN")
with self.assertRaisesRegex(FINALIZER.ContractError, "cannot encode canonical JSON"):
FINALIZER._json_bytes({"score": float("nan")})
@@ -1888,8 +1829,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
coverage = (self.scan_dir / "coverage.json").read_bytes()
report_markdown = (self.scan_dir / "report.md").read_bytes()
- with self.assertRaisesRegex(FINALIZER.ContractError, "schema pattern"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("schema pattern")
self.assertEqual(findings, (self.scan_dir / "findings.json").read_bytes())
self.assertEqual(coverage, (self.scan_dir / "coverage.json").read_bytes())
@@ -1919,8 +1859,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
external_report.write_text("unchanged\n", encoding="utf-8")
(self.scan_dir / "report.md").symlink_to(external_report)
- with self.assertRaisesRegex(FINALIZER.ContractError, "non-symlink path"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("non-symlink path")
self.assertEqual(external_report.read_text(encoding="utf-8"), "unchanged\n")
manifest = self.read_json("scan-manifest.json")
@@ -1932,8 +1871,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.write_scan()
report_markdown = (self.scan_dir / "report.md").read_bytes()
- with self.assertRaisesRegex(FINALIZER.ContractError, "schema type string"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("schema type string")
self.assertEqual(report_markdown, (self.scan_dir / "report.md").read_bytes())
manifest = self.read_json("scan-manifest.json")
@@ -1968,8 +1906,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
normalized_ref, [artifact["path"] for artifact in manifest["scan"]["artifacts"]]
)
receipt_path.write_text('{"status":"changed"}\n', encoding="utf-8")
- with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("sealed artifact changed")
def test_seals_import_source_without_claiming_coverage(self) -> None:
for with_surface in (False, True):
@@ -2022,18 +1959,14 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
FINALIZER.finalize_scan(self.scan_dir)
self.assertEqual((self.scan_dir / "scan-manifest.json").read_bytes(), sealed)
source_path.write_bytes(b"changed\n")
- with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("sealed artifact changed")
source_path.unlink()
with self.assertRaises(FINALIZER.ContractError):
FINALIZER.build_findings_export(self.scan_dir, "json")
source_path.write_bytes(source)
manifest["scan"]["artifacts"].remove(records[0])
self.write_json("scan-manifest.json", manifest)
- with self.assertRaisesRegex(
- FINALIZER.ContractError, "missing from sealed artifacts"
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("missing from sealed artifacts")
def test_rejects_missing_or_unsafe_import_source(self) -> None:
self.findings["findings"] = []
@@ -2059,8 +1992,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
target = source_path.with_name("retained.json")
target.write_bytes(b"{}\n")
source_path.symlink_to(target)
- with self.assertRaisesRegex(FINALIZER.ContractError, "non-symlink"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("non-symlink")
def test_verifies_legacy_aliased_receipt_ref(self) -> None:
receipt_ref = "artifacts/02_discovery/work_ledger.jsonl"
@@ -2094,8 +2026,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
]
self.write_json("scan-manifest.json", manifest)
- with self.assertRaisesRegex(FINALIZER.ContractError, "missing from sealed artifacts"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("missing from sealed artifacts")
with self.assertRaisesRegex(FINALIZER.ContractError, "missing from sealed artifacts"):
FINALIZER._prepare_scan_finalization(self.scan_dir, recover_unsealed=True)
@@ -2176,9 +2107,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
"algorithm": "codex-security/v0",
"primary": "codex-security/v0:sha256:example",
}
- self.write_scan()
-
- _, findings, _ = FINALIZER.finalize_scan(self.scan_dir)
+ _, findings, _ = self.write_sealed_scan()
finding = findings["findings"][0]
self.assertRegex(finding["findingId"], r"^csf_[a-f0-9]{24}$")
@@ -2344,16 +2273,11 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.assertEqual(self.read_json("coverage.json"), coverage)
def test_sealed_findings_keep_authored_identity_mismatches_strict(self) -> None:
- self.write_sealed_scan()
- findings = self.read_json("findings.json")
+ findings = self.seal_and_read_json("findings.json")
findings["findings"][0]["findingId"] = "csf_wrong"
self.rewrite_sealed_artifact("findings.json", findings)
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- "findingId: does not match derived fingerprint identity",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("findingId: does not match derived fingerprint identity")
def test_rejects_remote_credentials_and_query(self) -> None:
self.write_scan()
@@ -2365,10 +2289,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
with self.subTest(remote=remote):
self.manifest["scan"]["target"]["remote"] = remote
self.write_json("scan-manifest.json", self.manifest)
- with self.assertRaisesRegex(
- FINALIZER.ContractError, "must not contain credentials"
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("must not contain credentials")
def test_rejects_remote_unsafe_authority(self) -> None:
self.write_scan()
@@ -2388,10 +2309,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
with self.subTest(remote=remote):
self.manifest["scan"]["target"]["remote"] = remote
self.write_json("scan-manifest.json", self.manifest)
- with self.assertRaisesRegex(
- FINALIZER.ContractError, "expected a sanitized canonical absolute URL"
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected a sanitized canonical absolute URL")
def test_seals_remote_port_and_ipv6_authority(self) -> None:
for remote in (
@@ -2402,8 +2320,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
):
with self.subTest(remote=remote):
self.manifest["scan"]["target"]["remote"] = remote
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
manifest = self.read_json("scan-manifest.json")
self.assertEqual(manifest["scan"]["target"]["remote"], remote)
@@ -2416,9 +2333,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_allows_repository_root_scope(self) -> None:
self.manifest["scan"]["scope"]["includePaths"] = ["."]
self.coverage["includePaths"] = ["."]
- self.write_scan()
-
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
self.assertEqual(
self.read_json("scan-manifest.json")["scan"]["scope"]["includePaths"], ["."]
@@ -2443,8 +2358,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
"explanation": "Repository-relative evidence.",
}
]
- self.write_scan()
- manifest, findings, coverage = FINALIZER.finalize_scan(self.scan_dir)
+ manifest, findings, coverage = self.write_sealed_scan()
self.assertEqual(manifest["scan"]["scope"]["includePaths"], [path])
self.assertEqual(coverage["includePaths"], [path])
self.assertEqual(coverage["deferred"][0]["paths"], [path])
@@ -2464,8 +2378,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_rejects_missing_coverage_receipt(self) -> None:
self.coverage["surfaces"][0]["receiptRefs"] = ["artifacts/02_discovery/work_ledger.jsonl"]
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "inside the scan directory"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("inside the scan directory")
def test_rejects_coverage_receipt_refs_outside_artifacts(self) -> None:
for receipt_ref in (
@@ -2477,8 +2390,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
with self.subTest(receipt_ref=receipt_ref):
self.coverage["surfaces"][0]["receiptRefs"] = [receipt_ref]
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "under artifacts/"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("under artifacts/")
def test_rejects_symlink_receipt(self) -> None:
receipt_ref = "artifacts/02_discovery/work_ledger.jsonl"
@@ -2489,8 +2401,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
target_path = receipt_path.with_name("real_work_ledger.jsonl")
target_path.write_text('{"status":"reviewed"}\n', encoding="utf-8")
receipt_path.symlink_to(target_path)
- with self.assertRaisesRegex(FINALIZER.ContractError, "non-symlink"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("non-symlink")
def test_rejects_external_symlink_canonical_document(self) -> None:
self.write_scan()
@@ -2500,8 +2411,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
external_path.write_bytes(findings_path.read_bytes())
findings_path.unlink()
findings_path.symlink_to(external_path)
- with self.assertRaisesRegex(FINALIZER.ContractError, "inside the scan directory"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("inside the scan directory")
def test_finalizes_review_notes_and_discards_old_outcome_labels(self) -> None:
self.coverage["completeness"] = "partial"
@@ -2512,9 +2422,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
"reason": "Runtime verification remains unavailable.",
}
]
- self.write_scan()
-
- manifest, _, coverage = FINALIZER.finalize_scan(self.scan_dir)
+ manifest, _, coverage = self.write_sealed_scan()
self.assertEqual(manifest["scan"]["status"], "completed")
self.assertNotIn("completeness", coverage)
@@ -2538,11 +2446,9 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
for path in ("../../outside.ts", "/outside.ts", r"C:\outside.ts"):
with self.subTest(path=path):
self.write_code_evidence_scan(path)
- with self.assertRaisesRegex(
- FINALIZER.ContractError,
- r"codeEvidence\[0\]\.path: expected a safe repository-relative POSIX path",
- ):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(
+ r"codeEvidence\[0\]\.path: expected a safe repository-relative POSIX path"
+ )
def test_accepts_code_evidence_with_posix_colon_names(self) -> None:
for path in ("src/a:b.py", "report:summary.py"):
@@ -2574,8 +2480,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
{"id": "review", "reason": "Review is incomplete.", "paths": [path]}
]
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, r"coverage.*deferred"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails(r"coverage.*deferred")
def test_accepts_safe_code_evidence_and_deferred_paths(self) -> None:
self.findings["findings"][0]["codeEvidence"] = [
@@ -2614,8 +2519,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.coverage["deferred"] = [
{"id": "review", "reason": "Review is incomplete.", "paths": [path]}
]
- self.write_scan()
- _, findings, coverage = FINALIZER.finalize_scan(self.scan_dir)
+ _, findings, coverage = self.write_sealed_scan()
self.assertEqual(
findings["findings"][0]["codeEvidence"][0]["path"], "src/extract.py"
)
@@ -2624,27 +2528,22 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_rejects_coverage_scope_mismatch(self) -> None:
self.coverage["includePaths"] = ["other/"]
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "must match manifest scope"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("must match manifest scope")
def test_unsealed_contract_refs_are_replaced_with_canonical_refs(self) -> None:
self.manifest["scan"]["coverageRef"] = "not-the-coverage.json"
self.manifest["scan"]["findingsRef"] = "not-the-findings.json"
- self.write_scan()
-
- manifest, _, _ = FINALIZER.finalize_scan(self.scan_dir)
+ manifest, _, _ = self.write_sealed_scan()
self.assertEqual(manifest["scan"]["coverageRef"], "coverage.json")
self.assertEqual(manifest["scan"]["findingsRef"], "findings.json")
def test_sealed_contract_refs_remain_strict(self) -> None:
- self.write_sealed_scan()
- manifest = self.read_json("scan-manifest.json")
+ manifest = self.seal_and_read_json("scan-manifest.json")
manifest["scan"]["coverageRef"] = "not-the-coverage.json"
self.write_json("scan-manifest.json", manifest)
- with self.assertRaisesRegex(FINALIZER.ContractError, "expected 'coverage.json'"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected 'coverage.json'")
def test_completion_binding_populates_unsealed_workbench_envelope(self) -> None:
scan = self.manifest["scan"]
@@ -2720,8 +2619,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.assertEqual(coverage["excludePaths"], [])
def test_completion_binding_rejects_sealed_wrong_producer_without_rewriting(self) -> None:
- self.write_scan()
- manifest, _, _ = FINALIZER.finalize_scan(self.scan_dir)
+ manifest, _, _ = self.write_sealed_scan()
binding = self.completion_binding()
binding["startedAt"] = manifest["scan"]["startedAt"]
binding["completedAt"] = manifest["scan"]["completedAt"]
@@ -2784,9 +2682,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
{"question": 7},
42,
]
- self.write_scan()
-
- _, _, coverage = FINALIZER.finalize_scan(self.scan_dir)
+ _, _, coverage = self.write_sealed_scan()
self.assertEqual(
coverage["openQuestions"],
@@ -2802,16 +2698,13 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
def test_unsealed_non_array_open_questions_are_removed(self) -> None:
self.coverage["openQuestions"] = {"question": "not an array"}
- self.write_scan()
-
- _, _, coverage = FINALIZER.finalize_scan(self.scan_dir)
+ _, _, coverage = self.write_sealed_scan()
self.assertNotIn("openQuestions", coverage)
def test_sealed_coverage_mode_and_open_questions_remain_strict(self) -> None:
self.coverage["openQuestions"] = [{"question": "Still valid before sealing."}]
- self.write_sealed_scan()
- coverage = self.read_json("coverage.json")
+ coverage = self.seal_and_read_json("coverage.json")
coverage["mode"] = "scoped_path"
coverage["openQuestions"] = ["invalid sealed row"]
self.rewrite_sealed_artifact("coverage.json", coverage)
@@ -2824,16 +2717,13 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
coverage["mode"] = "repository"
self.rewrite_sealed_artifact("coverage.json", coverage)
- with self.assertRaisesRegex(FINALIZER.ContractError, "expected schema type object"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("expected schema type object")
def test_rejects_duplicate_artifact_paths(self) -> None:
- self.write_sealed_scan()
- manifest = self.read_json("scan-manifest.json")
+ manifest = self.seal_and_read_json("scan-manifest.json")
manifest["scan"]["artifacts"].append(copy.deepcopy(manifest["scan"]["artifacts"][0]))
self.write_json("scan-manifest.json", manifest)
- with self.assertRaisesRegex(FINALIZER.ContractError, "duplicate artifact path"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("duplicate artifact path")
def test_finalizes_no_findings_scan(self) -> None:
self.manifest["scan"]["artifacts"] = []
@@ -2854,8 +2744,7 @@ class FinalizeScanContractTest(FinalizerSarifCases, ScanFixtureTestCase):
self.manifest["scan"]["sealedAt"] = self.manifest["scan"]["completedAt"]
self.manifest["scan"]["artifacts"] = []
self.write_scan()
- with self.assertRaisesRegex(FINALIZER.ContractError, "requires artifact records"):
- FINALIZER.finalize_scan(self.scan_dir)
+ self.assert_finalization_fails("requires artifact records")
if __name__ == "__main__":
diff --git a/plugins/codex-security/tests/test_report_projection.py b/plugins/codex-security/tests/test_report_projection.py
index 2707419fe..b280294c7 100644
--- a/plugins/codex-security/tests/test_report_projection.py
+++ b/plugins/codex-security/tests/test_report_projection.py
@@ -48,6 +48,12 @@ def canonical_documents() -> tuple[dict[str, object], dict[str, object], dict[st
return manifest, findings, coverage
+def report_with_finding(**fields: object) -> str:
+ manifest, findings, coverage = canonical_documents()
+ findings["findings"][0].update(fields)
+ return PROJECTION.build_report_markdown(manifest, findings, coverage)
+
+
def test_projection_normalizes_structured_fields() -> None:
markdown = PROJECTION.build_report_markdown(*canonical_documents())
@@ -181,68 +187,64 @@ def test_projection_renders_inline_code_and_section_code_evidence() -> None:
@pytest.mark.parametrize("reference_key", ["evidenceRefs", "evidence_refs"])
def test_projection_renders_nested_attack_path_code_evidence(reference_key: str) -> None:
- manifest, findings, coverage = canonical_documents()
- finding = findings["findings"][0]
- finding["codeEvidence"] = [
- {
- "id": "archive-source",
- "label": "Attacker-controlled archive path",
- "path": "src/archive.py",
- "startLine": 20,
- "code": "entry_path = archive_entry.name",
- "explanation": "The archive controls the path.",
- },
- {
- "id": "archive-sink",
- "label": "Unchecked filesystem write",
- "path": "src/archive.py",
- "startLine": 41,
- "code": "destination.write_bytes(entry.read())",
- "explanation": "The unchecked path reaches the write.",
- },
- ]
- finding["attackPath"] = {
- "dataflow": {
- "summary": "An archive entry path reaches a filesystem write.",
- "evidenceRefs": [],
- "evidence_refs": ["archive-source"],
- },
- "reachability": {
- "summary": "An authenticated uploader can trigger extraction.",
- reference_key: ["archive-sink"],
+ markdown = report_with_finding(
+ codeEvidence=[
+ {
+ "id": "archive-source",
+ "label": "Attacker-controlled archive path",
+ "path": "src/archive.py",
+ "startLine": 20,
+ "code": "entry_path = archive_entry.name",
+ "explanation": "The archive controls the path.",
+ },
+ {
+ "id": "archive-sink",
+ "label": "Unchecked filesystem write",
+ "path": "src/archive.py",
+ "startLine": 41,
+ "code": "destination.write_bytes(entry.read())",
+ "explanation": "The unchecked path reaches the write.",
+ },
+ ],
+ attackPath={
+ "dataflow": {
+ "summary": "An archive entry path reaches a filesystem write.",
+ "evidenceRefs": [],
+ "evidence_refs": ["archive-source"],
+ },
+ "reachability": {
+ "summary": "An authenticated uploader can trigger extraction.",
+ reference_key: ["archive-sink"],
+ },
},
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ )
assert "entry_path = archive_entry.name" in markdown
assert "destination.write_bytes(entry.read())" in markdown
def test_projection_normalizes_scalar_evidence_references() -> None:
- manifest, findings, coverage = canonical_documents()
- finding = findings["findings"][0]
- finding["codeEvidence"] = [
- {"id": "root-source", "code": "root_source()"},
- {"id": "dataflow-source", "code": "dataflow_source()"},
- {"id": "reachability-source", "code": "reachability_source()"},
- ]
- finding["root_cause"] = {
- "summary": "The authorization check occurs after the write.",
- "evidence_refs": "root-source",
- }
- finding["attackPath"] = {
- "dataflow": {
- "summary": "An attacker-controlled value reaches the write.",
- "evidence_refs": "dataflow-source",
+ markdown = report_with_finding(
+ codeEvidence=[
+ {"id": "root-source", "code": "root_source()"},
+ {"id": "dataflow-source", "code": "dataflow_source()"},
+ {"id": "reachability-source", "code": "reachability_source()"},
+ ],
+ root_cause={
+ "summary": "The authorization check occurs after the write.",
+ "evidence_refs": "root-source",
},
- "reachability": {
- "summary": "An authenticated caller can reach the handler.",
- "evidence_refs": "reachability-source",
+ attackPath={
+ "dataflow": {
+ "summary": "An attacker-controlled value reaches the write.",
+ "evidence_refs": "dataflow-source",
+ },
+ "reachability": {
+ "summary": "An authenticated caller can reach the handler.",
+ "evidence_refs": "reachability-source",
+ },
},
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ )
assert "root_source()" in markdown
assert "dataflow_source()" in markdown
@@ -250,17 +252,16 @@ def test_projection_normalizes_scalar_evidence_references() -> None:
def test_projection_merges_transformations_across_data_flow_aliases() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["attackPath"] = {
- "dataFlow": {"transformations": ["decode archive entry", "parse *input*"]},
- "dataflow": {
- "summary": "request -> archive extraction -> filesystem write",
- "transformations": ["dispatch extraction", "decode archive entry"],
+ markdown = report_with_finding(
+ attackPath={
+ "dataFlow": {"transformations": ["decode archive entry", "parse *input*"]},
+ "dataflow": {
+ "summary": "request -> archive extraction -> filesystem write",
+ "transformations": ["dispatch extraction", "decode archive entry"],
+ },
+ "data_flow": {"transformations": None},
},
- "data_flow": {"transformations": None},
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ )
assert markdown.count("- decode archive entry") == 1
assert "- dispatch extraction" in markdown
@@ -268,22 +269,21 @@ def test_projection_merges_transformations_across_data_flow_aliases() -> None:
def test_projection_merges_top_level_and_reachability_preconditions() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["attackPath"] = {
- "preconditions": [
- "The service processes uploaded archives.",
- "The attacker can upload an archive.",
- ],
- "reachability": {
- "summary": "An authenticated uploader can trigger extraction.",
+ markdown = report_with_finding(
+ attackPath={
"preconditions": [
+ "The service processes uploaded archives.",
"The attacker can upload an archive.",
- "Automatic extraction is enabled.",
],
+ "reachability": {
+ "summary": "An authenticated uploader can trigger extraction.",
+ "preconditions": [
+ "The attacker can upload an archive.",
+ "Automatic extraction is enabled.",
+ ],
+ },
},
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ )
assert markdown.count("- The service processes uploaded archives.") == 1
assert markdown.count("- The attacker can upload an archive.") == 1
@@ -291,12 +291,11 @@ def test_projection_merges_top_level_and_reachability_preconditions() -> None:
def test_projection_uses_top_level_attack_path_summary_as_reachability_fallback() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["attackPath"] = {
- "summary": "An authenticated uploader can trigger parse_file to extract an archive."
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ attackPath={
+ "summary": "An authenticated uploader can trigger parse_file to extract an archive."
+ },
+ )
reachability = markdown.split("#### Reachability", 1)[1].split("#### Severity", 1)[0]
assert (
@@ -307,15 +306,14 @@ def test_projection_uses_top_level_attack_path_summary_as_reachability_fallback(
def test_projection_renders_typed_attack_path_steps() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["attackPath"] = {
- "steps": [
- "Upload an archive with a traversal entry.",
- "Trigger automatic extraction.",
- ]
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ attackPath={
+ "steps": [
+ "Upload an archive with a traversal entry.",
+ "Trigger automatic extraction.",
+ ]
+ },
+ )
dataflow = markdown.split("#### Dataflow", 1)[1].split("#### Reachability", 1)[0]
assert "Attack steps:" in dataflow
@@ -324,26 +322,25 @@ def test_projection_renders_typed_attack_path_steps() -> None:
def test_projection_renders_typed_assessments_and_validation_outcomes() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["validation"] = {
- "status": "validated",
- "disposition": "reported",
- "result": "The traversal write was confirmed.",
- }
- findings["findings"][0]["attackPath"] = {
- "impact": {
- "level": "high",
- "rationale": "The write can overwrite application files.",
- "why": "The destination escapes the extraction root.",
+ markdown = report_with_finding(
+ validation={
+ "status": "validated",
+ "disposition": "reported",
+ "result": "The traversal write was confirmed.",
},
- "likelihood": "Likely for authenticated uploaders.",
- "reachability": {
- "source": "Attacker-controlled archive entry.",
- "sink": "Unchecked filesystem write.",
+ attackPath={
+ "impact": {
+ "level": "high",
+ "rationale": "The write can overwrite application files.",
+ "why": "The destination escapes the extraction root.",
+ },
+ "likelihood": "Likely for authenticated uploaders.",
+ "reachability": {
+ "source": "Attacker-controlled archive entry.",
+ "sink": "Unchecked filesystem write.",
+ },
},
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ )
assert "- **Status:** validated" in markdown
assert "- **Disposition:** reported" in markdown
@@ -358,15 +355,14 @@ def test_projection_renders_typed_assessments_and_validation_outcomes() -> None:
def test_projection_renders_attack_path_context_lists() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["attackPath"] = {
- "assumptions": ["Automatic extraction is enabled."],
- "blindspots": ["A downstream sandbox was not exercised."],
- "controls": ["Archive uploads require authentication."],
- "limitations": ["The exploit was validated statically."],
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ attackPath={
+ "assumptions": ["Automatic extraction is enabled."],
+ "blindspots": ["A downstream sandbox was not exercised."],
+ "controls": ["Archive uploads require authentication."],
+ "limitations": ["The exploit was validated statically."],
+ },
+ )
reachability = markdown.split("#### Reachability", 1)[1].split("#### Severity", 1)[0]
assert "Assumptions:" in reachability
@@ -404,43 +400,38 @@ def test_projection_preserves_heading_like_source_evidence() -> None:
def test_projection_renders_string_root_cause() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["rootCause"] = (
- "The authorization check runs after the privileged `write`."
+ markdown = report_with_finding(
+ rootCause="The authorization check runs after the privileged `write`.",
)
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
-
assert (
"#### Root Cause\n\nThe authorization check runs after the privileged `write`." in markdown
)
def test_projection_merges_root_cause_aliases() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["rootCause"] = {"summary": ""}
- findings["findings"][0]["root_cause"] = {
- "summary": "The destination is not contained before the write.",
- "code": "destination.write_bytes(payload)",
- "language": "python",
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ rootCause={"summary": ""},
+ root_cause={
+ "summary": "The destination is not contained before the write.",
+ "code": "destination.write_bytes(payload)",
+ "language": "python",
+ },
+ )
assert "The destination is not contained before the write." in markdown
assert "```python\ndestination.write_bytes(payload)\n```" in markdown
def test_projection_ignores_malformed_root_cause_alias_fields() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["rootCause"] = {"summary": 42}
- findings["findings"][0]["root_cause"] = {
- "summary": "The valid legacy root cause.",
- "evidence_refs": ["legacy-root"],
- }
- findings["findings"][0]["code_evidence"] = [{"id": "legacy-root", "code": "legacy_root()"}]
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ rootCause={"summary": 42},
+ root_cause={
+ "summary": "The valid legacy root cause.",
+ "evidence_refs": ["legacy-root"],
+ },
+ code_evidence=[{"id": "legacy-root", "code": "legacy_root()"}],
+ )
assert "The valid legacy root cause." in markdown
assert "legacy_root()" in markdown
@@ -448,68 +439,58 @@ def test_projection_ignores_malformed_root_cause_alias_fields() -> None:
def test_projection_merges_scalar_and_list_root_cause_evidence_references() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["rootCause"] = {
- "summary": "The source reaches the write.",
- "evidenceRefs": ["canonical-root-source"],
- }
- findings["findings"][0]["root_cause"] = {"evidence_refs": "legacy-root-source"}
- findings["findings"][0]["codeEvidence"] = [
- {"id": "canonical-root-source", "code": "canonical_source()"}
- ]
- findings["findings"][0]["code_evidence"] = [
- {"id": "legacy-root-source", "code": "legacy_source()"}
- ]
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ rootCause={
+ "summary": "The source reaches the write.",
+ "evidenceRefs": ["canonical-root-source"],
+ },
+ root_cause={"evidence_refs": "legacy-root-source"},
+ codeEvidence=[{"id": "canonical-root-source", "code": "canonical_source()"}],
+ code_evidence=[{"id": "legacy-root-source", "code": "legacy_source()"}],
+ )
assert "canonical_source()" in markdown
assert "legacy_source()" in markdown
def test_projection_merges_embedded_evidence_across_root_cause_aliases() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["rootCause"] = {
- "summary": "The canonical root cause.",
- "codeEvidence": [{"id": "canonical-root", "code": "canonical_evidence()"}],
- }
- findings["findings"][0]["root_cause"] = {
- "codeEvidence": [{"id": "legacy-root", "code": "legacy_evidence()"}]
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ rootCause={
+ "summary": "The canonical root cause.",
+ "codeEvidence": [{"id": "canonical-root", "code": "canonical_evidence()"}],
+ },
+ root_cause={"codeEvidence": [{"id": "legacy-root", "code": "legacy_evidence()"}]},
+ )
assert "canonical_evidence()" in markdown
assert "legacy_evidence()" in markdown
def test_projection_treats_whitespace_root_cause_fields_as_unpopulated() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["rootCause"] = {
- "summary": " ",
- "code": "\t",
- "language": "text",
- }
- findings["findings"][0]["root_cause"] = {
- "summary": "The destination is not contained before the write.",
- "code": "destination.write_bytes(payload)",
- "language": "python",
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ rootCause={
+ "summary": " ",
+ "code": "\t",
+ "language": "text",
+ },
+ root_cause={
+ "summary": "The destination is not contained before the write.",
+ "code": "destination.write_bytes(payload)",
+ "language": "python",
+ },
+ )
assert "The destination is not contained before the write." in markdown
assert "```python\ndestination.write_bytes(payload)\n```" in markdown
def test_projection_ignores_malformed_data_flow_alias_values() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["attackPath"] = {
- "dataFlow": {"summary": ["invalid canonical value"]},
- "dataflow": {"summary": "request -> validated sink"},
- }
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ attackPath={
+ "dataFlow": {"summary": ["invalid canonical value"]},
+ "dataflow": {"summary": "request -> validated sink"},
+ },
+ )
assert "request -\\> validated sink" in markdown
@@ -686,10 +667,9 @@ def test_projection_rejects_unsafe_structural_hardening_portfolio() -> None:
def test_projection_escapes_markdown_link_syntax_in_finding_title() -> None:
- manifest, findings, coverage = canonical_documents()
- findings["findings"][0]["title"] = "Parser ](https://example.com) [boundary"
-
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ title="Parser ](https://example.com) [boundary",
+ )
assert "[Parser \\](https://example.com) \\[boundary](#finding-1)" in markdown
assert "[Parser ](https://example.com) [boundary](" not in markdown
@@ -1012,21 +992,20 @@ def test_projection_includes_surface_evidence_receipts() -> None:
def test_projection_preserves_identifier_text_and_code_path_spelling(
source_path: str, rendered_location: str
) -> None:
- manifest, findings, coverage = canonical_documents()
- finding = findings["findings"][0]
- finding["title"] = "__proto__ pollution"
- finding["validation"] = {"evidenceRefs": ["source"]}
- finding["codeEvidence"] = [
- {
- "id": "source",
- "label": "Source control",
- "path": source_path,
- "startLine": 7,
- "code": "handle(request)",
- "explanation": "A source-backed operation.",
- }
- ]
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ title="__proto__ pollution",
+ validation={"evidenceRefs": ["source"]},
+ codeEvidence=[
+ {
+ "id": "source",
+ "label": "Source control",
+ "path": source_path,
+ "startLine": 7,
+ "code": "handle(request)",
+ "explanation": "A source-backed operation.",
+ }
+ ],
+ )
assert "\\_\\_proto\\_\\_ pollution" in markdown
assert rendered_location in markdown
@@ -1046,17 +1025,16 @@ def test_projection_preserves_identifier_text_and_code_path_spelling(
def test_projection_normalizes_invalid_legacy_code_evidence_bounds(
bounds: dict[str, object], rendered_location: str
) -> None:
- manifest, findings, coverage = canonical_documents()
- finding = findings["findings"][0]
- finding["validation"] = {"evidenceRefs": ["source"]}
- finding["code_evidence"] = [
- {
- "id": "source",
- "label": "Source control",
- "path": "src/evidence.py",
- **bounds,
- "code": "handle(request)",
- }
- ]
- markdown = PROJECTION.build_report_markdown(manifest, findings, coverage)
+ markdown = report_with_finding(
+ validation={"evidenceRefs": ["source"]},
+ code_evidence=[
+ {
+ "id": "source",
+ "label": "Source control",
+ "path": "src/evidence.py",
+ **bounds,
+ "code": "handle(request)",
+ }
+ ],
+ )
assert rendered_location in markdown
diff --git a/plugins/codex-security/tests/test_threat_model_projection.py b/plugins/codex-security/tests/test_threat_model_projection.py
index 2c4041409..6a13d17ac 100644
--- a/plugins/codex-security/tests/test_threat_model_projection.py
+++ b/plugins/codex-security/tests/test_threat_model_projection.py
@@ -3,8 +3,6 @@ from __future__ import annotations
import io
import json
import os
-import subprocess
-import sys
import tempfile
import unittest
import unittest.mock
@@ -17,9 +15,16 @@ FINALIZER = fixtures.FINALIZER
class ThreatModelProjectionTest(ScanFixtureTestCase):
+ run_finalizer = fixtures.FinalizeScanContractTest.run_finalizer
setUp = fixtures.FinalizeScanContractTest.setUp
+ write_sealed_scan = fixtures.FinalizeScanContractTest.write_sealed_scan
write_scan = fixtures.FinalizeScanContractTest.write_scan
+ def assert_saved_model_error(self, source: Path, message: str) -> None:
+ for read_model in (FINALIZER.describe_threat_model, FINALIZER.build_threat_model_export):
+ with self.assertRaisesRegex(FINALIZER.ContractError, message):
+ read_model(source)
+
def test_saves_and_exports_markdown_threat_model_without_changing_authored_body(self) -> None:
body = "# Service Model\n\n| Asset | Owner |\n| --- | --- |\n| Queue | Service |\n\n```text\nA -> B\n```\n"
model = {
@@ -29,8 +34,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
"scope": {"includePaths": ["."], "excludePaths": ["vendor/"]},
}
self.manifest["scan"]["threatModel"] = model
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
document = (self.scan_dir / "threatmodel.md").read_bytes()
self.assertTrue(document.startswith(body.encode()))
self.assertIn(b"Model origin: provided", document)
@@ -125,8 +129,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
with self.subTest(extensions=extensions):
model = {"summary": "Existing structured model.", **extensions}
self.manifest["scan"]["threatModel"] = model
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
self.assertEqual(self.read_json("scan-manifest.json")["scan"]["threatModel"], model)
document = FINALIZER.build_threat_model_export(self.scan_dir)
self.assertIn(b"Existing structured model.", document)
@@ -143,8 +146,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
"content": "# Full model\n\nSource-backed detail.\n",
"summary": "A shorter summary.",
}
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
document = FINALIZER.build_threat_model_export(self.scan_dir)
self.assertTrue(document.startswith(b"# Full model\n\nSource-backed detail.\n"))
self.assertNotIn(b"A shorter summary.", document)
@@ -170,8 +172,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
**metadata,
}
self.manifest["scan"]["threatModel"] = model
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
sealed = (self.scan_dir / "scan-manifest.json").read_bytes()
document = FINALIZER.build_threat_model_export(self.scan_dir)
self.assertTrue(document.startswith(body.encode()))
@@ -219,18 +220,12 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
self.assertFalse((self.scan_dir / "threatmodel.md").exists())
def test_missing_threat_model_does_not_create_placeholder(self) -> None:
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
self.assertFalse((self.scan_dir / "threatmodel.md").exists())
with unittest.mock.patch.object(
FINALIZER, "_read_sealed_scan", wraps=FINALIZER._read_sealed_scan
) as read_sealed:
- for read_model in (
- FINALIZER.describe_threat_model,
- FINALIZER.build_threat_model_export,
- ):
- with self.assertRaisesRegex(FINALIZER.ContractError, "No saved threat model"):
- read_model(self.scan_dir)
+ self.assert_saved_model_error(self.scan_dir, "No saved threat model")
read_sealed.assert_not_called()
manifest = self.read_json("scan-manifest.json")
@@ -249,8 +244,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
"summary": " \n\t",
"assets": ["Stored records"],
}
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
FINALIZER.finalize_scan(self.scan_dir)
for document in (
(self.scan_dir / "report.md").read_text(),
@@ -274,10 +268,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
)
FINALIZER.finalize_scan(self.scan_dir)
self.assertFalse((self.scan_dir / "threatmodel.md").exists())
- with self.assertRaisesRegex(FINALIZER.ContractError, "No saved threat model"):
- FINALIZER.describe_threat_model(self.scan_dir)
- with self.assertRaisesRegex(FINALIZER.ContractError, "No saved threat model"):
- FINALIZER.build_threat_model_export(self.scan_dir)
+ self.assert_saved_model_error(self.scan_dir, "No saved threat model")
(self.scan_dir / "scan-manifest.json").unlink()
self.assertIn(
b"A separate policy model.", FINALIZER.build_threat_model_export(self.scan_dir)
@@ -309,12 +300,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
)
legacy = self.scan_dir / "threatmodel.md"
legacy.write_text("# Earlier model\n")
- for read_model in (
- FINALIZER.describe_threat_model,
- FINALIZER.build_threat_model_export,
- ):
- with self.assertRaisesRegex(FINALIZER.ContractError, r"scan-manifest\.json"):
- read_model(self.scan_dir)
+ self.assert_saved_model_error(self.scan_dir, r"scan-manifest\.json")
self.assertTrue(manifest.is_symlink())
self.assertEqual(legacy.read_text(), "# Earlier model\n")
manifest.unlink()
@@ -351,16 +337,14 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
def test_threat_model_export_validates_existing_scan_seal(self) -> None:
self.manifest["scan"]["threatModel"] = {"summary": "Queue ownership and boundaries."}
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
(self.scan_dir / "findings.json").write_text("{}")
with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
FINALIZER.build_threat_model_export(self.scan_dir)
def test_projection_preserves_existing_sealed_threat_model(self) -> None:
self.manifest["scan"]["threatModel"] = {"summary": "Canonical queue boundaries."}
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
original = b"# Historical model\n\nRetained sealed evidence.\n"
document = self.scan_dir / "threatmodel.md"
document.write_bytes(original)
@@ -373,18 +357,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
FINALIZER.finalize_scan(self.scan_dir)
self.assertEqual(document.read_bytes(), original)
- written = subprocess.run(
- [
- sys.executable,
- str(Path(FINALIZER.__file__)),
- "--scan-dir",
- str(self.scan_dir),
- "--write-threat-model",
- ],
- text=True,
- capture_output=True,
- check=False,
- )
+ written = self.run_finalizer("--write-threat-model")
self.assertEqual(written.returncode, 0, written.stderr)
self.assertEqual(document.read_bytes(), original)
self.assertEqual((self.scan_dir / "scan-manifest.json").read_bytes(), sealed_manifest)
@@ -555,14 +528,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
path.write_text(original)
legacy = source / "threatmodel.md"
legacy.write_bytes(body.encode())
- for read_model in (
- FINALIZER.describe_threat_model,
- FINALIZER.build_threat_model_export,
- ):
- with self.assertRaisesRegex(
- FINALIZER.ContractError, "threatModel: expected an object"
- ):
- read_model(source)
+ self.assert_saved_model_error(source, "threatModel: expected an object")
self.assertEqual(path.read_text(), original)
self.assertEqual(legacy.read_text(), body)
current = manifest["scan"] if filename == "scan-manifest.json" else manifest
@@ -585,50 +551,23 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
},
}
self.write_json("policy-draft.json", manifest)
- script = Path(FINALIZER.__file__)
- saved = subprocess.run(
- [sys.executable, str(script), "--scan-dir", str(self.scan_dir), "--write-threat-model"],
- text=True,
- capture_output=True,
- check=False,
- )
+ saved = self.run_finalizer("--write-threat-model")
self.assertEqual(saved.returncode, 0, saved.stderr)
self.assertIn("provisional", (self.scan_dir / "threatmodel.md").read_text())
manifest["status"] = "completed"
self.write_json("policy-draft.json", manifest)
- exported = subprocess.run(
- [
- sys.executable,
- str(script),
- "--scan-dir",
- str(self.scan_dir),
- "--export-artifact",
- "threat-model",
- ],
- text=True,
- capture_output=True,
- check=False,
- )
+ exported = self.run_finalizer("--export-artifact", "threat-model")
self.assertEqual(exported.returncode, 0, exported.stderr)
self.assertTrue(exported.stdout.startswith(manifest["threatModel"]["content"]))
self.assertNotIn("provisional", exported.stdout)
self.assertIn("Source: policy", exported.stdout)
destination = self.scan_dir / "exports" / "threatmodel.md"
- exported_with_metadata = subprocess.run(
- [
- sys.executable,
- str(script),
- "--scan-dir",
- str(self.scan_dir),
- "--export-artifact",
- "threat-model",
- "--export-output",
- str(destination),
- "--export-metadata",
- ],
- text=True,
- capture_output=True,
- check=False,
+ exported_with_metadata = self.run_finalizer(
+ "--export-artifact",
+ "threat-model",
+ "--export-output",
+ str(destination),
+ "--export-metadata",
)
self.assertEqual(exported_with_metadata.returncode, 0, exported_with_metadata.stderr)
metadata = json.loads(exported_with_metadata.stdout)
@@ -639,8 +578,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
def test_legacy_document_preserves_available_result_context(self) -> None:
body = "# Existing model\n\nReview the recorded service boundaries.\n"
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
legacy_path = self.scan_dir / "artifacts" / "01_context" / "threat_model.md"
legacy_path.parent.mkdir(parents=True)
legacy_path.write_bytes(body.encode())
@@ -666,9 +604,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
self.assertEqual(description["path"], str(legacy_path))
self.assertEqual(FINALIZER.build_threat_model_export(self.scan_dir), body.encode())
legacy_path.write_text(body + "\nChanged since sealing.\n")
- for read_model in (FINALIZER.describe_threat_model, FINALIZER.build_threat_model_export):
- with self.assertRaisesRegex(FINALIZER.ContractError, "sealed artifact changed"):
- read_model(self.scan_dir)
+ self.assert_saved_model_error(self.scan_dir, "sealed artifact changed")
with tempfile.TemporaryDirectory() as directory:
source = Path(directory).resolve()
@@ -690,8 +626,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
self.assertEqual(FINALIZER.build_threat_model_export(source), body.encode())
def test_sealed_scan_does_not_expose_unrecorded_legacy_models(self) -> None:
- self.write_scan()
- FINALIZER.finalize_scan(self.scan_dir)
+ self.write_sealed_scan()
manifest_bytes = (self.scan_dir / "scan-manifest.json").read_bytes()
body = b"# Unrecorded model\n"
for filename in (
@@ -704,12 +639,7 @@ class ThreatModelProjectionTest(ScanFixtureTestCase):
path = self.scan_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(body)
- for read_model in (
- FINALIZER.describe_threat_model,
- FINALIZER.build_threat_model_export,
- ):
- with self.assertRaisesRegex(FINALIZER.ContractError, "No saved threat model"):
- read_model(self.scan_dir)
+ self.assert_saved_model_error(self.scan_dir, "No saved threat model")
self.assertEqual(path.read_bytes(), body)
path.unlink()
self.assertEqual((self.scan_dir / "scan-manifest.json").read_bytes(), manifest_bytes)
diff --git a/plugins/codex-security/tests/test_threat_model_visualize.py b/plugins/codex-security/tests/test_threat_model_visualize.py
index 70f9ff8e8..8fd49d248 100644
--- a/plugins/codex-security/tests/test_threat_model_visualize.py
+++ b/plugins/codex-security/tests/test_threat_model_visualize.py
@@ -18,6 +18,22 @@ builder = importlib.util.module_from_spec(spec)
spec.loader.exec_module(builder)
+def run_builder(model_file, source, output):
+ return subprocess.run(
+ [
+ sys.executable,
+ str(Path(builder.__file__)),
+ str(model_file),
+ "--assessment",
+ str(source),
+ "--output",
+ str(output),
+ ],
+ text=True,
+ capture_output=True,
+ )
+
+
class EmbeddedHtml(HTMLParser):
"""Extract script and style bodies while preserving their exact contents."""
@@ -110,6 +126,9 @@ class ControlValidation(unittest.TestCase):
def check(self, model=None):
builder.validate(self.model if model is None else model, b"policy denies this caller")
+ def assert_invalid(self, expected, model=None):
+ self.assertRaisesRegex(ValueError, expected, self.check, model)
+
def effect(self):
return self.model["scenarios"][0]["controls"][0]["effect"]
@@ -130,19 +149,7 @@ class ControlValidation(unittest.TestCase):
if existing_html is not None:
output.write_text(existing_html, encoding="utf-8")
original_model = model_file.read_bytes()
- result = subprocess.run(
- [
- sys.executable,
- str(Path(builder.__file__)),
- str(model_file),
- "--assessment",
- str(source),
- "--output",
- str(output),
- ],
- text=True,
- capture_output=True,
- )
+ result = run_builder(model_file, source, output)
self.assertEqual(model_file.read_bytes(), original_model)
self.assertEqual(source.read_bytes(), assessment_bytes)
if error:
@@ -261,8 +268,7 @@ class ControlValidation(unittest.TestCase):
del target[path[-1]]
else:
target[path[-1]] = value
- with self.assertRaisesRegex(ValueError, message):
- self.check(model)
+ self.assert_invalid(message, model)
self.model["nodes"][0]["title"] = None
self.build_html(b"policy denies this caller", error="a.title must be a string")
@@ -293,19 +299,7 @@ class ControlValidation(unittest.TestCase):
)
elif alias == "hardlink":
output.hardlink_to(target)
- result = subprocess.run(
- [
- sys.executable,
- str(Path(builder.__file__)),
- str(inputs["model"]),
- "--assessment",
- str(inputs["assessment"]),
- "--output",
- str(output),
- ],
- text=True,
- capture_output=True,
- )
+ result = run_builder(inputs["model"], inputs["assessment"], output)
self.assertEqual(result.returncode, 2)
self.assertIn("output must not overwrite", result.stderr)
for name, path in inputs.items():
@@ -348,8 +342,7 @@ class ControlValidation(unittest.TestCase):
for key in path[:-1]:
target = target[key]
target[path[-1]] = value
- with self.assertRaisesRegex(ValueError, message):
- self.check(model)
+ self.assert_invalid(message, model)
# A 240 × 72 marker fits exactly at either corner of the 800 × 500 canvas.
for x, y in ((0, 0), (560, 428)):
self.effect()["marker"].update(x=x, y=y)
@@ -359,21 +352,18 @@ class ControlValidation(unittest.TestCase):
def test_unknown_effect_is_not_a_block(self):
self.effect()["basis"] = "unverified"
- with self.assertRaisesRegex(ValueError, "unverified effect cannot block"):
- self.check()
+ self.assert_invalid("unverified effect cannot block")
self.effect()["outcome"] = "unknown"
self.check()
def test_proposed_cannot_be_verified(self):
self.model["scenarios"][0]["controls"][0]["kind"] = "proposed"
self.effect()["basis"] = "verified"
- with self.assertRaisesRegex(ValueError, "hypothetical"):
- self.check()
+ self.assert_invalid("hypothetical")
def test_effect_requires_evidence(self):
self.model["scenarios"][0]["controls"][0]["evidence"] = []
- with self.assertRaisesRegex(ValueError, "require assessment evidence"):
- self.check()
+ self.assert_invalid("require assessment evidence")
def test_evidence_paths_require_an_array_of_strings(self):
evidence = self.model["evidence"][0]
@@ -388,15 +378,13 @@ class ControlValidation(unittest.TestCase):
evidence["paths"] = valid
self.check()
evidence["paths"] = ["not quoted"]
- with self.assertRaisesRegex(ValueError, "cited path not present"):
- self.check()
+ self.assert_invalid("cited path not present")
del evidence["paths"]
self.check()
def test_disconnected_steps_do_not_hide_an_independent_branch(self):
self.model["scenarios"][0]["steps"][1] = {"from": "a", "to": "b", "edge": "ab"}
- with self.assertRaisesRegex(ValueError, "continuous ordered path"):
- self.check()
+ self.assert_invalid("continuous ordered path")
def test_repeated_edge_cannot_hide_an_earlier_traversal(self):
self.model["edges"].append(
@@ -414,8 +402,19 @@ class ControlValidation(unittest.TestCase):
{"from": "b", "to": "a", "edge": "ba"},
{"from": "a", "to": "b", "edge": "ab"},
]
- with self.assertRaisesRegex(ValueError, "cannot reuse an edge"):
- self.check()
+ self.assert_invalid("cannot reuse an edge")
+
+ def add_edge(self, edge_id, source, destination):
+ self.model["edges"].append(
+ {
+ "id": edge_id,
+ "from": source,
+ "to": destination,
+ "points": [[0, 0], [20, 0]],
+ "labelX": 0,
+ "labelY": 0,
+ }
+ )
def routed(self):
s = self.model["scenarios"][0]
@@ -429,58 +428,35 @@ class ControlValidation(unittest.TestCase):
s = self.routed()
self.check()
s["routes"][0]["edges"] = ["bc", "ab"]
- with self.assertRaisesRegex(ValueError, "disconnected route"):
- self.check()
+ self.assert_invalid("disconnected route")
def test_route_cannot_omit_controlled_transition(self):
s = self.routed()
s["routes"][0]["edges"] = ["bc"]
- with self.assertRaisesRegex(ValueError, "steps in order"):
- self.check()
+ self.assert_invalid("steps in order")
def test_route_requires_evidence_and_explicit_focus(self):
s = self.routed()
del s["focusNodes"]
- with self.assertRaisesRegex(ValueError, "explicit focusNodes"):
- self.check()
+ self.assert_invalid("explicit focusNodes")
s["focusNodes"] = []
s["routes"][0]["evidence"] = []
- with self.assertRaisesRegex(ValueError, "route requires assessment evidence"):
- self.check()
+ self.assert_invalid("route requires assessment evidence")
def test_unattached_dependency_rejected(self):
s = self.routed()
self.model["nodes"].extend(
[{"id": n, "title": n, "x": 0, "y": 0, "w": 20, "h": 20} for n in "xy"]
)
- self.model["edges"].append(
- {
- "id": "xy",
- "from": "x",
- "to": "y",
- "points": [[0, 0], [20, 0]],
- "labelX": 0,
- "labelY": 0,
- }
- )
+ self.add_edge("xy", "x", "y")
s["routes"][0]["branches"] = [{"edges": ["xy"], "evidence": ["ev"]}]
- with self.assertRaisesRegex(ValueError, "unattached"):
- self.check()
+ self.assert_invalid("unattached")
def test_prerequisite_branch_must_identify_its_decision(self):
s = self.routed()
self.effect()["arrival"] = True
self.model["nodes"].append({"id": "d", "title": "d", "x": 0, "y": 0, "w": 20, "h": 20})
- self.model["edges"].append(
- {
- "id": "bd",
- "from": "b",
- "to": "d",
- "points": [[0, 0], [20, 0]],
- "labelX": 0,
- "labelY": 0,
- }
- )
+ self.add_edge("bd", "b", "d")
branch = {"edges": ["bd"], "evidence": ["ev"], "beforeStep": 1}
s["routes"][0]["branches"] = [branch]
self.check()
@@ -495,8 +471,7 @@ class ControlValidation(unittest.TestCase):
self.check() # b is an earlier, explicitly ordered prerequisite origin.
self.model["edges"][-1]["from"] = "c"
branch["beforeStep"] = 1
- with self.assertRaisesRegex(ValueError, "at or before its decision"):
- self.check()
+ self.assert_invalid("at or before its decision")
def test_decision_destination_prerequisite_requires_arrival(self):
s = self.routed()
@@ -515,10 +490,7 @@ class ControlValidation(unittest.TestCase):
if arrival is True:
self.check()
else:
- with self.assertRaisesRegex(
- ValueError, "destination requires arrival:true"
- ):
- self.check()
+ self.assert_invalid("destination requires arrival:true")
def test_prerequisite_arrival_guard_honors_control_scope(self):
s = self.routed()
@@ -541,24 +513,14 @@ class ControlValidation(unittest.TestCase):
control["sourceIds"] = ["origin"]
self.check() # This source cannot select the route with the destination branch.
control["sourceIds"] = ["second"]
- with self.assertRaisesRegex(ValueError, "destination requires arrival:true"):
- self.check()
+ self.assert_invalid("destination requires arrival:true")
control["routeIds"] = ["r"]
self.check() # The additional route restriction excludes that branch again.
def test_later_branch_guard_requires_one_unambiguous_step(self):
s = self.routed()
self.model["nodes"].append({"id": "d", "title": "d", "x": 0, "y": 0, "w": 20, "h": 20})
- self.model["edges"].append(
- {
- "id": "bd",
- "from": "b",
- "to": "d",
- "points": [[0, 0], [20, 0]],
- "labelX": 0,
- "labelY": 0,
- }
- )
+ self.add_edge("bd", "b", "d")
branch = {"edges": ["bd"], "evidence": ["ev"], "afterStep": 2}
s["routes"][0]["branches"] = [branch]
self.check()
@@ -571,8 +533,7 @@ class ControlValidation(unittest.TestCase):
self.check()
branch["afterStep"] = 1
branch["beforeStep"] = 1
- with self.assertRaisesRegex(ValueError, "cannot have both"):
- self.check()
+ self.assert_invalid("cannot have both")
def test_connector_warnings_distinguish_bent_legs_and_node_interiors(self):
ab, bc = self.model["edges"]
@@ -597,24 +558,20 @@ class ControlValidation(unittest.TestCase):
effect["arrival"] = True
self.check()
effect["arrival"] = "yes"
- with self.assertRaisesRegex(ValueError, "arrival must be a boolean"):
- self.check()
+ self.assert_invalid("arrival must be a boolean")
effect["arrival"] = True
effect["traffic"] = "continues"
- with self.assertRaisesRegex(ValueError, "arrival must be a boolean"):
- self.check()
+ self.assert_invalid("arrival must be a boolean")
del effect["arrival"]
for key in ("step", "edge", "marker"):
del effect[key]
self.model["scenarios"][0]["steps"] = []
self.check() # Data-only prevention need not invent a sequence.
effect["traffic"] = "maybe"
- with self.assertRaisesRegex(ValueError, "traffic must be stops or continues"):
- self.check()
+ self.assert_invalid("traffic must be stops or continues")
effect["traffic"] = "continues"
effect["outcome"] = "limits"
- with self.assertRaisesRegex(ValueError, "traffic must be stops or continues"):
- self.check()
+ self.assert_invalid("traffic must be stops or continues")
def test_builder_embeds_new_contract_and_assessment_in_standalone_html(self):
self.effect()["traffic"] = "continues"
@@ -732,31 +689,24 @@ class ControlValidation(unittest.TestCase):
if valid:
self.check()
else:
- with self.assertRaisesRegex(ValueError, "each applicable route"):
- self.check()
+ self.assert_invalid("each applicable route")
control.pop("routeIds")
control["sourceIds"] = ["origin"]
source.pop("routeIds")
- with self.assertRaisesRegex(ValueError, "each applicable route"):
- self.check()
+ self.assert_invalid("each applicable route")
# A detector attached to an edge only runs on routes that use it.
self.effect().pop("traffic")
self.effect()["outcome"] = "detects"
- with self.assertRaisesRegex(ValueError, "each applicable route"):
- self.check()
+ self.assert_invalid("each applicable route")
control["routeIds"] = ["r"]
self.check()
def test_prerequisite_on_a_revisited_decision_node_is_valid(self):
s = self.routed()
self.model["nodes"].append({"id": "d", "title": "d", "x": 0, "y": 0, "w": 20, "h": 20})
- self.model["edges"].extend(
- [
- {"id": i, "from": a, "to": b, "points": [[0, 0], [20, 0]], "labelX": 0, "labelY": 0}
- for i, a, b in [("cb", "c", "b"), ("bd", "b", "d")]
- ]
- )
+ self.add_edge("cb", "c", "b")
+ self.add_edge("bd", "b", "d")
s["steps"] = [
{"from": "b", "to": "c", "edge": "bc"},
{"from": "c", "to": "b", "edge": "cb"},
@@ -767,19 +717,14 @@ class ControlValidation(unittest.TestCase):
self.check()
# An earlier visit does not establish receipt of this later decision.
self.effect().update(step=2, edge="cb")
- with self.assertRaisesRegex(ValueError, "destination requires arrival:true"):
- self.check()
+ self.assert_invalid("destination requires arrival:true")
self.effect()["arrival"] = True
self.check()
def test_route_bridges_conceptual_steps(self):
s = self.routed()
- self.model["edges"].extend(
- [
- {"id": i, "from": a, "to": b, "points": [[0, 0], [20, 0]], "labelX": 0, "labelY": 0}
- for i, a, b in [("ba", "b", "a"), ("ac", "a", "c")]
- ]
- )
+ self.add_edge("ba", "b", "a")
+ self.add_edge("ac", "a", "c")
s["steps"][1] = {"from": "a", "to": "c", "edge": "ac"}
s["routes"][0]["edges"] = ["ab", "ba", "ac"]
self.effect()["step"] = 2
@@ -805,28 +750,23 @@ class ControlValidation(unittest.TestCase):
source["node"] = None
self.check()
source["node"] = "listener-that-is-not-present"
- with self.assertRaisesRegex(ValueError, "source node"):
- self.check()
+ self.assert_invalid("source node")
def test_source_requires_evidence_and_starting_authority(self):
source = self.origin()
source["evidence"] = []
- with self.assertRaisesRegex(ValueError, "source role requires assessment evidence"):
- self.check()
+ self.assert_invalid("source role requires assessment evidence")
source["evidence"] = ["ev"]
source["authority"] = ""
- with self.assertRaisesRegex(ValueError, "requires authority"):
- self.check()
+ self.assert_invalid("requires authority")
def test_source_basis_and_role_are_explicit(self):
source = self.origin()
source["basis"] = "inferred-safe"
- with self.assertRaisesRegex(ValueError, "explicit basis"):
- self.check()
+ self.assert_invalid("explicit basis")
source["basis"] = "unknown"
source["kind"] = "authenticated-therefore-benign"
- with self.assertRaisesRegex(ValueError, "invalid source role"):
- self.check()
+ self.assert_invalid("invalid source role")
def test_source_cannot_leak_to_unrelated_route(self):
self.routed()
@@ -834,8 +774,7 @@ class ControlValidation(unittest.TestCase):
source["routeIds"] = ["r"]
self.check()
source["routeIds"] = ["other-scenario-route"]
- with self.assertRaisesRegex(ValueError, "source routeIds"):
- self.check()
+ self.assert_invalid("source routeIds")
def test_privileged_component_cannot_stand_in_for_authorized_user(self):
self.model["participants"] = [
@@ -848,8 +787,7 @@ class ControlValidation(unittest.TestCase):
"evidence": ["ev"],
}
]
- with self.assertRaisesRegex(ValueError, "authorized user needs an actor"):
- self.check()
+ self.assert_invalid("authorized user needs an actor")
self.model["nodes"][1]["kind"] = "actor"
self.check()
@@ -857,8 +795,7 @@ class ControlValidation(unittest.TestCase):
source = self.origin()
scenario = self.model["scenarios"][0]
scenario["sources"].append(dict(source, id="other"))
- with self.assertRaisesRegex(ValueError, "explicit sourceMode"):
- self.check()
+ self.assert_invalid("explicit sourceMode")
scenario["sourceMode"] = "alternatives"
self.check()
@@ -868,31 +805,25 @@ class ControlValidation(unittest.TestCase):
scenario["sourceMode"] = "alternatives"
source["relatedNodes"] = []
source["relatedEdges"] = []
- with self.assertRaisesRegex(ValueError, "unordered views"):
- self.check()
+ self.assert_invalid("unordered views")
scenario["steps"] = []
scenario["controls"] = []
self.check()
source["relatedNodes"] = ["missing"]
- with self.assertRaisesRegex(ValueError, "unique resolved"):
- self.check()
+ self.assert_invalid("unique resolved")
source["relatedNodes"] = ["a", "a"]
- with self.assertRaisesRegex(ValueError, "unique resolved"):
- self.check()
+ self.assert_invalid("unique resolved")
source["relatedNodes"] = ["a"]
source["relatedEdges"] = ["missing"]
- with self.assertRaisesRegex(ValueError, "unique resolved"):
- self.check()
+ self.assert_invalid("unique resolved")
source["relatedEdges"] = ["ab"]
self.check()
scenario["sourceMode"] = "combined"
- with self.assertRaisesRegex(ValueError, "unordered views"):
- self.check()
+ self.assert_invalid("unordered views")
scenario["sourceMode"] = "alternatives"
scenario["focusNodes"] = []
scenario["routes"] = [{"id": "r", "label": "Flow", "edges": ["ab"], "evidence": ["ev"]}]
- with self.assertRaisesRegex(ValueError, "without routes"):
- self.check()
+ self.assert_invalid("without routes")
def test_unordered_effect_honors_each_source_relationship_override(self):
source = self.origin()
@@ -928,8 +859,7 @@ class ControlValidation(unittest.TestCase):
if valid:
self.check()
else:
- with self.assertRaisesRegex(ValueError, "effect edge"):
- self.check()
+ self.assert_invalid("effect edge")
def test_control_scope_requires_resolved_nonempty_unique_ids(self):
self.origin()
@@ -943,12 +873,10 @@ class ControlValidation(unittest.TestCase):
original = c[field]
for invalid in ([], ["missing"], original * 2, "not-array"):
c[field] = invalid
- with self.assertRaisesRegex(ValueError, "nonempty unique resolved"):
- self.check()
+ self.assert_invalid("nonempty unique resolved")
c[field] = original
scenario["sourceMode"] = "combined"
- with self.assertRaisesRegex(ValueError, "combined semantics"):
- self.check()
+ self.assert_invalid("combined semantics")
del c["sourceIds"]
self.check() # Combined sources may still have route-scoped controls.