Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: move finding compatibility checks into Python tests (#1407)

mldangelo-oai91cc959bdf27
diff --git a/plugins/codex-security/tests/test_finalize_scan_contract.py b/plugins/codex-security/tests/test_finalize_scan_contract.py
index 01479484a..366a93581 100644
--- a/plugins/codex-security/tests/test_finalize_scan_contract.py
+++ b/plugins/codex-security/tests/test_finalize_scan_contract.py
@@ -11,6 +11,8 @@ import sys
import tempfile
import threading
import unittest
+from collections.abc import Iterator
+from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
from unittest import mock
@@ -121,6 +123,12 @@ class FinalizeScanContractTest(ScanFixtureTestCase):
self.write_scan()
FINALIZER.finalize_scan(self.scan_dir)
+ def compatible_finding(self, findings: dict) -> dict:
+ original = copy.deepcopy(findings)
+ compatible = FINALIZER._legacy_sealed_findings_for_validation(findings)
+ self.assertEqual(findings, original)
+ return compatible["findings"][0]
+
def run_finalizer(self, *args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
@@ -203,6 +211,15 @@ The extraction root is not enforced.
raise AssertionError(f"missing sealed artifact: {relative_path}")
self.write_json("scan-manifest.json", manifest)
+ @contextmanager
+ def preserving_sealed_findings(self, findings: dict) -> Iterator[None]:
+ self.rewrite_sealed_artifact("findings.json", findings)
+ sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
+ yield
+ _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
+ self.assertEqual(accepted, findings)
+ self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
+
def test_finalize_seals_manifest_deterministically(self) -> None:
self.write_scan()
@@ -425,20 +442,31 @@ The extraction root is not enforced.
}
]
findings["findings"][0]["code_evidence"] = [
- {"id": "shared-source", "code": "legacy_source()"}
+ {"id": "shared-source", "code": "legacy_source()"},
+ {"id": "legacy-duplicate", "code": "first_legacy_source()"},
+ {"id": "legacy-duplicate", "code": "second_legacy_source()"},
]
+ findings["findings"][0]["rootCause"] = {
+ "summary": "Canonical root cause.",
+ "code": "canonical_root()",
+ "language": "python",
+ }
findings["findings"][0]["validation"] = {"evidence_refs": ["legacy-validation-evidence"]}
findings["findings"][0]["attackPath"] = {
"evidence_refs": ["legacy-attack-evidence"],
"dataFlow": {"evidenceRefs": ["legacy-missing-evidence"]},
}
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_findings = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_findings)
+ with self.preserving_sealed_findings(findings):
+ compatible = self.compatible_finding(findings)
+ self.assertEqual(compatible["rootCause"], findings["findings"][0]["rootCause"])
+ self.assertEqual(
+ compatible["code_evidence"],
+ [{"id": "legacy-duplicate", "code": "first_legacy_source()"}],
+ )
+ self.assertEqual(compatible["validation"], {"evidence_refs": []})
+ self.assertEqual(
+ compatible["attackPath"], {"evidence_refs": [], "dataFlow": {"evidenceRefs": []}}
+ )
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_malformed_legacy_evidence_references(self) -> None:
@@ -447,19 +475,26 @@ The extraction root is not enforced.
findings["findings"][0]["code_evidence"] = [
{"id": "legacy-source", "code": "legacy_source()"}
]
+ findings["findings"][0]["rootCause"] = {
+ "summary": "Canonical root cause.",
+ "evidenceRefs": ["legacy-source", "missing-source"],
+ }
findings["findings"][0]["validation"] = {
"evidenceRefs": [None, "", 42, "legacy-source", "missing-source"]
}
findings["findings"][0]["attackPath"] = {
"dataflow": {"evidence_refs": [None, "", 42, "legacy-source", "missing-source"]}
}
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_findings = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_findings)
+ with self.preserving_sealed_findings(findings):
+ compatible = self.compatible_finding(findings)
+ self.assertEqual(
+ compatible["rootCause"],
+ {"summary": "Canonical root cause.", "evidenceRefs": ["legacy-source"]},
+ )
+ self.assertEqual(compatible["validation"], {"evidenceRefs": ["legacy-source"]})
+ self.assertEqual(
+ compatible["attackPath"], {"dataflow": {"evidence_refs": ["legacy-source"]}}
+ )
def test_unsealed_scan_rejects_malformed_legacy_evidence_references(self) -> None:
self.findings["findings"][0]["validation"] = {"evidenceRefs": [None]}
@@ -478,13 +513,8 @@ The extraction root is not enforced.
{"id": "legacy-duplicate", "code": "first_legacy_source()"},
{"id": "legacy-duplicate", "code": "second_legacy_source()"},
]
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_findings = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_findings)
+ with self.preserving_sealed_findings(findings):
+ pass
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_non_string_legacy_validation_scalars(self) -> None:
@@ -494,13 +524,8 @@ The extraction root is not enforced.
"method": [],
"summary": {"legacy": True},
}
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_findings = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_findings)
+ with self.preserving_sealed_findings(findings):
+ self.assertEqual(self.compatible_finding(findings)["validation"], {})
@pytest.mark.cross_platform
def test_sealed_rerun_accepts_formerly_free_form_finding_details(self) -> None:
@@ -508,17 +533,17 @@ The extraction root is not enforced.
findings = self.read_json("findings.json")
findings["findings"][0]["validation"] = {
"evidence": {"kind": "trace"},
+ "counterEvidence": [None, "The mitigation was checked."],
}
findings["findings"][0]["attackPath"] = {
"steps": {"first": "upload"},
}
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_findings = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_findings)
+ with self.preserving_sealed_findings(findings):
+ compatible = self.compatible_finding(findings)
+ self.assertEqual(
+ compatible["validation"], {"counterEvidence": ["The mitigation was checked."]}
+ )
+ self.assertEqual(compatible["attackPath"], {})
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_blank_legacy_attack_path_details(self) -> None:
@@ -535,13 +560,8 @@ The extraction root is not enforced.
"likelihood",
)
}
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_findings = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_findings)
+ with self.preserving_sealed_findings(findings):
+ self.assertEqual(self.compatible_finding(findings)["attackPath"], {})
@pytest.mark.cross_platform
def test_sealed_rerun_rejects_nullable_canonical_evidence_catalog(self) -> None:
@@ -550,6 +570,7 @@ The extraction root is not enforced.
findings["findings"][0]["codeEvidence"] = None
self.rewrite_sealed_artifact("findings.json", findings)
+ self.assertIsNone(self.compatible_finding(findings)["codeEvidence"])
with self.assertRaisesRegex(
FINALIZER.ContractError,
r"codeEvidence: expected (?:an |schema type )?array",
@@ -1622,6 +1643,11 @@ The extraction root is not enforced.
data_flow_key: "request -> filesystem write",
"reachability": "An authenticated uploader can reach extraction.",
}
+ findings["findings"][0]["validation"] = {
+ "assertions": ["The destination escapes the extraction root."],
+ "evidence": "The archive entry path is not contained.",
+ "limitations": ["The upload route was not exercised dynamically."],
+ }
report = FINALIZER._generate_report_projection(
self.manifest, findings, self.coverage
@@ -1629,6 +1655,9 @@ The extraction root is not enforced.
self.assertIn("request -\\> filesystem write", report)
self.assertIn("An authenticated uploader can reach extraction.", report)
+ self.assertIn("The destination escapes the extraction root.", report)
+ self.assertIn("The archive entry path is not contained.", report)
+ self.assertIn("The upload route was not exercised dynamically.", report)
@pytest.mark.cross_platform
def test_report_projection_prefers_populated_data_flow_alias(self) -> None:
@@ -1768,30 +1797,15 @@ The extraction root is not enforced.
"summary": "",
}
findings["findings"][0]["root_cause"] = None
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(
- accepted["findings"][0]["validation"]["assertions"],
- "The destination escapes the extraction root.",
- )
- self.assertEqual(accepted["findings"][0]["validation"]["limitations"], "")
- self.assertIsNone(accepted["findings"][0]["validation"]["counterEvidence"])
- self.assertEqual(accepted["findings"][0]["validation"]["method"], "")
- self.assertEqual(accepted["findings"][0]["validation"]["status"], "")
- self.assertEqual(accepted["findings"][0]["validation"]["summary"], "")
- self.assertEqual(accepted["findings"][0]["validation"]["disposition"], "")
- self.assertEqual(accepted["findings"][0]["validation"]["result"], "")
- self.assertIsNone(accepted["findings"][0]["attackPath"]["dataFlow"])
- self.assertEqual(accepted["findings"][0]["attackPath"]["dataflow"]["summary"], "")
- self.assertEqual(accepted["findings"][0]["attackPath"]["impact"]["level"], "")
- self.assertIsNone(accepted["findings"][0]["attackPath"]["likelihood"]["level"])
- self.assertIsNone(accepted["findings"][0]["attackPath"]["reachability"])
- self.assertEqual(accepted["findings"][0]["attackPath"]["summary"], "")
- self.assertIsNone(accepted["findings"][0]["root_cause"])
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
+ with self.preserving_sealed_findings(findings):
+ compatible = self.compatible_finding(findings)
+ self.assertEqual(
+ compatible["validation"],
+ {"assertions": ["The destination escapes the extraction root."]},
+ )
+ self.assertEqual(
+ compatible["attackPath"], {"dataflow": {}, "impact": {}, "likelihood": {}}
+ )
@pytest.mark.cross_platform
def test_sealed_rerun_rejects_malformed_canonical_root_cause(self) -> None:
@@ -1814,25 +1828,15 @@ The extraction root is not enforced.
self.write_sealed_scan()
findings = self.read_json("findings.json")
findings["findings"][0]["code_evidence"] = None
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertIsNone(accepted["findings"][0]["code_evidence"])
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
+ with self.preserving_sealed_findings(findings):
+ self.assertNotIn("code_evidence", self.compatible_finding(findings))
def test_sealed_rerun_accepts_empty_legacy_root_cause(self) -> None:
self.write_sealed_scan()
findings = self.read_json("findings.json")
findings["findings"][0]["root_cause"] = ""
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted["findings"][0]["root_cause"], "")
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
+ with self.preserving_sealed_findings(findings):
+ pass
@pytest.mark.cross_platform
def test_sealed_rerun_ignores_malformed_legacy_evidence_rows(self) -> None:
@@ -1847,13 +1851,11 @@ The extraction root is not enforced.
{"id": "empty-code", "code": ""},
{"id": "legacy-source", "code": "legacy_source()"},
]
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(accepted, findings)
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
+ with self.preserving_sealed_findings(findings):
+ self.assertEqual(
+ self.compatible_finding(findings)["code_evidence"],
+ [{"id": "legacy-source", "code": "legacy_source()"}],
+ )
def test_unsealed_scan_rejects_malformed_legacy_evidence_rows(self) -> None:
self.findings["findings"][0]["code_evidence"] = [None]
@@ -1895,20 +1897,8 @@ The extraction root is not enforced.
"dataflow": ["archive entry path", "filesystem write"],
"reachability": ["authenticated uploader"],
}
- self.rewrite_sealed_artifact("findings.json", findings)
- sealed_bytes = (self.scan_dir / "findings.json").read_bytes()
-
- _, accepted, _ = FINALIZER.finalize_scan(self.scan_dir)
-
- self.assertEqual(
- accepted["findings"][0]["attackPath"]["dataflow"],
- ["archive entry path", "filesystem write"],
- )
- self.assertEqual(
- accepted["findings"][0]["attackPath"]["reachability"],
- ["authenticated uploader"],
- )
- self.assertEqual((self.scan_dir / "findings.json").read_bytes(), sealed_bytes)
+ with self.preserving_sealed_findings(findings):
+ self.assertEqual(self.compatible_finding(findings)["attackPath"], {})
def test_non_standard_json_numbers_are_rejected(self) -> None:
self.findings["findings"][0]["validation"] = {"score": float("nan")}
@@ -2242,11 +2232,10 @@ The extraction root is not enforced.
self.assertEqual(prepared[3]["findings"][0]["summary"], "SECOND")
self.assertIsNone(prepared[3]["findings"][0]["root_cause"])
- self.assertTrue(
- any(
- "retained stronger duplicate logical finding" in warning
- for warning in warnings
- )
+ self.assertEqual(prepared[3]["findings"][0][evidence_field], [evidence])
+ self.assertEqual(
+ warnings,
+ ["Recovered finding 2: retained stronger duplicate logical finding."],
)
@pytest.mark.cross_platform
@@ -2280,11 +2269,9 @@ The extraction root is not enforced.
)
self.assertEqual(prepared[3]["findings"][0]["summary"], "SECOND")
- self.assertTrue(
- any(
- "retained stronger duplicate logical finding" in warning
- for warning in warnings
- )
+ self.assertEqual(
+ warnings,
+ ["Recovered finding 2: retained stronger duplicate logical finding."],
)
def test_recovery_rejects_malformed_severity_change_condition_lists(self) -> None:
diff --git a/plugins/codex-security/tests/test_report_projection.py b/plugins/codex-security/tests/test_report_projection.py
index 070e41e8c..b531c0a82 100644
--- a/plugins/codex-security/tests/test_report_projection.py
+++ b/plugins/codex-security/tests/test_report_projection.py
@@ -106,7 +106,8 @@ def test_projection_renders_inline_code_and_section_code_evidence() -> None:
assert "The runtime path inserts `local` without reusing the startup check." in markdown
-def test_projection_renders_nested_attack_path_code_evidence() -> None:
[email protected]("reference_key", ["evidenceRefs", "evidence_refs"])
+def test_projection_renders_nested_attack_path_code_evidence(reference_key: str) -> None:
manifest, findings, coverage = canonical_documents()
finding = findings["findings"][0]
finding["codeEvidence"] = [
@@ -135,7 +136,7 @@ def test_projection_renders_nested_attack_path_code_evidence() -> None:
},
"reachability": {
"summary": "An authenticated uploader can trigger extraction.",
- "evidence_refs": ["archive-sink"],
+ reference_key: ["archive-sink"],
},
}
diff --git a/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts b/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts
index b5af235dd..15bd6c8ce 100644
--- a/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts
+++ b/sdk/typescript/tests-ts/plugin-finding-detail-contract.test.ts
@@ -4,7 +4,6 @@ import { describe, expect, test } from "bun:test";
import Ajv from "ajv";
import Ajv2020 from "ajv/dist/2020.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
-import { runPython } from "./support/python-probe.js";
import { readJson as readJsonFile } from "./support/json.js";
import { initializeMcpClient } from "./support/mcp-client.js";
@@ -94,28 +93,6 @@ const stringAssessmentInput = {
],
};
-function projectFindingDetails(details: JsonObject): string {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "examples = plugin / 'examples' / 'completed-scan'",
- "manifest, findings, coverage = [json.loads((examples / name).read_text()) for name in ('scan-manifest.json', 'findings.json', 'coverage.json')]",
- "findings['findings'][0].update(json.loads(sys.argv[2]))",
- "projection = runpy.run_path(str(plugin / 'scripts' / 'report_projection.py'))",
- "print(projection['build_report_markdown'](manifest, findings, coverage))",
- ].join("\n");
- const result = runPython(python!, [
- "-c",
- script,
- PLUGIN_ROOT,
- JSON.stringify(details),
- ]);
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- return new TextDecoder().decode(result.stdout);
-}
-
const readJson = readJsonFile<JsonObject>;
async function startMcp() {
@@ -235,772 +212,4 @@ describe("bundled plugin finding detail contracts", () => {
);
}
});
-
- test.each(["dataFlow", "dataflow", "data_flow"] as const)(
- "projects scalar finding details for the %s alias",
- (dataFlowKey) => {
- const report = projectFindingDetails({
- attackPath: {
- [dataFlowKey]: "request -> archive extraction -> filesystem write",
- reachability: "An authenticated uploader can trigger extraction.",
- },
- validation: {
- assertions: ["The destination escapes the extraction root."],
- evidence: "The archive entry path is not contained.",
- limitations: ["The upload route was not exercised dynamically."],
- },
- });
-
- expect(report).toContain(
- "request -\\> archive extraction -\\> filesystem write",
- );
- expect(report).toContain(
- "An authenticated uploader can trigger extraction.",
- );
- expect(report).toContain("The destination escapes the extraction root.");
- expect(report).toContain("The archive entry path is not contained.");
- expect(report).toContain(
- "The upload route was not exercised dynamically.",
- );
- },
- );
-
- test("projects the populated data-flow alias", () => {
- const report = projectFindingDetails({
- attackPath: {
- dataFlow: { producerExtension: true },
- dataflow: {
- summary:
- "request -> populated lowercase dataflow -> filesystem write",
- },
- },
- });
-
- expect(report).toContain(
- "request -\\> populated lowercase dataflow -\\> filesystem write",
- );
- });
-
- test("ignores malformed data-flow alias values", () => {
- const report = projectFindingDetails({
- attackPath: {
- dataFlow: { summary: ["invalid canonical value"] },
- dataflow: { summary: "request -> validated sink" },
- },
- });
-
- expect(report).toContain("request -\\> validated sink");
- });
-
- test("merges transformations across data-flow aliases", () => {
- const report = projectFindingDetails({
- attackPath: {
- dataFlow: {
- transformations: ["decode archive entry", "parse *input*"],
- },
- dataflow: {
- summary: "request -> archive extraction -> filesystem write",
- transformations: ["dispatch extraction", "decode archive entry"],
- },
- data_flow: { transformations: null },
- },
- });
-
- expect(report).toContain("- decode archive entry");
- expect(report).toContain("- dispatch extraction");
- expect(report.match(/- decode archive entry/gu)).toHaveLength(1);
- expect(report.split(/\r?\n/u)).toContain("- parse \\*input\\*");
- });
-
- test("merges top-level and reachability preconditions", () => {
- const report = projectFindingDetails({
- attackPath: {
- preconditions: [
- "The service processes uploaded archives.",
- "The attacker can upload an archive.",
- ],
- reachability: {
- summary: "An authenticated uploader can trigger extraction.",
- preconditions: [
- "The attacker can upload an archive.",
- "Automatic extraction is enabled.",
- ],
- },
- },
- });
-
- expect(
- report.match(/- The service processes uploaded archives\./gu),
- ).toHaveLength(1);
- expect(
- report.match(/- The attacker can upload an archive\./gu),
- ).toHaveLength(1);
- expect(report.match(/- Automatic extraction is enabled\./gu)).toHaveLength(
- 1,
- );
- });
-
- test("projects the top-level attack-path summary as reachability", () => {
- const report = projectFindingDetails({
- attackPath: {
- summary: "An authenticated uploader can trigger archive extraction.",
- },
- });
-
- const reachability = report
- .split("#### Reachability", 2)[1]!
- .split("#### Severity", 1)[0]!;
- expect(reachability).toContain(
- "An authenticated uploader can trigger archive extraction.",
- );
- expect(reachability).not.toContain("Reachability was not recorded");
- });
-
- test("projects typed attack-path steps", () => {
- const report = projectFindingDetails({
- attackPath: {
- steps: [
- "Upload an archive with a traversal entry.",
- "Trigger automatic extraction.",
- ],
- },
- });
-
- const dataflow = report
- .split("#### Dataflow", 2)[1]!
- .split("#### Reachability", 1)[0]!;
- expect(dataflow).toContain("Attack steps:");
- expect(dataflow).toContain("- Upload an archive with a traversal entry.");
- expect(dataflow).toContain("- Trigger automatic extraction.");
- });
-
- test("projects typed assessments and validation outcomes", () => {
- const report = projectFindingDetails({
- validation: {
- status: "validated",
- disposition: "reported",
- result: "The traversal write was confirmed.",
- },
- attackPath: {
- impact: {
- level: "high",
- rationale: "The write can overwrite application files.",
- why: "The destination escapes the extraction root.",
- },
- likelihood: "Likely for authenticated uploaders.",
- reachability: {
- source: "Attacker-controlled archive entry.",
- sink: "Unchecked filesystem write.",
- },
- },
- });
-
- expect(report).toContain("- **Status:** validated");
- expect(report).toContain("- **Disposition:** reported");
- expect(report).toContain(
- "- **Result:** The traversal write was confirmed.",
- );
- expect(report).toContain(
- "- **Source:** Attacker-controlled archive entry.",
- );
- expect(report).toContain("- **Sink:** Unchecked filesystem write.");
- expect(report).toContain("Impact assessment:");
- expect(report).toContain("- **Level:** high");
- expect(report).toContain(
- "- **Rationale:** The write can overwrite application files.",
- );
- expect(report).toContain(
- "- **Why:** The destination escapes the extraction root.",
- );
- expect(report).toContain(
- "**Likelihood assessment:** Likely for authenticated uploaders.",
- );
- });
-
- test("projects attack-path context lists", () => {
- const report = projectFindingDetails({
- attackPath: {
- assumptions: ["Automatic extraction is enabled."],
- blindspots: ["A downstream sandbox was not exercised."],
- controls: ["Archive uploads require authentication."],
- limitations: ["The exploit was validated statically."],
- },
- });
-
- const reachability = report
- .split("#### Reachability", 2)[1]!
- .split("#### Severity", 1)[0]!;
- expect(reachability).toContain("Assumptions:");
- expect(reachability).toContain("- Automatic extraction is enabled.");
- expect(reachability).toContain("Existing controls:");
- expect(reachability).toContain("- Archive uploads require authentication.");
- expect(reachability).toContain("Blind spots:");
- expect(reachability).toContain("- A downstream sandbox was not exercised.");
- expect(reachability).toContain("Limitations:");
- expect(reachability).toContain("- The exploit was validated statically.");
- });
-
- test("merges root-cause aliases in generated reports", () => {
- const report = projectFindingDetails({
- rootCause: { summary: "" },
- root_cause: {
- summary: "The destination is not contained before the write.",
- code: "destination.write_bytes(payload)",
- language: "python",
- },
- }).replaceAll("\r\n", "\n");
-
- expect(report).toContain(
- "The destination is not contained before the write.",
- );
- expect(report).toContain(
- "```python\ndestination.write_bytes(payload)\n```",
- );
- });
-
- test("merges scalar and list root-cause evidence references", () => {
- const report = projectFindingDetails({
- rootCause: {
- evidenceRefs: ["canonical-root-source"],
- summary: "The source reaches the write.",
- },
- root_cause: { evidence_refs: "legacy-root-source" },
- codeEvidence: [
- { id: "canonical-root-source", code: "canonical_source()" },
- ],
- code_evidence: [{ id: "legacy-root-source", code: "legacy_source()" }],
- });
-
- expect(report).toContain("canonical_source()");
- expect(report).toContain("legacy_source()");
- });
-
- test("merges embedded evidence across root-cause aliases", () => {
- const report = projectFindingDetails({
- rootCause: {
- summary: "The canonical root cause.",
- codeEvidence: [{ id: "canonical-root", code: "canonical_evidence()" }],
- },
- root_cause: {
- codeEvidence: [{ id: "legacy-root", code: "legacy_evidence()" }],
- },
- });
-
- expect(report).toContain("canonical_evidence()");
- expect(report).toContain("legacy_evidence()");
- });
-
- test("treats whitespace root-cause fields as unpopulated", () => {
- const report = projectFindingDetails({
- rootCause: { summary: " ", code: "\t", language: "text" },
- root_cause: {
- summary: "The destination is not contained before the write.",
- code: "destination.write_bytes(payload)",
- language: "python",
- },
- }).replaceAll("\r\n", "\n");
-
- expect(report).toContain(
- "The destination is not contained before the write.",
- );
- expect(report).toContain(
- "```python\ndestination.write_bytes(payload)\n```",
- );
- });
-
- test("projects code evidence referenced by nested attack-path details", () => {
- const report = projectFindingDetails({
- attackPath: {
- dataflow: {
- evidenceRefs: [],
- evidence_refs: ["archive-source"],
- summary: "An archive entry path reaches a filesystem write.",
- },
- reachability: {
- evidenceRefs: ["archive-sink"],
- summary: "An authenticated uploader can trigger extraction.",
- },
- },
- codeEvidence: [
- {
- code: "entry_path = archive_entry.name",
- explanation: "The archive controls the path.",
- id: "archive-source",
- label: "Attacker-controlled archive path",
- path: "src/archive.py",
- startLine: 20,
- },
- {
- code: "destination.write_bytes(entry.read())",
- explanation: "The unchecked path reaches the write.",
- id: "archive-sink",
- label: "Unchecked filesystem write",
- path: "src/archive.py",
- startLine: 41,
- },
- ],
- });
-
- expect(report).toContain("entry_path = archive_entry.name");
- expect(report).toContain("destination.write_bytes(entry.read())");
- });
-
- test("normalizes scalar code-evidence references in generated reports", () => {
- const report = projectFindingDetails({
- root_cause: {
- summary: "The authorization check occurs after the write.",
- evidence_refs: "root-source",
- },
- attackPath: {
- dataflow: {
- summary: "An attacker-controlled value reaches the write.",
- evidence_refs: "dataflow-source",
- },
- reachability: {
- summary: "An authenticated caller can reach the handler.",
- evidence_refs: "reachability-source",
- },
- },
- codeEvidence: [
- { id: "root-source", code: "root_source()" },
- { id: "dataflow-source", code: "dataflow_source()" },
- { id: "reachability-source", code: "reachability_source()" },
- ],
- });
-
- expect(report).toContain("root_source()");
- expect(report).toContain("dataflow_source()");
- expect(report).toContain("reachability_source()");
- });
-
- test("rejects unknown code evidence referenced by nested attack-path details", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "finding = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())['findings'][0]",
- "finding['attackPath'] = {'dataflow': {'evidenceRefs': ['missing-evidence']}}",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "try:",
- " finalizer['_validate_finding'](finding, 'findings[0]')",
- "except finalizer['ContractError'] as error:",
- " print(error)",
- "else:",
- " print('accepted')",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(new TextDecoder().decode(result.stdout)).toContain(
- "attackPath.dataflow.evidenceRefs: unknown code-evidence ids: missing-evidence",
- );
- });
-
- test("accepts nested references to the legacy code evidence catalog", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "finding = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())['findings'][0]",
- "finding.pop('codeEvidence', None)",
- "finding['code_evidence'] = [{'id': 'legacy-source', 'code': 'entry_path = archive_entry.name'}]",
- "finding['attackPath'] = {'dataflow': {'evidence_refs': ['legacy-source']}}",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "finalizer['_validate_finding'](finding, 'findings[0]')",
- "print('accepted')",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(new TextDecoder().decode(result.stdout)).toContain("accepted");
- });
-
- test("rejects duplicate IDs across code evidence catalogs", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "finding = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())['findings'][0]",
- "finding['codeEvidence'] = [{'id': 'shared-source', 'code': 'canonical_source()'}]",
- "finding['code_evidence'] = [{'id': 'shared-source', 'code': 'conflicting_legacy_source()'}]",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "try:",
- " finalizer['_validate_finding'](finding, 'findings[0]')",
- "except finalizer['ContractError'] as error:",
- " print(error)",
- "else:",
- " print('accepted')",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(new TextDecoder().decode(result.stdout)).toContain(
- "code_evidence[0].id: duplicate code-evidence id",
- );
- });
-
- test("keeps legacy sealed evidence references compatible", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "findings = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())",
- "findings['findings'][0]['codeEvidence'] = [{'id': 'shared-source', 'label': 'Canonical source', 'path': 'src/extract.py', 'startLine': 41, 'code': 'canonical_source()', 'explanation': 'Canonical snippet.'}]",
- "findings['findings'][0]['code_evidence'] = [",
- " {'id': 'shared-source', 'code': 'legacy_source()'},",
- " {'id': 'legacy-duplicate', 'code': 'first_legacy_source()'},",
- " {'id': 'legacy-duplicate', 'code': 'second_legacy_source()'},",
- "]",
- "findings['findings'][0]['validation'] = {'evidence_refs': ['legacy-validation-evidence'], 'counterEvidence': None, 'limitations': '', 'method': [], 'status': '', 'summary': {}, 'disposition': '', 'result': ''}",
- "findings['findings'][0]['rootCause'] = {'summary': 'Canonical root cause.', 'code': 'canonical_root()', 'language': 'python'}",
- "findings['findings'][0]['attackPath'] = {'evidence_refs': ['legacy-attack-evidence'], 'dataFlow': '', 'dataflow': {'summary': '', 'source': None, 'sink': None, 'outcome': None, 'evidenceRefs': ['legacy-missing-evidence']}, 'impact': {'level': '', 'rationale': None, 'why': None}, 'likelihood': {'level': None, 'rationale': None, 'why': None}, 'reachability': None, 'summary': ''}",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "compatible = finalizer['_legacy_sealed_findings_for_validation'](findings)",
- "finalizer['_validate_finding'](compatible['findings'][0], 'findings[0]')",
- "finalizer['validate_against_schema'](compatible, plugin / 'schemas' / 'findings.schema.json')",
- "print(json.dumps({'originalDataFlow': findings['findings'][0]['attackPath']['dataFlow'], 'compatibleHasDataFlow': 'dataFlow' in compatible['findings'][0]['attackPath'], 'originalNested': findings['findings'][0]['attackPath']['dataflow']['evidenceRefs'], 'compatibleNested': compatible['findings'][0]['attackPath']['dataflow']['evidenceRefs'], 'originalReachability': findings['findings'][0]['attackPath']['reachability'], 'compatibleHasReachability': 'reachability' in compatible['findings'][0]['attackPath'], 'originalAttack': findings['findings'][0]['attackPath']['evidence_refs'], 'compatibleAttack': compatible['findings'][0]['attackPath']['evidence_refs'], 'originalValidation': findings['findings'][0]['validation']['evidence_refs'], 'compatibleValidation': compatible['findings'][0]['validation']['evidence_refs'], 'compatibleHasCounterEvidence': 'counterEvidence' in compatible['findings'][0]['validation'], 'compatibleHasDisposition': 'disposition' in compatible['findings'][0]['validation'], 'compatibleHasLimitations': 'limitations' in compatible['findings'][0]['validation'], 'compatibleHasMethod': 'method' in compatible['findings'][0]['validation'], 'compatibleHasResult': 'result' in compatible['findings'][0]['validation'], 'compatibleHasRootCause': 'rootCause' in compatible['findings'][0], 'compatibleHasStatus': 'status' in compatible['findings'][0]['validation'], 'compatibleHasSummary': 'summary' in compatible['findings'][0]['validation'], 'originalMethod': findings['findings'][0]['validation']['method'], 'originalRootCauseSummary': findings['findings'][0]['rootCause']['summary'], 'originalSummary': findings['findings'][0]['validation']['summary'], 'originalLegacyCatalog': findings['findings'][0]['code_evidence'], 'compatibleLegacyCatalog': compatible['findings'][0]['code_evidence']}))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- compatibleAttack: [],
- compatibleLegacyCatalog: [
- { code: "first_legacy_source()", id: "legacy-duplicate" },
- ],
- compatibleHasDataFlow: false,
- compatibleHasReachability: false,
- compatibleNested: [],
- compatibleValidation: [],
- compatibleHasCounterEvidence: false,
- compatibleHasDisposition: false,
- compatibleHasLimitations: false,
- compatibleHasMethod: false,
- compatibleHasResult: false,
- compatibleHasRootCause: true,
- compatibleHasStatus: false,
- compatibleHasSummary: false,
- originalAttack: ["legacy-attack-evidence"],
- originalLegacyCatalog: [
- { code: "legacy_source()", id: "shared-source" },
- { code: "first_legacy_source()", id: "legacy-duplicate" },
- { code: "second_legacy_source()", id: "legacy-duplicate" },
- ],
- originalMethod: [],
- originalDataFlow: "",
- originalNested: ["legacy-missing-evidence"],
- originalReachability: null,
- originalRootCauseSummary: "Canonical root cause.",
- originalSummary: {},
- originalValidation: ["legacy-validation-evidence"],
- });
- });
-
- test("normalizes formerly free-form sealed details for validation", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "findings = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())",
- "finding = findings['findings'][0]",
- "finding['validation'] = {'evidence': {'kind': 'trace'}, 'counterEvidence': [None, 'The mitigation was checked.']}",
- "finding['attackPath'] = {'steps': {'first': 'upload'}}",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "compatible = finalizer['_legacy_sealed_findings_for_validation'](findings)",
- "finalizer['_validate_finding'](compatible['findings'][0], 'findings[0]')",
- "finalizer['validate_against_schema'](compatible, plugin / 'schemas' / 'findings.schema.json')",
- "print(json.dumps({'originalValidation': finding['validation'], 'originalAttackPath': finding['attackPath'], 'compatibleValidation': compatible['findings'][0]['validation'], 'compatibleAttackPath': compatible['findings'][0]['attackPath']}))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- originalValidation: {
- evidence: { kind: "trace" },
- counterEvidence: [null, "The mitigation was checked."],
- },
- originalAttackPath: { steps: { first: "upload" } },
- compatibleValidation: {
- counterEvidence: ["The mitigation was checked."],
- },
- compatibleAttackPath: {},
- });
- });
-
- test("rejects malformed canonical root causes during sealed validation", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "findings = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())",
- "findings['findings'][0]['rootCause'] = {'summary': []}",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "compatible = finalizer['_legacy_sealed_findings_for_validation'](findings)",
- "try:",
- " finalizer['validate_against_schema'](compatible, plugin / 'schemas' / 'findings.schema.json')",
- "except finalizer['ContractError'] as error:",
- " print(error)",
- "else:",
- " print('accepted')",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(new TextDecoder().decode(result.stdout)).toContain(
- "rootCause.summary",
- );
- });
-
- test("ranks legacy code evidence during interrupted recovery", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import copy, json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "examples = plugin / 'examples' / 'completed-scan'",
- "manifest = json.loads((examples / 'scan-manifest.json').read_text())",
- "example = json.loads((examples / 'findings.json').read_text())['findings'][0]",
- "first = copy.deepcopy(example)",
- "first.pop('codeEvidence', None)",
- "first['summary'] = 'FIRST'",
- "second = copy.deepcopy(first)",
- "second['summary'] = 'SECOND'",
- "second['code_evidence'] = [{'id': 'legacy-source', 'code': 'legacy_source()'}]",
- "second['root_cause'] = None",
- "findings = {'scanId': manifest['scan']['id'], 'findings': [first, second]}",
- "warnings = []",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "finalizer['_recover_unsealed_findings'](manifest, findings, plugin / 'schemas', examples, warnings)",
- "print(json.dumps({'summary': findings['findings'][0]['summary'], 'evidence': findings['findings'][0].get('code_evidence'), 'rootCause': findings['findings'][0].get('root_cause'), 'warnings': warnings}))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- summary: "SECOND",
- evidence: [{ id: "legacy-source", code: "legacy_source()" }],
- rootCause: null,
- warnings: [
- "Recovered finding 2: retained stronger duplicate logical finding.",
- ],
- });
- });
-
- test("ranks embedded root-cause evidence during interrupted recovery", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import copy, json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "examples = plugin / 'examples' / 'completed-scan'",
- "manifest = json.loads((examples / 'scan-manifest.json').read_text())",
- "example = json.loads((examples / 'findings.json').read_text())['findings'][0]",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "results = {}",
- "for name, detail in {'embedded': {'codeEvidence': [{'id': 'embedded-root', 'code': 'embedded_root()'}]}, 'legacy-code': {'code': 'legacy_root()'}}.items():",
- " first = copy.deepcopy(example)",
- " first.pop('codeEvidence', None)",
- " first['summary'] = 'FIRST'",
- " second = copy.deepcopy(first)",
- " second['summary'] = 'SECOND'",
- " second['rootCause'] = {'summary': 'Richer root cause', **detail}",
- " findings = {'scanId': manifest['scan']['id'], 'findings': [first, second]}",
- " warnings = []",
- " finalizer['_recover_unsealed_findings'](manifest, findings, plugin / 'schemas', examples, warnings)",
- " results[name] = {'summary': findings['findings'][0]['summary'], 'warnings': warnings}",
- "print(json.dumps(results))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- embedded: {
- summary: "SECOND",
- warnings: [
- "Recovered finding 2: retained stronger duplicate logical finding.",
- ],
- },
- "legacy-code": {
- summary: "SECOND",
- warnings: [
- "Recovered finding 2: retained stronger duplicate logical finding.",
- ],
- },
- });
- });
-
- test("ignores malformed legacy evidence references in sealed validation", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "findings = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())",
- "finding = findings['findings'][0]",
- "finding['code_evidence'] = [{'id': 'legacy-source', 'code': 'legacy_source()'}]",
- "finding['rootCause'] = {'summary': 'Canonical root cause.', 'evidenceRefs': ['legacy-source', 'missing-source']}",
- "finding['validation'] = {'evidenceRefs': [None, '', 42, 'legacy-source', 'missing-source']}",
- "finding['attackPath'] = {'dataflow': {'evidence_refs': [None, '', 42, 'legacy-source', 'missing-source']}}",
- "original = json.dumps(findings, sort_keys=True)",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "compatible = finalizer['_legacy_sealed_findings_for_validation'](findings)",
- "finalizer['_validate_finding'](compatible['findings'][0], 'findings[0]')",
- "print(json.dumps({",
- " 'originalUnchanged': json.dumps(findings, sort_keys=True) == original,",
- " 'originalRootCause': finding['rootCause'],",
- " 'compatibleRootCause': compatible['findings'][0]['rootCause'],",
- " 'originalValidation': finding['validation']['evidenceRefs'],",
- " 'compatibleValidation': compatible['findings'][0]['validation']['evidenceRefs'],",
- " 'originalDataflow': finding['attackPath']['dataflow']['evidence_refs'],",
- " 'compatibleDataflow': compatible['findings'][0]['attackPath']['dataflow']['evidence_refs'],",
- "}))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- originalUnchanged: true,
- originalRootCause: {
- summary: "Canonical root cause.",
- evidenceRefs: ["legacy-source", "missing-source"],
- },
- compatibleRootCause: {
- summary: "Canonical root cause.",
- evidenceRefs: ["legacy-source"],
- },
- originalValidation: [null, "", 42, "legacy-source", "missing-source"],
- compatibleValidation: ["legacy-source"],
- originalDataflow: [null, "", 42, "legacy-source", "missing-source"],
- compatibleDataflow: ["legacy-source"],
- });
- });
-
- test("drops unsupported legacy attack-path sequences from sealed validation", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "findings = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())",
- "findings['findings'][0]['attackPath'] = {'dataflow': ['source', 'sink'], 'reachability': ['authenticated uploader']}",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "compatible = finalizer['_legacy_sealed_findings_for_validation'](findings)",
- "finalizer['validate_against_schema'](compatible, plugin / 'schemas' / 'findings.schema.json')",
- "print(json.dumps({'original': findings['findings'][0]['attackPath'], 'compatible': compatible['findings'][0]['attackPath']}))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- original: {
- dataflow: ["source", "sink"],
- reachability: ["authenticated uploader"],
- },
- compatible: {},
- });
- });
-
- test("keeps nullable legacy sealed evidence catalogs compatible", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import copy, json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "example = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())['findings'][0]",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "results = {}",
- "for field in ('codeEvidence', 'code_evidence'):",
- " finding = copy.deepcopy(example)",
- " finding[field] = None",
- " compatible = finalizer['_legacy_sealed_findings_for_validation']({'findings': [finding]})",
- " try:",
- " finalizer['_validate_finding'](compatible['findings'][0], 'findings[0]')",
- " except finalizer['ContractError'] as error:",
- " outcome = str(error)",
- " else:",
- " outcome = 'accepted'",
- " results[field] = {'outcome': outcome, 'present': field in compatible['findings'][0]} ",
- "print(json.dumps(results))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- codeEvidence: {
- outcome: "findings[0].codeEvidence: expected an array",
- present: true,
- },
- code_evidence: { outcome: "accepted", present: false },
- });
- });
-
- test("ignores malformed legacy evidence rows in sealed validation", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "findings = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())",
- "findings['findings'][0]['code_evidence'] = [None, 'legacy source', {}, {'id': '', 'code': 'empty_id()'}, {'id': 'missing-code'}, {'id': 'empty-code', 'code': ''}, {'id': 'legacy-source', 'code': 'legacy_source()'}]",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "compatible = finalizer['_legacy_sealed_findings_for_validation'](findings)",
- "finalizer['_validate_finding'](compatible['findings'][0], 'findings[0]')",
- "print(json.dumps({'original': findings['findings'][0]['code_evidence'], 'compatible': compatible['findings'][0]['code_evidence']}))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
- original: [
- null,
- "legacy source",
- {},
- { id: "", code: "empty_id()" },
- { id: "missing-code" },
- { id: "empty-code", code: "" },
- { id: "legacy-source", code: "legacy_source()" },
- ],
- compatible: [{ id: "legacy-source", code: "legacy_source()" }],
- });
- });
-
- test("projects canonical and legacy code evidence into safe SARIF locations", () => {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const script = [
- "import json, pathlib, runpy, sys",
- "plugin = pathlib.Path(sys.argv[1])",
- "finding = json.loads((plugin / 'examples' / 'completed-scan' / 'findings.json').read_text())['findings'][0]",
- "finding['codeEvidence'] = [{'id': 'canonical-source', 'path': 'src/canonical.py', 'startLine': 12, 'code': 'canonical_source()'}]",
- "finding['code_evidence'] = [",
- " {'id': 'legacy-sink', 'path': 'src/legacy.py', 'startLine': 37, 'endLine': 39, 'code': 'legacy_sink()'},",
- " {'id': 'legacy-null-end', 'path': 'src/null_end.py', 'startLine': 48, 'endLine': None, 'code': 'null_end()'},",
- " {'id': 'legacy-reversed-end', 'path': 'src/reversed_end.py', 'startLine': 59, 'endLine': 58, 'code': 'reversed_end()'},",
- " {'id': 'legacy-text-end', 'path': 'src/text_end.py', 'startLine': 70, 'endLine': '71', 'code': 'text_end()'},",
- " {'id': 'legacy-zero-start', 'path': 'src/zero.py', 'startLine': 0, 'code': 'zero_start()'},",
- " {'id': 'legacy-unsafe-path', 'path': '../outside.py', 'startLine': 81, 'code': 'unsafe_path()'},",
- "]",
- "finalizer = runpy.run_path(str(plugin / 'scripts' / 'finalize_scan_contract.py'))",
- "result = finalizer['_sarif_result'](finding, 0)",
- "regions = {location['physicalLocation']['artifactLocation']['uri']: location['physicalLocation']['region'] for location in result['locations']}",
- "print(json.dumps(regions, sort_keys=True))",
- ].join("\n");
- const result = runPython(python!, ["-c", script, PLUGIN_ROOT]);
-
- expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- expect(JSON.parse(new TextDecoder().decode(result.stdout))).toMatchObject({
- "src/canonical.py": { startLine: 12, endLine: 12 },
- "src/legacy.py": { startLine: 37, endLine: 39 },
- "src/null_end.py": { startLine: 48, endLine: 48 },
- "src/reversed_end.py": { startLine: 59, endLine: 59 },
- "src/text_end.py": { startLine: 70, endLine: 70 },
- });
- expect(
- JSON.parse(new TextDecoder().decode(result.stdout)),
- ).not.toHaveProperty("src/zero.py");
- expect(
- JSON.parse(new TextDecoder().decode(result.stdout)),
- ).not.toHaveProperty("../outside.py");
- });
});