Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: consolidate finding preview regression coverage (#1606)

Michael D'Angeload22cbef6e6e
diff --git a/plugins/codex-security/tests/test_finding_preview.py b/plugins/codex-security/tests/test_finding_preview.py
index 18efdee79..09832214e 100644
--- a/plugins/codex-security/tests/test_finding_preview.py
+++ b/plugins/codex-security/tests/test_finding_preview.py
@@ -2,12 +2,15 @@ from __future__ import annotations
import json
import runpy
+from copy import deepcopy
from pathlib import Path
import pytest
FINDING_PREVIEW_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "finding_preview.py"
+pytestmark = pytest.mark.cross_platform
+
def test_compact_finding_fields_survive_verbose_code_evidence() -> None:
preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
@@ -169,6 +172,33 @@ def test_bounded_finding_details_normalizes_scalar_attack_path_assessments() ->
assert "likelihood" not in missing["attackPath"]
+def test_nested_attack_path_string_arrays_survive_without_relaxing_section_depth() -> None:
+ preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
+ original = {
+ "attackPath": {
+ "dataflow": {
+ "evidenceRefs": ["source-to-sink"],
+ "nested": {"evidenceRefs": ["must remain depth-limited"]},
+ },
+ "reachability": {"preconditions": ["The handler is reachable."]},
+ }
+ }
+ finding = deepcopy(original)
+
+ bounded = preview["bounded_finding_details"](finding)
+
+ assert bounded == {
+ "attackPath": {
+ "dataflow": {
+ "evidenceRefs": ["source-to-sink"],
+ "nested": {"evidenceRefs": [None]},
+ },
+ "reachability": {"preconditions": ["The handler is reachable."]},
+ }
+ }
+ assert finding == original
+
+
def test_bounded_finding_details_merges_root_cause_aliases() -> None:
preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
original = {
diff --git a/sdk/typescript/tests-ts/finding-preview.test.ts b/sdk/typescript/tests-ts/finding-preview.test.ts
index b3392bd0f..3fcb319cc 100644
--- a/sdk/typescript/tests-ts/finding-preview.test.ts
+++ b/sdk/typescript/tests-ts/finding-preview.test.ts
@@ -1,17 +1,21 @@
import { join } from "node:path";
-import { describe, expect, test } from "bun:test";
+import { expect, test } from "bun:test";
import { PLUGIN_ROOT } from "./plugin-root.js";
-function projectFindingDetails(original: Record<string, unknown>) {
+test("bundled finding previews normalize details without changing stored findings", () => {
const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");
expect(python).not.toBeNull();
-
+ const original = {
+ attackPath: { likelihood: "medium" },
+ code_evidence: [{ id: "source", code: "synthetic_source()" }],
+ rootCause: { summary: "Synthetic root cause." },
+ };
const program = [
"import json, sys",
"sys.path.insert(0, sys.argv[1])",
"from finding_preview import bounded_finding_details",
"original = json.loads(sys.stdin.read())",
- "projected = {name: bounded_finding_details(details) for name, details in original.items()}",
+ "projected = bounded_finding_details(original)",
"print(json.dumps({'projected': projected, 'original': original}))",
].join("\n");
const result = Bun.spawnSync(
@@ -24,207 +28,11 @@ function projectFindingDetails(original: Record<string, unknown>) {
);
expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
- return JSON.parse(new TextDecoder().decode(result.stdout));
-}
-
-describe("bundled finding previews", () => {
- test("normalizes attack-path assessments without changing stored finding details", () => {
- const original = {
- scalar: {
- attackPath: {
- impact: "Native memory corruption is possible.",
- likelihood: "medium",
- },
- },
- structured: {
- attackPath: {
- impact: { level: "low", rationale: "Synthetic assessment." },
- likelihood: null,
- },
- },
- absentAssessments: {
- attackPath: { narrative: "Synthetic attack path." },
- },
- absentAttackPath: {
- rootCause: { summary: "Synthetic root cause." },
- },
- bothEvidenceAliases: {
- codeEvidence: [{ id: "canonical", code: "canonical_source()" }],
- code_evidence: [{ id: "legacy", code: "legacy_source()" }],
- rootCause: { summary: "Synthetic root cause." },
- },
- bothRootCauseAliases: {
- rootCause: { code: "SELECT * FROM users" },
- root_cause: {
- code: "os.system(user_input)",
- evidence_refs: ["legacy-source"],
- language: "python",
- summary: "The destination is not contained.",
- },
- },
- invalidLegacyEvidenceFields: {
- code_evidence: [
- {
- id: "legacy-source",
- code: "dangerous_call()",
- startLine: 0,
- endLine: "12",
- label: 7,
- role: { kind: "sink" },
- },
- ],
- },
- malformedCanonicalRootCause: {
- rootCause: { summary: 42 },
- root_cause: {
- summary: "The valid legacy root cause.",
- evidence_refs: ["legacy-root"],
- },
- code_evidence: [{ id: "legacy-root", code: "legacy_root()" }],
- },
- };
- expect(projectFindingDetails(original)).toEqual({
- projected: {
- scalar: {
- attackPath: {
- impact: { rationale: "Native memory corruption is possible." },
- likelihood: { level: "medium" },
- },
- },
- structured: original.structured,
- absentAssessments: original.absentAssessments,
- absentAttackPath: original.absentAttackPath,
- bothEvidenceAliases: {
- codeEvidence: [
- { id: "canonical", code: "canonical_source()" },
- { id: "legacy", code: "legacy_source()" },
- ],
- rootCause: { summary: "Synthetic root cause." },
- },
- bothRootCauseAliases: {
- rootCause: {
- code: "SELECT * FROM users",
- evidenceRefs: ["legacy-source"],
- summary: "The destination is not contained.",
- },
- },
- invalidLegacyEvidenceFields: {
- code_evidence: [{ id: "legacy-source", code: "dangerous_call()" }],
- },
- malformedCanonicalRootCause: {
- rootCause: {
- summary: "The valid legacy root cause.",
- evidenceRefs: ["legacy-root"],
- },
- code_evidence: [{ id: "legacy-root", code: "legacy_root()" }],
- },
- },
- original,
- });
- });
-
- test("preserves nested attack-path string arrays without relaxing section depth", () => {
- const original = {
- supported: {
- attackPath: {
- dataflow: { evidenceRefs: ["source-to-sink"] },
- reachability: { preconditions: ["The handler is reachable."] },
- },
- },
- tooDeep: {
- attackPath: {
- dataflow: {
- nested: { evidenceRefs: ["must remain depth-limited"] },
- },
- },
- },
- };
-
- expect(projectFindingDetails(original)).toEqual({
- projected: {
- supported: original.supported,
- tooDeep: {
- attackPath: {
- dataflow: {
- nested: { evidenceRefs: [null] },
- },
- },
- },
- },
- original,
- });
- });
-
- test("preserves counter-evidence under the validation preview budget", () => {
- const original = {
- finding: {
- validation: {
- evidence: ["x".repeat(20_000)],
- summary: `The traversal was reproduced. ${"x".repeat(20_000)}`,
- method: "focused extraction test",
- evidenceRefs: ["evidence-0"],
- futureMetadata: "x".repeat(20_000),
- counterEvidence: ["Known mitigations remain unverified."],
- },
- },
- };
-
- const result = projectFindingDetails(original);
-
- expect(result.projected.finding.validation.counterEvidence).toEqual([
- "Known mitigations remain unverified.",
- ]);
- expect(result.original).toEqual(original);
- });
-
- test("deduplicates evidence before applying the preview limit", () => {
- const original = {
- finding: {
- codeEvidence: [
- { id: "shared", code: "canonical_shared()" },
- { id: "shared", code: "duplicate_shared()" },
- { id: "canonical-two", code: "canonical_two()" },
- { id: "canonical-three", code: "canonical_three()" },
- ],
- code_evidence: [
- { id: "shared", code: "legacy_shared()" },
- { id: "legacy-four", code: "legacy_four()" },
- { id: "legacy-five", code: "legacy_five()" },
- ],
- },
- };
-
- const result = projectFindingDetails(original);
-
- expect(
- result.projected.finding.codeEvidence.map(
- (item: { id: string }) => item.id,
- ),
- ).toEqual(["shared", "canonical-two", "canonical-three", "legacy-four"]);
- expect(result.projected.finding.codeEvidence[0].code).toBe(
- "canonical_shared()",
- );
- expect(result.original).toEqual(original);
- });
-
- test("filters malformed evidence before applying the preview limit", () => {
- const original = {
- finding: {
- code_evidence: [
- null,
- "junk",
- {},
- { id: "empty", code: "" },
- { id: "valid", code: "valid_source()" },
- ],
- },
- };
-
- const result = projectFindingDetails(original);
-
- expect(result.projected.finding.code_evidence).toEqual([
- { id: "valid", code: "valid_source()" },
- ]);
- expect(result.original).toEqual(original);
+ expect(JSON.parse(new TextDecoder().decode(result.stdout))).toEqual({
+ projected: {
+ ...original,
+ attackPath: { likelihood: { level: "medium" } },
+ },
+ original,
});
});