Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(sdk): simplify findings fixtures and replay checks (#1975)

Michael D'Angeloc362983fa6de
diff --git a/plugins/codex-security/tests/test_sarif_projection.py b/plugins/codex-security/tests/test_sarif_projection.py
index e146a8f85..a4032c3d9 100644
--- a/plugins/codex-security/tests/test_sarif_projection.py
+++ b/plugins/codex-security/tests/test_sarif_projection.py
@@ -25,6 +25,33 @@ class SarifProjectionTest(unittest.TestCase):
self.findings = json.loads((EXAMPLE_DIR / "findings.json").read_text())
self.finding = self.findings["findings"][0]
+ def test_sarif_preserves_alert_identity_when_finding_text_changes(self) -> None:
+ finding = self.finding
+ finding["extensions"] = {"candidateId": "candidate-example"}
+ run = FINALIZER.build_sarif(self.manifest, self.findings)["runs"][0]
+ rule = run["tool"]["driver"]["rules"][0]
+ result = run["results"][0]
+ self.assertEqual(rule["shortDescription"]["text"], rule["name"])
+ self.assertIn(finding["remediation"], rule["help"]["text"])
+ for field in ("findingId", "occurrenceId"):
+ self.assertEqual(result["properties"][field], finding[field])
+ self.assertEqual(result["properties"]["candidateId"], "candidate-example")
+ for text in (
+ finding["title"],
+ finding["summary"],
+ *finding["remediationTests"],
+ *finding["preventiveControls"],
+ ):
+ self.assertIn(text, result["message"]["text"])
+
+ finding.update(
+ title="A new title", summary="A changed summary", remediation="A different fix."
+ )
+ changed = FINALIZER.build_sarif(self.manifest, self.findings)["runs"][0]
+ self.assertEqual(changed["tool"]["driver"]["rules"][0]["name"], rule["name"])
+ for field in ("ruleId", "ruleIndex", "locations", "partialFingerprints", "properties"):
+ self.assertEqual(changed["results"][0][field], result[field])
+
def test_sarif_merges_shared_rule_metadata_without_mixing_results(self) -> None:
first = copy.deepcopy(self.finding)
first["severity"] = {"level": "low"}
@@ -59,6 +86,7 @@ class SarifProjectionTest(unittest.TestCase):
},
)
results = {result["properties"]["occurrenceId"]: result for result in run["results"]}
+ self.assertEqual(results[first["occurrenceId"]]["level"], "note")
for own, other in ((first, second), (second, first)):
result = results[own["occurrenceId"]]
self.assertIn(own["remediation"], rules[0]["help"]["markdown"])
@@ -83,10 +111,12 @@ class SarifProjectionTest(unittest.TestCase):
finding["severity"] = {"level": level}
if score is not None:
finding["severity"].update(score=score, scoringSystem="CVSS:3.1")
- finding["taxonomy"]["cwe"] = []
- properties = FINALIZER._sarif_rule(finding["ruleId"], [finding])["properties"]
- self.assertEqual(properties.get("security-severity"), expected)
- self.assertEqual(properties["tags"], ["path-traversal", "security"])
+ finding["taxonomy"] = {"category": "sql-injection", "cwe": []}
+ run = FINALIZER.build_sarif(self.manifest, {"findings": [finding]})["runs"][0]
+ properties = {"tags": ["security", "sql-injection"]}
+ if expected is not None:
+ properties["security-severity"] = expected
+ self.assertEqual(run["tool"]["driver"]["rules"][0]["properties"], properties)
def test_sarif_normalizes_long_cwe_identifiers_as_text(self) -> None:
for label, digits, tag in (
diff --git a/sdk/typescript/src/deduplication/scan.ts b/sdk/typescript/src/deduplication/scan.ts
index f095f67cf..d13d6637f 100644
--- a/sdk/typescript/src/deduplication/scan.ts
+++ b/sdk/typescript/src/deduplication/scan.ts
@@ -26,6 +26,7 @@ import {
type DeduplicationReviewer,
} from "./deduplication-reviewer.js";
import { FindingsClient, type FindingsRequest } from "../findings-client.js";
+import type { waitForRetry } from "./retry.js";
import type {
FindingSearchScope,
FindingSourceSnapshot,
@@ -104,6 +105,7 @@ type DeduplicateScanDependencies = Partial<SavedScanDependencies> & {
reviewRunner?: Pick<CodexReviewRunner, "run">;
resolveReviewEnvironment?: typeof comparisonEnvironment;
fetch?: FindingsRequest;
+ waitForRetry?: typeof waitForRetry;
};
/** @internal */
@@ -212,6 +214,7 @@ async function deduplicateResolvedScan(
options.findingsUrl!,
options.signal,
dependencies.fetch,
+ { wait: dependencies.waitForRetry },
);
const workflow =
options.workflowId === undefined
diff --git a/sdk/typescript/stryker.config.json b/sdk/typescript/stryker.config.json
index 452fbc763..3dacb0ff9 100644
--- a/sdk/typescript/stryker.config.json
+++ b/sdk/typescript/stryker.config.json
@@ -2,7 +2,7 @@
"$schema": "./node_modules/@stryker-mutator/core/schema/stryker-schema.json",
"testRunner": "command",
"commandRunner": {
- "command": "bun test --timeout 30000 ./tests-ts/cost-model.property.test.ts ./tests-ts/errors.test.ts ./tests-ts/errors.property.test.ts ./tests-ts/worker-progress.test.ts ./tests-ts/worker-progress.property.test.ts"
+ "command": "bun test --timeout 30000 ./tests-ts/cost-model.property.test.ts ./tests-ts/errors.test.ts ./tests-ts/worker-progress.test.ts ./tests-ts/worker-progress.property.test.ts"
},
"mutate": ["src/cost-model.ts", "src/errors.ts", "src/worker-progress.ts"],
"coverageAnalysis": "off",
diff --git a/sdk/typescript/tests-ts/cloud-publish.test.ts b/sdk/typescript/tests-ts/cloud-publish.test.ts
index dcba30e77..891398565 100644
--- a/sdk/typescript/tests-ts/cloud-publish.test.ts
+++ b/sdk/typescript/tests-ts/cloud-publish.test.ts
@@ -318,24 +318,6 @@ describe("Cloud publication", () => {
expect(finding!.occurrenceId).toMatch(/^occ_[a-f0-9]{24}$/);
});
- test("keeps separate CSV occurrences with the same source finding ID", async () => {
- const second = csvRow.replace(
- "occ_e79cb19591e696572a1c22be",
- "occ_111111111111111111111111",
- );
- const path = await csvFixture(`${csvHeader}\n${csvRow}\n${second}\n`);
- const result = await publishFindingsCsvToCloud(path, { dryRun: true });
- expect(result.findingCount).toBe(2);
- expect(
- new Set(result.findings!.map((finding) => finding.occurrenceId)).size,
- ).toBe(2);
- expect(
- new Set(result.findings!.map((finding) => finding.findingId)).size,
- ).toBe(2);
- const repeated = await publishFindingsCsvToCloud(path, { dryRun: true });
- expect(repeated.findings).toEqual(result.findings);
- });
-
test("accepts the candidate_id column from a Deep scan export", async () => {
const path = await csvFixture(
`${csvHeader.replace("finding_id,", "finding_id,candidate_id,")}\n${csvRow.replace("csf_852f90d6e1177502ff113d4a,", "csf_852f90d6e1177502ff113d4a,candidate-001,")}\n`,
@@ -569,24 +551,6 @@ describe("Cloud publication", () => {
});
});
- test("honors a Cloud publication URL override", async () => {
- const { scan, environment } = await fixture();
- const publishUrl =
- "https://chatgpt-staging.com/backend-api/aardvark/cli/findings";
- let requestedUrl: string | undefined;
- await publishScanToCloud(scan, {
- environment: {
- ...environment,
- CODEX_SECURITY_CLOUD_PUBLISH_URL: publishUrl,
- },
- fetch: async (url) => {
- requestedUrl = String(url);
- return Response.json(receipt);
- },
- });
- expect(requestedUrl).toBe(publishUrl);
- });
-
test("uses the shared Cloud API base while preserving the publication URL override", async () => {
const { scan, environment } = await fixture();
const cloudBase = "https://cloud.example.test/pilot/backend-api/aardvark";
@@ -595,23 +559,21 @@ describe("Cloud publication", () => {
urls.push(String(url));
return Response.json(receipt);
};
- await publishScanToCloud(scan, {
- environment: {
- ...environment,
- CODEX_SECURITY_CLOUD_BASE_URL: `${cloudBase}/`,
- },
- fetch,
- });
const publishUrl = "https://other-cloud.example.test/publish";
- await publishScanToCloud(scan, {
- environment: {
- ...environment,
+ for (const settings of [
+ { CODEX_SECURITY_CLOUD_BASE_URL: `${cloudBase}/` },
+ { CODEX_SECURITY_CLOUD_PUBLISH_URL: publishUrl },
+ {
CODEX_SECURITY_CLOUD_BASE_URL: cloudBase,
CODEX_SECURITY_CLOUD_PUBLISH_URL: publishUrl,
},
- fetch,
- });
- expect(urls).toEqual([`${cloudBase}/cli/findings`, publishUrl]);
+ ]) {
+ await publishScanToCloud(scan, {
+ environment: { ...environment, ...settings },
+ fetch,
+ });
+ }
+ expect(urls).toEqual([`${cloudBase}/cli/findings`, publishUrl, publishUrl]);
});
test("rejects a remote HTTP Cloud base before reading credentials or sending a scan", async () => {
diff --git a/sdk/typescript/tests-ts/contract.property.test.ts b/sdk/typescript/tests-ts/contract.property.test.ts
new file mode 100644
index 000000000..f344f8831
--- /dev/null
+++ b/sdk/typescript/tests-ts/contract.property.test.ts
@@ -0,0 +1,108 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { afterEach, describe, expect, test } from "bun:test";
+import fc from "fast-check";
+import { ContractValidationError, loadContract } from "../src/index.js";
+import type { ScanManifest } from "../src/models.js";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { createContractTestFixtures } from "./support/contract-fixture.js";
+import { sha256 } from "./support/finding-identity.js";
+import { readJson, writeJson } from "./support/json.js";
+import { propertyOptions } from "./support/property.js";
+
+const { copyExample, cleanup } = createContractTestFixtures();
+afterEach(cleanup);
+
+describe("canonical scan contract", () => {
+ test("rejects byte changes to any sealed binary artifact", async () => {
+ const scanDir = await copyExample();
+ const artifactPath = join(scanDir, "artifacts", "synthetic.bin");
+ const manifestPath = join(scanDir, "scan-manifest.json");
+ const manifest = await readJson<ScanManifest>(manifestPath);
+ await mkdir(dirname(artifactPath), { recursive: true });
+
+ await fc.assert(
+ fc.asyncProperty(
+ fc.uint8Array({ minLength: 1, maxLength: 256 }),
+ fc.nat(),
+ fc.integer({ min: 1, max: 255 }),
+ async (bytes, offset, difference) => {
+ await writeFile(artifactPath, bytes);
+ await writeJson(manifestPath, {
+ ...manifest,
+ scan: {
+ ...manifest.scan,
+ artifacts: [
+ ...manifest.scan.artifacts,
+ {
+ path: "artifacts/synthetic.bin",
+ sha256: sha256(bytes),
+ mediaType: "application/octet-stream",
+ },
+ ],
+ },
+ });
+ await loadContract(scanDir, { pluginRoot: PLUGIN_ROOT });
+ const changed = Uint8Array.from(bytes);
+ changed[offset % bytes.length]! ^= difference;
+ await writeFile(artifactPath, changed);
+ await expect(
+ loadContract(scanDir, { pluginRoot: PLUGIN_ROOT }),
+ ).rejects.toBeInstanceOf(ContractValidationError);
+ },
+ ),
+ {
+ ...propertyOptions,
+ numRuns: Number(process.env["CODEX_SECURITY_PROPERTY_RUNS"] ?? "20"),
+ },
+ );
+ });
+
+ test("rejects non-relative artifact paths before accepting a seal", async () => {
+ const scanDir = await copyExample();
+ const manifestPath = join(scanDir, "scan-manifest.json");
+ const manifest = await readJson<ScanManifest>(manifestPath);
+ const prefixes = ["../", "/", "C:/", "artifacts/../", "artifacts\\"];
+ await mkdir(join(scanDir, "artifacts"), { recursive: true });
+ await fc.assert(
+ fc.asyncProperty(
+ fc.stringMatching(/^[a-z]{1,16}$/u),
+ fc.constantFrom(...prefixes),
+ async (name, prefix) => {
+ const filename = `artifact-${name}`;
+ const bytes = Buffer.from(name);
+ const artifact = {
+ path: `artifacts/${filename}`,
+ sha256: sha256(bytes),
+ mediaType: "application/octet-stream",
+ };
+ const scan = {
+ ...manifest.scan,
+ artifacts: [...manifest.scan.artifacts, artifact],
+ };
+ await writeFile(join(scanDir, "artifacts", filename), bytes);
+ await writeJson(manifestPath, { ...manifest, scan });
+ await loadContract(scanDir, { pluginRoot: PLUGIN_ROOT });
+
+ artifact.path = `${prefix}${filename}`;
+ await writeJson(manifestPath, { ...manifest, scan });
+ const rejected = loadContract(scanDir, { pluginRoot: PLUGIN_ROOT });
+ await expect(rejected).rejects.toBeInstanceOf(
+ ContractValidationError,
+ );
+ await expect(rejected).rejects.toThrow(
+ /safe scan-relative POSIX path|schema validation failed \(pattern/u,
+ );
+ },
+ ),
+ {
+ ...propertyOptions,
+ numRuns: Number(process.env["CODEX_SECURITY_PROPERTY_RUNS"] ?? "20"),
+ examples: [
+ ...prefixes.map((prefix): [string, string] => ["synthetic", prefix]),
+ ["con", "C:/"],
+ ],
+ },
+ );
+ });
+});
diff --git a/sdk/typescript/tests-ts/contract.test.ts b/sdk/typescript/tests-ts/contract.test.ts
index 43fe20d5b..48b444a65 100644
--- a/sdk/typescript/tests-ts/contract.test.ts
+++ b/sdk/typescript/tests-ts/contract.test.ts
@@ -14,33 +14,21 @@ import {
} from "node:fs/promises";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, test, mock, spyOn } from "bun:test";
-import fc from "fast-check";
import { ContractValidationError, loadContract } from "../src/index.js";
import { readScanFile, sameCheckedFileDevice } from "../src/contract.js";
import type { NormalizedTarget, ScanExpectation } from "../src/index.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { runPython } from "./support/python-probe.js";
-import { propertyOptions } from "./support/property.js";
-import { createApiTestFixtures } from "./support/temporary-directories.js";
+import { createContractTestFixtures } from "./support/contract-fixture.js";
import { readJson as readJsonFile, writeJson } from "./support/json.js";
import { runTestInSubprocess } from "./support/test-subprocess.js";
const EXAMPLE = join(PLUGIN_ROOT, "examples", "completed-scan");
-const { temporaryDirectory, cleanup } = createApiTestFixtures(
- "codex-security-contract-",
- false,
-);
+const { temporaryDirectory, cleanup, copyExample } =
+ createContractTestFixtures();
afterEach(cleanup);
-async function copyExample(): Promise<string> {
- const root = await temporaryDirectory();
- const scanDir = join(root, "scan");
- await cp(EXAMPLE, scanDir, { recursive: true });
- if (process.platform !== "win32") await chmod(scanDir, 0o700);
- return scanDir;
-}
-
const readJson = readJsonFile<Record<string, any>>;
async function reseal(scanDir: string): Promise<void> {
@@ -151,98 +139,6 @@ describe("canonical scan contract", () => {
).rejects.toBeInstanceOf(ContractValidationError);
});
- test("rejects byte changes to any sealed binary artifact", async () => {
- const scanDir = await copyExample();
- const artifactPath = join(scanDir, "artifacts", "synthetic.bin");
- const manifestPath = join(scanDir, "scan-manifest.json");
- const manifest = await readJson(manifestPath);
- await mkdir(dirname(artifactPath), { recursive: true });
-
- await fc.assert(
- fc.asyncProperty(
- fc.uint8Array({ minLength: 1, maxLength: 256 }),
- fc.nat(),
- fc.integer({ min: 1, max: 255 }),
- async (bytes, offset, difference) => {
- await writeFile(artifactPath, bytes);
- await writeJson(manifestPath, {
- ...manifest,
- scan: {
- ...manifest["scan"],
- artifacts: [
- ...manifest["scan"]["artifacts"],
- {
- path: "artifacts/synthetic.bin",
- sha256: sha256(bytes),
- mediaType: "application/octet-stream",
- },
- ],
- },
- });
- await loadContract(scanDir, { pluginRoot: PLUGIN_ROOT });
- const changed = Uint8Array.from(bytes);
- changed[offset % bytes.length]! ^= difference;
- await writeFile(artifactPath, changed);
- await expect(
- loadContract(scanDir, { pluginRoot: PLUGIN_ROOT }),
- ).rejects.toBeInstanceOf(ContractValidationError);
- },
- ),
- {
- ...propertyOptions,
- numRuns: Number(process.env["CODEX_SECURITY_PROPERTY_RUNS"] ?? "20"),
- },
- );
- });
-
- test("rejects non-relative artifact paths before accepting a seal", async () => {
- const scanDir = await copyExample();
- const manifestPath = join(scanDir, "scan-manifest.json");
- const manifest = await readJson(manifestPath);
- const prefixes = ["../", "/", "C:/", "artifacts/../", "artifacts\\"];
- await mkdir(join(scanDir, "artifacts"), { recursive: true });
- await fc.assert(
- fc.asyncProperty(
- fc.stringMatching(/^[a-z]{1,16}$/u),
- fc.constantFrom(...prefixes),
- async (name, prefix) => {
- const filename = `artifact-${name}`;
- const bytes = Buffer.from(name);
- const artifact = {
- path: `artifacts/${filename}`,
- sha256: sha256(bytes),
- mediaType: "application/octet-stream",
- };
- const scan = {
- ...manifest["scan"],
- artifacts: [...manifest["scan"]["artifacts"], artifact],
- };
- await writeFile(join(scanDir, "artifacts", filename), bytes);
- await writeJson(manifestPath, { ...manifest, scan });
- await loadContract(scanDir, { pluginRoot: PLUGIN_ROOT });
-
- artifact.path = `${prefix}${filename}`;
- await writeJson(manifestPath, { ...manifest, scan });
- const rejected = loadContract(scanDir, { pluginRoot: PLUGIN_ROOT });
- await expect(rejected).rejects.toBeInstanceOf(
- ContractValidationError,
- );
- await expect(rejected).rejects.toThrow(
- /safe scan-relative POSIX path|schema validation failed \(pattern/u,
- );
- },
- ),
- {
- ...propertyOptions,
- numRuns: Number(process.env["CODEX_SECURITY_PROPERTY_RUNS"] ?? "20"),
- examples: [
- ...prefixes.map((prefix): [string, string] => ["synthetic", prefix]),
- ["con", "C:/"],
- ],
- },
- );
- });
-
test("ships a completed example that passes tracking preflight", () => {
const result = runPython(Bun.which("python3") ?? "python", [
join(PLUGIN_ROOT, "scripts", "validate_tracking_source.py"),
diff --git a/sdk/typescript/tests-ts/errors.property.test.ts b/sdk/typescript/tests-ts/errors.property.test.ts
deleted file mode 100644
index 8f6d2a18f..000000000
--- a/sdk/typescript/tests-ts/errors.property.test.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import fc from "fast-check";
-import { errorMessage } from "../src/errors.js";
-import { propertyOptions } from "./support/property.js";
-
-describe("error-message invariants", () => {
- test("preserves arbitrary diagnostic text exactly", () => {
- fc.assert(
- fc.property(fc.string(), (message) => {
- expect(errorMessage(message)).toBe(message);
- expect(errorMessage(new Error(message))).toBe(message);
- }),
- propertyOptions,
- );
- });
-});
diff --git a/sdk/typescript/tests-ts/errors.test.ts b/sdk/typescript/tests-ts/errors.test.ts
index 656a4e907..b59e2ffd2 100644
--- a/sdk/typescript/tests-ts/errors.test.ts
+++ b/sdk/typescript/tests-ts/errors.test.ts
@@ -70,6 +70,12 @@ describe("error messages", () => {
expect(errorMessage(message)).toBe(message);
});
+ test("preserves whitespace and Unicode in diagnostics", () => {
+ const message = " first line\nsecond line: café 雪 ";
+ expect(errorMessage(new Error(message))).toBe(message);
+ expect(errorMessage(message)).toBe(message);
+ });
+
test("formats non-error values without parsing them", () => {
expect(errorMessage(42)).toBe("42");
expect(errorMessage(null)).toBe("null");
diff --git a/sdk/typescript/tests-ts/external-findings-local-limits.test.ts b/sdk/typescript/tests-ts/external-findings-local-limits.test.ts
index c90ffea73..1111e7337 100644
--- a/sdk/typescript/tests-ts/external-findings-local-limits.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-local-limits.test.ts
@@ -10,6 +10,10 @@ import type {
ExternalFindingEvidence,
Provider,
} from "../src/external-import-models.js";
+import {
+ temporaryDirectory,
+ removeTemporaryDirectory,
+} from "./support/temporary-directories.js";
import { localFindingsFixture } from "./external-findings-local-helpers.js";
import { endorFinding, endorSarif } from "./support/endor-import-fixtures.js";
import {
@@ -184,46 +188,26 @@ const formats: [string, Provider, () => string][] = [
];
test.each(formats)(
- "%s persists large local evidence while Cloud retains its size limit",
+ "%s retains large evidence locally and excludes it from Cloud",
async (_name, provider, contents) => {
- const fixture = await localFindingsFixture({ remote: repositoryUrl });
+ const directory = await temporaryDirectory("scanner-limits-");
try {
- const file = join(fixture.root, "scanner-report");
+ const file = join(directory, "scanner-report");
await writeFile(file, contents());
- const options = {
- provider,
- repository: fixture.repository,
- sourceKey: "synthetic-project",
- sourceRepositoryUrl: repositoryUrl,
- };
- const dependencies = { environment: fixture.environment };
+ const options = { provider, sourceRepositoryUrl: repositoryUrl };
const cloud = await readExternalFindings(file, options);
expect(cloud.findings.findings).toEqual([]);
expect(cloud.findings.excluded).toHaveLength(1);
expect(cloud.findings.excluded[0]!.reason).toContain("256 KiB");
- const prepared = await prepareLocalExternalImport(
- file,
- options,
- dependencies,
- );
- expect(prepared.preview.excluded).toEqual([]);
- expect(prepared.preview.findings).toHaveLength(1);
+ const local = await readExternalFindings(file, options, "local");
+ expect(local.findings.excluded).toEqual([]);
+ expect(local.findings.findings).toHaveLength(1);
expect(
- JSON.stringify(prepared.preview.findings[0]!.evidence.source_data),
+ JSON.stringify(local.findings.findings[0]!.evidence.source_data),
).toContain(retainedEvidence);
- expect((await prepared.import()).counts).toEqual({
- created: 1,
- updated: 0,
- unchanged: 0,
- });
- const stored = await listLocalExternalFindings(options, dependencies);
- expect(stored).toHaveLength(1);
- expect(stored[0]!.evidence).toEqual(
- prepared.preview.findings[0]!.evidence,
- );
} finally {
- await fixture.dispose();
+ await removeTemporaryDirectory(directory);
}
},
);
@@ -232,6 +216,7 @@ let nested: Record<string, unknown> = { leaf: "retained" };
for (let index = 0; index < 24; index++) nested = { child: nested };
const evidenceLimits: [string, Partial<ExternalFindingEvidence>][] = [
+ ["raw evidence size", { source_data: { retainedEvidence } }],
["UTF-8 field bytes", { title: "é".repeat(257) }],
["description length", { description: "x".repeat(65537) }],
[
@@ -255,45 +240,55 @@ const evidenceLimits: [string, Partial<ExternalFindingEvidence>][] = [
["JSON depth", { source_data: nested }],
];
-test.each(evidenceLimits)(
- "local imports retain %s above Cloud's contract",
- async (_name, fields) => {
- const fixture = await localFindingsFixture({ remote: repositoryUrl });
- try {
- const input = {
- ...normalized(),
- evidence: { ...normalized().evidence, ...fields },
- };
- const file = join(fixture.root, "normalized.jsonl");
- await writeFile(file, JSON.stringify(input));
- const options = {
- provider: "wiz" as const,
- repository: fixture.repository,
- sourceKey: "synthetic-project",
- };
- const dependencies = { environment: fixture.environment };
- const prepared = await prepareLocalExternalImport(
- file,
- options,
- dependencies,
- );
- expect(prepared.preview.excluded).toEqual([]);
- const imported = await prepared.import();
- expect(imported.counts.created).toBe(1);
- const stored = await listLocalExternalFindings(options, dependencies);
- expect(stored[0]!.evidence).toEqual(
- prepared.preview.findings[0]!.evidence,
+test("local SQLite preserves each oversized evidence field and Cloud rejects each one before authentication", async () => {
+ const fixture = await localFindingsFixture({ remote: repositoryUrl });
+ try {
+ const inputs = evidenceLimits.map(([id, fields]) => ({
+ source_finding_id: id,
+ evidence: { ...normalized().evidence, ...fields },
+ }));
+ const file = join(fixture.root, "normalized.jsonl");
+ await writeFile(
+ file,
+ inputs.map((record) => JSON.stringify(record)).join("\n"),
+ );
+ const options = {
+ provider: "wiz" as const,
+ repository: fixture.repository,
+ sourceKey: "synthetic-project",
+ };
+ const dependencies = { environment: fixture.environment };
+ const prepared = await prepareLocalExternalImport(
+ file,
+ options,
+ dependencies,
+ );
+ expect(prepared.preview.excluded).toEqual([]);
+ expect((await prepared.import()).counts.created).toBe(inputs.length);
+ const stored = new Map(
+ (await listLocalExternalFindings(options, dependencies)).map(
+ (finding) => [finding.sourceFindingId, finding.evidence],
+ ),
+ );
+ expect(stored.size).toBe(inputs.length);
+ for (const { source_finding_id, evidence } of prepared.preview.findings)
+ expect(stored.get(source_finding_id)).toEqual(evidence);
+
+ const cloud = await readExternalFindings(file, options);
+ expect(cloud.findings.findings).toEqual([]);
+ expect(
+ cloud.findings.excluded.map((item) => item.source_finding_id),
+ ).toEqual(inputs.map((input) => input.source_finding_id));
+ const credentials = mock(async () => {
+ throw new Error("Unexpected credential access");
+ });
+ for (const [index, input] of inputs.entries()) {
+ expect(stored.get(input.source_finding_id)).toMatchObject(
+ evidenceLimits[index]![1],
);
- expect(stored[0]!.evidence).toMatchObject(fields);
expect(() =>
validateExternalEvidence(structuredClone(input.evidence)),
).toThrow();
- const cloud = await readExternalFindings(file, options);
- expect(cloud.findings.findings).toEqual([]);
- expect(cloud.findings.excluded).toHaveLength(1);
- const credentials = mock(async () => {
- throw new Error("Unexpected credential access");
- });
await expect(
prepareNormalizedExternalPublication(
{ read: 1, findings: [input], excluded: [] },
@@ -301,12 +296,12 @@ test.each(evidenceLimits)(
{ credentials },
),
).rejects.toThrow();
- expect(credentials).not.toHaveBeenCalled();
- } finally {
- await fixture.dispose();
}
- },
-);
+ expect(credentials).not.toHaveBeenCalled();
+ } finally {
+ await fixture.dispose();
+ }
+});
test.each(["wiz", "endor"] as const)(
"%s local imports preserve source identities longer than Cloud accepts",
diff --git a/sdk/typescript/tests-ts/external-findings-local.test.ts b/sdk/typescript/tests-ts/external-findings-local.test.ts
index 09fc8be6e..ebcee431c 100644
--- a/sdk/typescript/tests-ts/external-findings-local.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-local.test.ts
@@ -1914,29 +1914,4 @@ describe("local normalized scanner findings", () => {
}
expect(existsSync(stateDirectory)).toBe(false);
});
-
- test("does not impose Cloud field-size limits on normalized local evidence", async () => {
- const { repository, environment } = await fixture();
- const data = input();
- data.findings[0]!.evidence.description = "x".repeat(65537);
- data.findings[0]!.evidence.locations = Array.from(
- { length: 101 },
- (_, index) => ({ path: `src/file-${index}.ts` }),
- );
- const result = await (
- await prepareNormalizedLocalImport(
- data,
- { repository, ...source },
- { environment },
- )
- ).import();
- expect(
- (
- await listLocalExternalFindings(
- { repository, id: result.findings[0]!.id },
- { environment },
- )
- )[0]!.evidence,
- ).toEqual(data.findings[0]!.evidence);
- });
});
diff --git a/sdk/typescript/tests-ts/finding-workflow-integration.test.ts b/sdk/typescript/tests-ts/finding-workflow-integration.test.ts
index e0790e4c8..035a726e4 100644
--- a/sdk/typescript/tests-ts/finding-workflow-integration.test.ts
+++ b/sdk/typescript/tests-ts/finding-workflow-integration.test.ts
@@ -384,39 +384,52 @@ test("reuses publication after dedupe failure and persists a successful empty du
expect(await readFile(join(scanDir, "findings.json"), "utf8")).toBe(original);
});
-test("an empty scan completes publication and dedupe and remains retrievable", async () => {
- const { scanDir, environment, document, history } = await fixture();
- document.findings = [];
- const content = JSON.stringify(document);
- await writeFile(join(scanDir, "findings.json"), content);
- const manifestPath = join(scanDir, "scan-manifest.json");
- const manifest = JSON.parse(
- await readFile(manifestPath, "utf8"),
- ) as ScanManifest;
- manifest.scan.artifacts!.find(
- (artifact) => artifact.path === "findings.json",
- )!.sha256 = sha256(content);
- await writeFile(manifestPath, JSON.stringify(manifest));
+test("an empty workflow preserves its result across cancellation at completion and on cached lookup", async () => {
+ const { environment, document, history } = await fixtureWithFindings(0);
const options = {
workflowId: "empty-scan",
findingsUrl: "http://synthetic.test",
};
-
- const fetch = mock(async (_url: URL, init: RequestInit) => {
- expect(JSON.parse(init.body as string).findings).toEqual([]);
- return Response.json([]);
- });
- const result = await deduplicateScanInternal(document.scanId, options, {
- environment,
- runWorkbench: history,
- fetch,
- });
- expect(result).toEqual({
+ const result = {
scanId: document.scanId,
uniqueFindingIds: [],
duplicateGroups: [],
- deduplicationStatus: "completed",
+ deduplicationStatus: "completed" as const,
+ };
+ const fetch = mock(async (url: URL, init: RequestInit) => {
+ expect(url.pathname).toBe("/v1/bulk/findings");
+ expect(JSON.parse(init.body as string).findings).toEqual([]);
+ return Response.json([]);
});
+ const workflow = new FindingWorkflow(options.workflowId, environment);
+ for (const action of ["complete", "begin"]) {
+ const controller = new AbortController();
+ const reason = new Error(`Canceled after dedupe ${action}`);
+ await expect(
+ deduplicateScanInternal(
+ document.scanId,
+ { ...options, signal: controller.signal },
+ {
+ environment,
+ fetch,
+ runWorkbench: async (args, input) => {
+ const response = await history(args, input);
+ const payload = input ? JSON.parse(input) : {};
+ if (payload.stage === "dedupe" && payload.action === action)
+ controller.abort(reason);
+ return response;
+ },
+ },
+ ),
+ ).rejects.toThrow(reason.message);
+ expect((await workflow.get())?.stages).toMatchObject({
+ publish: {
+ status: "completed",
+ result: { findingIds: [], findingCount: 0 },
+ },
+ dedupe: { status: "completed", result },
+ });
+ }
expect(
await deduplicateScanInternal(document.scanId, options, {
environment,
@@ -425,13 +438,6 @@ test("an empty scan completes publication and dedupe and remains retrievable", a
}),
).toEqual(result);
expect(fetch).toHaveBeenCalledTimes(1);
- expect(
- (await new FindingWorkflow(options.workflowId, environment).get())?.stages
- .publish,
- ).toMatchObject({
- status: "completed",
- result: { findingIds: [], findingCount: 0 },
- });
});
test.each(["acknowledged", "interrupted", "without workflow"])(
@@ -502,66 +508,6 @@ test.each(["acknowledged", "interrupted", "without workflow"])(
},
);
-test.each(["complete", "cached"])(
- "observes final cancellation for a %s workflow without group writes",
- async (phase) => {
- const { environment, document, history } = await fixtureWithFindings(0);
- const controller = new AbortController();
- const reason = new Error(
- "Synthetic cancellation after workflow completion",
- );
- const options = {
- workflowId: "empty-canceled",
- findingsUrl: "http://synthetic.test",
- };
- const fetch = mock(findingFetch(document));
- if (phase === "cached")
- await deduplicateScanInternal(document.scanId, options, {
- environment,
- runWorkbench: history,
- fetch,
- });
- await expect(
- deduplicateScanInternal(
- document.scanId,
- { ...options, signal: controller.signal },
- {
- environment,
- fetch,
- runWorkbench: async (args, input) => {
- const result = await history(args, input);
- const payload = input ? JSON.parse(input) : {};
- if (
- payload.stage === "dedupe" &&
- payload.action === (phase === "cached" ? "begin" : "complete")
- )
- controller.abort(reason);
- return result;
- },
- },
- ),
- ).rejects.toThrow(reason.message);
- const saved = (await new FindingWorkflow(
- options.workflowId,
- environment,
- ).get())!.stages.dedupe;
- expect(saved).toMatchObject({
- status: "completed",
- result: { duplicateGroups: [] },
- });
- expect(fetch).toHaveBeenCalledTimes(1);
- expect(fetch.mock.calls[0]![0].pathname).toBe("/v1/bulk/findings");
- expect<unknown>(
- await deduplicateScanInternal(document.scanId, options, {
- environment,
- runWorkbench: history,
- fetch,
- }),
- ).toEqual(saved.result);
- expect(fetch).toHaveBeenCalledTimes(1);
- },
-);
-
test("does not write workflow metadata into sealed artifacts", async () => {
const { scanDir, environment } = await fixture();
await expect(
diff --git a/sdk/typescript/tests-ts/finding-workflow-publication.test.ts b/sdk/typescript/tests-ts/finding-workflow-publication.test.ts
index 7cd15d91b..ef455a1f0 100644
--- a/sdk/typescript/tests-ts/finding-workflow-publication.test.ts
+++ b/sdk/typescript/tests-ts/finding-workflow-publication.test.ts
@@ -384,6 +384,7 @@ test.each(["before-post", "before-write", "lost-ack", "lost-completion"])(
runWorkbench: history,
reviewer,
fetch,
+ waitForRetry: async () => {},
};
await expect(
deduplicateScanInternal(document.scanId, options, dependencies),
diff --git a/sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json b/sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json
new file mode 100644
index 000000000..74eb8553b
--- /dev/null
+++ b/sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json
@@ -0,0 +1,82 @@
+{
+ "accountId": "synthetic-account",
+ "requests": [
+ {
+ "request_id": "4d815058-28ce-4c37-9c59-36188c3372be",
+ "repository": {
+ "id": "cloud-repository",
+ "repo_connector_id": "github",
+ "environment_id": "environment-1",
+ "reset_marker": "generation-1"
+ },
+ "source": {
+ "provider": "wiz",
+ "source_key": "synthetic-tenant/wiz"
+ },
+ "items": [
+ {
+ "client_id": "item-1",
+ "source_finding_id": "unicode-evidence",
+ "expected_version": 0,
+ "evidence": {
+ "title": "Unicode vendor metadata",
+ "severity": "high",
+ "source_data": {
+ "é": "precomposed",
+ "é": "combining",
+ "Z": 1,
+ "a": 2,
+ "ä": 3
+ },
+ "description": null,
+ "url": null,
+ "advisory_ids": [],
+ "packages": [],
+ "locations": [],
+ "branch": null,
+ "code_revision": null,
+ "source_scan_id": null,
+ "image_digests": [],
+ "source_updated_at": null
+ }
+ }
+ ]
+ }
+ ],
+ "receipts": [
+ {
+ "id": "4d815058-28ce-4c37-9c59-36188c3372be",
+ "repository": {
+ "id": "cloud-repository",
+ "repo_connector_id": "github",
+ "environment_id": "environment-1",
+ "reset_marker": "generation-1"
+ },
+ "source": {
+ "provider": "wiz",
+ "source_key": "synthetic-tenant/wiz"
+ },
+ "actor": "synthetic-user",
+ "created_at": 1,
+ "item_count": 1,
+ "counts": {
+ "created": 1,
+ "updated": 0,
+ "unchanged": 0,
+ "error": 0
+ },
+ "results": [
+ {
+ "client_id": "item-1",
+ "outcome": "created",
+ "source_report_id": "aif_4fb0fb6973a0ba27504f384bf21a7408bfca3a4b281cc7d11012e8cac270d591",
+ "observation_id": "aio_85b5b4200540ca44d31f8b8ca9ee8314461d55b8d94f5e97b35a832da08d9b81",
+ "canonical_finding_id": "acf_4fb0fb6973a0ba27504f384bf21a7408",
+ "version": 1,
+ "error": null
+ }
+ ],
+ "object": "security.finding_import"
+ }
+ ]
+}
diff --git a/sdk/typescript/tests-ts/local-dedupe.test.ts b/sdk/typescript/tests-ts/local-dedupe.test.ts
index 1d22a9c8e..65c3fe0d4 100644
--- a/sdk/typescript/tests-ts/local-dedupe.test.ts
+++ b/sdk/typescript/tests-ts/local-dedupe.test.ts
@@ -5,7 +5,7 @@ import {
setTimeout as delay,
setImmediate as nextTurn,
} from "node:timers/promises";
-import { readFile, writeFile, mkdir, chmod } from "node:fs/promises";
+import { readFile, writeFile, mkdir, chmod, unlink } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, expect, mock, spyOn, test } from "bun:test";
import {
@@ -32,6 +32,7 @@ import { screeningPairSlot } from "../src/deduplication/deduplication-reviewer.j
import { rejecting } from "./support/errors.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import type { Finding } from "../src/models.js";
+import type { JsonObject } from "../src/config.js";
import type { CodexReview } from "../src/deduplication/codex-review.js";
import { comparisonEnvironment } from "../src/scan-comparison.js";
import { reviewSettingsDigest } from "../src/deduplication/checkpointed-review.js";
@@ -319,101 +320,160 @@ test("saved-scan dedupe uses configured Codex models and efforts for both review
]);
});
-test.each(["environment", "configuration"])(
- "workflow ignores native SQLite writes but detects source changes (%s)",
- async (selection) => {
- const f = await fixture();
- const sqliteHome = join(f.repository, "runtime", "native");
- await mkdir(f.environment.CODEX_HOME, { recursive: true });
- if (selection === "configuration")
- await writeFile(
- join(f.environment.CODEX_HOME, "config.toml"),
- `sqlite_home = ${JSON.stringify(sqliteHome)}\n`,
- );
- await writeFile(join(f.repository, "source.py"), "original source\n");
- await writeFile(join(f.repository, ".gitignore"), "runtime/\nsource.py\n");
- const anchor = f.document.findings[0]!;
- await f.store.insert(
- [
- {
- finding: {
- ...anchor,
- findingId: "csf_neighbor",
- fingerprints: { ...anchor.fingerprints, primary: "neighbor" },
- },
- embedding: { model: EMBEDDING_MODEL, vector },
+test("workflow ignores native SQLite writes on replay but detects source changes", async () => {
+ const f = await fixture();
+ let sqliteHome = join(f.repository, "runtime", "from-config");
+ const configPath = join(f.environment.CODEX_HOME, "config.toml");
+ await mkdir(f.environment.CODEX_HOME, { recursive: true });
+ await writeFile(configPath, `sqlite_home = ${JSON.stringify(sqliteHome)}\n`);
+ await writeFile(join(f.repository, "source.py"), "original source\n");
+ await writeFile(join(f.repository, ".gitignore"), "runtime/\nsource.py\n");
+ const anchor = f.document.findings[0]!;
+ await f.store.insert(
+ [
+ {
+ finding: {
+ ...anchor,
+ findingId: "csf_neighbor",
+ fingerprints: { ...anchor.fingerprints, primary: "neighbor" },
},
- ],
- f.targetId,
- );
- const environment = {
- ...f.environment,
- OPENAI_API_KEY: "synthetic-review-key",
- ...(selection === "environment" ? { CODEX_SQLITE_HOME: sqliteHome } : {}),
- };
- let calls = 0;
- let mutateSource = false;
- let loseAcknowledgement = true;
- const options = {
- repository: f.repository,
- workflowId: `native-state-${selection}`,
- embedding: f.embedding,
- };
- const dependencies = {
- environment,
- runWorkbench: async (args: readonly string[], input?: string) => {
- const result = await runWorkbench(f.options, args, input);
- if (args[0] === "store-dedupe-groups" && loseAcknowledgement) {
- loseAcknowledgement = false;
- throw new Error("Lost native-state group acknowledgement");
- }
- return result;
+ embedding: { model: EMBEDDING_MODEL, vector },
},
- reviewRunner: {
- async run<T>(review: CodexReview<T>): Promise<T> {
- calls++;
- await mkdir(sqliteHome, { recursive: true });
+ ],
+ f.targetId,
+ );
+ const environment = {
+ ...f.environment,
+ OPENAI_API_KEY: "synthetic-review-key",
+ };
+ const snapshotPaths: unknown[] = [];
+ let calls = 0;
+ let mutateSource = false;
+ let loseAcknowledgement = true;
+ const options = {
+ repository: f.repository,
+ workflowId: "native-state",
+ embedding: f.embedding,
+ };
+ const dependencies = {
+ environment,
+ runWorkbench: async (args: readonly string[], input?: string) => {
+ if (args[0] === "finding-workflow") {
+ const { action, privateStatePaths } = JSON.parse(input!);
+ if (action === "source") snapshotPaths.push(privateStatePaths);
+ }
+ const result = await runWorkbench(f.options, args, input);
+ if (args[0] === "store-dedupe-groups" && loseAcknowledgement) {
+ loseAcknowledgement = false;
+ throw new Error("Lost native-state group acknowledgement");
+ }
+ return result;
+ },
+ reviewRunner: {
+ async run<T>(review: CodexReview<T>): Promise<T> {
+ calls++;
+ await mkdir(sqliteHome, { recursive: true });
+ await writeFile(
+ join(sqliteHome, "state_5.sqlite"),
+ `synthetic native state ${calls}`,
+ );
+ if (mutateSource && review.stage === "screening")
await writeFile(
- join(sqliteHome, "state_5.sqlite"),
- `synthetic native state ${calls}`,
- );
- if (mutateSource)
- await writeFile(
- join(f.repository, "source.py"),
- "changed source\n",
- );
- const decision = { decision: "SAME", rationale: "Same control" };
- return review.validate(
- review.stage === "screening"
- ? { decisions: { "pair-1": decision } }
- : decision,
+ join(sqliteHome, "state_5.sqlite-wal"),
+ "review WAL activity",
);
- },
+ if (mutateSource && review.stage === "pair-review")
+ await writeFile(join(f.repository, "source.py"), "changed source\n");
+ const decision = { decision: "SAME", rationale: "Same control" };
+ return review.validate(
+ review.stage === "screening"
+ ? { decisions: { "pair-1": decision } }
+ : decision,
+ );
},
- };
- await expect(
- deduplicateScanDirectoryInternal(f.scanDir, options, dependencies),
- ).rejects.toThrow("Lost native-state group acknowledgement");
- expect(calls).toBe(2);
- await writeFile(
- join(sqliteHome, "state_5.sqlite-wal"),
- "later native activity",
- );
- expect(
- (await deduplicateScanDirectoryInternal(f.scanDir, options, dependencies))
- .deduplicationStatus,
- ).toBe("completed");
- expect(calls).toBe(2);
- mutateSource = true;
- await expect(
- deduplicateScanDirectoryInternal(
- f.scanDir,
- { ...options, workflowId: `${options.workflowId}-source-change` },
- dependencies,
- ),
- ).rejects.toThrow("Source changed during deduplication");
- },
-);
+ },
+ };
+ await expect(
+ deduplicateScanDirectoryInternal(f.scanDir, options, dependencies),
+ ).rejects.toThrow("Lost native-state group acknowledgement");
+ expect(calls).toBe(2);
+ expect(snapshotPaths).toContainEqual([sqliteHome]);
+ snapshotPaths.length = 0;
+ await writeFile(
+ join(sqliteHome, "state_5.sqlite-wal"),
+ "later native activity",
+ );
+ expect(
+ (await deduplicateScanDirectoryInternal(f.scanDir, options, dependencies))
+ .deduplicationStatus,
+ ).toBe("completed");
+ expect(calls).toBe(2);
+ expect(snapshotPaths).toContainEqual([sqliteHome]);
+ await unlink(configPath);
+ sqliteHome = join(f.repository, "runtime", "from-environment");
+ mutateSource = true;
+ await expect(
+ deduplicateScanDirectoryInternal(
+ f.scanDir,
+ { ...options, workflowId: `${options.workflowId}-source-change` },
+ {
+ ...dependencies,
+ environment: { ...environment, CODEX_SQLITE_HOME: sqliteHome },
+ },
+ ),
+ ).rejects.toThrow("Source changed during deduplication");
+ expect(snapshotPaths).toContainEqual([sqliteHome]);
+ // Screening wrote native state; the pair review then changed repository source.
+ expect(calls).toBe(4);
+});
+
+// A fresh workflow with one cached neighbor lets provider cases observe the
+// actual review and checkpoint requests without starting a workbench process.
+function freshProviderReviewResponse(
+ action: string,
+ repository: string,
+ finding: Finding,
+ neighbor: Finding,
+): JsonObject {
+ switch (action) {
+ case "prepare":
+ return {
+ cacheKeys: {
+ [finding.findingId]: "anchor",
+ [neighbor.findingId]: "neighbor",
+ },
+ findingsToEmbed: [],
+ };
+ case "neighbors":
+ return JSON.parse(
+ JSON.stringify({ finding, potentialDuplicates: [neighbor] }),
+ );
+ case "begin":
+ case "get":
+ return { workflow: { stages: { dedupe: { status: "running" } } } };
+ case "source":
+ return {
+ source: {
+ repository,
+ revision: "unversioned",
+ refsDigest: sha256(""),
+ // Workbench source snapshot of the fixture's empty repository.
+ content:
+ "codex-security-snapshot/v1:sha256:75886c9285f7b38118ac1acf8d567d6ae3308d694babd55d09436d65620f2b1e",
+ },
+ };
+ case "get-review":
+ return { review: null };
+ case "bind":
+ case "complete":
+ case "save-review":
+ case "prepare-dedupe":
+ case "commit":
+ return {};
+ default:
+ throw new Error(`Unexpected provider review request: ${action}`);
+ }
+}
test.each([
"stored-login",
@@ -457,19 +517,17 @@ test.each([
: 'model = "stored-model"\nmodel_reasoning_effort = "high"\n',
);
const anchor = f.document.findings[0]!;
- await f.store.insert(
- [
- {
- finding: {
- ...anchor,
- findingId: "csf_neighbor",
- fingerprints: { ...anchor.fingerprints, primary: "neighbor" },
- },
- embedding: { model: EMBEDDING_MODEL, vector },
- },
- ],
- f.targetId,
- );
+ const neighbor = {
+ ...anchor,
+ findingId: "csf_neighbor",
+ fingerprints: { ...anchor.fingerprints, primary: "neighbor" },
+ };
+ const useDatabase = selection === "stored-login";
+ if (useDatabase)
+ await f.store.insert(
+ [{ finding: neighbor, embedding: { model: EMBEDDING_MODEL, vector } }],
+ f.targetId,
+ );
const environment = {
...f.environment,
...(selection === "environment-key"
@@ -483,17 +541,23 @@ test.each([
}));
const calls: string[][] = [];
let selectedEnvironment: NodeJS.ProcessEnv = environment;
- const bindings: {
- model: string;
- effort: string;
- settingsDigest: string;
+ const workflowId = `effective-${selection}`;
+ const checkpoints: {
+ key: string;
+ result: unknown;
+ binding: {
+ stage: string;
+ model: string;
+ effort: string;
+ settingsDigest: string;
+ };
}[] = [];
await deduplicateScanDirectoryInternal(
f.scanDir,
{
repository: f.repository,
embedding: f.embedding,
- workflowId: `effective-${selection}`,
+ workflowId,
},
{
environment,
@@ -507,9 +571,16 @@ test.each([
return selectedEnvironment as Record<string, string>;
},
runWorkbench: async (args, input) => {
- const payload = input ? JSON.parse(input) : {};
- if (payload.action === "save-review") bindings.push(payload.binding);
- return runWorkbench(f.options, args, input);
+ const payload = JSON.parse(input!);
+ if (payload.action === "save-review") checkpoints.push(payload);
+ return useDatabase
+ ? runWorkbench(f.options, args, input)
+ : freshProviderReviewResponse(
+ payload.action,
+ f.repository,
+ anchor,
+ neighbor,
+ );
},
reviewRunner: {
async run<T>(review: CodexReview<T>): Promise<T> {
@@ -530,25 +601,31 @@ test.each([
},
},
);
- const settings =
+ const { model, effort } =
selection === "stored-login"
- ? ["stored-model", "high"]
- : ["ambient-model", "medium"];
+ ? { model: "stored-model", effort: "high" }
+ : { model: "ambient-model", effort: "medium" };
expect(calls).toEqual([
- ["screening", ...settings],
- ["pair-review", ...settings],
- ]);
- expect(bindings.map(({ model, effort }) => [model, effort])).toEqual([
- settings,
- settings,
+ ["screening", model, effort],
+ ["pair-review", model, effort],
]);
const digest = await reviewSettingsDigest(selectedEnvironment, {
- model: settings[0]!,
- model_reasoning_effort: settings[1]!,
+ model,
+ model_reasoning_effort: effort,
});
- expect(bindings.every((binding) => binding.settingsDigest === digest)).toBe(
- true,
+ expect(checkpoints.map(({ binding }) => binding)).toMatchObject(
+ ["screening", "pair-review"].map((stage) => ({
+ stage,
+ model,
+ effort,
+ settingsDigest: digest,
+ })),
);
+ if (useDatabase) {
+ const workflow = new FindingWorkflow(workflowId, environment);
+ for (const { key, result } of checkpoints)
+ expect(await workflow.getReview(key)).toEqual(result);
+ }
expect(accountStatus).toHaveBeenCalledTimes(
["stored-login", "managed-defaults"].includes(selection) ? 1 : 0,
);
diff --git a/sdk/typescript/tests-ts/publication-store.test.ts b/sdk/typescript/tests-ts/publication-store.test.ts
index 53a668746..1c92b2e06 100644
--- a/sdk/typescript/tests-ts/publication-store.test.ts
+++ b/sdk/typescript/tests-ts/publication-store.test.ts
@@ -1,3 +1,4 @@
+import { Database, type SQLQueryBindings } from "bun:sqlite";
import { once } from "node:events";
import { createHash, randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
@@ -134,53 +135,85 @@ function seedPublicationScan(
fixture: PublicationFixture,
publication: PreparedScanPublication,
): void {
- const workspaceId = randomUUID();
- const seed = runNodePython(fixture.python, [
- "-c",
- [
- "import json, sqlite3, sys",
- "database, workspace_id, publication = sys.argv[1], sys.argv[2], json.loads(sys.argv[3])",
- "connection = sqlite3.connect(database)",
- "connection.execute('PRAGMA foreign_keys = ON')",
- "timestamp = '2026-08-01T00:00:00Z'",
- "connection.execute('INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)', (workspace_id, timestamp, timestamp))",
- "connection.execute('INSERT INTO scans (id, workspace_id, target_path, target_revision, scope, mode, scan_dir, status, phase, started_at, completed_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (publication['scanId'], workspace_id, publication['scanDirectory'], 'example-revision', '.', 'standard', publication['scanDirectory'], 'complete', 'reporting', timestamp, timestamp, timestamp, timestamp))",
- "for issue in publication['issues']:",
- " connection.execute('INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO NOTHING', (issue['findingId'], 'fingerprint-' + issue['findingId'], 'example-rule', issue['findingId'], timestamp, timestamp))",
- " connection.execute('INSERT INTO finding_occurrences (id, finding_id, scan_id, title, summary, severity, confidence, remediation, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (issue['occurrenceId'], issue['findingId'], publication['scanId'], issue['title'], 'example summary', 'high', 'high', 'example remediation', timestamp))",
- "connection.commit()",
- "connection.close()",
- ].join("\n"),
+ using database = new Database(
join(fixture.stateDirectory, "workbench.sqlite3"),
- workspaceId,
- JSON.stringify(publication),
- ]);
- expect(seed.status, seed.stderr).toBe(0);
+ );
+ database.run("PRAGMA foreign_keys = ON");
+ database.transaction(() => {
+ const workspaceId = randomUUID();
+ const timestamp = "2026-08-01T00:00:00Z";
+ database.run(
+ "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?1, ?2, ?2)",
+ [workspaceId, timestamp],
+ );
+ database.run(
+ `INSERT INTO scans
+ (id, workspace_id, target_path, target_revision, scope, mode, scan_dir, status,
+ phase, started_at, completed_at, created_at, updated_at)
+ VALUES (?1, ?2, ?3, 'example-revision', '.', 'standard', ?3, 'complete',
+ 'reporting', ?4, ?4, ?4, ?4)`,
+ [publication.scanId, workspaceId, publication.scanDirectory, timestamp],
+ );
+ for (const issue of publication.issues) {
+ database.run(
+ `INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, created_at, updated_at)
+ VALUES (?1, 'fingerprint-' || ?1, 'example-rule', ?1, ?2, ?2)
+ ON CONFLICT(id) DO NOTHING`,
+ [issue.findingId, timestamp],
+ );
+ database.run(
+ `INSERT INTO finding_occurrences
+ (id, finding_id, scan_id, title, summary, severity, confidence, remediation, created_at)
+ VALUES (?1, ?2, ?3, ?4, 'example summary', 'high', 'high', 'example remediation', ?5)`,
+ [
+ issue.occurrenceId,
+ issue.findingId,
+ publication.scanId,
+ issue.title,
+ timestamp,
+ ],
+ );
+ }
+ })();
}
-function databaseRows(
+function databaseRows<Row = Record<string, unknown>>(
fixture: PublicationFixture,
query: string,
- values: readonly unknown[] = [],
-): Record<string, unknown>[] {
- const result = runNodePython(fixture.python, [
- "-c",
- [
- "import json, sqlite3, sys",
- "connection = sqlite3.connect(sys.argv[1])",
- "connection.row_factory = sqlite3.Row",
- "cursor = connection.execute(sys.argv[2], json.loads(sys.argv[3]))",
- "rows = [dict(row) for row in cursor.fetchall()] if cursor.description else []",
- "connection.commit()",
- "connection.close()",
- "print(json.dumps(rows))",
- ].join("\n"),
+ values: SQLQueryBindings[] = [],
+): Row[] {
+ using database = new Database(
join(fixture.stateDirectory, "workbench.sqlite3"),
- query,
- JSON.stringify(values),
- ]);
- expect(result.status, result.stderr).toBe(0);
- return JSON.parse(result.stdout) as Record<string, unknown>[];
+ );
+ return database.query<Row, SQLQueryBindings[]>(query).all(...values);
+}
+
+async function mockStoreFixture() {
+ const stateDirectory = await temporaryDirectory();
+ const scanDirectory = join(stateDirectory, "scan");
+ await mkdir(scanDirectory);
+ await writeFile(join(stateDirectory, "workbench.sqlite3"), "");
+ const publication: PreparedScanPublication = {
+ scanId: SCAN_ID,
+ scanDirectory,
+ destination: {
+ type: "linear",
+ teamId: "team-example",
+ projectId: "project-example",
+ },
+ issues: [
+ {
+ findingId: "finding-1",
+ occurrenceId: "occurrence-1",
+ title: "Synthetic finding",
+ description: "Synthetic evidence.",
+ },
+ ],
+ };
+ return {
+ publication,
+ environment: { ...process.env, CODEX_SECURITY_STATE_DIR: stateDirectory },
+ };
}
function publishedIssue(
@@ -289,14 +322,14 @@ describe("read-only publication history", () => {
});
test("forwards inspection cancellation through Python discovery and the shared workbench", async () => {
- const fixture = await publicationFixture();
+ const fixture = await mockStoreFixture();
const controller = new AbortController();
const reason = new Error("Synthetic inspection cancellation.");
const inspecting = Promise.withResolvers<void>();
const python = spyOn(runtime, "resolvePluginPython").mockImplementation(
async (options) => {
expect(options?.signal).toBe(controller.signal);
- return fixture.python;
+ return "unused";
},
);
const workbench = spyOn(runtime, "runWorkbench").mockImplementation(
@@ -346,7 +379,11 @@ describe("read-only publication history", () => {
});
test("rejects inspection responses for another destination or occurrence", async () => {
- const fixture = await publicationFixture({ count: 1 });
+ const fixture = await mockStoreFixture();
+ using _runtime = spyOn(
+ runtime,
+ "resolveWorkbenchRuntime",
+ ).mockResolvedValue(["unused", PLUGIN_ROOT]);
const receipt = {
...publishedIssue(fixture.publication, 0),
scanId: SCAN_ID,
@@ -382,7 +419,11 @@ describe("read-only publication history", () => {
});
test("rejects a persisted receipt that differs from the accepted issue", async () => {
- const fixture = await publicationFixture({ count: 1 });
+ const fixture = await mockStoreFixture();
+ using _runtime = spyOn(
+ runtime,
+ "resolveWorkbenchRuntime",
+ ).mockResolvedValue(["unused", PLUGIN_ROOT]);
const issue = publishedIssue(fixture.publication, 0, "EXAMPLE-101");
const workbench = spyOn(runtime, "runWorkbench").mockResolvedValue({
scanId: SCAN_ID,
@@ -411,11 +452,11 @@ describe("read-only publication history", () => {
expect(existsSync(missing.stateDirectory)).toBe(false);
const fixture = await publicationFixture();
- const version = databaseRows(
+ const version = databaseRows<{ version: number }>(
fixture,
"SELECT version FROM schema_migrations WHERE name = ?",
["share finding issue receipts across trackers"],
- )[0]!["version"];
+ )[0]!.version;
databaseRows(fixture, "DROP TABLE finding_issue_receipts");
databaseRows(fixture, "DELETE FROM schema_migrations WHERE version >= ?", [
version,
@@ -691,9 +732,9 @@ describe("persisted finding publication associations", () => {
original.findingId,
original.occurrenceId,
fixture.publication.destination.teamId,
- fixture.publication.destination.projectId,
+ fixture.publication.destination.projectId ?? null,
original.issueIdentifier,
- original.url,
+ original.url ?? null,
],
);
const before = await readFile(database);
diff --git a/sdk/typescript/tests-ts/sarif.test.ts b/sdk/typescript/tests-ts/sarif.test.ts
deleted file mode 100644
index 4e56745a0..000000000
--- a/sdk/typescript/tests-ts/sarif.test.ts
+++ /dev/null
@@ -1,155 +0,0 @@
-import { runNodePython } from "./support/python-probe.js";
-import { readFile } from "node:fs/promises";
-import { join } from "node:path";
-import { describe, expect, test } from "bun:test";
-import type { Finding, FindingsDocument } from "../src/models.js";
-import { PLUGIN_ROOT } from "./plugin-root.js";
-
-const example = join(PLUGIN_ROOT, "examples", "completed-scan");
-const manifest = JSON.parse(
- await readFile(join(example, "scan-manifest.json"), "utf8"),
-);
-const document = JSON.parse(
- await readFile(join(example, "findings.json"), "utf8"),
-) as FindingsDocument;
-
-function finding(overrides: Partial<Finding> = {}): Finding {
- return { ...document.findings[0]!, ...overrides };
-}
-
-function buildSarif(findings: Finding[]) {
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
- const result = runNodePython(
- python!,
- [
- "-c",
- [
- "import json, sys",
- "sys.path.insert(0, sys.argv[1])",
- "from finalize_scan_contract import build_sarif",
- "payload = json.load(sys.stdin)",
- "print(json.dumps(build_sarif(payload['manifest'], payload['findings'])['runs'][0]))",
- ].join("\n"),
- join(PLUGIN_ROOT, "scripts"),
- ],
- {
- input: JSON.stringify({ manifest, findings: { ...document, findings } }),
- },
- );
- expect(result.status, result.stderr).toBe(0);
- return JSON.parse(result.stdout);
-}
-
-describe("SARIF presentation", () => {
- test("keeps rule names and alert identity independent of finding text", () => {
- const source = finding({
- extensions: { candidateId: "candidate-example" },
- });
- const run = buildSarif([source]);
- const rule = run.tool.driver.rules[0];
- const result = run.results[0];
- expect(rule.name).not.toBe(source.ruleId);
- expect(rule.shortDescription.text).toBe(rule.name);
- expect(rule.help.text).toContain(source.remediation);
- expect(result.properties).toMatchObject({
- findingId: source.findingId,
- occurrenceId: source.occurrenceId,
- candidateId: "candidate-example",
- });
- for (const text of [
- source.title,
- source.summary,
- ...source.remediationTests!,
- ...source.preventiveControls!,
- ]) {
- expect(result.message.text).toContain(text);
- }
-
- const changed = buildSarif([
- {
- ...source,
- title: "A new title",
- summary: "A changed summary",
- remediation: "A different fix.",
- },
- ]);
- expect(changed.tool.driver.rules[0].name).toBe(rule.name);
- for (const key of [
- "ruleId",
- "ruleIndex",
- "locations",
- "partialFingerprints",
- "properties",
- ]) {
- expect(changed.results[0][key]).toEqual(result[key]);
- }
- });
-
- test("merges shared rule metadata without mixing result remediation", () => {
- const first = finding({
- occurrenceId: `occ_${"1".repeat(24)}`,
- severity: { level: "low" },
- remediation: "Apply the first control.",
- });
- const second = finding({
- occurrenceId: `occ_${"2".repeat(24)}`,
- severity: { level: "critical", score: 9.7, scoringSystem: "CVSS:3.1" },
- taxonomy: {
- category: "archive-extraction",
- cwe: ["cwe-022", "CWE-23", "unknown"],
- },
- remediation: "Apply the second control.",
- });
- const run = buildSarif([first, second]);
- expect(buildSarif([second, first])).toEqual(run);
- expect(run.tool.driver.rules).toHaveLength(1);
- const rule = run.tool.driver.rules[0];
- expect(rule.properties).toEqual({
- "security-severity": "9.7",
- tags: [
- "archive-extraction",
- "external/cwe/cwe-022",
- "external/cwe/cwe-023",
- "path-traversal",
- "security",
- ],
- });
- expect(rule.help.markdown).toContain(first.remediation);
- expect(rule.help.markdown).toContain(second.remediation);
- for (const [index, own, other] of [
- [0, first, second],
- [1, second, first],
- ] as const) {
- const result = run.results[index];
- expect(result.ruleIndex).toBe(0);
- expect(result.message.text).toContain(own.remediation);
- expect(result.message.text).not.toContain(other.remediation);
- expect(result.properties.severity).toBe(own.severity.level);
- }
- expect(run.results[0].level).toBe("note");
- });
-
- test.each([
- ["critical", undefined, "9.5"],
- ["high", undefined, "8.0"],
- ["medium", undefined, "5.0"],
- ["low", undefined, "2.0"],
- ["informational", undefined, undefined],
- ["high", 0, undefined],
- ["high", 6.25, "6.25"],
- ["critical", 10, "10"],
- ] as const)("maps %s severity with score %p", (level, score, expected) => {
- const severity =
- score === undefined
- ? { level }
- : { level, score, scoringSystem: "CVSS:3.1" };
- const run = buildSarif([
- finding({ severity, taxonomy: { category: "sql-injection", cwe: [] } }),
- ]);
- expect(run.tool.driver.rules[0].properties).toEqual({
- tags: ["security", "sql-injection"],
- ...(expected === undefined ? {} : { "security-severity": expected }),
- });
- });
-});
diff --git a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts
index de22cb8a8..0fdc36501 100644
--- a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts
+++ b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts
@@ -348,117 +348,112 @@ function sarifWithRepositories(urls: string[]) {
};
}
-for (const format of ["SARIF", "SARIF gzip"]) {
- test.each([
- ["unknown host", "git.example.test", "with remote"],
- ["GitHub", "github.com", "with remote"],
- ["known GitHub custom host", "git.example.test", "with remote"],
- ["GitHub", "github.com", "with uppercase remote"],
- ["known GitHub custom host", "git.example.test", "with uppercase remote"],
- ["unknown host", "git.example.test", "without remote"],
- ["GitHub", "github.com", "without remote"],
- ["known GitHub custom host", "git.example.test", "without remote"],
- ])(
- `Semgrep ${format} preserves original repository references on %s (%s, %s)`,
- async (platform, host, checkout) => {
- const sourceRepositoryUrl = `https://${host}/team/project`;
- const originalUrl = `https://${host}/Team/Project`;
- const fixture = await localFindingsFixture(
- checkout === "without remote"
- ? {}
- : {
- remote: `git@${host}:${checkout === "with uppercase remote" ? "Team/Project" : "team/project"}.git`,
- },
- );
- try {
- const file = join(fixture.root, "semgrep-report");
- const options = {
- repository: fixture.repository,
- provider: "semgrep" as const,
- sourceKey: "example-project:main",
- sourceRepositoryUrl,
- };
- const dependencies = {
- environment: fixture.environment,
- ...(platform === "known GitHub custom host"
- ? {
- sourceRepositories: [
- { url: originalUrl, platform: "github" as const },
- ],
- }
- : {}),
- };
- const database = join(fixture.stateDirectory, "workbench.sqlite3");
- async function writeReport(url: string) {
- // Keep both refs: checking only the first would miss the case-distinct one.
- const input = sarifWithRepositories([sourceRepositoryUrl, url]);
- const contents = JSON.stringify(input);
- await writeFile(
- file,
- format.endsWith("gzip") ? gzipSync(contents) : contents,
- );
- return input;
- }
- await writeReport(originalUrl);
- if (platform === "unknown host") {
- await expect(
- prepareLocalExternalImport(file, options, dependencies),
- ).rejects.toThrow("does not match");
- await expect(access(database)).rejects.toThrow();
- await writeReport(sourceRepositoryUrl);
- }
- const parsed = await readSemgrepFindings(file, { sourceRepositoryUrl });
- const prepared = await prepareLocalExternalImport(
- file,
- options,
- dependencies,
- );
- expect(prepared.preview.findings).toEqual(parsed.findings);
- expect((await prepared.import()).counts).toEqual({
- created: 1,
- updated: 0,
- unchanged: 0,
- });
- const stored = await listLocalExternalFindings(options, dependencies);
- expect(stored[0]!.evidence).toEqual(parsed.findings[0]!.evidence);
- expect(stored[0]!.evidence.details?.repository?.url).toBe(
- sourceRepositoryUrl,
- );
- expect(stored[0]!.evidence.source_data?.["runContext"]).toMatchObject({
- versionControlProvenance: [
- { repositoryUri: sourceRepositoryUrl },
- {
- repositoryUri:
- platform === "unknown host" ? sourceRepositoryUrl : originalUrl,
- },
- ],
- });
- expect(
- (
- await (
- await prepareLocalExternalImport(file, options, dependencies)
- ).import()
- ).counts,
- ).toEqual({ created: 0, updated: 0, unchanged: 1 });
- const before = await readFile(database);
- await writeReport(
- platform === "unknown host"
- ? originalUrl
- : `https://${host}/other/project`,
+test.each([
+ ["unknown host", "git.example.test", "with remote"],
+ ["GitHub", "github.com", "with remote"],
+ ["known GitHub custom host", "git.example.test", "with remote"],
+ ["GitHub", "github.com", "with uppercase remote"],
+ ["known GitHub custom host", "git.example.test", "with uppercase remote"],
+ ["unknown host", "git.example.test", "without remote"],
+ ["GitHub", "github.com", "without remote"],
+ ["known GitHub custom host", "git.example.test", "without remote"],
+])(
+ "Semgrep SARIF preserves original repository references on %s (%s, %s)",
+ async (platform, host, checkout) => {
+ const sourceRepositoryUrl = `https://${host}/team/project`;
+ const originalUrl = `https://${host}/Team/Project`;
+ const fixture = await localFindingsFixture(
+ checkout === "without remote"
+ ? {}
+ : {
+ remote: `git@${host}:${checkout === "with uppercase remote" ? "Team/Project" : "team/project"}.git`,
+ },
+ );
+ try {
+ const file = join(fixture.root, "semgrep-report");
+ const options = {
+ repository: fixture.repository,
+ provider: "semgrep" as const,
+ sourceKey: "example-project:main",
+ sourceRepositoryUrl,
+ };
+ const dependencies = {
+ environment: fixture.environment,
+ ...(platform === "known GitHub custom host"
+ ? {
+ sourceRepositories: [
+ { url: originalUrl, platform: "github" as const },
+ ],
+ }
+ : {}),
+ };
+ const database = join(fixture.stateDirectory, "workbench.sqlite3");
+ async function writeReport(url: string, compressed = false) {
+ // Keep both refs: checking only the first would miss the case-distinct one.
+ const contents = JSON.stringify(
+ sarifWithRepositories([sourceRepositoryUrl, url]),
);
+ await writeFile(file, compressed ? gzipSync(contents) : contents);
+ }
+ await writeReport(originalUrl, true);
+ if (platform === "unknown host") {
await expect(
prepareLocalExternalImport(file, options, dependencies),
).rejects.toThrow("does not match");
- expect(await readFile(database)).toEqual(before);
- expect(await listLocalExternalFindings(options, dependencies)).toEqual(
- stored,
- );
- } finally {
- await fixture.dispose();
+ await expect(access(database)).rejects.toThrow();
+ await writeReport(sourceRepositoryUrl);
}
- },
- );
-}
+ const parsed = await readSemgrepFindings(file, { sourceRepositoryUrl });
+ const prepared = await prepareLocalExternalImport(
+ file,
+ options,
+ dependencies,
+ );
+ expect(prepared.preview.findings).toEqual(parsed.findings);
+ expect((await prepared.import()).counts).toEqual({
+ created: 1,
+ updated: 0,
+ unchanged: 0,
+ });
+ const stored = await listLocalExternalFindings(options, dependencies);
+ expect(stored[0]!.evidence).toEqual(parsed.findings[0]!.evidence);
+ expect(stored[0]!.evidence.details?.repository?.url).toBe(
+ sourceRepositoryUrl,
+ );
+ expect(stored[0]!.evidence.source_data?.["runContext"]).toMatchObject({
+ versionControlProvenance: [
+ { repositoryUri: sourceRepositoryUrl },
+ {
+ repositoryUri:
+ platform === "unknown host" ? sourceRepositoryUrl : originalUrl,
+ },
+ ],
+ });
+ expect(
+ (
+ await (
+ await prepareLocalExternalImport(file, options, dependencies)
+ ).import()
+ ).counts,
+ ).toEqual({ created: 0, updated: 0, unchanged: 1 });
+ const before = await readFile(database);
+ await writeReport(
+ platform === "unknown host"
+ ? originalUrl
+ : `https://${host}/other/project`,
+ );
+ await expect(
+ prepareLocalExternalImport(file, options, dependencies),
+ ).rejects.toThrow("does not match");
+ expect(await readFile(database)).toEqual(before);
+ expect(await listLocalExternalFindings(options, dependencies)).toEqual(
+ stored,
+ );
+ } finally {
+ await fixture.dispose();
+ }
+ },
+);
test("normalized Semgrep SDK imports retain original SARIF repository metadata through the explicit dependency", async () => {
const sourceRepositoryUrl = "https://git.example.test/team/project";
@@ -501,68 +496,61 @@ test("normalized Semgrep SDK imports retain original SARIF repository metadata t
}
});
-test.each(["JSON", "JSON gzip"])(
- "Semgrep %s uses the caller source URL without treating rule metadata as repository provenance",
- async (format) => {
- const sourceRepositoryUrl = "https://git.example.test/Team/Project";
- const fixture = await localFindingsFixture({
- remote: "[email protected]:Team/Project.git",
- });
- try {
- const input = semgrepJson();
- const first = input.results[0]!;
- const report = {
- ...input,
- results: [
- {
- ...first,
- extra: {
- ...first.extra,
- metadata: {
- ...first.extra.metadata,
- source: "https://rules.example.test/Other/Repository",
- },
+test("Semgrep gzipped JSON uses the caller source URL without treating rule metadata as repository provenance", async () => {
+ const sourceRepositoryUrl = "https://git.example.test/Team/Project";
+ const fixture = await localFindingsFixture({
+ remote: "[email protected]:Team/Project.git",
+ });
+ try {
+ const input = semgrepJson();
+ const first = input.results[0]!;
+ const report = {
+ ...input,
+ results: [
+ {
+ ...first,
+ extra: {
+ ...first.extra,
+ metadata: {
+ ...first.extra.metadata,
+ source: "https://rules.example.test/Other/Repository",
},
},
- ],
- };
- const file = join(fixture.root, "semgrep-report");
- const contents = JSON.stringify(report);
- await writeFile(
- file,
- format.endsWith("gzip") ? gzipSync(contents) : contents,
- );
- const options = {
- repository: fixture.repository,
- provider: "semgrep" as const,
- sourceKey: "example-project:main",
- sourceRepositoryUrl,
- };
- const dependencies = { environment: fixture.environment };
- const parsed = await readSemgrepFindings(file, { sourceRepositoryUrl });
- const prepared = await prepareLocalExternalImport(
+ },
+ ],
+ };
+ const file = join(fixture.root, "semgrep-report");
+ await writeFile(file, gzipSync(JSON.stringify(report)));
+ const options = {
+ repository: fixture.repository,
+ provider: "semgrep" as const,
+ sourceKey: "example-project:main",
+ sourceRepositoryUrl,
+ };
+ const dependencies = { environment: fixture.environment };
+ const parsed = await readSemgrepFindings(file, { sourceRepositoryUrl });
+ const prepared = await prepareLocalExternalImport(
+ file,
+ options,
+ dependencies,
+ );
+ expect((await prepared.import()).counts.created).toBe(1);
+ const stored = await listLocalExternalFindings(options, dependencies);
+ expect(stored[0]!.evidence).toEqual(parsed.findings[0]!.evidence);
+ await expect(
+ prepareLocalExternalImport(
file,
- options,
+ {
+ ...options,
+ sourceRepositoryUrl: "https://git.example.test/team/project",
+ },
dependencies,
- );
- expect((await prepared.import()).counts.created).toBe(1);
- const stored = await listLocalExternalFindings(options, dependencies);
- expect(stored[0]!.evidence).toEqual(parsed.findings[0]!.evidence);
- await expect(
- prepareLocalExternalImport(
- file,
- {
- ...options,
- sourceRepositoryUrl: "https://git.example.test/team/project",
- },
- dependencies,
- ),
- ).rejects.toThrow("does not match");
- expect(await listLocalExternalFindings(options, dependencies)).toEqual(
- stored,
- );
- } finally {
- await fixture.dispose();
- }
- },
-);
+ ),
+ ).rejects.toThrow("does not match");
+ expect(await listLocalExternalFindings(options, dependencies)).toEqual(
+ stored,
+ );
+ } finally {
+ await fixture.dispose();
+ }
+});
diff --git a/sdk/typescript/tests-ts/support/contract-fixture.ts b/sdk/typescript/tests-ts/support/contract-fixture.ts
new file mode 100644
index 000000000..da642208d
--- /dev/null
+++ b/sdk/typescript/tests-ts/support/contract-fixture.ts
@@ -0,0 +1,18 @@
+import { chmod } from "node:fs/promises";
+import { join } from "node:path";
+import { copyCompletedScanFixture } from "../plugin-root.js";
+import { createApiTestFixtures } from "./temporary-directories.js";
+
+export function createContractTestFixtures() {
+ const { temporaryDirectory, cleanup } = createApiTestFixtures(
+ "codex-security-contract-",
+ false,
+ );
+ async function copyExample(): Promise<string> {
+ const scanDir = join(await temporaryDirectory(), "scan");
+ await copyCompletedScanFixture(scanDir);
+ if (process.platform !== "win32") await chmod(scanDir, 0o700);
+ return scanDir;
+ }
+ return { temporaryDirectory, cleanup, copyExample };
+}
diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts
index 1ba8d182d..9ad9f1efc 100644
--- a/sdk/typescript/tests-ts/wiz-findings.test.ts
+++ b/sdk/typescript/tests-ts/wiz-findings.test.ts
@@ -32,6 +32,7 @@ import {
} from "../src/wiz-findings.js";
import { capture, dependencies } from "./cli-fixtures.js";
import { createCliTest } from "./support/cli-run.js";
+import previewCheckpoint from "./fixtures/wiz-preview-checkpoint.json" with { type: "json" };
import roundtripFixtures from "./fixtures/wiz-details-roundtrip.json" with { type: "json" };
const directories: string[] = [];
@@ -520,7 +521,6 @@ async function cloudFixture(payload: unknown) {
const state: {
failReadback: boolean;
readbackDetails?: ExternalFindingEvidence["details"];
- readbackEvidence?: ExternalFindingEvidence;
} = { failReadback: false };
const transport = async (
url: string,
@@ -560,8 +560,6 @@ async function cloudFixture(payload: unknown) {
const report = [...reports.values()].find(
(item) => item.id === target.pathname.split("/").at(-1),
);
- if (report && state.readbackEvidence)
- return json({ ...report, evidence: state.readbackEvidence });
return json(
report && state.readbackDetails
? {
@@ -1413,7 +1411,11 @@ test.each(["named collection", "normalized JSONL"])(
severity: "HIGH",
vulnerableAsset: { id: "synthetic-asset" },
}));
- const f = await fixture(envelope("vulnerabilityFindings", records, false));
+ const f = await fixture(
+ format === "named collection"
+ ? envelope("vulnerabilityFindings", records, false)
+ : {},
+ );
if (format === "normalized JSONL")
await writeFile(
f.file,
@@ -1486,214 +1488,172 @@ test.each(["named collection", "normalized JSONL"])(
},
);
-test.each(["migrate", "distinct", "duplicate"])(
- "Unicode readback resumes historical checkpoints (%s)",
- async (scenario) => {
- const input = {
- source_finding_id: "unicode-evidence",
- evidence: {
- title: "Unicode vendor metadata",
- severity: "high",
- source_data: { é: "precomposed", "e\u0301": "combining", Z: 1, a: 2 },
- },
- };
- const f = await cloudFixture([input]);
- const deps = {
- ...f.deps,
- environment: {
- ...f.environment,
- CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL,
- },
- };
- const prepared = await prepareExternalPublication(f.file, f.options, deps);
- f.state.failReadback = true;
- let failure: ExternalPublicationError | undefined;
- try {
- await prepared.publish();
- } catch (error) {
- expect(error).toBeInstanceOf(ExternalPublicationError);
- failure = error as ExternalPublicationError;
+// Captured after the pre-release writer at a431663b5e1 received a receipt and
+// readback failed. Its filenames depended on deployment and host collation.
+async function previewCheckpointFixture(base = DEFAULT_CLOUD_BASE_URL) {
+ const request = previewCheckpoint.requests[0]!;
+ const item = request.items[0]!;
+ const { title, severity, source_data } = item.evidence;
+ const f = await cloudFixture([
+ {
+ source_finding_id: item.source_finding_id,
+ evidence: { title, severity, source_data },
+ },
+ ]);
+ // Seed the injected Cloud transport with the observation in the saved receipt.
+ await f.deps.fetch(`${base}/finding_imports`, {
+ method: "POST",
+ body: JSON.stringify(request),
+ });
+ const directory = join(
+ f.environment.CODEX_SECURITY_STATE_DIR,
+ "external-finding-publications",
+ );
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ return {
+ ...f,
+ directory,
+ prepare: () =>
+ prepareExternalPublication(f.file, f.options, {
+ ...f.deps,
+ environment: { ...f.environment, CODEX_SECURITY_CLOUD_BASE_URL: base },
+ }),
+ };
+}
+
+const savedPreview = JSON.stringify(previewCheckpoint);
+
+// Use the host's collation to recover the original filename in any locale.
+function localPreviewFilename() {
+ const request = previewCheckpoint.requests[0]!;
+ const selection = request.items.map(({ source_finding_id, evidence }) => ({
+ source_finding_id,
+ evidence,
+ }));
+ const identity = [
+ previewCheckpoint.accountId,
+ request.repository.id,
+ request.repository.repo_connector_id,
+ request.source,
+ selection,
+ ];
+ return `${legacyCheckpointKey(identity)}.pending.json`;
+}
+
+function legacyCheckpointKey(identity: unknown[]) {
+ const encoded = JSON.stringify(identity, (_key, child: unknown) =>
+ child && typeof child === "object" && !Array.isArray(child)
+ ? Object.fromEntries(
+ Object.entries(child).sort(([a], [b]) => a.localeCompare(b)),
+ )
+ : child,
+ );
+ return hash("sha256", encoded);
+}
+
+test.each(["distinct", "duplicate"] as const)(
+ "a migrated preview checkpoint completes readback with a %s older file",
+ async (kind) => {
+ const f = await previewCheckpointFixture();
+ const oldName = localPreviewFilename();
+ const oldPath = join(f.directory, oldName);
+ await writeFile(oldPath, savedPreview);
+ const migrated = await f.prepare();
+ expect(migrated.preview.resumed).toBe(true);
+ expect<unknown[]>(migrated.preview.requests).toEqual(
+ previewCheckpoint.requests,
+ );
+ const pendingNames = (await readdir(f.directory)).filter((name) =>
+ name.endsWith(".pending.json"),
+ );
+ expect(pendingNames).toHaveLength(1);
+ expect(pendingNames).not.toContain(oldName);
+ const pendingPath = join(f.directory, pendingNames[0]!);
+ expect(await readFile(pendingPath, "utf8")).toBe(savedPreview);
+
+ // A crash can leave both names. Only an identical request can be removed.
+ const old = structuredClone(previewCheckpoint);
+ if (kind === "distinct") {
+ old.requests[0]!.request_id = "synthetic-older-request";
+ old.receipts = [];
}
- const pendingPath = failure!.result.savedSubmission!;
- const historical = JSON.parse(await readFile(pendingPath, "utf8"));
- delete historical.inputProjection;
- const saved = JSON.stringify(historical);
- await writeFile(pendingPath, saved);
- // Frozen from the original publisher's production checkpoint format.
- const historicalPath = join(
- f.environment.CODEX_SECURITY_STATE_DIR,
- "external-finding-publications",
- "ddeb8ec0be8765f89ef5aeb759dde7753d7ff72f8b8bb25f89a591b045ad40ae.pending.json",
+ const older = JSON.stringify(old);
+ await writeFile(oldPath, older);
+ const resumed = await f.prepare();
+ expect<unknown[]>(resumed.preview.requests).toEqual(
+ previewCheckpoint.requests,
);
- expect(pendingPath).not.toBe(historicalPath);
- if (scenario !== "migrate") {
- const legacy = JSON.parse(saved);
- if (scenario === "distinct") {
- legacy.requests[0].request_id = "synthetic-older-request";
- legacy.receipts = [];
- }
- await writeFile(historicalPath, JSON.stringify(legacy));
+ expect(await readFile(pendingPath, "utf8")).toBe(savedPreview);
+ if (kind === "duplicate") {
+ await expect(readFile(oldPath, "utf8")).rejects.toMatchObject({
+ code: "ENOENT",
+ });
} else {
- await rename(pendingPath, historicalPath);
+ expect(await readFile(oldPath, "utf8")).toBe(older);
}
- const evidence = structuredClone(prepared.preview.findings[0]!.evidence);
- evidence.source_data = Object.fromEntries(
- Object.entries(evidence.source_data!).reverse(),
+
+ const report = f.reports.get("unicode-evidence")!;
+ report.evidence.source_data = Object.fromEntries(
+ Object.entries(report.evidence.source_data!).reverse(),
);
- f.state.readbackEvidence = evidence;
- f.state.failReadback = false;
- const retry = await prepareExternalPublication(f.file, f.options, deps);
- expect(retry.preview.resumed).toBe(true);
- expect(retry.preview.requests).toEqual(prepared.preview.requests);
- expect(await readFile(pendingPath, "utf8")).toBe(saved);
- expect(
- (
- await readdir(
- join(
- f.environment.CODEX_SECURITY_STATE_DIR,
- "external-finding-publications",
- ),
- )
- ).includes(historicalPath.split(/[\\/]/u).at(-1)!),
- ).toBe(scenario === "distinct");
- expect((await retry.publish()).verified).toBe(1);
- expect(f.posts).toHaveLength(1);
- expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual(
- input.evidence.source_data,
+ const beforePublish = f.calls.length;
+ expect((await resumed.publish()).verified).toBe(1);
+ expect(f.calls.slice(beforePublish)).toContainEqual(
+ expect.stringContaining(`/source_reports/${report.id}`),
);
- if (scenario === "duplicate")
- expect(
- (await prepareExternalPublication(f.file, f.options, deps)).preview
- .resumed,
- ).toBe(false);
+ expect(f.posts).toHaveLength(1);
+ if (kind === "distinct") {
+ expect(await readFile(oldPath, "utf8")).toBe(older);
+ } else {
+ expect((await f.prepare()).preview.resumed).toBe(false);
+ }
},
);
+// Pick a filename the historical writer produced under a different collation.
+function foreignFilename(englishKey: string, swedishKey: string) {
+ return `${"ä".localeCompare("Z") < 0 ? swedishKey : englishKey}.pending.json`;
+}
+const hostedForeignName = foreignFilename(
+ "cde1eadb671511a7a309b2d76cf3e6f95fbb452e86543547d841be981002fbcc",
+ "253163ba65ed2f1929f28c25e3d129e111ec119851244e8dcbe32a1bb4ed952a",
+);
+const customForeignName = foreignFilename(
+ "2dcaa1dba62bf389461d0a02692f34ec3f3119bb4aacb8b21502466ae6ce0d33",
+ "d64d6d01223834e06cf10a7b71fa31c730c3f3c5d08cc189f6a0d5ab6d3b6b3f",
+);
+const alternateBase = "https://cloud.example.test";
+
test.each([
- {
- phase: "prepare",
- originalBase: DEFAULT_CLOUD_BASE_URL,
- targetBase: DEFAULT_CLOUD_BASE_URL,
- },
- {
- phase: "publish",
- originalBase: DEFAULT_CLOUD_BASE_URL,
- targetBase: DEFAULT_CLOUD_BASE_URL,
- },
- {
- phase: "prepare",
- originalBase: "https://cloud.example.test",
- targetBase: "https://cloud.example.test",
- },
- {
- phase: "prepare",
- originalBase: DEFAULT_CLOUD_BASE_URL,
- targetBase: "https://cloud.example.test",
- },
-])(
- "unidentified legacy checkpoints stop before upload (%j)",
- async ({ phase, originalBase, targetBase }) => {
- const f = await cloudFixture([
- {
- source_finding_id: "legacy-locale-evidence",
- evidence: {
- title: "Original evidence",
- severity: "high",
- source_data: { z: 1, ä: 2, Z: 3, a: 4, é: 5 },
- },
- },
- ]);
- const originalDeps = {
- ...f.deps,
- environment: {
- ...f.environment,
- CODEX_SECURITY_CLOUD_BASE_URL: originalBase,
- },
- };
- const original = await prepareExternalPublication(
- f.file,
- f.options,
- originalDeps,
- );
- f.state.failReadback = true;
- let originalPath = "";
- try {
- await original.publish();
- } catch (error) {
- expect(error).toBeInstanceOf(ExternalPublicationError);
- originalPath = (error as ExternalPublicationError).result
- .savedSubmission!;
- }
- const saved = JSON.parse(await readFile(originalPath, "utf8"));
- delete saved.inputProjection;
- const serialized = JSON.stringify(saved);
- const identity = [
- ...(originalBase === DEFAULT_CLOUD_BASE_URL ? [] : [originalBase]),
- original.preview.accountId,
- original.preview.destination.id,
- original.preview.destination.repo_connector_id,
- original.preview.source,
- original.preview.findings,
- ];
- const historicalKey = (locale?: string) => {
- const compare = new Intl.Collator(locale).compare;
- return hash(
- "sha256",
- JSON.stringify(identity, (_key, child: unknown) =>
- child !== null && typeof child === "object" && !Array.isArray(child)
- ? Object.fromEntries(
- Object.entries(child).sort(([left], [right]) =>
- compare(left, right),
- ),
- )
- : child,
- ),
- );
- };
- const currentName = originalPath.split(/[\\/]/u).at(-1)!;
- const legacyKey = ["en-US", "sv-SE"]
- .map(historicalKey)
- .find(
- (key) =>
- key !== historicalKey() && `${key}.pending.json` !== currentName,
- );
- expect(legacyKey).toBeDefined();
- const legacyPath = join(
- f.environment.CODEX_SECURITY_STATE_DIR,
- "external-finding-publications",
- `${legacyKey}.pending.json`,
- );
- await rm(originalPath);
- const newer = f.reports.get("legacy-locale-evidence")!;
- newer.version += 1;
+ ["hosted, another locale", DEFAULT_CLOUD_BASE_URL, hostedForeignName, false],
+ [
+ "custom, another locale after preview",
+ alternateBase,
+ customForeignName,
+ true,
+ ],
+ [
+ "custom, recognizable hosted filename",
+ alternateBase,
+ localPreviewFilename(),
+ false,
+ ],
+] as const)(
+ "an unidentified preview checkpoint stops before upload: %s",
+ async (_name, target, filename, afterPreview) => {
+ const f = await previewCheckpointFixture(target);
+ const path = join(f.directory, filename);
+ const newer = f.reports.get("unicode-evidence")!;
+ newer.version++;
newer.evidence.description = "Newer Cloud evidence";
- f.state.failReadback = false;
- const targetDeps = {
- ...f.deps,
- environment: {
- ...f.environment,
- CODEX_SECURITY_CLOUD_BASE_URL: targetBase,
- },
- };
- if (phase === "publish") {
- const fresh = await prepareExternalPublication(
- f.file,
- f.options,
- targetDeps,
- );
- await writeFile(legacyPath, serialized);
- await expect(fresh.publish()).rejects.toThrow(
- "original deployment and locale",
- );
- } else {
- await writeFile(legacyPath, serialized);
- await expect(
- prepareExternalPublication(f.file, f.options, targetDeps),
- ).rejects.toThrow("original deployment and locale");
- }
+ const fresh = afterPreview ? await f.prepare() : undefined;
+ await writeFile(path, savedPreview);
+ await expect(fresh ? fresh.publish() : f.prepare()).rejects.toThrow(
+ "original deployment and locale",
+ );
expect(f.posts).toHaveLength(1);
expect(newer.evidence.description).toBe("Newer Cloud evidence");
- expect(await readFile(legacyPath, "utf8")).toBe(serialized);
+ expect(await readFile(path, "utf8")).toBe(savedPreview);
},
);
@@ -2457,19 +2417,7 @@ test.each([
original.preview.source,
original.preview.findings,
];
- // Reproduce the previous checkpoint encoding, before code-unit key ordering.
- const oldKey = hash(
- "sha256",
- JSON.stringify(identity, (_key, child) =>
- child !== null && typeof child === "object" && !Array.isArray(child)
- ? Object.fromEntries(
- Object.keys(child)
- .sort((left, right) => left.localeCompare(right))
- .map((key) => [key, child[key]]),
- )
- : child,
- ),
- );
+ const oldKey = legacyCheckpointKey(identity);
const oldPath = join(
f.environment.CODEX_SECURITY_STATE_DIR,
"external-finding-publications",
@@ -2671,14 +2619,12 @@ test("concurrent legacy inventory resumers retain the acknowledged request after
original.preview.destination.repo_connector_id,
original.preview.source,
];
- const encode = (value: unknown, legacy: boolean) =>
+ const encode = (value: unknown) =>
JSON.stringify(value, (_key, child) =>
child !== null && typeof child === "object" && !Array.isArray(child)
? Object.fromEntries(
Object.keys(child)
- .sort(
- legacy ? (left, right) => left.localeCompare(right) : undefined,
- )
+ .sort()
.map((key) => [key, child[key]]),
)
: child,
@@ -2689,18 +2635,15 @@ test("concurrent legacy inventory resumers retain the acknowledged request after
);
saved.inputProjection.inputDigest = hash(
"sha256",
- encode(historical.legacyIdentity, false),
+ encode(historical.legacyIdentity),
);
await writeFile(originalPath, JSON.stringify(saved));
- const oldKey = hash(
- "sha256",
- encode([...identity, original.preview.findings], true),
- );
+ const oldKey = legacyCheckpointKey([...identity, original.preview.findings]);
const oldPath = join(directory, `${oldKey}.pending.json`);
if (oldPath !== originalPath) await rename(originalPath, oldPath);
await writeFile(f.file, JSON.stringify(payload));
const { currentIdentity } = await readVendorFindingsForPublication(f.file);
- const key = hash("sha256", encode([...identity, currentIdentity], false));
+ const key = hash("sha256", encode([...identity, currentIdentity]));
const pendingPath = join(directory, `${key}.pending.json`);
const lockPath = join(directory, `${key}.lock.sqlite`);
expect(pendingPath).not.toBe(oldPath);