Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

perf(imports): avoid repeated repository and Snyk project checks (#2187)

Michael D'Angelob79a0a8cfe59
diff --git a/docs/scanner-imports/semgrep.md b/docs/scanner-imports/semgrep.md
index 5a69c9598..faee32692 100644
--- a/docs/scanner-imports/semgrep.md
+++ b/docs/scanner-imports/semgrep.md
@@ -219,6 +219,33 @@ any scope retained by the original native records. A caller-managed row can
lack the original collection scope. Changing supplemental display metadata on
a retry cannot change the approved request bytes, account or destination.
+To preview a Cloud publication in the SDK, append this to the script that
+created `selected`. Set `repository` to the verified URL of the destination.
+
+```js
+import {
+ normalizeSemgrepPlatformFindings,
+ prepareNormalizedExternalPublication,
+} from "@openai/codex-security";
+
+const repository = "https://github.com/example/project";
+const findings = normalizeSemgrepPlatformFindings(
+ selected,
+ repository,
+ "cloud",
+);
+const publication = await prepareNormalizedExternalPublication(findings, {
+ provider: "semgrep",
+ repository,
+ sourceKey: "example/project/release/platform",
+});
+console.log(JSON.stringify(publication.preview, null, 2));
+```
+
+Review the destination, findings and exclusions. To publish the approved
+selection, call `await publication.publish()` on the prepared object. Reuse
+the same source key for subsequent imports.
+
## Collect a Platform project and branch
With a Web API token in `SEMGREP_APP_TOKEN`, collect all findings returned for a
diff --git a/sdk/typescript/src/external-findings-import.ts b/sdk/typescript/src/external-findings-import.ts
index e0ff7e119..2180eabdc 100644
--- a/sdk/typescript/src/external-findings-import.ts
+++ b/sdk/typescript/src/external-findings-import.ts
@@ -8,6 +8,7 @@ import {
readExternalFindings,
} from "./external-findings-read.js";
import {
+ distinctSourceRepositories,
prepareLocalImport,
type LocalExternalImportDependencies,
} from "./external-findings-local.js";
@@ -27,8 +28,8 @@ export async function prepareLocalExternalImport(
dependencies.signal?.throwIfAborted();
requireExternalImports(dependencies.environment);
const settings = snapshotImportOptions(options);
- const readerSources = snapshotSourceRepositories(
- dependencies.sourceRepositories,
+ const readerSources = distinctSourceRepositories(
+ snapshotSourceRepositories(dependencies.sourceRepositories),
);
const parsed = await abortable(
() => readExternalFindings(path, settings, "local", readerSources),
@@ -53,6 +54,7 @@ export async function prepareLocalExternalImport(
cloudRepositoryUrlKey(url) === cloudRepositoryUrlKey(source),
);
const key = knownGitHub ? cloudRepositoryUrlKey : repositoryUrlKey;
+ sourceRepositories = distinctSourceRepositories(sourceRepositories);
for (const reference of sourceRepositories) {
if (key(reference.url) !== key(source))
throw new Error(
diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts
index 4f3a86d62..b58c327f2 100644
--- a/sdk/typescript/src/external-findings-local.ts
+++ b/sdk/typescript/src/external-findings-local.ts
@@ -728,16 +728,33 @@ export async function prepareNormalizedLocalImport(
);
}
+// A scanner can repeat the same repository reference for every finding in a run.
+export function distinctSourceRepositories(
+ sources: readonly ExternalSourceRepository[],
+) {
+ const seen = new Map<ExternalSourceRepository["platform"], Set<string>>();
+ return sources.filter(({ url, platform }) => {
+ const urls = seen.get(platform) ?? new Set<string>();
+ if (urls.has(url)) return false;
+ urls.add(url);
+ seen.set(platform, urls);
+ return true;
+ });
+}
+
/** @internal Prepare evidence using settings captured by either local import entrypoint. */
export async function prepareLocalImport(
input: ExternalFindings & Partial<SnykIssuesFindings>,
options: NormalizedLocalImportOptions & { sourceRepositoryUrl?: string },
dependencies: LocalExternalImportDependencies,
- readerSources = snapshotSourceRepositories(dependencies.sourceRepositories),
- parsedSources: readonly ExternalSourceRepository[] = [],
+ readerSources = distinctSourceRepositories(
+ snapshotSourceRepositories(dependencies.sourceRepositories),
+ ),
+ parsedInput: readonly ExternalSourceRepository[] = [],
fetchRemotes?: readonly ConfiguredFetchRemote[],
): Promise<PreparedLocalExternalImport> {
requireIdentity(options.sourceKey, "Source key");
+ const parsedSources = distinctSourceRepositories(parsedInput);
const current = snapshotNormalizedFindings(input, checkStoredNumber);
// The JSON snapshot omits inherited or non-enumerable SDK reader properties.
if (!("snykRestContext" in current) && "snykRestContext" in input)
diff --git a/sdk/typescript/src/semgrep-findings.ts b/sdk/typescript/src/semgrep-findings.ts
index 326b498e4..204fca298 100644
--- a/sdk/typescript/src/semgrep-findings.ts
+++ b/sdk/typescript/src/semgrep-findings.ts
@@ -222,6 +222,8 @@ export function normalizeSemgrepSarif(
const document = record(input, "Semgrep SARIF");
if (document["version"] !== "2.1.0")
throw new Error("Expected Semgrep SARIF 2.1.0.");
+ const repositoryKey = cloudRepositoryUrlKey(repositoryUrl);
+ const verifiedUrls = new Set([repositoryUrl]);
const entries = array(document["runs"], "SARIF runs").flatMap((value) => {
const run = record(value, "SARIF run");
for (const entry of array(
@@ -231,13 +233,12 @@ export function normalizeSemgrepSarif(
const declared = text(
record(entry, "Repository provenance")["repositoryUri"],
);
- if (
- declared &&
- cloudRepositoryUrlKey(declared) !== cloudRepositoryUrlKey(repositoryUrl)
- )
+ if (!declared || verifiedUrls.has(declared)) continue;
+ if (cloudRepositoryUrlKey(declared) !== repositoryKey)
throw new Error(
"SARIF source repository does not match the declared source repository URL.",
);
+ verifiedUrls.add(declared);
}
const tool = record(run["tool"], "SARIF tool");
const driver = record(tool["driver"], "SARIF driver");
diff --git a/sdk/typescript/src/snyk-rest-findings.ts b/sdk/typescript/src/snyk-rest-findings.ts
index f0acc4b09..9261c90cf 100644
--- a/sdk/typescript/src/snyk-rest-findings.ts
+++ b/sdk/typescript/src/snyk-rest-findings.ts
@@ -1023,7 +1023,7 @@ function bindSnykFinding(
finding: ExternalFinding,
references: readonly ExternalSourceRepository[],
originals: ReturnType<typeof snykRowEvidence>,
- report?: Resource,
+ report?: { fields: Resource; asset: ReturnType<typeof savedContext> },
retain?: (finding: ExternalFinding, source: Resource) => void,
) {
const retained = record(
@@ -1043,7 +1043,10 @@ function bindSnykFinding(
throw new Error(
"A normalized Snyk row cannot claim both v1 and v2 native representations.",
);
- const document = readSnykRestDocument({ ...retained, format }, report);
+ const document = readSnykRestDocument(
+ { ...retained, format },
+ report?.fields,
+ );
const issue =
(format === SNYK_REST_ISSUES_FORMAT ? originals?.v2?.issue : undefined) ??
parseHostedScannerJson(
@@ -1052,13 +1055,21 @@ function bindSnykFinding(
// A selected issue can retain any of its original monitor fields. Fill only those it omitted.
let projected = false;
if (report && (originals?.v1?.issue || originals?.v2?.issue)) {
- for (const [field, value] of Object.entries(report)) {
+ for (const [field, value] of Object.entries(report.fields)) {
if (document[field] !== undefined) continue;
document[field] = value;
projected = true;
}
}
- const asset = savedContext(document, references);
+ // Complete v2 selections normally carry the same verified monitor and native assets on every row.
+ const asset =
+ format === SNYK_REST_ISSUES_FORMAT &&
+ report &&
+ Object.entries(report.fields).every(
+ ([key, value]) => document[key] === value,
+ )
+ ? report.asset
+ : savedContext(document, references);
const { source, repository } = asset;
const { id: issueId, attributes } = selectedIssue(
scopedIssue(issue, source),
@@ -1384,9 +1395,12 @@ export function snykRestImportRepository(
project !== undefined || rows.some(({ originals }) => originals);
if (!hosted && !declaredCloudRepository) return;
const context = project && {
- ...project.source,
- projectJson: JSON.stringify(project.project),
- targetJson: JSON.stringify(project.target),
+ asset: project,
+ fields: {
+ ...project.source,
+ projectJson: JSON.stringify(project.project),
+ targetJson: JSON.stringify(project.target),
+ },
};
const retainedIssues: { finding: ExternalFinding; issue: Resource }[] = [];
const retainedRows: Resource[] = [];
diff --git a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts
index 360c8a993..72fedbe10 100644
--- a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts
+++ b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts
@@ -295,8 +295,9 @@ function sarifWithRepositories(urls: string[]) {
runs: [
{
...input.runs[0]!,
- versionControlProvenance: urls.map((repositoryUri) => ({
+ versionControlProvenance: urls.map((repositoryUri, index) => ({
repositoryUri,
+ branch: index === 0 ? "main" : "release",
})),
},
],
@@ -376,8 +377,9 @@ test.each([
);
expect(stored[0]!.evidence.source_data?.["runContext"]).toMatchObject({
versionControlProvenance: [
- { repositoryUri: sourceRepositoryUrl },
+ { repositoryUri: sourceRepositoryUrl, branch: "main" },
{
+ branch: "release",
repositoryUri:
platform === "unknown host" ? sourceRepositoryUrl : originalUrl,
},
diff --git a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts
index 5e9b09c1e..3f9b0da1d 100644
--- a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts
+++ b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts
@@ -715,6 +715,25 @@ test("a selection checks every caller-supplied source before saving API findings
}
});
+test("a valid reader hint cannot hide a URL object with the same JSON spelling", async () => {
+ await using fixture = await localReportFixture(url);
+ await fixture.writeJson(selectedRepository(url, url));
+ await expect(
+ prepareLocalExternalImport(
+ fixture.file,
+ { ...platformSource, repository: fixture.repository },
+ {
+ environment: fixture.environment,
+ sourceRepositories: [
+ { url },
+ // @ts-expect-error JavaScript callers can supply a URL instead of its string.
+ { url: new URL(url) },
+ ],
+ },
+ ),
+ ).rejects.toThrow();
+});
+
test("each named Git remote contributes only its active fetch URL", async () => {
await using fixture = await localReportFixture(
"https://github.com/example/other",
diff --git a/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts b/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts
index 1098e34f6..57f14bada 100644
--- a/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts
+++ b/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts
@@ -2,6 +2,7 @@ import { expect, mock, test } from "bun:test";
import { readSnykFindings } from "../src/snyk-findings.js";
import { snykJson, snykSarif } from "./support/snyk-import-fixtures.js";
import { prepareNormalizedExternalPublication } from "../src/external-findings-publish.js";
+import type { ExternalFindings } from "../src/external-findings.js";
import { prepareNormalizedLocalImport } from "../src/external-findings-local.js";
import {
normalizeSnykIssues,
@@ -741,7 +742,31 @@ test("rows from a different Snyk monitor cannot use the report's source key even
await using f = await snykLocalFixture({
remote: report.sourceRepositoryUrl,
});
+ const prepareCloud = (input: ExternalFindings) =>
+ prepareNormalizedExternalPublication(
+ input,
+ {
+ provider: "snyk",
+ sourceKey: f.sourceKey,
+ repository: report.sourceRepositoryUrl,
+ },
+ cloudCatalog(f.environment, report.sourceRepositoryUrl),
+ );
const original = report.findings[0]!;
+ const sibling = normalizeSnykIssues(replace(snykIssue(false).id))
+ .findings[0]!;
+ const selected = { ...report, read: 2, findings: [original, sibling] };
+ expect(
+ (await (await f.prepareNormalized(selected)).import()).counts.created,
+ ).toBe(2);
+ const stored = await f.read();
+ const { projectJson, targetJson } = original.evidence.source_data!;
+ for (const row of stored)
+ expect(row.evidence.source_data).toMatchObject({ projectJson, targetJson });
+ sibling.evidence.source_data!["apiVersion"] = "unsupported";
+ await expect(prepareCloud(selected)).rejects.toThrow("API version");
+ await expect(f.prepareNormalized(selected)).rejects.toThrow("API version");
+
for (const other of [
replace(capture.organizationId),
replace(capture.projectId),
@@ -761,17 +786,7 @@ test("rows from a different Snyk monitor cannot use the report's source key even
await expect(f.prepareNormalized(input)).rejects.toThrow(
"Snyk project scope",
);
- await expect(
- prepareNormalizedExternalPublication(
- input,
- {
- provider: "snyk",
- sourceKey: f.sourceKey,
- repository: report.sourceRepositoryUrl,
- },
- cloudCatalog(f.environment, report.sourceRepositoryUrl),
- ),
- ).rejects.toThrow("Snyk project scope");
+ await expect(prepareCloud(input)).rejects.toThrow("Snyk project scope");
}
}
const saved = structuredClone(original);
@@ -812,7 +827,7 @@ test("rows from a different Snyk monitor cannot use the report's source key even
expect(() =>
checkSaved(replace("github-cloud-app", "github-server-app")),
).toThrow("conflicting targets or integrations");
- expect(await f.read()).toHaveLength(0);
+ expect(await f.read()).toEqual(stored);
});
test("a sparse approved Snyk record still checks its retained repository URLs", () => {
diff --git a/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts b/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts
index 581a307bc..399334997 100644
--- a/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts
+++ b/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts
@@ -1,5 +1,4 @@
import { expect, test } from "bun:test";
-import { execFileSync } from "node:child_process";
import { appendFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { prepareNormalizedLocalImport } from "../src/external-findings-local.js";
@@ -30,13 +29,7 @@ test("Snyk checks the remote helper used by fetch when GIT_CONFIG selects a diff
const config = join(fixture.root, "empty-config");
await writeFile(config, "");
for (const helper of ["other", ""]) {
- execFileSync("git", [
- "config",
- "--file",
- join(fixture.repository, ".git", "config"),
- "remote.origin.vcs",
- helper,
- ]);
+ fixture.git("config", "remote.origin.vcs", helper);
await expect(
prepare(fixture, remote, { GIT_CONFIG: config }),
).rejects.toThrow("configured remote");
@@ -59,10 +52,7 @@ test.skipIf(process.platform === "win32")(
await expect(prepare(fixture, reported)).rejects.toThrow(
"configured remote",
);
- execFileSync("git", ["remote", "add", "upstream", reported], {
- cwd: fixture.repository,
- env: fixture.environment,
- });
+ fixture.git("remote", "add", "upstream", reported);
expect((await prepare(fixture, reported)).preview.findings).toHaveLength(1);
},
);
@@ -72,13 +62,6 @@ test("a later fetch URL on the same Git remote cannot bind a Snyk project", asyn
await using fixture = await snykLocalFixture({
remote: selected + "-different",
});
- execFileSync("git", [
- "config",
- "--file",
- join(fixture.repository, ".git", "config"),
- "--add",
- "remote.origin.url",
- selected,
- ]);
+ fixture.git("config", "--add", "remote.origin.url", selected);
await expect(prepare(fixture, selected)).rejects.toThrow("configured remote");
});