OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
perf(imports): avoid repeated repository and Snyk project checks (#2187)
Michael D'Angelob79a0a8cfe59
diff --git a/docs/scanner-imports/semgrep.md b/docs/scanner-imports/semgrep.mdindex 5a69c9598..faee32692 100644--- a/docs/scanner-imports/semgrep.md+++ b/docs/scanner-imports/semgrep.md@@ -219,6 +219,33 @@ any scope retained by the original native records. A caller-managed row canlack the original collection scope. Changing supplemental display metadata ona retry cannot change the approved request bytes, account or destination.+To preview a Cloud publication in the SDK, append this to the script that+created `selected`. Set `repository` to the verified URL of the destination.++```js+import {+ normalizeSemgrepPlatformFindings,+ prepareNormalizedExternalPublication,+} from "@openai/codex-security";++const repository = "https://github.com/example/project";+const findings = normalizeSemgrepPlatformFindings(+ selected,+ repository,+ "cloud",+);+const publication = await prepareNormalizedExternalPublication(findings, {+ provider: "semgrep",+ repository,+ sourceKey: "example/project/release/platform",+});+console.log(JSON.stringify(publication.preview, null, 2));+```++Review the destination, findings and exclusions. To publish the approved+selection, call `await publication.publish()` on the prepared object. Reuse+the same source key for subsequent imports.+## Collect a Platform project and branchWith a Web API token in `SEMGREP_APP_TOKEN`, collect all findings returned for adiff --git a/sdk/typescript/src/external-findings-import.ts b/sdk/typescript/src/external-findings-import.tsindex e0ff7e119..2180eabdc 100644--- a/sdk/typescript/src/external-findings-import.ts+++ b/sdk/typescript/src/external-findings-import.ts@@ -8,6 +8,7 @@ import {readExternalFindings,} from "./external-findings-read.js";import {+ distinctSourceRepositories,prepareLocalImport,type LocalExternalImportDependencies,} from "./external-findings-local.js";@@ -27,8 +28,8 @@ export async function prepareLocalExternalImport(dependencies.signal?.throwIfAborted();requireExternalImports(dependencies.environment);const settings = snapshotImportOptions(options);- const readerSources = snapshotSourceRepositories(- dependencies.sourceRepositories,+ const readerSources = distinctSourceRepositories(+ snapshotSourceRepositories(dependencies.sourceRepositories),);const parsed = await abortable(() => readExternalFindings(path, settings, "local", readerSources),@@ -53,6 +54,7 @@ export async function prepareLocalExternalImport(cloudRepositoryUrlKey(url) === cloudRepositoryUrlKey(source),);const key = knownGitHub ? cloudRepositoryUrlKey : repositoryUrlKey;+ sourceRepositories = distinctSourceRepositories(sourceRepositories);for (const reference of sourceRepositories) {if (key(reference.url) !== key(source))throw new Error(diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.tsindex 4f3a86d62..b58c327f2 100644--- a/sdk/typescript/src/external-findings-local.ts+++ b/sdk/typescript/src/external-findings-local.ts@@ -728,16 +728,33 @@ export async function prepareNormalizedLocalImport();}+// A scanner can repeat the same repository reference for every finding in a run.+export function distinctSourceRepositories(+ sources: readonly ExternalSourceRepository[],+) {+ const seen = new Map<ExternalSourceRepository["platform"], Set<string>>();+ return sources.filter(({ url, platform }) => {+ const urls = seen.get(platform) ?? new Set<string>();+ if (urls.has(url)) return false;+ urls.add(url);+ seen.set(platform, urls);+ return true;+ });+}+/** @internal Prepare evidence using settings captured by either local import entrypoint. */export async function prepareLocalImport(input: ExternalFindings & Partial<SnykIssuesFindings>,options: NormalizedLocalImportOptions & { sourceRepositoryUrl?: string },dependencies: LocalExternalImportDependencies,- readerSources = snapshotSourceRepositories(dependencies.sourceRepositories),- parsedSources: readonly ExternalSourceRepository[] = [],+ readerSources = distinctSourceRepositories(+ snapshotSourceRepositories(dependencies.sourceRepositories),+ ),+ parsedInput: readonly ExternalSourceRepository[] = [],fetchRemotes?: readonly ConfiguredFetchRemote[],): Promise<PreparedLocalExternalImport> {requireIdentity(options.sourceKey, "Source key");+ const parsedSources = distinctSourceRepositories(parsedInput);const current = snapshotNormalizedFindings(input, checkStoredNumber);// The JSON snapshot omits inherited or non-enumerable SDK reader properties.if (!("snykRestContext" in current) && "snykRestContext" in input)diff --git a/sdk/typescript/src/semgrep-findings.ts b/sdk/typescript/src/semgrep-findings.tsindex 326b498e4..204fca298 100644--- a/sdk/typescript/src/semgrep-findings.ts+++ b/sdk/typescript/src/semgrep-findings.ts@@ -222,6 +222,8 @@ export function normalizeSemgrepSarif(const document = record(input, "Semgrep SARIF");if (document["version"] !== "2.1.0")throw new Error("Expected Semgrep SARIF 2.1.0.");+ const repositoryKey = cloudRepositoryUrlKey(repositoryUrl);+ const verifiedUrls = new Set([repositoryUrl]);const entries = array(document["runs"], "SARIF runs").flatMap((value) => {const run = record(value, "SARIF run");for (const entry of array(@@ -231,13 +233,12 @@ export function normalizeSemgrepSarif(const declared = text(record(entry, "Repository provenance")["repositoryUri"],);- if (- declared &&- cloudRepositoryUrlKey(declared) !== cloudRepositoryUrlKey(repositoryUrl)- )+ if (!declared || verifiedUrls.has(declared)) continue;+ if (cloudRepositoryUrlKey(declared) !== repositoryKey)throw new Error("SARIF source repository does not match the declared source repository URL.",);+ verifiedUrls.add(declared);}const tool = record(run["tool"], "SARIF tool");const driver = record(tool["driver"], "SARIF driver");diff --git a/sdk/typescript/src/snyk-rest-findings.ts b/sdk/typescript/src/snyk-rest-findings.tsindex f0acc4b09..9261c90cf 100644--- a/sdk/typescript/src/snyk-rest-findings.ts+++ b/sdk/typescript/src/snyk-rest-findings.ts@@ -1023,7 +1023,7 @@ function bindSnykFinding(finding: ExternalFinding,references: readonly ExternalSourceRepository[],originals: ReturnType<typeof snykRowEvidence>,- report?: Resource,+ report?: { fields: Resource; asset: ReturnType<typeof savedContext> },retain?: (finding: ExternalFinding, source: Resource) => void,) {const retained = record(@@ -1043,7 +1043,10 @@ function bindSnykFinding(throw new Error("A normalized Snyk row cannot claim both v1 and v2 native representations.",);- const document = readSnykRestDocument({ ...retained, format }, report);+ const document = readSnykRestDocument(+ { ...retained, format },+ report?.fields,+ );const issue =(format === SNYK_REST_ISSUES_FORMAT ? originals?.v2?.issue : undefined) ??parseHostedScannerJson(@@ -1052,13 +1055,21 @@ function bindSnykFinding(// A selected issue can retain any of its original monitor fields. Fill only those it omitted.let projected = false;if (report && (originals?.v1?.issue || originals?.v2?.issue)) {- for (const [field, value] of Object.entries(report)) {+ for (const [field, value] of Object.entries(report.fields)) {if (document[field] !== undefined) continue;document[field] = value;projected = true;}}- const asset = savedContext(document, references);+ // Complete v2 selections normally carry the same verified monitor and native assets on every row.+ const asset =+ format === SNYK_REST_ISSUES_FORMAT &&+ report &&+ Object.entries(report.fields).every(+ ([key, value]) => document[key] === value,+ )+ ? report.asset+ : savedContext(document, references);const { source, repository } = asset;const { id: issueId, attributes } = selectedIssue(scopedIssue(issue, source),@@ -1384,9 +1395,12 @@ export function snykRestImportRepository(project !== undefined || rows.some(({ originals }) => originals);if (!hosted && !declaredCloudRepository) return;const context = project && {- ...project.source,- projectJson: JSON.stringify(project.project),- targetJson: JSON.stringify(project.target),+ asset: project,+ fields: {+ ...project.source,+ projectJson: JSON.stringify(project.project),+ targetJson: JSON.stringify(project.target),+ },};const retainedIssues: { finding: ExternalFinding; issue: Resource }[] = [];const retainedRows: Resource[] = [];diff --git a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts b/sdk/typescript/tests-ts/semgrep-findings-local.test.tsindex 360c8a993..72fedbe10 100644--- a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts+++ b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts@@ -295,8 +295,9 @@ function sarifWithRepositories(urls: string[]) {runs: [{...input.runs[0]!,- versionControlProvenance: urls.map((repositoryUri) => ({+ versionControlProvenance: urls.map((repositoryUri, index) => ({repositoryUri,+ branch: index === 0 ? "main" : "release",})),},],@@ -376,8 +377,9 @@ test.each([);expect(stored[0]!.evidence.source_data?.["runContext"]).toMatchObject({versionControlProvenance: [- { repositoryUri: sourceRepositoryUrl },+ { repositoryUri: sourceRepositoryUrl, branch: "main" },{+ branch: "release",repositoryUri:platform === "unknown host" ? sourceRepositoryUrl : originalUrl,},diff --git a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.tsindex 5e9b09c1e..3f9b0da1d 100644--- a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts+++ b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts@@ -715,6 +715,25 @@ test("a selection checks every caller-supplied source before saving API findings}});+test("a valid reader hint cannot hide a URL object with the same JSON spelling", async () => {+ await using fixture = await localReportFixture(url);+ await fixture.writeJson(selectedRepository(url, url));+ await expect(+ prepareLocalExternalImport(+ fixture.file,+ { ...platformSource, repository: fixture.repository },+ {+ environment: fixture.environment,+ sourceRepositories: [+ { url },+ // @ts-expect-error JavaScript callers can supply a URL instead of its string.+ { url: new URL(url) },+ ],+ },+ ),+ ).rejects.toThrow();+});+test("each named Git remote contributes only its active fetch URL", async () => {await using fixture = await localReportFixture("https://github.com/example/other",diff --git a/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts b/sdk/typescript/tests-ts/snyk-rest-boundaries.test.tsindex 1098e34f6..57f14bada 100644--- a/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts+++ b/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts@@ -2,6 +2,7 @@ import { expect, mock, test } from "bun:test";import { readSnykFindings } from "../src/snyk-findings.js";import { snykJson, snykSarif } from "./support/snyk-import-fixtures.js";import { prepareNormalizedExternalPublication } from "../src/external-findings-publish.js";+import type { ExternalFindings } from "../src/external-findings.js";import { prepareNormalizedLocalImport } from "../src/external-findings-local.js";import {normalizeSnykIssues,@@ -741,7 +742,31 @@ test("rows from a different Snyk monitor cannot use the report's source key evenawait using f = await snykLocalFixture({remote: report.sourceRepositoryUrl,});+ const prepareCloud = (input: ExternalFindings) =>+ prepareNormalizedExternalPublication(+ input,+ {+ provider: "snyk",+ sourceKey: f.sourceKey,+ repository: report.sourceRepositoryUrl,+ },+ cloudCatalog(f.environment, report.sourceRepositoryUrl),+ );const original = report.findings[0]!;+ const sibling = normalizeSnykIssues(replace(snykIssue(false).id))+ .findings[0]!;+ const selected = { ...report, read: 2, findings: [original, sibling] };+ expect(+ (await (await f.prepareNormalized(selected)).import()).counts.created,+ ).toBe(2);+ const stored = await f.read();+ const { projectJson, targetJson } = original.evidence.source_data!;+ for (const row of stored)+ expect(row.evidence.source_data).toMatchObject({ projectJson, targetJson });+ sibling.evidence.source_data!["apiVersion"] = "unsupported";+ await expect(prepareCloud(selected)).rejects.toThrow("API version");+ await expect(f.prepareNormalized(selected)).rejects.toThrow("API version");+for (const other of [replace(capture.organizationId),replace(capture.projectId),@@ -761,17 +786,7 @@ test("rows from a different Snyk monitor cannot use the report's source key evenawait expect(f.prepareNormalized(input)).rejects.toThrow("Snyk project scope",);- await expect(- prepareNormalizedExternalPublication(- input,- {- provider: "snyk",- sourceKey: f.sourceKey,- repository: report.sourceRepositoryUrl,- },- cloudCatalog(f.environment, report.sourceRepositoryUrl),- ),- ).rejects.toThrow("Snyk project scope");+ await expect(prepareCloud(input)).rejects.toThrow("Snyk project scope");}}const saved = structuredClone(original);@@ -812,7 +827,7 @@ test("rows from a different Snyk monitor cannot use the report's source key evenexpect(() =>checkSaved(replace("github-cloud-app", "github-server-app")),).toThrow("conflicting targets or integrations");- expect(await f.read()).toHaveLength(0);+ expect(await f.read()).toEqual(stored);});test("a sparse approved Snyk record still checks its retained repository URLs", () => {diff --git a/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts b/sdk/typescript/tests-ts/snyk-rest-git-config.test.tsindex 581a307bc..399334997 100644--- a/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts+++ b/sdk/typescript/tests-ts/snyk-rest-git-config.test.ts@@ -1,5 +1,4 @@import { expect, test } from "bun:test";-import { execFileSync } from "node:child_process";import { appendFile, writeFile } from "node:fs/promises";import { join } from "node:path";import { prepareNormalizedLocalImport } from "../src/external-findings-local.js";@@ -30,13 +29,7 @@ test("Snyk checks the remote helper used by fetch when GIT_CONFIG selects a diffconst config = join(fixture.root, "empty-config");await writeFile(config, "");for (const helper of ["other", ""]) {- execFileSync("git", [- "config",- "--file",- join(fixture.repository, ".git", "config"),- "remote.origin.vcs",- helper,- ]);+ fixture.git("config", "remote.origin.vcs", helper);await expect(prepare(fixture, remote, { GIT_CONFIG: config }),).rejects.toThrow("configured remote");@@ -59,10 +52,7 @@ test.skipIf(process.platform === "win32")(await expect(prepare(fixture, reported)).rejects.toThrow("configured remote",);- execFileSync("git", ["remote", "add", "upstream", reported], {- cwd: fixture.repository,- env: fixture.environment,- });+ fixture.git("remote", "add", "upstream", reported);expect((await prepare(fixture, reported)).preview.findings).toHaveLength(1);},);@@ -72,13 +62,6 @@ test("a later fetch URL on the same Git remote cannot bind a Snyk project", asynawait using fixture = await snykLocalFixture({remote: selected + "-different",});- execFileSync("git", [- "config",- "--file",- join(fixture.repository, ".git", "config"),- "--add",- "remote.origin.url",- selected,- ]);+ fixture.git("config", "--add", "remote.origin.url", selected);await expect(prepare(fixture, selected)).rejects.toThrow("configured remote");});