OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
feat(sdk): deduplicate sealed scan directories (#779)
Kyle Browncda0b3f9c3cf
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.mdindex 074ab7a3d..1ea8a36b7 100644--- a/sdk/typescript/README.md+++ b/sdk/typescript/README.md@@ -1175,6 +1175,22 @@ const result = await deduplicateScan("scan_example_001", {console.log(result.duplicateGroups);```+For a complete, sealed scan directory that is not registered in local scan+history, provide the repository checkout separately:++```typescript+import { deduplicateScanDirectory } from "@openai/codex-security";++const result = await deduplicateScanDirectory("/path/to/completed-scan", {+ repository: "/path/to/repository",+ findingsUrl: "http://127.0.0.1:3000",+ // expectedScanId: "scan_example_001",+ // allRepositories: true,+ // signal: controller.signal,+});+console.log(result.duplicateGroups);+```+The CLI and SDK return the same result:```json@@ -1518,7 +1534,9 @@ The local workflow lives under `src/deduplication/`. `FindingDeduplicator`receives a candidate API client and a `DeduplicationReviewer`, keeping groupingseparate from HTTP and model transport. `CodexDeduplicationReviewer` owns promptsand result validation; `CodexReviewRunner` owns app-server sessions and cleanup.-`deduplicateScan` validates saved scan artifacts before running the workflow.+`deduplicateScan` validates saved scan artifacts before running the workflow;+`deduplicateScanDirectory` performs the same validation for an explicit sealed+scan directory without consulting local scan history.The SDK reuses the existing Codex runtime and credentials without additionalruntime dependencies.diff --git a/sdk/typescript/src/deduplication/scan.ts b/sdk/typescript/src/deduplication/scan.tsindex 2cb58652b..3f5dec78c 100644--- a/sdk/typescript/src/deduplication/scan.ts+++ b/sdk/typescript/src/deduplication/scan.ts@@ -30,17 +30,26 @@ import {CheckpointedReviewRunner,reviewSettingsDigest,} from "./checkpointed-review.js";+import { normalizeRepository } from "../targets.js";export interface DeduplicateScanOptions {/** Resume the named local findings workflow, including custom publication. */workflowId?: string;/** Findings API base URL. The scan's findings must already be indexed there. */findingsUrl: string;- /** Search all repositories instead of the saved scan's targetId. Defaults to false. */+ /** Search all repositories instead of the scan's targetId. Defaults to false. */allRepositories?: boolean;signal?: AbortSignal;}+export interface DeduplicateScanDirectoryOptions+ extends DeduplicateScanOptions {+ /** Local repository checkout used to review duplicate candidates. */+ repository: string;+ /** Require the sealed artifacts to belong to this scan. */+ expectedScanId?: string;+}+export interface DeduplicateScanResult extends DeduplicationResult {scanId: string;}@@ -53,16 +62,48 @@ export async function deduplicateScan(return await deduplicateScanInternal(scanId, options);}+/** Review a complete, sealed scan directory without resolving local scan history. */+export async function deduplicateScanDirectory(+ scanDirectory: string,+ options: DeduplicateScanDirectoryOptions,+): Promise<DeduplicateScanResult> {+ return await deduplicateScanDirectoryInternal(scanDirectory, options);+}++type DeduplicateScanDependencies = Partial<SavedScanDependencies> & {+ environment?: NodeJS.ProcessEnv;+ reviewer?: DeduplicationReviewer;+ reviewRunner?: Pick<CodexReviewRunner, "run">;+ fetch?: FindingsRequest;+};++/** @internal */+export async function deduplicateScanDirectoryInternal(+ scanDirectory: string,+ options: DeduplicateScanDirectoryOptions,+ dependencies: DeduplicateScanDependencies = {},+): Promise<DeduplicateScanResult> {+ options.signal?.throwIfAborted();+ const repository = await normalizeRepository(+ options.repository,+ options.signal,+ );+ return await deduplicateResolvedScan(+ scanDirectory,+ repository,+ options.expectedScanId,+ options,+ dependencies,+ await bundledPluginRoot(),+ true,+ );+}+/** @internal */export async function deduplicateScanInternal(scanId: string,options: DeduplicateScanOptions,- dependencies: Partial<SavedScanDependencies> & {- environment?: NodeJS.ProcessEnv;- reviewer?: DeduplicationReviewer;- reviewRunner?: Pick<CodexReviewRunner, "run">;- fetch?: FindingsRequest;- } = {},+ dependencies: DeduplicateScanDependencies = {},): Promise<DeduplicateScanResult> {options.signal?.throwIfAborted();const environment = dependencies.environment ?? process.env;@@ -90,14 +131,36 @@ export async function deduplicateScanInternal();}),});- const { contract, scanDirectory } = await loadContractWithScanDirectory(+ return await deduplicateResolvedScan(scan.scanDir,+ scan["targetPath"] as string,+ scan.scanId,+ options,+ dependencies,+ pluginRoot,+ false,+ );+}++async function deduplicateResolvedScan(+ selectedDirectory: string,+ repositoryPath: string,+ expectedScanId: string | undefined,+ options: DeduplicateScanOptions,+ dependencies: DeduplicateScanDependencies,+ pluginRoot: string,+ bindRepository: boolean,+): Promise<DeduplicateScanResult> {+ const environment = dependencies.environment ?? process.env;+ const { contract, scanDirectory } = await loadContractWithScanDirectory(+ selectedDirectory,{pluginRoot,- expectedScanId: scan.scanId,+ expectedScanId,signal: options.signal,},);+ const scanId = contract.manifest.scan.id;const client = new FindingsClient(options.findingsUrl,options.signal,@@ -121,7 +184,8 @@ export async function deduplicateScanInternal(if (workflow) {await workflow.protectArtifacts(scanDirectory);await workflow.bind({- scanId: scan.scanId,+ ...(bindRepository ? { repositoryPath } : {}),+ scanId,scanDir: scanDirectory,artifactDigest: workflowDigest(contract),destination: workflowDestination(options.findingsUrl),@@ -133,7 +197,7 @@ export async function deduplicateScanInternal({findingsUrl: options.findingsUrl,workflowId: options.workflowId,- expectedScanId: scan.scanId,+ expectedScanId: scanId,signal: options.signal,},{@@ -159,13 +223,13 @@ export async function deduplicateScanInternal(environment,undefined,options.signal,- scan["targetPath"] as string,+ repositoryPath,);const checkpoints = workflow? new CheckpointedReviewRunner(workflow,runner,- await workflow.sourceSnapshot(scan["targetPath"] as string),+ await workflow.sourceSnapshot(repositoryPath),scope,await reviewSettingsDigest(environment),)@@ -184,7 +248,7 @@ export async function deduplicateScanInternal();await checkpoints?.assertSourceUnchanged();options.signal?.throwIfAborted();- const result: DeduplicateScanResult = { scanId: scan.scanId, ...reviewed };+ const result: DeduplicateScanResult = { scanId, ...reviewed };await workflow?.prepareDedupe(result, { groups: result.duplicateGroups });await client.storeDedupeGroups(result.duplicateGroups);return result;diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.tsindex c964670de..72c728843 100644--- a/sdk/typescript/src/index.ts+++ b/sdk/typescript/src/index.ts@@ -71,8 +71,12 @@ export type {PublishScanToCustomOptions,CustomPublicationResult,} from "./custom-publish.js";-export { deduplicateScan } from "./deduplication/scan.js";+export {+ deduplicateScan,+ deduplicateScanDirectory,+} from "./deduplication/scan.js";export type {+ DeduplicateScanDirectoryOptions,DeduplicateScanOptions,DeduplicateScanResult,} from "./deduplication/scan.js";diff --git a/sdk/typescript/tests-ts/finding-deduplication.test.ts b/sdk/typescript/tests-ts/finding-deduplication.test.tsindex 78ea87488..4cb35de3a 100644--- a/sdk/typescript/tests-ts/finding-deduplication.test.ts+++ b/sdk/typescript/tests-ts/finding-deduplication.test.ts@@ -17,7 +17,11 @@ import {} from "../src/deduplication/deduplication-reviewer.js";import { CodexSecurityError } from "../src/errors.js";import { FindingsClient } from "../src/findings-client.js";-import { deduplicateScanInternal } from "../src/deduplication/scan.js";+import { deduplicateScanDirectory } from "../src/index.js";+import {+ deduplicateScanDirectoryInternal,+ deduplicateScanInternal,+} from "../src/deduplication/scan.js";import { PLUGIN_ROOT } from "./plugin-root.js";import type { JsonObject } from "../src/config.js";@@ -497,6 +501,75 @@ test("resolves a saved scan and retrieves its IDs without uploading or modifying}});+test("deduplicates an explicit sealed scan directory without reading scan history", async () => {+ const directory = await mkdtemp(join(tmpdir(), "dedupe-directory-"));+ const repository = await mkdtemp(join(tmpdir(), "dedupe-repository-"));+ try {+ await cp(join(PLUGIN_ROOT, "examples/completed-scan"), directory, {+ recursive: true,+ });+ if (process.platform !== "win32") await chmod(directory, 0o700);+ const original = await readFile(join(directory, "findings.json"), "utf8");+ const commands: string[][] = [];+ const requests: string[] = [];+ const result = await deduplicateScanDirectoryInternal(+ directory,+ {+ repository,+ expectedScanId: "scan_example_001",+ findingsUrl: "http://synthetic.test/api",+ },+ {+ runWorkbench: async (args) => {+ commands.push([...args]);+ throw new Error("Explicit scan directories must not read history");+ },+ fetch: async (url, options) => {+ requests.push(String(url));+ expect(options?.method).toBeUndefined();+ return Response.json({+ finding: document.findings[0],+ potentialDuplicates: [],+ });+ },+ reviewer: {+ async screen() {+ throw new Error("No review for an empty neighborhood");+ },+ async reviewPair() {+ throw new Error("No pair to review");+ },+ },+ },+ );+ expect(result).toEqual({+ scanId: "scan_example_001",+ uniqueFindingIds: document.findings.map((finding) => finding.findingId),+ duplicateGroups: [],+ deduplicationStatus: "completed",+ });+ expect(commands).toEqual([]);+ expect(requests).toEqual([+ `http://synthetic.test/api/v1/finding/${document.findings[0]!.findingId}/potential-duplicates?repositoryId=target_sha256_example`,+ ]);+ expect(await readFile(join(directory, "findings.json"), "utf8")).toBe(+ original,+ );+ await expect(+ deduplicateScanDirectory(directory, {+ repository,+ expectedScanId: "scan_other",+ findingsUrl: "http://synthetic.test/api",+ }),+ ).rejects.toThrow("do not match selected scan");+ } finally {+ await Promise.all([+ rm(directory, { recursive: true, force: true }),+ rm(repository, { recursive: true, force: true }),+ ]);+ }+});+test("lookup failures and cancellation never produce a completed uniqueness result", async () => {for (const status of [404, 502]) {const client = new FindingsClient(diff --git a/sdk/typescript/tests-ts/finding-workflow-publication.test.ts b/sdk/typescript/tests-ts/finding-workflow-publication.test.tsindex 163bc9c63..a9f203ce5 100644--- a/sdk/typescript/tests-ts/finding-workflow-publication.test.ts+++ b/sdk/typescript/tests-ts/finding-workflow-publication.test.ts@@ -1,9 +1,10 @@import { readFile } from "node:fs/promises";-import { join } from "node:path";+import { join, relative } from "node:path";import { expect, test } from "bun:test";import type { JsonObject } from "../src/config.js";import { publishScanToCustomInternal } from "../src/custom-publish.js";import {+ deduplicateScanDirectoryInternal,deduplicateScanInternal,type DeduplicateScanResult,} from "../src/deduplication/scan.js";@@ -37,6 +38,122 @@ function publicationBinding(];}+test("deduplicates an external scan through its bound workflow without reading scan history", async () => {+ await using fixture = await workflowFixture();+ const { scanDir, repository, environment, document } = fixture;+ const id = "external-directory";+ const options = {+ workflowId: id,+ findingsUrl: "http://synthetic.test/service",+ repository: relative(process.cwd(), repository),+ expectedScanId: document.scanId,+ allRepositories: true,+ };+ const result: DeduplicateScanResult = {+ scanId: document.scanId,+ uniqueFindingIds: document.findings.map((finding) => finding.findingId),+ duplicateGroups: [],+ deduplicationStatus: "completed",+ };+ const source = {+ repository,+ revision: "synthetic-revision",+ refsDigest: "synthetic-refs",+ content: "synthetic-content",+ };+ const binding = {+ scanId: document.scanId,+ scanDir,+ artifactDigest: expect.stringMatching(/^[a-f0-9]{64}$/),+ destination: "http://synthetic.test/service/",+ };+ const workbench = scriptedWorkbench([+ {+ request: {+ id,+ action: "bind",+ binding: {+ repositoryPath: repository,+ ...binding,+ scope: { allRepositories: true },+ },+ },+ },+ { request: { id, action: "complete", stage: "scan", result: null } },+ { request: { id, action: "bind", binding } },+ { request: { id, action: "complete", stage: "scan", result: null } },+ {+ request: { id, action: "begin", stage: "publish" },+ response: {+ workflow: {+ stages: {+ publish: {+ status: "completed",+ result: {+ findingIds: document.findings.map(+ (finding) => finding.findingId,+ ),+ },+ },+ },+ },+ },+ },+ {+ request: { id, action: "begin", stage: "dedupe" },+ response: { workflow: { stages: { dedupe: { status: "running" } } } },+ },+ {+ request: { id, action: "get" },+ response: { workflow: { stages: { dedupe: { status: "running" } } } },+ },+ { request: { id, action: "source", repository }, response: { source } },+ { request: { id, action: "source", repository }, response: { source } },+ {+ request: {+ id,+ action: "prepare-dedupe",+ stage: "dedupe",+ result,+ pendingWrite: { groups: [] },+ },+ },+ { request: { id, action: "complete", stage: "dedupe", result } },+ ]);+ const requests: string[] = [];+ expect(+ await deduplicateScanDirectoryInternal(scanDir, options, {+ environment,+ runWorkbench: (args, input) =>+ workbench.run(+ { environment, pluginRoot: PLUGIN_ROOT, python: "unused" },+ args,+ input,+ ),+ fetch: async (url, init) => {+ requests.push(String(url));+ expect(init.method).toBeUndefined();+ return Response.json({+ finding: document.findings[0],+ potentialDuplicates: [],+ });+ },+ reviewer: {+ async screen() {+ throw new Error("No review for an empty neighborhood");+ },+ async reviewPair() {+ throw new Error("No pair to review");+ },+ },+ }),+ ).toEqual(result);+ expect(requests).toEqual([+ `http://synthetic.test/service/v1/finding/${document.findings[0]!.findingId}/potential-duplicates?allRepositories=true`,+ ]);+ workbench.assertDone();+});+test("failed publication records its error, dry-run does not advance it, and retry records the receipt", async () => {await using fixture = await workflowFixture();const { scanDir, environment, document } = fixture;