Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

feat(sdk): deduplicate sealed scan directories (#779)

Kyle Browncda0b3f9c3cf
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index 074ab7a3d..1ea8a36b7 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -1175,6 +1175,22 @@ const result = await deduplicateScan("scan_example_001", {
console.log(result.duplicateGroups);
```
+For a complete, sealed scan directory that is not registered in local scan
+history, provide the repository checkout separately:
+
+```typescript
+import { deduplicateScanDirectory } from "@openai/codex-security";
+
+const result = await deduplicateScanDirectory("/path/to/completed-scan", {
+ repository: "/path/to/repository",
+ findingsUrl: "http://127.0.0.1:3000",
+ // expectedScanId: "scan_example_001",
+ // allRepositories: true,
+ // signal: controller.signal,
+});
+console.log(result.duplicateGroups);
+```
+
The CLI and SDK return the same result:
```json
@@ -1518,7 +1534,9 @@ The local workflow lives under `src/deduplication/`. `FindingDeduplicator`
receives a candidate API client and a `DeduplicationReviewer`, keeping grouping
separate from HTTP and model transport. `CodexDeduplicationReviewer` owns prompts
and result validation; `CodexReviewRunner` owns app-server sessions and cleanup.
-`deduplicateScan` validates saved scan artifacts before running the workflow.
+`deduplicateScan` validates saved scan artifacts before running the workflow;
+`deduplicateScanDirectory` performs the same validation for an explicit sealed
+scan directory without consulting local scan history.
The SDK reuses the existing Codex runtime and credentials without additional
runtime dependencies.
diff --git a/sdk/typescript/src/deduplication/scan.ts b/sdk/typescript/src/deduplication/scan.ts
index 2cb58652b..3f5dec78c 100644
--- a/sdk/typescript/src/deduplication/scan.ts
+++ b/sdk/typescript/src/deduplication/scan.ts
@@ -30,17 +30,26 @@ import {
CheckpointedReviewRunner,
reviewSettingsDigest,
} from "./checkpointed-review.js";
+import { normalizeRepository } from "../targets.js";
export interface DeduplicateScanOptions {
/** Resume the named local findings workflow, including custom publication. */
workflowId?: string;
/** Findings API base URL. The scan's findings must already be indexed there. */
findingsUrl: string;
- /** Search all repositories instead of the saved scan's targetId. Defaults to false. */
+ /** Search all repositories instead of the scan's targetId. Defaults to false. */
allRepositories?: boolean;
signal?: AbortSignal;
}
+export interface DeduplicateScanDirectoryOptions
+ extends DeduplicateScanOptions {
+ /** Local repository checkout used to review duplicate candidates. */
+ repository: string;
+ /** Require the sealed artifacts to belong to this scan. */
+ expectedScanId?: string;
+}
+
export interface DeduplicateScanResult extends DeduplicationResult {
scanId: string;
}
@@ -53,16 +62,48 @@ export async function deduplicateScan(
return await deduplicateScanInternal(scanId, options);
}
+/** Review a complete, sealed scan directory without resolving local scan history. */
+export async function deduplicateScanDirectory(
+ scanDirectory: string,
+ options: DeduplicateScanDirectoryOptions,
+): Promise<DeduplicateScanResult> {
+ return await deduplicateScanDirectoryInternal(scanDirectory, options);
+}
+
+type DeduplicateScanDependencies = Partial<SavedScanDependencies> & {
+ environment?: NodeJS.ProcessEnv;
+ reviewer?: DeduplicationReviewer;
+ reviewRunner?: Pick<CodexReviewRunner, "run">;
+ fetch?: FindingsRequest;
+};
+
+/** @internal */
+export async function deduplicateScanDirectoryInternal(
+ scanDirectory: string,
+ options: DeduplicateScanDirectoryOptions,
+ dependencies: DeduplicateScanDependencies = {},
+): Promise<DeduplicateScanResult> {
+ options.signal?.throwIfAborted();
+ const repository = await normalizeRepository(
+ options.repository,
+ options.signal,
+ );
+ return await deduplicateResolvedScan(
+ scanDirectory,
+ repository,
+ options.expectedScanId,
+ options,
+ dependencies,
+ await bundledPluginRoot(),
+ true,
+ );
+}
+
/** @internal */
export async function deduplicateScanInternal(
scanId: string,
options: DeduplicateScanOptions,
- dependencies: Partial<SavedScanDependencies> & {
- environment?: NodeJS.ProcessEnv;
- reviewer?: DeduplicationReviewer;
- reviewRunner?: Pick<CodexReviewRunner, "run">;
- fetch?: FindingsRequest;
- } = {},
+ dependencies: DeduplicateScanDependencies = {},
): Promise<DeduplicateScanResult> {
options.signal?.throwIfAborted();
const environment = dependencies.environment ?? process.env;
@@ -90,14 +131,36 @@ export async function deduplicateScanInternal(
);
}),
});
- const { contract, scanDirectory } = await loadContractWithScanDirectory(
+ return await deduplicateResolvedScan(
scan.scanDir,
+ scan["targetPath"] as string,
+ scan.scanId,
+ options,
+ dependencies,
+ pluginRoot,
+ false,
+ );
+}
+
+async function deduplicateResolvedScan(
+ selectedDirectory: string,
+ repositoryPath: string,
+ expectedScanId: string | undefined,
+ options: DeduplicateScanOptions,
+ dependencies: DeduplicateScanDependencies,
+ pluginRoot: string,
+ bindRepository: boolean,
+): Promise<DeduplicateScanResult> {
+ const environment = dependencies.environment ?? process.env;
+ const { contract, scanDirectory } = await loadContractWithScanDirectory(
+ selectedDirectory,
{
pluginRoot,
- expectedScanId: scan.scanId,
+ expectedScanId,
signal: options.signal,
},
);
+ const scanId = contract.manifest.scan.id;
const client = new FindingsClient(
options.findingsUrl,
options.signal,
@@ -121,7 +184,8 @@ export async function deduplicateScanInternal(
if (workflow) {
await workflow.protectArtifacts(scanDirectory);
await workflow.bind({
- scanId: scan.scanId,
+ ...(bindRepository ? { repositoryPath } : {}),
+ scanId,
scanDir: scanDirectory,
artifactDigest: workflowDigest(contract),
destination: workflowDestination(options.findingsUrl),
@@ -133,7 +197,7 @@ export async function deduplicateScanInternal(
{
findingsUrl: options.findingsUrl,
workflowId: options.workflowId,
- expectedScanId: scan.scanId,
+ expectedScanId: scanId,
signal: options.signal,
},
{
@@ -159,13 +223,13 @@ export async function deduplicateScanInternal(
environment,
undefined,
options.signal,
- scan["targetPath"] as string,
+ repositoryPath,
);
const checkpoints = workflow
? new CheckpointedReviewRunner(
workflow,
runner,
- await workflow.sourceSnapshot(scan["targetPath"] as string),
+ await workflow.sourceSnapshot(repositoryPath),
scope,
await reviewSettingsDigest(environment),
)
@@ -184,7 +248,7 @@ export async function deduplicateScanInternal(
);
await checkpoints?.assertSourceUnchanged();
options.signal?.throwIfAborted();
- const result: DeduplicateScanResult = { scanId: scan.scanId, ...reviewed };
+ const result: DeduplicateScanResult = { scanId, ...reviewed };
await workflow?.prepareDedupe(result, { groups: result.duplicateGroups });
await client.storeDedupeGroups(result.duplicateGroups);
return result;
diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts
index c964670de..72c728843 100644
--- a/sdk/typescript/src/index.ts
+++ b/sdk/typescript/src/index.ts
@@ -71,8 +71,12 @@ export type {
PublishScanToCustomOptions,
CustomPublicationResult,
} from "./custom-publish.js";
-export { deduplicateScan } from "./deduplication/scan.js";
+export {
+ deduplicateScan,
+ deduplicateScanDirectory,
+} from "./deduplication/scan.js";
export type {
+ DeduplicateScanDirectoryOptions,
DeduplicateScanOptions,
DeduplicateScanResult,
} from "./deduplication/scan.js";
diff --git a/sdk/typescript/tests-ts/finding-deduplication.test.ts b/sdk/typescript/tests-ts/finding-deduplication.test.ts
index 78ea87488..4cb35de3a 100644
--- a/sdk/typescript/tests-ts/finding-deduplication.test.ts
+++ b/sdk/typescript/tests-ts/finding-deduplication.test.ts
@@ -17,7 +17,11 @@ import {
} from "../src/deduplication/deduplication-reviewer.js";
import { CodexSecurityError } from "../src/errors.js";
import { FindingsClient } from "../src/findings-client.js";
-import { deduplicateScanInternal } from "../src/deduplication/scan.js";
+import { deduplicateScanDirectory } from "../src/index.js";
+import {
+ deduplicateScanDirectoryInternal,
+ deduplicateScanInternal,
+} from "../src/deduplication/scan.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import type { JsonObject } from "../src/config.js";
@@ -497,6 +501,75 @@ test("resolves a saved scan and retrieves its IDs without uploading or modifying
}
});
+test("deduplicates an explicit sealed scan directory without reading scan history", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "dedupe-directory-"));
+ const repository = await mkdtemp(join(tmpdir(), "dedupe-repository-"));
+ try {
+ await cp(join(PLUGIN_ROOT, "examples/completed-scan"), directory, {
+ recursive: true,
+ });
+ if (process.platform !== "win32") await chmod(directory, 0o700);
+ const original = await readFile(join(directory, "findings.json"), "utf8");
+ const commands: string[][] = [];
+ const requests: string[] = [];
+ const result = await deduplicateScanDirectoryInternal(
+ directory,
+ {
+ repository,
+ expectedScanId: "scan_example_001",
+ findingsUrl: "http://synthetic.test/api",
+ },
+ {
+ runWorkbench: async (args) => {
+ commands.push([...args]);
+ throw new Error("Explicit scan directories must not read history");
+ },
+ fetch: async (url, options) => {
+ requests.push(String(url));
+ expect(options?.method).toBeUndefined();
+ return Response.json({
+ finding: document.findings[0],
+ potentialDuplicates: [],
+ });
+ },
+ reviewer: {
+ async screen() {
+ throw new Error("No review for an empty neighborhood");
+ },
+ async reviewPair() {
+ throw new Error("No pair to review");
+ },
+ },
+ },
+ );
+ expect(result).toEqual({
+ scanId: "scan_example_001",
+ uniqueFindingIds: document.findings.map((finding) => finding.findingId),
+ duplicateGroups: [],
+ deduplicationStatus: "completed",
+ });
+ expect(commands).toEqual([]);
+ expect(requests).toEqual([
+ `http://synthetic.test/api/v1/finding/${document.findings[0]!.findingId}/potential-duplicates?repositoryId=target_sha256_example`,
+ ]);
+ expect(await readFile(join(directory, "findings.json"), "utf8")).toBe(
+ original,
+ );
+ await expect(
+ deduplicateScanDirectory(directory, {
+ repository,
+ expectedScanId: "scan_other",
+ findingsUrl: "http://synthetic.test/api",
+ }),
+ ).rejects.toThrow("do not match selected scan");
+ } finally {
+ await Promise.all([
+ rm(directory, { recursive: true, force: true }),
+ rm(repository, { recursive: true, force: true }),
+ ]);
+ }
+});
+
test("lookup failures and cancellation never produce a completed uniqueness result", async () => {
for (const status of [404, 502]) {
const client = new FindingsClient(
diff --git a/sdk/typescript/tests-ts/finding-workflow-publication.test.ts b/sdk/typescript/tests-ts/finding-workflow-publication.test.ts
index 163bc9c63..a9f203ce5 100644
--- a/sdk/typescript/tests-ts/finding-workflow-publication.test.ts
+++ b/sdk/typescript/tests-ts/finding-workflow-publication.test.ts
@@ -1,9 +1,10 @@
import { readFile } from "node:fs/promises";
-import { join } from "node:path";
+import { join, relative } from "node:path";
import { expect, test } from "bun:test";
import type { JsonObject } from "../src/config.js";
import { publishScanToCustomInternal } from "../src/custom-publish.js";
import {
+ deduplicateScanDirectoryInternal,
deduplicateScanInternal,
type DeduplicateScanResult,
} from "../src/deduplication/scan.js";
@@ -37,6 +38,122 @@ function publicationBinding(
];
}
+test("deduplicates an external scan through its bound workflow without reading scan history", async () => {
+ await using fixture = await workflowFixture();
+ const { scanDir, repository, environment, document } = fixture;
+ const id = "external-directory";
+ const options = {
+ workflowId: id,
+ findingsUrl: "http://synthetic.test/service",
+ repository: relative(process.cwd(), repository),
+ expectedScanId: document.scanId,
+ allRepositories: true,
+ };
+ const result: DeduplicateScanResult = {
+ scanId: document.scanId,
+ uniqueFindingIds: document.findings.map((finding) => finding.findingId),
+ duplicateGroups: [],
+ deduplicationStatus: "completed",
+ };
+ const source = {
+ repository,
+ revision: "synthetic-revision",
+ refsDigest: "synthetic-refs",
+ content: "synthetic-content",
+ };
+ const binding = {
+ scanId: document.scanId,
+ scanDir,
+ artifactDigest: expect.stringMatching(/^[a-f0-9]{64}$/),
+ destination: "http://synthetic.test/service/",
+ };
+ const workbench = scriptedWorkbench([
+ {
+ request: {
+ id,
+ action: "bind",
+ binding: {
+ repositoryPath: repository,
+ ...binding,
+ scope: { allRepositories: true },
+ },
+ },
+ },
+ { request: { id, action: "complete", stage: "scan", result: null } },
+ { request: { id, action: "bind", binding } },
+ { request: { id, action: "complete", stage: "scan", result: null } },
+ {
+ request: { id, action: "begin", stage: "publish" },
+ response: {
+ workflow: {
+ stages: {
+ publish: {
+ status: "completed",
+ result: {
+ findingIds: document.findings.map(
+ (finding) => finding.findingId,
+ ),
+ },
+ },
+ },
+ },
+ },
+ },
+ {
+ request: { id, action: "begin", stage: "dedupe" },
+ response: { workflow: { stages: { dedupe: { status: "running" } } } },
+ },
+ {
+ request: { id, action: "get" },
+ response: { workflow: { stages: { dedupe: { status: "running" } } } },
+ },
+ { request: { id, action: "source", repository }, response: { source } },
+ { request: { id, action: "source", repository }, response: { source } },
+ {
+ request: {
+ id,
+ action: "prepare-dedupe",
+ stage: "dedupe",
+ result,
+ pendingWrite: { groups: [] },
+ },
+ },
+ { request: { id, action: "complete", stage: "dedupe", result } },
+ ]);
+ const requests: string[] = [];
+ expect(
+ await deduplicateScanDirectoryInternal(scanDir, options, {
+ environment,
+ runWorkbench: (args, input) =>
+ workbench.run(
+ { environment, pluginRoot: PLUGIN_ROOT, python: "unused" },
+ args,
+ input,
+ ),
+ fetch: async (url, init) => {
+ requests.push(String(url));
+ expect(init.method).toBeUndefined();
+ return Response.json({
+ finding: document.findings[0],
+ potentialDuplicates: [],
+ });
+ },
+ reviewer: {
+ async screen() {
+ throw new Error("No review for an empty neighborhood");
+ },
+ async reviewPair() {
+ throw new Error("No pair to review");
+ },
+ },
+ }),
+ ).toEqual(result);
+ expect(requests).toEqual([
+ `http://synthetic.test/service/v1/finding/${document.findings[0]!.findingId}/potential-duplicates?allRepositories=true`,
+ ]);
+ workbench.assertDone();
+});
+
test("failed publication records its error, dry-run does not advance it, and retry records the receipt", async () => {
await using fixture = await workflowFixture();
const { scanDir, environment, document } = fixture;