Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor!: prepare local deduplication in Node (#1840)

Michael D'Angelo1503a508778d
diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts
index 0993c9db8..e25974558 100644
--- a/plugins/codex-security/mcp-app/helpers-main.ts
+++ b/plugins/codex-security/mcp-app/helpers-main.ts
@@ -23,7 +23,11 @@ import { validateTrackingSourceCommand } from "./src/helpers/validate-tracking-s
import { deepReviewInputCommand } from "./src/helpers/deep-review-input";
import { inventoryCommand } from "./src/helpers/generate-inventory";
import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes";
-import { escapeControls, stringifyJson } from "./src/helpers/json";
+import {
+ escapeControls,
+ requireFiniteNumbers,
+ stringifyJson,
+} from "./src/helpers/json";
import { decodeUtf8 } from "./src/helpers/utf8";
// Importing the bundled parser does not invoke its CLI adapter.
@@ -68,6 +72,8 @@ function runHelper(): void {
const workbenchUsage: Record<string, string> = {
"database-info":
"Usage: database-info (reads a JSON absolute state-directory string from stdin)",
+ "local-dedupe":
+ "Usage: local-dedupe\nReads a JSON object from stdin with an absolute stateDirectory and a prepare or embed payload.",
"store-findings":
"Usage: store-findings\nReads a JSON object from stdin with an absolute stateDirectory and payload.entries containing finding and embedding records; payload.repositoryId is optional.",
"list-stored-findings":
@@ -145,6 +151,8 @@ function runHelper(): void {
values["scan-id"] as string | undefined,
);
}
+ // Preparation commits before response serialization, as other workbench writes do.
+ if (command === "local-dedupe") requireFiniteNumbers(result);
console.log(
stringifyJson(result, 0).replace(/[\p{Cc}\p{Cf}]/gu, (character) =>
character
diff --git a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts
index 7c199d010..3bff94588 100644
--- a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts
@@ -11,13 +11,8 @@ import { parseArgs } from "node:util";
import { mkdir, readFile, writeFile } from "./helper-files";
import { expandHome, windowsFiles, windowsJoin } from "./resolve-security-md";
import { decodePosixBytes, encodePosixPath } from "./posix-path";
-import { stringifyJson } from "./json";
-import {
- compare,
- print,
- reportCommandError,
- ArgumentError,
-} from "./rank-worklists";
+import { compareCodePoints, stringifyJson } from "./json";
+import { print, reportCommandError, ArgumentError } from "./rank-worklists";
import { decodeUtf8 } from "./utf8";
import {
ancestors,
@@ -432,7 +427,7 @@ export async function generateInventory(
return { count: unique.length, output };
}
const rows = [...rankRows.values()]
- .sort((left, right) => compare(left.path, right.path))
+ .sort((left, right) => compareCodePoints(left.path, right.path))
.map((row) =>
Buffer.from(
stringifyJson(row, 0).replace(
diff --git a/plugins/codex-security/mcp-app/src/helpers/json.ts b/plugins/codex-security/mcp-app/src/helpers/json.ts
index c39898d1e..573edc3fc 100644
--- a/plugins/codex-security/mcp-app/src/helpers/json.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/json.ts
@@ -1,5 +1,14 @@
export { isRecord as object } from "../record.ts";
+export function compareCodePoints(left: string, right: string): number {
+ const a = Array.from(left, (character) => character.codePointAt(0)!);
+ const b = Array.from(right, (character) => character.codePointAt(0)!);
+ for (let index = 0; index < Math.min(a.length, b.length); index++) {
+ if (a[index] !== b[index]) return a[index]! - b[index]!;
+ }
+ return a.length - b.length;
+}
+
export function requireFiniteNumbers(value: unknown): void {
if (typeof value === "number" && !Number.isFinite(value))
throw new TypeError(
diff --git a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts
index a14f0ee2c..7f3a403d0 100644
--- a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts
@@ -8,7 +8,7 @@ import {
writeFile,
} from "./helper-files";
import { encodePosixPath } from "./posix-path";
-import { formatDiagnostic, object, parseJson } from "./json";
+import { compareCodePoints, formatDiagnostic, object, parseJson } from "./json";
import { expandHome } from "./resolve-security-md";
export interface RankRow {
@@ -17,15 +17,6 @@ export interface RankRow {
preview?: string;
}
const blank = (value: string) => value.trim().length === 0;
-export function compare(left: string, right: string): number {
- const a = Array.from(left, (character) => character.codePointAt(0)!);
- const b = Array.from(right, (character) => character.codePointAt(0)!);
- for (let index = 0; index < Math.min(a.length, b.length); index++) {
- if (a[index] !== b[index]) return a[index]! - b[index]!;
- }
- return a.length - b.length;
-}
-
function missingNames(
expected: Iterable<string>,
actual: Iterable<string>,
@@ -33,7 +24,7 @@ function missingNames(
const present = new Set(actual);
return Array.from(expected)
.filter((name) => !present.has(name))
- .sort(compare);
+ .sort(compareCodePoints);
}
export function loadRankRows(path: string): RankRow[] {
@@ -83,7 +74,7 @@ export function requireUniquePaths(rows: RankRow[], label: string): void {
}
if (duplicates.size)
throw new Error(
- `${label} contains duplicate paths: ${formatDiagnostic([...duplicates].sort(compare))}`,
+ `${label} contains duplicate paths: ${formatDiagnostic([...duplicates].sort(compareCodePoints))}`,
);
}
diff --git a/plugins/codex-security/mcp-app/src/workbench/commands.ts b/plugins/codex-security/mcp-app/src/workbench/commands.ts
index c91b9704f..c86a732b6 100644
--- a/plugins/codex-security/mcp-app/src/workbench/commands.ts
+++ b/plugins/codex-security/mcp-app/src/workbench/commands.ts
@@ -1,5 +1,6 @@
import { parseJson } from "../helpers/json";
import { openWorkbenchDatabase, workbenchDatabasePath } from "./database";
+import { localDedupe, type LocalDedupeRequest } from "./local-dedupe";
import {
listStoredFindings,
storeFindings,
@@ -88,6 +89,7 @@ export async function workbenchCommand(
workbenchDatabasePath(stateDirectory),
{
deferred:
+ command !== "local-dedupe" &&
command !== "store-findings" &&
command !== "store-external-findings" &&
command !== "store-dedupe-groups" &&
@@ -95,6 +97,12 @@ export async function workbenchCommand(
},
);
try {
+ if (command === "local-dedupe")
+ return localDedupe(
+ database,
+ payload as LocalDedupeRequest,
+ new Date().toISOString(),
+ );
if (command === "store-external-findings")
return storeExternalFindings(
database,
diff --git a/plugins/codex-security/mcp-app/src/workbench/local-dedupe.ts b/plugins/codex-security/mcp-app/src/workbench/local-dedupe.ts
new file mode 100644
index 000000000..d18c41a00
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/workbench/local-dedupe.ts
@@ -0,0 +1,397 @@
+import { createHash } from "node:crypto";
+import type { DatabaseSync, SQLOutputValue } from "node:sqlite";
+import { resolvedPathText } from "../helpers/helper-files";
+import {
+ compareCodePoints,
+ parseJson,
+ requireFiniteNumbers,
+ stringifyJson,
+} from "../helpers/json";
+import { decodeUtf8 } from "../helpers/utf8";
+import { requireSqliteText } from "./database";
+import type { Finding } from "./findings";
+import { stableTargetId } from "./migrations";
+import { transaction } from "./transaction";
+
+interface Preparation {
+ action: "prepare";
+ space: string;
+ repositoryPath: string;
+ anchorRepositoryId: string;
+ repositoryId?: string;
+ findings: readonly Finding[];
+}
+
+interface EmbeddingBatch {
+ action: "embed";
+ space: string;
+ model: string;
+ dimensions: number | bigint | boolean;
+ entries: readonly {
+ findingId: string;
+ cacheKey: string;
+ embedding: {
+ model: string;
+ vector: readonly (number | bigint | boolean)[];
+ };
+ }[];
+}
+
+export type LocalDedupeRequest = Preparation | EmbeddingBatch;
+
+class LocalDedupeError extends Error {}
+
+// Historical SQLite rows can contain BLOBs despite their TEXT affinity. Decode
+// TEXT explicitly so Node does not silently replace invalid text.
+function column(expression: string, name: string): string {
+ return `CASE WHEN typeof(${expression}) = 'text' THEN CAST(${expression} AS BLOB)
+ ELSE ${expression} END AS ${name}, typeof(${expression}) AS ${name}_type`;
+}
+
+function databaseEncoding(database: DatabaseSync): string {
+ // CAST(TEXT AS BLOB) returns bytes in SQLite's database encoding.
+ return database.prepare("PRAGMA encoding").get()!.encoding as string;
+}
+
+function columnReader(encoding: string) {
+ const decoder =
+ encoding === "UTF-8"
+ ? undefined
+ : new TextDecoder(encoding, { fatal: true, ignoreBOM: true });
+ return (
+ row: Record<string, SQLOutputValue>,
+ name: string,
+ ): SQLOutputValue => {
+ const stored = row[name]!;
+ if (row[`${name}_type`] !== "text") return stored;
+ const bytes = stored as Uint8Array;
+ return decoder ? decoder.decode(bytes) : decodeUtf8(Buffer.from(bytes));
+ };
+}
+
+function cacheKey(space: string, document: SQLOutputValue): string {
+ if (space === undefined)
+ throw new TypeError("Local deduplication requires an embedding space.");
+ let text: string;
+ if (document instanceof Uint8Array) {
+ // Existing BLOB cache keys hash the bytes representation, not decoded JSON.
+ const quote = document.includes(39) && !document.includes(34) ? 34 : 39;
+ text = `b${String.fromCharCode(quote)}`;
+ for (const byte of document) {
+ if (byte === quote || byte === 92)
+ text += `\\${String.fromCharCode(byte)}`;
+ else if (byte === 9) text += "\\t";
+ else if (byte === 10) text += "\\n";
+ else if (byte === 13) text += "\\r";
+ else if (byte < 32 || byte >= 127)
+ text += `\\x${byte.toString(16).padStart(2, "0")}`;
+ else text += String.fromCharCode(byte);
+ }
+ text += String.fromCharCode(quote);
+ } else {
+ text = document === null ? "None" : String(document);
+ }
+ requireSqliteText([space, text]);
+ return createHash("sha256").update(`${space}\0${text}`).digest("hex");
+}
+
+function parseStoredJson(text: string): unknown {
+ if (!text.includes("NaN") && !text.includes("Infinity"))
+ return parseJson(text);
+
+ // Python's json.loads accepted three extra number literals. Index bare values
+ // so JSON.parse still validates the structure and decodes strings normally.
+ const values: unknown[] = [];
+ try {
+ const indexed = text.replace(
+ /"(?:[^"\\]|\\[\s\S])*"?|[^\s"[\]{},:]+/gu,
+ (token) => {
+ if (token.startsWith('"')) return token;
+ const value =
+ token === "NaN" || token === "Infinity" || token === "-Infinity"
+ ? Number(token)
+ : parseJson(token);
+ return String(values.push(value) - 1);
+ },
+ );
+ return JSON.parse(indexed, (_key, value: unknown) =>
+ typeof value === "number" ? values[value] : value,
+ );
+ } catch (error) {
+ // Report malformed JSON against the original document.
+ if (error instanceof SyntaxError) return parseJson(text);
+ throw error;
+ }
+}
+
+function parseBody(document: SQLOutputValue): unknown {
+ if (typeof document === "string") return parseStoredJson(document);
+ if (!(document instanceof Uint8Array))
+ throw new TypeError("Stored finding JSON must be text or bytes.");
+ const bytes = Buffer.from(document);
+ let text: string;
+ try {
+ // json.loads historically accepted UTF-8, UTF-16 and UTF-32 BLOB bodies.
+ if (
+ bytes.subarray(0, 4).equals(Buffer.from([0, 0, 0xfe, 0xff])) ||
+ bytes.subarray(0, 4).equals(Buffer.from([0xff, 0xfe, 0, 0])) ||
+ (bytes.length >= 4 && bytes[0] === 0 && bytes[1] === 0) ||
+ (bytes.length >= 4 && bytes[1] === 0 && bytes[2] === 0 && bytes[3] === 0)
+ ) {
+ const little = bytes[0] !== 0;
+ if (bytes.length % 4) throw new Error("Truncated UTF-32 finding JSON.");
+ text = "";
+ for (let offset = 0; offset < bytes.length; offset += 4)
+ text += String.fromCodePoint(
+ little ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset),
+ );
+ } else if (
+ bytes.subarray(0, 2).equals(Buffer.from([0xfe, 0xff])) ||
+ bytes.subarray(0, 2).equals(Buffer.from([0xff, 0xfe])) ||
+ (bytes.length >= 2 && (bytes[0] === 0 || bytes[1] === 0))
+ ) {
+ if (bytes.length % 2) throw new Error("Truncated UTF-16 finding JSON.");
+ const little = bytes[0] !== 0 && bytes[0] !== 0xfe;
+ text = (little ? bytes : bytes.swap16()).toString("utf16le");
+ } else {
+ text = decodeUtf8(bytes);
+ }
+ } catch (error) {
+ throw new LocalDedupeError((error as Error).message);
+ }
+ return parseStoredJson(text.startsWith("\ufeff") ? text.slice(1) : text);
+}
+
+function orderedFinding(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(orderedFinding);
+ if (value === null || typeof value !== "object") return value;
+ // Python sorted response keys recursively; preserve the SDK embedding input.
+ return Object.fromEntries(
+ Object.entries(value)
+ .sort(([left], [right]) => compareCodePoints(left, right))
+ .map(([key, item]) => [key, orderedFinding(item)]),
+ );
+}
+
+function prepare(
+ database: DatabaseSync,
+ payload: Preparation,
+ timestamp: string,
+) {
+ const encoding = databaseEncoding(database);
+ const value = columnReader(encoding);
+ // A sealed scan selects finding IDs, not permission to select another corpus.
+ const target = resolvedPathText(payload.repositoryPath, false);
+ requireSqliteText([target]);
+ const registered = database
+ .prepare(
+ `SELECT ${column("id", "id")} FROM security_targets WHERE current_path = ?`,
+ )
+ .get(target);
+ const targetId =
+ registered === undefined ? stableTargetId(target) : value(registered, "id");
+ if (
+ payload.anchorRepositoryId !== targetId ||
+ (payload.repositoryId !== undefined && payload.repositoryId !== targetId)
+ )
+ throw new LocalDedupeError("target_mismatch");
+ if (payload.findings.length === 0)
+ return {
+ cacheKeys: new Map<SQLOutputValue, string>(),
+ findingsToEmbed: [],
+ };
+
+ const existing = database.prepare(
+ `SELECT ${["fingerprint", "rule_id", "identity_anchor", "identity_instance", "details_json"].map((name) => column(name, name)).join(", ")}
+ FROM findings WHERE id = ?`,
+ );
+ const insert = database.prepare(
+ `INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, identity_instance,
+ details_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO UPDATE SET details_json = excluded.details_json,
+ updated_at = excluded.updated_at`,
+ );
+ const link = database.prepare(
+ "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
+ );
+ for (const finding of payload.findings) {
+ const identity = [
+ finding.fingerprints.primary,
+ finding.ruleId,
+ finding.identity.anchor,
+ finding.identity.instance ?? null,
+ ] as const;
+ requireSqliteText([finding.findingId]);
+ const current = existing.get(finding.findingId);
+ const values =
+ current &&
+ [
+ "fingerprint",
+ "rule_id",
+ "identity_anchor",
+ "identity_instance",
+ "details_json",
+ ].map((name) => value(current, name));
+ if (
+ values &&
+ values.slice(0, 4).some((value, index) => value !== identity[index])
+ )
+ throw new LocalDedupeError("finding_conflict");
+ // Historical input must never overwrite the current body or its cache key.
+ if (values === undefined || values[4] === null) {
+ requireSqliteText([...identity, timestamp]);
+ try {
+ requireFiniteNumbers(finding);
+ } catch (error) {
+ throw new LocalDedupeError((error as Error).message);
+ }
+ const body = stringifyJson(finding, 0);
+ // SQLite replaces these code points when transcoding bound UTF-8 to UTF-16.
+ const details =
+ encoding === "UTF-8"
+ ? body
+ : body
+ .replaceAll("\uFFFE", "\\ufffe")
+ .replaceAll("\uFFFF", "\\uffff");
+ insert.run(finding.findingId, ...identity, details, timestamp, timestamp);
+ }
+ link.run(payload.anchorRepositoryId, finding.findingId);
+ }
+
+ const rows = database.prepare(
+ `SELECT ${column("findings.id", "id")}, ${column("findings.details_json", "details")},
+ ${column("embeddings.cache_key", "cache_key")} FROM findings
+ LEFT JOIN local_finding_embeddings AS embeddings ON findings.id = embeddings.finding_id
+ ${
+ payload.repositoryId === undefined
+ ? ""
+ : "WHERE EXISTS (SELECT 1 FROM finding_repositories WHERE finding_id = findings.id AND repository_id = ?)"
+ }
+ ORDER BY findings.created_at, findings.id`,
+ );
+ const cacheKeys = new Map<SQLOutputValue, string>();
+ const findingsToEmbed: unknown[] = [];
+ for (const row of rows.iterate(
+ ...(payload.repositoryId === undefined ? [] : [payload.repositoryId]),
+ )) {
+ const [id, document, savedKey] = ["id", "details", "cache_key"].map(
+ (name) => value(row, name),
+ );
+ if (document === null) throw new LocalDedupeError("finding_not_indexed");
+ const key = cacheKey(payload.space, document!);
+ cacheKeys.set(id!, key);
+ // A cache hit does not parse or rewrite the stored document.
+ if (key !== savedKey)
+ findingsToEmbed.push(orderedFinding(parseBody(document!)));
+ }
+ return { cacheKeys, findingsToEmbed };
+}
+
+function validVector(
+ vector: EmbeddingBatch["entries"][number]["embedding"]["vector"],
+): boolean {
+ let maximum = 0;
+ const numbers: number[] = [];
+ for (const value of vector) {
+ if (!["number", "bigint", "boolean"].includes(typeof value)) return false;
+ const number = Number(value);
+ if (typeof value === "bigint" && !Number.isFinite(number))
+ throw new RangeError("int too large to convert to float");
+ if (!Number.isFinite(number)) return false;
+ maximum = Math.max(maximum, Math.abs(number));
+ numbers.push(number);
+ }
+ if (maximum === 0) return false;
+ let sum = 0;
+ let correction = 0;
+ for (const number of numbers) {
+ const scaled = number / maximum;
+ const term = scaled * scaled - correction;
+ const next = sum + term;
+ correction = next - sum - term;
+ sum = next;
+ }
+ if (maximum * Math.sqrt(sum) < Number.MAX_VALUE / 2) return true;
+ // Near overflow, a rounded sqrt/product can reject a norm that rounds to
+ // MAX_VALUE. Compare exact squares in units of 2^-2148 with the overflow midpoint.
+ const boundary = ((1n << 2098n) - (1n << 2044n)) ** 2n;
+ const bytes = Buffer.allocUnsafe(8);
+ let squares = 0n;
+ for (const number of numbers) {
+ bytes.writeDoubleBE(number);
+ const bits = bytes.readBigUInt64BE();
+ const exponent = Number((bits >> 52n) & 0x7ffn);
+ const significand =
+ (bits & ((1n << 52n) - 1n)) | (exponent === 0 ? 0n : 1n << 52n);
+ squares +=
+ (significand * significand) <<
+ BigInt(exponent === 0 ? 0 : 2 * (exponent - 1));
+ if (squares >= boundary) return false;
+ }
+ return true;
+}
+
+function embed(database: DatabaseSync, payload: EmbeddingBatch) {
+ const value = columnReader(databaseEncoding(database));
+ const current = database.prepare(
+ `SELECT ${column("details_json", "details")} FROM findings WHERE id = ?`,
+ );
+ const save = database.prepare(
+ `INSERT INTO local_finding_embeddings (finding_id, model, vector_json, cache_key)
+ VALUES (?, ?, ?, ?) ON CONFLICT(finding_id) DO UPDATE SET
+ model = excluded.model, vector_json = excluded.vector_json, cache_key = excluded.cache_key`,
+ );
+ for (const entry of payload.entries) {
+ requireSqliteText([entry.findingId]);
+ const row = current.get(entry.findingId);
+ if (
+ row === undefined ||
+ cacheKey(payload.space, value(row, "details")) !== entry.cacheKey
+ )
+ throw new LocalDedupeError("finding_changed");
+ const { model, vector } = entry.embedding;
+ if (
+ model !== payload.model ||
+ !["number", "bigint", "boolean"].includes(typeof payload.dimensions) ||
+ vector.length !== Number(payload.dimensions) ||
+ !validVector(vector)
+ )
+ throw new LocalDedupeError("embedding_failed");
+ requireSqliteText([model, entry.cacheKey]);
+ save.run(entry.findingId, model, stringifyJson(vector, 0), entry.cacheKey);
+ }
+ return {};
+}
+
+export function localDedupe(
+ database: DatabaseSync,
+ payload: LocalDedupeRequest,
+ timestamp: string,
+) {
+ try {
+ const result = transaction(database, "BEGIN IMMEDIATE", () => {
+ if (payload.action === "prepare")
+ return prepare(database, payload, timestamp);
+ if (payload.action === "embed") return embed(database, payload);
+ throw new LocalDedupeError("invalid_request");
+ });
+ if ("cacheKeys" in result && result.cacheKeys instanceof Map) {
+ // Unsupported JSON object keys fail at the response boundary after commit.
+ for (const key of result.cacheKeys.keys())
+ if (key instanceof Uint8Array)
+ throw new TypeError("Finding cache keys cannot contain bytes.");
+ return { ...result, cacheKeys: Object.fromEntries(result.cacheKeys) };
+ }
+ return result;
+ } catch (error) {
+ if (error instanceof LocalDedupeError || error instanceof SyntaxError)
+ return { error: error.message };
+ if (
+ error instanceof Error &&
+ "errcode" in error &&
+ (Number(error.errcode) & 0xff) === 19
+ )
+ return { error: "finding_conflict" };
+ throw error;
+ }
+}
diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts
index f619eb30b..d462c80b3 100644
--- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts
+++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts
@@ -129,6 +129,15 @@ function repairStableTargets(database: DatabaseSync): boolean {
return true;
}
+export function stableTargetId(path: string): string {
+ return (
+ "target_sha256_" +
+ createHash("sha256")
+ .update("local-workspace\0" + path)
+ .digest("hex")
+ );
+}
+
function backfillTargets(database: DatabaseSync): void {
const targets = database
.prepare(
@@ -145,11 +154,7 @@ function backfillTargets(database: DatabaseSync): void {
const path = String(row.target_path);
let id = lookup.get(path)?.id;
if (id === undefined) {
- id =
- "target_sha256_" +
- createHash("sha256")
- .update("local-workspace\0" + path)
- .digest("hex");
+ id = stableTargetId(path);
const timestamp = new Date().toISOString();
insert.run(id, path, basename(path), timestamp, timestamp);
}
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_local_dedupe.ts b/plugins/codex-security/mcp-app/tests/test_workbench_local_dedupe.ts
new file mode 100644
index 000000000..2c67c43f8
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_local_dedupe.ts
@@ -0,0 +1,567 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { DatabaseSync } from "node:sqlite";
+import { after, test, type TestContext } from "node:test";
+import { importSource } from "./import-module.ts";
+import { createTemporaryDirectories } from "./support/temporary-directories.ts";
+import type * as Local from "../src/workbench/local-dedupe.ts";
+import type * as Migrations from "../src/workbench/migrations.ts";
+import type { Finding } from "../src/workbench/findings.ts";
+
+const { localDedupe } = (await importSource("src/workbench/local-dedupe.ts", {
+ define: {
+ // Native bindings are installed beside the generated server bundle.
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/server.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+})) as typeof Local;
+const { applyMigrations } = (await importSource(
+ "src/workbench/migrations.ts",
+)) as typeof Migrations;
+const temporary = createTemporaryDirectories(true);
+after(() => temporary.cleanup());
+const timestamp = "2026-01-01T00:00:00Z";
+const space = "synthetic-space";
+
+async function fixture(
+ t: TestContext,
+ encoding: "UTF-8" | "UTF-16le" | "UTF-16be" = "UTF-8",
+) {
+ const root = await temporary.create("local-dedupe-");
+ const database = new DatabaseSync(":memory:");
+ t.after(() => database.close());
+ database.exec(`PRAGMA encoding = '${encoding}'`);
+ database.exec("PRAGMA foreign_keys = ON");
+ applyMigrations(database);
+ const targetId = `target_sha256_${createHash("sha256").update(`local-workspace\0${root}`).digest("hex")}`;
+ const prepare = (
+ findings: Finding[],
+ overrides: Partial<
+ Extract<Local.LocalDedupeRequest, { action: "prepare" }>
+ > = {},
+ ) =>
+ localDedupe(
+ database,
+ {
+ action: "prepare",
+ space,
+ repositoryPath: root,
+ anchorRepositoryId: targetId,
+ repositoryId: targetId,
+ findings,
+ ...overrides,
+ },
+ timestamp,
+ );
+ const embed = (
+ entries: Extract<Local.LocalDedupeRequest, { action: "embed" }>["entries"],
+ ) =>
+ localDedupe(
+ database,
+ { action: "embed", space, model: "synthetic", dimensions: 2, entries },
+ timestamp,
+ );
+ return { root, database, targetId, prepare, embed };
+}
+
+function finding(id: string): Finding {
+ return {
+ findingId: id,
+ fingerprints: { primary: `fingerprint-${id}` },
+ ruleId: "synthetic-rule",
+ identity: { anchor: `anchor-${id}` },
+ title: `Finding ${id} λ`,
+ evidence: { diagnostic: "complete\0message", count: 9007199254740993n },
+ };
+}
+
+function prepared(result: ReturnType<typeof localDedupe>) {
+ assert.ok("cacheKeys" in result && "findingsToEmbed" in result);
+ return result as {
+ cacheKeys: Record<string, string>;
+ findingsToEmbed: Finding[];
+ };
+}
+
+function entry(result: ReturnType<typeof prepared>, id: string) {
+ return {
+ findingId: id,
+ cacheKey: result.cacheKeys[id]!,
+ embedding: { model: "synthetic", vector: [1, 0] },
+ };
+}
+
+function cacheKey(body: string) {
+ return createHash("sha256").update(`${space}\0${body}`).digest("hex");
+}
+
+function snapshot(database: DatabaseSync) {
+ return [
+ "findings",
+ "finding_repositories",
+ "local_finding_embeddings",
+ "finding_embeddings",
+ ].map((table) => database.prepare(`SELECT * FROM ${table} ORDER BY 1`).all());
+}
+
+test("preparation retains current body bytes, large integers and independent service embeddings", async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ const original = finding("__proto__");
+ prepared(prepare([original]));
+ const body =
+ '{ "evidence":{"count":9007199254740993,"diagnostic":"complete\\u0000message"},' +
+ '"title":"Current \\u03bb evidence","identity":{"anchor":"anchor-__proto__"},' +
+ '"ruleId":"synthetic-rule","fingerprints":{"primary":"fingerprint-__proto__"},"findingId":"__proto__" }';
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = ?")
+ .run(body, original.findingId);
+ const first = prepared(prepare([original]));
+ assert.equal(first.cacheKeys[original.findingId], cacheKey(body));
+ assert.deepEqual(first.findingsToEmbed, [
+ { ...original, title: "Current λ evidence" },
+ ]);
+ assert.deepEqual(Object.keys(first.findingsToEmbed[0]!), [
+ "evidence",
+ "findingId",
+ "fingerprints",
+ "identity",
+ "ruleId",
+ "title",
+ ]);
+ assert.deepEqual(embed([entry(first, original.findingId)]), {});
+ database
+ .prepare(
+ "INSERT INTO finding_embeddings VALUES (?, 'synthetic', '[0, 1]', NULL)",
+ )
+ .run(original.findingId);
+ assert.deepEqual(prepare([original]), {
+ cacheKeys: first.cacheKeys,
+ findingsToEmbed: [],
+ });
+ assert.equal(
+ database.prepare("SELECT details_json FROM findings").get()!.details_json,
+ body,
+ );
+ assert.equal(
+ database.prepare("SELECT vector_json FROM finding_embeddings").get()!
+ .vector_json,
+ "[0, 1]",
+ );
+ const changedSpace = prepared(
+ prepare([original], { space: "another-space" }),
+ );
+ assert.notEqual(changedSpace.cacheKeys[original.findingId], cacheKey(body));
+ assert.deepEqual(changedSpace.findingsToEmbed, first.findingsToEmbed);
+});
+
+for (const encoding of ["UTF-8", "UTF-16le", "UTF-16be"] as const) {
+ test(`${encoding} preserves distinct noncharacter keys in stored finding JSON`, async (t) => {
+ const { database, prepare } = await fixture(t, encoding);
+ const keys = { "\uFFFE": "first", "\uFFFF": "second" };
+ const original = { ...finding("synthetic"), extensions: keys };
+ const { findingsToEmbed } = prepared(prepare([original]));
+ assert.deepEqual(findingsToEmbed, [original]);
+ const { bytes } = database
+ .prepare("SELECT CAST(details_json AS BLOB) AS bytes FROM findings")
+ .get()!;
+ const stored = new TextDecoder(encoding).decode(bytes as Uint8Array);
+ assert.deepEqual(JSON.parse(stored).extensions, keys);
+ if (encoding === "UTF-8") {
+ for (const key of Object.keys(keys)) assert.ok(stored.includes(key));
+ }
+ });
+}
+
+for (const encoding of ["UTF-16le", "UTF-16be"] as const) {
+ test(`${encoding} databases reuse the same text and cache keys across preparation and embedding`, async (t) => {
+ const { database, prepare, embed } = await fixture(t, encoding);
+ assert.equal(database.prepare("PRAGMA encoding").get()!.encoding, encoding);
+ const original = finding("synthetic");
+ const first = prepared(prepare([original]));
+ assert.deepEqual(first.findingsToEmbed, [original]);
+ const body = database.prepare("SELECT details_json FROM findings").get()!
+ .details_json;
+ assert.equal(first.cacheKeys.synthetic, cacheKey(String(body)));
+ assert.deepEqual(embed([entry(first, "synthetic")]), {});
+ assert.deepEqual(prepare([original]), {
+ cacheKeys: first.cacheKeys,
+ findingsToEmbed: [],
+ });
+ });
+}
+
+test("a missing embedding space rolls back when deriving a cache key", async (t) => {
+ const { database, prepare } = await fixture(t);
+ const before = snapshot(database);
+ assert.deepEqual(prepare([], { space: undefined }), {
+ cacheKeys: {},
+ findingsToEmbed: [],
+ });
+ assert.throws(
+ () => prepare([finding("synthetic")], { space: undefined }),
+ /embedding space/i,
+ );
+ assert.deepEqual(snapshot(database), before);
+ assert.equal(
+ prepared(prepare([finding("synthetic")], { space: "undefined" }))
+ .findingsToEmbed.length,
+ 1,
+ );
+});
+
+for (const [encoding, expectedKey] of [
+ ["UTF-8", "41841dc6a8e60f506e90d97fb9599f6cd6ddba109c24bd6ca2593ca72fb8f646"],
+ [
+ "UTF-16LE",
+ "23ba87b2a052c2b711dc909ad478346d1e253701cecf6f4b82b62d4cab9597c1",
+ ],
+ [
+ "UTF-32LE",
+ "c4e623bd52874215c18d910e333ad041eaef16e11ca5b307c1a0a51cb061ab3e",
+ ],
+] as const) {
+ test(`${encoding} BLOB bodies retain Python cache identity and stored bytes`, async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ prepared(prepare([finding("old")]));
+ const text = '{"findingId":"old","title":"BLOB body"}';
+ const body =
+ encoding === "UTF-8"
+ ? Buffer.from(text)
+ : encoding === "UTF-16LE"
+ ? Buffer.concat([
+ Buffer.from([255, 254]),
+ Buffer.from(text, "utf16le"),
+ ])
+ : Buffer.from([
+ 255,
+ 254,
+ 0,
+ 0,
+ ...Array.from(Buffer.from(text)).flatMap((byte) => [
+ byte,
+ 0,
+ 0,
+ 0,
+ ]),
+ ]);
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")
+ .run(body);
+ const first = prepared(prepare([finding("old")]));
+ // Frozen Python SHA256 values use its bytes representation, not decoded JSON.
+ assert.equal(first.cacheKeys.old, expectedKey);
+ assert.notEqual(first.cacheKeys.old, cacheKey(text));
+ assert.deepEqual(first.findingsToEmbed, [JSON.parse(text)]);
+ assert.deepEqual(embed([entry(first, "old")]), {});
+ assert.deepEqual(prepare([finding("old")]), {
+ cacheKeys: first.cacheKeys,
+ findingsToEmbed: [],
+ });
+ const row = database
+ .prepare(
+ "SELECT typeof(details_json) AS storage, details_json FROM findings",
+ )
+ .get()!;
+ assert.equal(row.storage, "blob");
+ assert.deepEqual(Buffer.from(row.details_json as Uint8Array), body);
+ });
+}
+
+test("a malformed cached BLOB body is reused without decoding and permits later inserts", async (t) => {
+ const { database, prepare } = await fixture(t);
+ prepared(prepare([finding("old")]));
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")
+ .run(Buffer.from("{"));
+ const key = cacheKey("b'{'");
+ database
+ .prepare(
+ "INSERT INTO local_finding_embeddings VALUES ('old', 'synthetic', '[1,0]', ?)",
+ )
+ .run(key);
+ const result = prepared(prepare([finding("new")]));
+ assert.equal(result.cacheKeys.old, key);
+ assert.deepEqual(result.findingsToEmbed, [finding("new")]);
+ assert.ok(database.prepare("SELECT id FROM findings WHERE id = 'new'").get());
+});
+
+test("malformed UTF-8 TEXT is rejected before committing new findings", async (t) => {
+ const { database, prepare } = await fixture(t);
+ prepared(prepare([finding("old")]));
+ const body = Buffer.concat([
+ Buffer.from('{"note":"'),
+ Buffer.from([255]),
+ Buffer.from('"}'),
+ ]);
+ database
+ .prepare(
+ "UPDATE findings SET details_json = CAST(? AS TEXT) WHERE id = 'old'",
+ )
+ .run(body);
+ const before = snapshot(database);
+ assert.throws(() => prepare([finding("new")]), /utf-8/iu);
+ assert.deepEqual(snapshot(database), before);
+ assert.equal(
+ database.prepare("SELECT hex(details_json) AS bytes FROM findings").get()!
+ .bytes,
+ body.toString("hex").toUpperCase(),
+ );
+});
+
+test("preparation indexes legacy missing bodies and invalidates changed local vectors", async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ const original = finding("legacy");
+ database
+ .prepare(
+ `INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, created_at, updated_at)
+ VALUES ('legacy', 'fingerprint-legacy', 'synthetic-rule', 'anchor-legacy', ?, ?)`,
+ )
+ .run(timestamp, timestamp);
+ const first = prepared(prepare([original]));
+ assert.deepEqual(first.findingsToEmbed, [original]);
+ assert.deepEqual(embed([entry(first, "legacy")]), {});
+ const body = String(
+ database.prepare("SELECT details_json FROM findings").get()!.details_json,
+ );
+ assert.equal(first.cacheKeys.legacy, cacheKey(body));
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'legacy'")
+ .run(body.replace("Finding legacy", "Updated legacy"));
+ assert.equal(
+ database
+ .prepare("SELECT COUNT(*) AS total FROM local_finding_embeddings")
+ .get()!.total,
+ 0,
+ );
+ assert.equal(
+ prepared(prepare([original])).findingsToEmbed[0]!.title,
+ "Updated legacy λ",
+ );
+});
+
+test("repository scope preserves creation order, cached keys and empty-input behavior", async (t) => {
+ const { database, targetId, prepare, embed } = await fixture(t);
+ const other = prepared(prepare([finding("other")]));
+ const first = prepared(prepare([finding("first")]));
+ prepared(prepare([finding("second")]));
+ database
+ .prepare("DELETE FROM finding_repositories WHERE finding_id = 'other'")
+ .run();
+ database
+ .prepare(
+ "INSERT INTO finding_repositories VALUES ('other-target', 'other')",
+ )
+ .run();
+ database
+ .prepare("UPDATE findings SET created_at = '2025' WHERE id = 'second'")
+ .run();
+ assert.deepEqual(embed([entry(first, "first")]), {});
+ const scoped = prepared(prepare([finding("first")]));
+ assert.deepEqual(Object.keys(scoped.cacheKeys), ["second", "first"]);
+ assert.deepEqual(scoped.findingsToEmbed, [finding("second")]);
+ const all = prepared(
+ prepare([finding("first")], { repositoryId: undefined }),
+ );
+ assert.deepEqual(Object.keys(all.cacheKeys), ["second", "first", "other"]);
+ assert.equal(all.cacheKeys.other, other.cacheKeys.other);
+ assert.equal(all.cacheKeys.first, first.cacheKeys.first);
+ assert.deepEqual(prepare([], { repositoryId: undefined }), {
+ cacheKeys: {},
+ findingsToEmbed: [],
+ });
+ assert.deepEqual(prepare([], { anchorRepositoryId: `${targetId}-wrong` }), {
+ error: "target_mismatch",
+ });
+});
+
+test("registered target identity is authoritative and a later identity conflict rolls back the batch", async (t) => {
+ const { root, database, prepare } = await fixture(t);
+ database
+ .prepare(
+ "INSERT INTO security_targets (id, current_path, display_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
+ )
+ .run("registered-target", root, "Synthetic target", timestamp, timestamp);
+ assert.deepEqual(prepare([finding("old")]), { error: "target_mismatch" });
+ const scope = {
+ anchorRepositoryId: "registered-target",
+ repositoryId: "registered-target",
+ };
+ prepared(prepare([finding("old")], scope));
+ database.prepare("DELETE FROM finding_repositories").run();
+ const before = snapshot(database);
+ const conflicting = finding("old");
+ conflicting.identity.anchor = "different-anchor";
+ assert.deepEqual(prepare([finding("new"), conflicting], scope), {
+ error: "finding_conflict",
+ });
+ assert.deepEqual(snapshot(database), before);
+});
+
+for (const failure of [
+ "stale",
+ "model",
+ "dimensions",
+ "zero",
+ "nonfinite",
+ "overflow-norm",
+] as const) {
+ test(`a later ${failure} embedding rolls back earlier replacements`, async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ const first = prepared(prepare([finding("one"), finding("two")]));
+ assert.deepEqual(embed([entry(first, "one")]), {});
+ const before = snapshot(database);
+ const replacement = entry(first, "one");
+ replacement.embedding.vector = [0, 1];
+ const invalid = entry(first, "two");
+ if (failure === "stale") invalid.cacheKey = "previous-input";
+ else if (failure === "model") invalid.embedding.model = "another-model";
+ else
+ invalid.embedding.vector = {
+ dimensions: [1],
+ zero: [0, 0],
+ nonfinite: [Number.NaN, 0],
+ "overflow-norm": [Number.MAX_VALUE, Number.MAX_VALUE],
+ }[failure];
+ assert.deepEqual(embed([replacement, invalid]), {
+ error: failure === "stale" ? "finding_changed" : "embedding_failed",
+ });
+ assert.deepEqual(snapshot(database), before);
+ });
+}
+
+test("a changed document rejects an in-flight embedding and retains the previous batch", async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ const first = prepared(prepare([finding("one"), finding("two")]));
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'two'")
+ .run('{"findingId":"two","title":"Newer evidence"}');
+ const before = snapshot(database);
+ assert.deepEqual(embed([entry(first, "one"), entry(first, "two")]), {
+ error: "finding_changed",
+ });
+ assert.deepEqual(snapshot(database), before);
+});
+
+test("finite integer, boolean and subnormal vector components keep their stored values", async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ const first = prepared(prepare([finding("one")]));
+ const saved = entry(first, "one");
+ for (const [vector, stored] of [
+ [[true, 9007199254740993n], "[true,9007199254740993]"],
+ [[Number.MIN_VALUE, 0], "[5e-324,0]"],
+ [[Number.MAX_VALUE, 0], "[1.7976931348623157e+308,0]"],
+ ] as const) {
+ assert.deepEqual(
+ embed([{ ...saved, embedding: { model: "synthetic", vector } }]),
+ {},
+ );
+ assert.equal(
+ database
+ .prepare("SELECT vector_json FROM local_finding_embeddings")
+ .get()!.vector_json,
+ stored,
+ );
+ }
+ const before = snapshot(database);
+ assert.throws(
+ () =>
+ embed([
+ {
+ ...saved,
+ embedding: { model: "synthetic", vector: [10n ** 400n, 0] },
+ },
+ ]),
+ /int too large to convert to float/u,
+ );
+ assert.deepEqual(snapshot(database), before);
+});
+
+test("vector norm admits the largest finite result and rejects its overflowing neighbor", async (t) => {
+ const { database, prepare, embed } = await fixture(t);
+ const first = prepared(prepare([finding("one")]));
+ const saved = entry(first, "one");
+ const accepted = 1.2711610061536462e308;
+ assert.deepEqual(
+ embed([
+ {
+ ...saved,
+ embedding: { model: "synthetic", vector: [accepted, accepted] },
+ },
+ ]),
+ {},
+ );
+ const before = snapshot(database);
+ const rejected = 1.2711610061536464e308;
+ assert.deepEqual(
+ embed([
+ {
+ ...saved,
+ embedding: { model: "synthetic", vector: [rejected, rejected] },
+ },
+ ]),
+ { error: "embedding_failed" },
+ );
+ assert.deepEqual(snapshot(database), before);
+});
+
+test("cache hits skip body parsing while malformed cache misses roll back inserted findings", async (t) => {
+ const { database, prepare } = await fixture(t);
+ prepared(prepare([finding("old")]));
+ const body = '{"evidence":[NaN,]}';
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")
+ .run(body);
+ database
+ .prepare(
+ "INSERT INTO local_finding_embeddings VALUES ('old', 'synthetic', '[1, 0]', ?)",
+ )
+ .run(cacheKey(body));
+ assert.deepEqual(prepare([finding("old")]), {
+ cacheKeys: { old: cacheKey(body) },
+ findingsToEmbed: [],
+ });
+ database.prepare("DELETE FROM local_finding_embeddings").run();
+ const before = snapshot(database);
+ const failed = prepare([finding("new")]);
+ assert.ok("error" in failed);
+ assert.match(String(failed.error), /JSON|position|property|end/iu);
+ assert.deepEqual(snapshot(database), before);
+});
+
+test("legacy non-finite numbers reach the response after preparation commits", async (t) => {
+ const { database, prepare } = await fixture(t);
+ prepared(prepare([finding("old")]));
+ database
+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")
+ .run(
+ String.raw`{"findingId":"old","evidence":[1e400,NaN,Infinity,-Infinity,9007199254740993,{"NaN":"quoted \"Infinity\""}]}`,
+ );
+ const result = prepared(prepare([finding("new")]));
+ assert.deepEqual(
+ result.findingsToEmbed.find((item) => item.findingId === "old")!.evidence,
+ [
+ Infinity,
+ NaN,
+ Infinity,
+ -Infinity,
+ 9007199254740993n,
+ { NaN: 'quoted "Infinity"' },
+ ],
+ );
+ assert.ok(database.prepare("SELECT id FROM findings WHERE id = 'new'").get());
+});
+
+test("an incomplete corpus body rolls back newly indexed findings", async (t) => {
+ const { database, prepare } = await fixture(t);
+ prepared(prepare([finding("old")]));
+ database.prepare("UPDATE findings SET details_json = NULL").run();
+ const before = snapshot(database);
+ assert.deepEqual(prepare([finding("new")]), {
+ error: "finding_not_indexed",
+ });
+ assert.deepEqual(snapshot(database), before);
+});
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index 65ddc8422..d1b0c1650 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -96,7 +96,6 @@
"scripts/workbench_feedback.py",
"scripts/workbench_finding_index.py",
"scripts/workbench_finding_workflows.py",
- "scripts/workbench_local_dedupe.py",
"scripts/workbench_native_indexes.py",
"scripts/workbench_progress.py",
"scripts/workbench_publication.py",
diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py
index 08b7636a1..1c206b242 100644
--- a/plugins/codex-security/scripts/workbench_cli.py
+++ b/plugins/codex-security/scripts/workbench_cli.py
@@ -401,7 +401,6 @@ def parse_args(description: str, *, inspection_only: bool = False) -> argparse.N
subparsers.add_parser("database-info")
subparsers.add_parser("finding-workflow")
- subparsers.add_parser("local-dedupe")
args = parser.parse_args()
if getattr(args, "user_context_stdin", False):
args.user_context = sys.stdin.buffer.read().decode("utf-8")
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index e8b36ad18..e86164bb0 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -77,7 +77,6 @@ from workbench_constants import (
from workbench_feedback import get_scan_feedback
from workbench_finding_index import index_findings
from workbench_finding_workflows import finding_workflow, register_workflow_scan
-from workbench_local_dedupe import local_dedupe
from workbench_scan_start import (
archive_scan,
insert_running_scan,
@@ -3261,8 +3260,6 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None:
)
elif args.command == "database-info":
result = {"databasePath": str(database_path())}
- elif args.command == "local-dedupe":
- result = local_dedupe(connection, json.load(sys.stdin), now())
elif args.command == "finding-workflow":
result = finding_workflow(connection, json.load(sys.stdin), now())
else:
diff --git a/plugins/codex-security/scripts/workbench_local_dedupe.py b/plugins/codex-security/scripts/workbench_local_dedupe.py
deleted file mode 100644
index 9f0c7f322..000000000
--- a/plugins/codex-security/scripts/workbench_local_dedupe.py
+++ /dev/null
@@ -1,136 +0,0 @@
-"""Prepare and cache embeddings without publishing findings to an HTTP service."""
-
-from __future__ import annotations
-
-import argparse
-import hashlib
-import json
-import math
-import sqlite3
-import sys
-from pathlib import Path
-from typing import Any
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from workbench_finding_index import upsert_finding
-from workbench_target_state import stable_target_id
-
-
-def cache_key(space: str, document: str) -> str:
- return hashlib.sha256(f"{space}\0{document}".encode()).hexdigest()
-
-
-def local_dedupe(
- connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str
-) -> dict[str, Any]:
- action = payload["action"]
- try:
- with connection:
- connection.execute("BEGIN IMMEDIATE")
- if action == "prepare":
- return prepare(connection, payload, timestamp)
- if action == "embed":
- for entry in payload["entries"]:
- row = connection.execute(
- "SELECT details_json FROM findings WHERE id = ?", (entry["findingId"],)
- ).fetchone()
- if (
- row is None
- or cache_key(payload["space"], row["details_json"]) != entry["cacheKey"]
- ):
- raise ValueError("finding_changed")
- embedding = entry["embedding"]
- vector = embedding["vector"]
- if (
- embedding["model"] != payload["model"]
- or len(vector) != payload["dimensions"]
- or not all(isinstance(v, (int, float)) and math.isfinite(v) for v in vector)
- or not math.isfinite(math.hypot(*vector))
- or math.hypot(*vector) == 0
- ):
- raise ValueError("embedding_failed")
- connection.execute(
- "INSERT INTO local_finding_embeddings (finding_id, model, vector_json, cache_key) "
- "VALUES (?, ?, ?, ?) ON CONFLICT(finding_id) DO UPDATE SET "
- "model = excluded.model, vector_json = excluded.vector_json, "
- "cache_key = excluded.cache_key",
- (
- entry["findingId"],
- embedding["model"],
- json.dumps(vector, allow_nan=False),
- entry["cacheKey"],
- ),
- )
- return {}
- raise ValueError("invalid_request")
- except sqlite3.IntegrityError:
- return {"error": "finding_conflict"}
- except ValueError as exc:
- return {"error": str(exc)}
-
-
-def prepare(
- connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str
-) -> dict[str, Any]:
- # Artifact seals establish consistency, not permission to select another local corpus.
- target = Path(payload["repositoryPath"]).resolve()
- registered = connection.execute(
- "SELECT id FROM security_targets WHERE current_path = ?", (str(target),)
- ).fetchone()
- target_id = registered["id"] if registered is not None else stable_target_id(target)
- if (
- payload["anchorRepositoryId"] != target_id
- or payload.get("repositoryId", target_id) != target_id
- ):
- raise ValueError("target_mismatch")
- if not payload["findings"]:
- return {"cacheKeys": {}, "findingsToEmbed": []}
- # A sealed historical scan selects logical IDs; never overwrite their current bodies.
- for finding in payload["findings"]:
- existing = connection.execute(
- "SELECT fingerprint, rule_id, identity_anchor, identity_instance, details_json "
- "FROM findings WHERE id = ?",
- (finding["findingId"],),
- ).fetchone()
- identity = (
- finding["fingerprints"]["primary"],
- finding["ruleId"],
- finding["identity"]["anchor"],
- finding["identity"].get("instance"),
- )
- if existing is not None and tuple(existing)[:4] != identity:
- raise sqlite3.IntegrityError("Conflicting finding identity")
- if existing is None or existing["details_json"] is None:
- upsert_finding(connection, finding, timestamp, payload["anchorRepositoryId"])
- else:
- connection.execute(
- "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
- (payload["anchorRepositoryId"], finding["findingId"]),
- )
- repository_id = payload.get("repositoryId")
- rows = connection.execute(
- "SELECT findings.id, findings.details_json, embeddings.cache_key FROM findings "
- "LEFT JOIN local_finding_embeddings AS embeddings ON findings.id = embeddings.finding_id "
- + (
- "WHERE EXISTS (SELECT 1 FROM finding_repositories WHERE finding_id = findings.id "
- "AND repository_id = ?) "
- if repository_id is not None
- else ""
- )
- + "ORDER BY findings.created_at, findings.id",
- (repository_id,) if repository_id is not None else (),
- )
- cache_keys = {}
- findings_to_embed = []
- for row in rows:
- if row["details_json"] is None:
- raise ValueError("finding_not_indexed")
- key = cache_key(payload["space"], row["details_json"])
- cache_keys[row["id"]] = key
- if key != row["cache_key"]:
- findings_to_embed.append(json.loads(row["details_json"]))
- return {"cacheKeys": cache_keys, "findingsToEmbed": findings_to_embed}
-
-
-if __name__ == "__main__":
- argparse.ArgumentParser(description=__doc__).parse_args()
diff --git a/plugins/codex-security/tests/test_workbench_local_dedupe.py b/plugins/codex-security/tests/test_workbench_local_dedupe.py
index 607031a9d..b09236365 100644
--- a/plugins/codex-security/tests/test_workbench_local_dedupe.py
+++ b/plugins/codex-security/tests/test_workbench_local_dedupe.py
@@ -1,119 +1,11 @@
from __future__ import annotations
-import json
import sqlite3
from contextlib import closing
-from pathlib import Path
-
-import pytest
TIMESTAMP = "2026-01-01T00:00:00Z"
-def finding(index: int) -> dict:
- source = Path(__file__).resolve().parents[1] / "examples/legacy-completed-scan/findings.json"
- result = json.loads(source.read_text())["findings"][0]
- result["findingId"] = f"finding-{index}"
- result["fingerprints"]["primary"] = f"fingerprint-{index}"
- return result
-
-
-def request(api, connection, action, **values):
- return api["local_dedupe"](
- connection,
- {
- "action": action,
- "space": "synthetic-v1",
- "model": "synthetic",
- "dimensions": 2,
- **values,
- },
- TIMESTAMP,
- )
-
-
-def prepare(api, connection, findings, **values):
- repository_path = str(Path(__file__).resolve().parent)
- with connection:
- target_id = api["ensure_security_target"](connection, repository_path)
- return request(
- api,
- connection,
- "prepare",
- findings=findings,
- anchorRepositoryId=target_id,
- repositoryId=target_id,
- repositoryPath=repository_path,
- **values,
- )
-
-
-def entry(prepared, index=0, **values):
- finding_id = prepared["findingsToEmbed"][index]["findingId"]
- return {
- "findingId": finding_id,
- "cacheKey": prepared["cacheKeys"][finding_id],
- "embedding": {"model": "synthetic", "vector": [1, 0]},
- **values,
- }
-
-
-def test_cache_reuse_survives_service_embedding_writes(workbench_api, workbench_db):
- first = prepare(workbench_api, workbench_db, [finding(1)])
- assert first["findingsToEmbed"] == [finding(1)]
- assert request(workbench_api, workbench_db, "embed", entries=[entry(first)]) == {}
- assert prepare(workbench_api, workbench_db, [finding(1)]) == {
- "cacheKeys": first["cacheKeys"],
- "findingsToEmbed": [],
- }
- assert prepare(workbench_api, workbench_db, [finding(1)], space="synthetic-v2")[
- "findingsToEmbed"
- ] == [finding(1)]
- with workbench_db:
- workbench_db.execute(
- "INSERT INTO finding_embeddings (finding_id, model, vector_json) VALUES (?, ?, ?)",
- ("finding-1", "synthetic", "[0, 1]"),
- )
- assert prepare(workbench_api, workbench_db, [finding(1)])["findingsToEmbed"] == []
-
-
-def test_preparation_returns_all_cache_keys_but_only_uncached_bodies(workbench_api, workbench_db):
- first = prepare(workbench_api, workbench_db, [finding(1)])
- request(workbench_api, workbench_db, "embed", entries=[entry(first)])
- mixed = prepare(workbench_api, workbench_db, [finding(1), finding(2)])
- assert mixed["findingsToEmbed"] == [finding(2)]
- assert set(mixed["cacheKeys"]) == {"finding-1", "finding-2"}
- assert mixed["cacheKeys"]["finding-1"] == first["cacheKeys"]["finding-1"]
- assert prepare(workbench_api, workbench_db, []) == {"cacheKeys": {}, "findingsToEmbed": []}
-
-
[email protected]("failure", ["stale", "dimensions", "zero", "nonfinite"])
-def test_embedding_batch_rolls_back_on_invalid_or_stale_input(workbench_api, workbench_db, failure):
- prepared = prepare(workbench_api, workbench_db, [finding(1), finding(2)])
- second = entry(prepared, 1)
- if failure == "stale":
- second["cacheKey"] = "old-input"
- else:
- second["embedding"]["vector"] = {
- "dimensions": [1],
- "zero": [0, 0],
- "nonfinite": [float("nan"), 0],
- }[failure]
- result = request(workbench_api, workbench_db, "embed", entries=[entry(prepared), second])
- assert result == {"error": "finding_changed" if failure == "stale" else "embedding_failed"}
- assert workbench_db.execute("SELECT COUNT(*) FROM local_finding_embeddings").fetchone()[0] == 0
-
-
-def test_preparation_conflict_rolls_back_new_findings(workbench_api, workbench_db):
- prepare(workbench_api, workbench_db, [finding(1)])
- conflicting = finding(1)
- conflicting["identity"]["anchor"] = "different-anchor"
- assert prepare(workbench_api, workbench_db, [finding(2), conflicting]) == {
- "error": "finding_conflict"
- }
- assert workbench_db.execute("SELECT COUNT(*) FROM findings").fetchone()[0] == 1
-
-
def test_embedding_cache_migration_preserves_existing_vectors(workbench_api, tmp_path):
with closing(sqlite3.connect(tmp_path / "legacy.sqlite3")) as connection:
connection.row_factory = sqlite3.Row
@@ -171,18 +63,3 @@ def test_local_cache_migration_keeps_service_and_local_rows_separate(workbench_a
"SELECT finding_id, cache_key FROM local_finding_embeddings"
)
] == [("local", "cached-input")]
-
-
-def test_finding_body_update_invalidates_local_search_and_pending_groups(
- workbench_api, workbench_db
-):
- prepared = prepare(workbench_api, workbench_db, [finding(1)])
- request(workbench_api, workbench_db, "embed", entries=[entry(prepared)])
- changed = {**finding(1), "title": "Updated evidence from another scan"}
- with workbench_db:
- workbench_db.execute(
- "UPDATE findings SET details_json = ? WHERE id = ?",
- (json.dumps(changed, sort_keys=True), "finding-1"),
- )
- assert workbench_db.execute("SELECT COUNT(*) FROM local_finding_embeddings").fetchone()[0] == 0
- assert prepare(workbench_api, workbench_db, [finding(1)])["findingsToEmbed"] == [changed]
diff --git a/sdk/typescript/docs/findings-service.md b/sdk/typescript/docs/findings-service.md
index 4d6d26416..fab1dc1a4 100644
--- a/sdk/typescript/docs/findings-service.md
+++ b/sdk/typescript/docs/findings-service.md
@@ -399,13 +399,23 @@ and JSON on stdin. For example, `request.json` selects the existing database:
Use an absolute Windows path on Windows. The retained helper commands accept
these `payload` fields:
-| Command | Payload |
-| --------------------------- | ------------------------------------------------------------------------------------------------------- |
-| `store-findings` | `entries` containing finding and embedding records; optional `repositoryId`. |
-| `list-stored-findings` | Positive integer `limit` and non-negative integer `offset`. |
-| `find-potential-duplicates` | `findingId` and `scope`: either `{ "repositoryId": "REPOSITORY_ID" }` or `{ "allRepositories": true }`. |
-| `store-dedupe-groups` | `groups`, an array of finding-ID arrays. |
-| `list-dedupe-groups` | `findingId`. |
+| Command | Payload |
+| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `store-findings` | `entries` containing finding and embedding records; optional `repositoryId`. |
+| `list-stored-findings` | Positive integer `limit` and non-negative integer `offset`. |
+| `find-potential-duplicates` | `findingId` and `scope`: either `{ "repositoryId": "REPOSITORY_ID" }` or `{ "allRepositories": true }`. |
+| `store-dedupe-groups` | `groups`, an array of finding-ID arrays. |
+| `list-dedupe-groups` | `findingId`. |
+| `local-dedupe` | `action: "prepare"` with `findings`, `repositoryPath`, `anchorRepositoryId`, optional `repositoryId`, and `space`; or `action: "embed"` with `entries`, `space`, `model`, and `dimensions`. |
+
+Local embedding preparation and cache writes now use the bundled Node helper
+instead of `workbench_db.py local-dedupe`. Each embed entry contains `findingId`,
+the preparation response's `cacheKey`, and an `embedding` with `model` and
+`vector`. Preparation returns `cacheKeys` and `findingsToEmbed`; embedding returns
+an empty object. These operations use Node even when `PYTHON` is configured.
+Explicitly selected external plugins retain their Python workbench route.
+Saved-scan lookup and export still require Python, so the complete saved-scan
+deduplication workflow continues to need it.
Retained scan commands such as `workbench_db.py list-findings` keep their existing
interface.
diff --git a/sdk/typescript/src/deduplication/local.ts b/sdk/typescript/src/deduplication/local.ts
index ce7a44368..ff7df1e46 100644
--- a/sdk/typescript/src/deduplication/local.ts
+++ b/sdk/typescript/src/deduplication/local.ts
@@ -8,7 +8,7 @@ import type {
import { CodexSecurityError } from "../errors.js";
import { workflowDigest } from "../finding-workflow.js";
import {
- resolveWorkbenchRuntime,
+ bundledPluginRoot,
workbenchEnvironment,
runWorkbench,
type WorkbenchCommandOptions,
@@ -36,7 +36,7 @@ export interface FindingEmbeddingBinding {
/** Direct workbench adapter. Importing it starts no HTTP or MCP server. */
export class LocalDeduplication {
- private options?: Promise<WorkbenchCommandOptions>;
+ private options?: Promise<Omit<WorkbenchCommandOptions, "python">>;
private cacheKeys: Record<string, string> = {};
private readonly space: string;
private readonly embedding: FindingEmbeddingBinding;
@@ -137,14 +137,9 @@ export class LocalDeduplication {
this.signal?.throwIfAborted();
this.options ??= (async () => {
const environment = workbenchEnvironment(this.environment);
- const [python, pluginRoot] = await resolveWorkbenchRuntime({
- environment,
- protectedRoot: this.repositoryPath,
- signal: this.signal,
- });
return {
- python,
- pluginRoot,
+ pluginRoot: await bundledPluginRoot(),
+ stateDirectory: environment.CODEX_SECURITY_STATE_DIR,
environment,
signal: this.signal,
protectedRoot: [
diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts
index a56e7030d..f1bf4d7ae 100644
--- a/sdk/typescript/src/runtime.ts
+++ b/sdk/typescript/src/runtime.ts
@@ -1670,24 +1670,30 @@ export async function runWorkbench(
): Promise<JsonObject> {
const script = join(options.pluginRoot, "scripts", "workbench_db.py");
const signal = options.signal;
+ // Explicitly selected external plugins retain their own workbench protocol.
+ const nativeLocalDedupe =
+ args[0] === "local-dedupe" &&
+ (await sameFile(options.pluginRoot, await bundledPluginRoot()));
const run = async (
arguments_: readonly string[],
input?: string,
archiveHandshake = false,
): Promise<string> => {
- const native = [
- "database-info",
- "store-findings",
- "list-stored-findings",
- "store-external-findings",
- "list-external-findings",
- "find-potential-duplicates",
- "store-dedupe-groups",
- "list-dedupe-groups",
- "dashboard",
- "severity-classification",
- "read-severity-classification",
- ].includes(arguments_[0] ?? "");
+ const native =
+ nativeLocalDedupe ||
+ [
+ "database-info",
+ "store-findings",
+ "list-stored-findings",
+ "store-external-findings",
+ "list-external-findings",
+ "find-potential-duplicates",
+ "store-dedupe-groups",
+ "list-dedupe-groups",
+ "dashboard",
+ "severity-classification",
+ "read-severity-classification",
+ ].includes(arguments_[0] ?? "");
const node =
native && process.versions["bun"]
? await resolveTrustedExecutable("node", options.environment, [
@@ -1713,12 +1719,8 @@ export async function runWorkbench(
? (options.stateDirectory ??
codexSecurityStateDirectory(options.environment))
: undefined;
- // Workbench text fields can contain NUL or exceed OS command-line limits.
- const framedArguments = !native;
- let program = archiveHandshake ? ARCHIVE_REGISTRATION_PROGRAM : undefined;
- if (framedArguments)
- program =
- WORKBENCH_ARGUMENTS_PROGRAM + (program ?? WORKBENCH_SCRIPT_PROGRAM);
+ // Validate one JSON document before framing its unchanged source as payload.
+ if (nativeLocalDedupe && input !== undefined) JSON.parse(input);
const result = await runCodexCommand(
{ command },
native
@@ -1728,28 +1730,34 @@ export async function runWorkbench(
"-X",
"utf8",
"-B",
- ...(program === undefined ? [] : ["-c", program]),
+ "-c",
+ WORKBENCH_ARGUMENTS_PROGRAM +
+ (archiveHandshake
+ ? ARCHIVE_REGISTRATION_PROGRAM
+ : WORKBENCH_SCRIPT_PROGRAM),
script,
- ...(framedArguments ? [] : arguments_),
],
pluginHelperEnvironment(node?.environment ?? options.environment),
// The SDK owns configuration normalization; the helper receives its resolved location.
- native
- ? JSON.stringify(
- arguments_[0] === "database-info"
- ? stateDirectory
- : {
- stateDirectory,
- payload: input === undefined ? undefined : JSON.parse(input),
- },
- )
- : input,
+ nativeLocalDedupe
+ ? `{"stateDirectory":${JSON.stringify(stateDirectory)},"payload":${input ?? "null"}}`
+ : native
+ ? JSON.stringify(
+ arguments_[0] === "database-info"
+ ? stateDirectory
+ : {
+ stateDirectory,
+ payload:
+ input === undefined ? undefined : JSON.parse(input),
+ },
+ )
+ : input,
signal,
archiveHandshake,
- // Match native argv's UTF-8 encoding for the private argument frame.
- framedArguments
- ? `${JSON.stringify(arguments_.map((argument) => argument.toWellFormed()))}\n`
- : undefined,
+ // Frame Python arguments on stdin to preserve NUL and long diagnostics.
+ native
+ ? undefined
+ : `${JSON.stringify(arguments_.map((argument) => argument.toWellFormed()))}\n`,
);
if (!result.success) {
throw new Error(
diff --git a/sdk/typescript/tests-ts/local-dedupe-routing.test.ts b/sdk/typescript/tests-ts/local-dedupe-routing.test.ts
new file mode 100644
index 000000000..b04bcd341
--- /dev/null
+++ b/sdk/typescript/tests-ts/local-dedupe-routing.test.ts
@@ -0,0 +1,365 @@
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { afterEach, expect, mock, spyOn, test } from "bun:test";
+import { Tiktoken } from "js-tiktoken/lite";
+import cl100kBase from "js-tiktoken/ranks/cl100k_base";
+import { LocalDeduplication } from "../src/deduplication/local.js";
+import type { Finding, FindingsDocument } from "../src/models.js";
+import { resolvePluginPython, runWorkbench } from "../src/runtime.js";
+import {
+ EMBEDDING_DIMENSIONS,
+ EMBEDDING_MODEL,
+} from "../src/server/embeddings.js";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { sha256 } from "./support/finding-identity.js";
+import { createTemporaryDirectories } from "./support/temporary-directories.js";
+
+const directories = createTemporaryDirectories(true);
+afterEach(directories.cleanup);
+
+async function fixture() {
+ const root = await directories.create("local-dedupe-routing-");
+ const repository = join(root, "repository");
+ await mkdir(repository);
+ const repositoryId = `target_sha256_${sha256(`local-workspace\0${repository}`)}`;
+ const document = JSON.parse(
+ await readFile(
+ join(PLUGIN_ROOT, "examples/completed-scan/findings.json"),
+ "utf8",
+ ),
+ ) as FindingsDocument;
+ const finding = document.findings[0]!;
+ const stateDirectory = join(root, "state with spaces");
+ const options = {
+ pluginRoot: PLUGIN_ROOT,
+ environment: process.env,
+ python: join(root, "missing-python"),
+ stateDirectory,
+ protectedRoot: repository,
+ };
+ const prepare = {
+ action: "prepare",
+ repositoryPath: repository,
+ anchorRepositoryId: repositoryId,
+ repositoryId,
+ space: "synthetic-embedding-space",
+ model: "synthetic-embedding",
+ dimensions: 2,
+ findings: [finding],
+ };
+ return { options, prepare, finding, stateDirectory };
+}
+
+test.skipIf(process.platform === "win32")(
+ "uses the configured state path when resolution exposes a trailing filename space",
+ async () => {
+ const { options, prepare, stateDirectory } = await fixture();
+ const selected = `${stateDirectory} `;
+ const local = new LocalDeduplication(
+ { ...options.environment, CODEX_SECURITY_STATE_DIR: `${selected}/.` },
+ { repositoryId: prepare.repositoryId },
+ prepare.repositoryPath,
+ );
+ await local.prepare([], prepare.repositoryId);
+ expect(existsSync(join(selected, "workbench.sqlite3"))).toBe(true);
+ expect(existsSync(join(stateDirectory, "workbench.sqlite3"))).toBe(false);
+ },
+);
+
+test("bundled local dedupe retains raw integer bytes and SDK number values without Python", async () => {
+ const { options, prepare, finding } = await fixture();
+ finding.extensions = {
+ integer: "9007199254740993",
+ text: "synthetic evidence \0 λ",
+ };
+ const input = JSON.stringify(prepare).replace(
+ '"integer":"9007199254740993"',
+ '"integer":9007199254740993',
+ );
+ const prepared = await runWorkbench(options, ["local-dedupe"], input);
+ expect(prepared["findingsToEmbed"] as unknown).toEqual([
+ {
+ ...finding,
+ extensions: { ...finding.extensions, integer: 9007199254740992 },
+ },
+ ]);
+ const cacheKeys = prepared["cacheKeys"] as Record<string, string>;
+ expect(Object.keys(cacheKeys)).toEqual([finding.findingId]);
+ const stored = spawnSync(
+ "node",
+ [
+ "-e",
+ `const { DatabaseSync } = require("node:sqlite");
+const database = new DatabaseSync(process.argv[1]);
+console.log(database.prepare("SELECT details_json FROM findings WHERE id = ?").get(process.argv[2]).details_json);
+database.close();`,
+ join(options.stateDirectory, "workbench.sqlite3"),
+ finding.findingId,
+ ],
+ { encoding: "utf8" },
+ );
+ expect(stored.status, stored.stderr).toBe(0);
+ const storedBody = stored.stdout.trim();
+ expect(storedBody).toMatch(/"integer":\s*9007199254740993/u);
+ expect(cacheKeys[finding.findingId]).toBe(
+ sha256(`${prepare.space}\0${storedBody}`),
+ );
+
+ expect(
+ await runWorkbench(
+ options,
+ ["local-dedupe"],
+ JSON.stringify({
+ action: "embed",
+ space: prepare.space,
+ model: prepare.model,
+ dimensions: prepare.dimensions,
+ entries: [
+ {
+ findingId: finding.findingId,
+ cacheKey: cacheKeys[finding.findingId],
+ embedding: { model: prepare.model, vector: [1, 0] },
+ },
+ ],
+ }),
+ ),
+ ).toEqual({});
+ expect(await runWorkbench(options, ["local-dedupe"], input)).toEqual({
+ cacheKeys,
+ findingsToEmbed: [],
+ });
+ const largeFinding = {
+ ...finding,
+ extensions: { ...finding.extensions, integer: 9007199254740992 },
+ } as Finding;
+ const embed = mock(async (batch: readonly Finding[]) => {
+ expect(batch).toEqual([largeFinding]);
+ return batch.map(() => ({ model: prepare.model, vector: [1, 0] }));
+ });
+ const local = new LocalDeduplication(
+ {
+ ...options.environment,
+ CODEX_SECURITY_STATE_DIR: options.stateDirectory,
+ PYTHON: options.python,
+ },
+ { repositoryId: prepare.repositoryId },
+ prepare.repositoryPath,
+ undefined,
+ undefined,
+ {
+ model: prepare.model,
+ dimensions: prepare.dimensions,
+ cacheNamespace: "synthetic-adapter",
+ embedder: { embed },
+ },
+ );
+ await local.prepare([largeFinding], prepare.repositoryId);
+ await local.prepare([largeFinding], prepare.repositoryId);
+ expect(embed).toHaveBeenCalledTimes(1);
+});
+
+test("default embeddings retain historical key order and SDK number values and reuse their cache", async () => {
+ const { options, prepare, finding } = await fixture();
+ await runWorkbench(options, ["local-dedupe"], JSON.stringify(prepare));
+ const storedBody =
+ `{"z":true,"findingId":${JSON.stringify(finding.findingId)},` +
+ '"extensions":{"😀":"astral","z":{"z":2,"a":1},"10":"ten","2":"two","integer":9007199254740993,' +
+ '"\\ue000":"BMP","a":[{"z":4,"a":3}]},"a":false}';
+ const stored = spawnSync(
+ "node",
+ [
+ "-e",
+ `const { DatabaseSync } = require("node:sqlite");
+const database = new DatabaseSync(process.argv[1]);
+database.prepare("UPDATE findings SET details_json = ? WHERE id = ?").run(process.argv[3], process.argv[2]);
+database.close();`,
+ join(options.stateDirectory, "workbench.sqlite3"),
+ finding.findingId,
+ storedBody,
+ ],
+ { encoding: "utf8" },
+ );
+ expect(stored.status, stored.stderr).toBe(0);
+ const expectedInput = JSON.stringify({
+ a: false,
+ extensions: {
+ "2": "two",
+ "10": "ten",
+ a: [{ a: 3, z: 4 }],
+ integer: 9007199254740992,
+ z: { a: 1, z: 2 },
+ "\ue000": "BMP",
+ "😀": "astral",
+ },
+ findingId: finding.findingId,
+ z: true,
+ });
+ const encoding = new Tiktoken(cl100kBase);
+ const request = spyOn(globalThis, "fetch").mockResolvedValue(
+ Response.json({
+ model: EMBEDDING_MODEL,
+ data: [
+ {
+ index: 0,
+ embedding: [1, ...Array<number>(EMBEDDING_DIMENSIONS - 1).fill(0)],
+ },
+ ],
+ }),
+ );
+ try {
+ const local = new LocalDeduplication(
+ {
+ ...options.environment,
+ CODEX_SECURITY_STATE_DIR: options.stateDirectory,
+ PYTHON: options.python,
+ OPENAI_API_KEY: "synthetic-key",
+ },
+ { repositoryId: prepare.repositoryId },
+ prepare.repositoryPath,
+ );
+ await local.prepare([finding], prepare.repositoryId);
+ expect(request).toHaveBeenCalledTimes(1);
+ const body = JSON.parse(String(request.mock.calls[0]![1]!.body)) as {
+ input: number[][];
+ };
+ expect(body.input).toHaveLength(1);
+ expect(body.input[0]).toEqual(encoding.encode(expectedInput, [], []));
+ expect(encoding.decode(body.input[0]!)).toBe(expectedInput);
+ await local.prepare([finding], prepare.repositoryId);
+ expect(request).toHaveBeenCalledTimes(1);
+ } finally {
+ request.mockRestore();
+ }
+});
+
+test("pre-aborted native local dedupe does not initialize its database", async () => {
+ const { options, prepare, stateDirectory } = await fixture();
+ await expect(
+ runWorkbench(
+ { ...options, signal: AbortSignal.abort() },
+ ["local-dedupe"],
+ JSON.stringify(prepare),
+ ),
+ ).rejects.toMatchObject({ name: "AbortError" });
+ expect(existsSync(join(stateDirectory, "workbench.sqlite3"))).toBe(false);
+});
+
+test("malformed local dedupe input cannot replace the selected state directory", async () => {
+ const { options, prepare, stateDirectory } = await fixture();
+ const otherState = join(stateDirectory, "..", "alternate state");
+ const input =
+ JSON.stringify(prepare) + ',"stateDirectory":' + JSON.stringify(otherState);
+ await expect(
+ runWorkbench(options, ["local-dedupe"], input),
+ ).rejects.toMatchObject({ cause: expect.any(SyntaxError) });
+ expect(existsSync(stateDirectory)).toBe(false);
+ expect(existsSync(otherState)).toBe(false);
+});
+
+test("explicit external plugins retain their Python local dedupe protocol", async () => {
+ const { options, stateDirectory } = await fixture();
+ const pluginRoot = join(stateDirectory, "external-plugin");
+ await mkdir(join(pluginRoot, "scripts"), { recursive: true });
+ await writeFile(
+ join(pluginRoot, "scripts", "workbench_db.py"),
+ `import json, sys
+request = json.load(sys.stdin)
+print(json.dumps({"action": request["action"], "arguments": sys.argv[1:]}))
+`,
+ );
+ const python = await resolvePluginPython({
+ environment: options.environment,
+ protectedRoot: options.protectedRoot,
+ });
+ expect(
+ await runWorkbench(
+ { ...options, pluginRoot, python },
+ ["local-dedupe"],
+ '{"action":"embed","entries":[]}',
+ ),
+ ).toEqual({ action: "embed", arguments: ["local-dedupe"] });
+});
+
+for (const failure of [
+ "1e400",
+ "NaN",
+ "Infinity",
+ "-Infinity",
+ "BLOB finding ID",
+]) {
+ test(`native local dedupe commits before reporting a stored ${failure} response error`, async () => {
+ const { options, prepare, finding, stateDirectory } = await fixture();
+ await runWorkbench(options, ["local-dedupe"], JSON.stringify(prepare));
+ const database = join(stateDirectory, "workbench.sqlite3");
+ function sql(script: string, ...args: string[]) {
+ const result = spawnSync(
+ "node",
+ [
+ "-e",
+ `const { DatabaseSync } = require("node:sqlite");
+const database = new DatabaseSync(process.argv[1]);
+${script}
+database.close();`,
+ database,
+ ...args,
+ ],
+ { encoding: "utf8" },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ return result.stdout;
+ }
+ if (failure === "BLOB finding ID") {
+ sql(
+ `
+database.exec("DELETE FROM finding_repositories");
+database.prepare("UPDATE findings SET id = CAST(id AS BLOB) WHERE id = ?")
+ .run(process.argv[2]);
+`,
+ finding.findingId,
+ );
+ } else {
+ sql(
+ 'database.prepare("UPDATE findings SET details_json = ? WHERE id = ?").run(...process.argv.slice(2));',
+ `{"evidence":[${failure}]}`,
+ finding.findingId,
+ );
+ }
+ const newId = `${finding.findingId}-new`;
+ const payload = {
+ ...prepare,
+ repositoryId: undefined,
+ findings: [
+ {
+ ...finding,
+ findingId: newId,
+ fingerprints: {
+ ...finding.fingerprints,
+ primary: `${finding.fingerprints.primary}-new`,
+ },
+ },
+ ],
+ };
+ const result = spawnSync(
+ "node",
+ [join(PLUGIN_ROOT, "mcp", "helpers.mjs"), "local-dedupe"],
+ {
+ input: JSON.stringify({ stateDirectory, payload }),
+ encoding: "utf8",
+ },
+ );
+ expect(result.status).toBe(1);
+ expect(result.stdout).toBe("");
+ if (failure === "BLOB finding ID")
+ expect(result.stderr).toMatch(/key|blob|bytes/iu);
+ else expect(result.stderr).toContain("non-finite");
+ expect(
+ sql(
+ `console.log(database.prepare("SELECT COUNT(*) AS total FROM findings f JOIN finding_repositories r ON r.finding_id = f.id WHERE f.id = ? AND r.repository_id = ?").get(...process.argv.slice(2)).total);`,
+ newId,
+ prepare.anchorRepositoryId,
+ ).trim(),
+ ).toBe("1");
+ });
+}
diff --git a/sdk/typescript/tests-ts/local-dedupe.test.ts b/sdk/typescript/tests-ts/local-dedupe.test.ts
index 65c3fe0d4..35b9b74d4 100644
--- a/sdk/typescript/tests-ts/local-dedupe.test.ts
+++ b/sdk/typescript/tests-ts/local-dedupe.test.ts
@@ -1,10 +1,7 @@
import { setFindingIdentity, sha256 } from "./support/finding-identity.js";
import { existsSync } from "node:fs";
import { getEventListeners } from "node:events";
-import {
- setTimeout as delay,
- setImmediate as nextTurn,
-} from "node:timers/promises";
+import { setImmediate as nextTurn } from "node:timers/promises";
import { readFile, writeFile, mkdir, chmod, unlink } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, expect, mock, spyOn, test } from "bun:test";
@@ -43,7 +40,9 @@ const vector = [1, ...Array<number>(EMBEDDING_DIMENSIONS - 1).fill(0)];
for (const mode of ["direct", "fresh workflow", "resumed workflow"]) {
test.skipIf(process.platform === "win32")(
- `directory dedupe rejects repository Python before probing it from another directory (${mode})`,
+ mode === "direct"
+ ? "direct directory dedupe uses native helpers without probing repository Python"
+ : `directory dedupe rejects repository Python before probing it from another directory (${mode})`,
async () => {
const f = await fixture();
const options = {
@@ -66,19 +65,31 @@ for (const mode of ["direct", "fresh workflow", "resumed workflow"]) {
);
await chmod(python, 0o700);
expect(process.cwd()).not.toBe(f.repository);
- await expect(
- deduplicateScanDirectoryInternal(f.scanDir, options, {
- environment: {
- ...f.environment,
- PYTHON: python,
- TEST_PYTHON_PROBE: marker,
- },
- runWorkbench: rejecting("Repository Python reached the workbench"),
- reviewer: emptyNeighborhoodReviewer(),
- }),
- ).rejects.toThrow("PYTHON interpreter is unavailable or unusable");
+ const result = deduplicateScanDirectoryInternal(f.scanDir, options, {
+ environment: {
+ ...f.environment,
+ PYTHON: python,
+ TEST_PYTHON_PROBE: marker,
+ },
+ ...(mode === "direct"
+ ? {}
+ : {
+ runWorkbench: rejecting(
+ "Repository Python reached the workbench",
+ ),
+ }),
+ reviewer: emptyNeighborhoodReviewer(),
+ });
+ if (mode === "direct") {
+ expect((await result).deduplicationStatus).toBe("completed");
+ expect(f.embed).toHaveBeenCalledTimes(1);
+ } else {
+ await expect(result).rejects.toThrow(
+ "PYTHON interpreter is unavailable or unusable",
+ );
+ expect(f.embed).not.toHaveBeenCalled();
+ }
expect(existsSync(marker)).toBe(false);
- expect(f.embed).not.toHaveBeenCalled();
},
);
}
@@ -1232,43 +1243,6 @@ test("local workbench calls receive cancellation and cannot report success after
expect(actions).toEqual(["prepare", "embed", "neighbors", "commit"]);
});
-test.skipIf(process.platform === "win32")(
- "local preparation cancels an active Python probe",
- async () => {
- const f = await fixture();
- const python = join(f.root, "slow-python");
- const marker = join(f.root, "probe-started");
- await writeFile(
- python,
- '#!/bin/sh\nprintf started > "$TEST_PYTHON_PROBE"\nexec sleep 10\n',
- );
- await chmod(python, 0o700);
- const controller = new AbortController();
- try {
- const local = new LocalDeduplication(
- { ...f.environment, PYTHON: python, TEST_PYTHON_PROBE: marker },
- { repositoryId: f.targetId },
- f.repository,
- controller.signal,
- undefined,
- f.embedding,
- );
- const outcome = local
- .prepare(f.document.findings, f.targetId)
- .catch((error: unknown) => error);
- const completion = outcome.then((error) => ({ error }));
- while (!existsSync(marker)) {
- const exited = await Promise.race([delay(10), completion]);
- if (exited) throw exited.error;
- }
- controller.abort();
- expect(await outcome).toMatchObject({ name: "AbortError" });
- } finally {
- controller.abort();
- }
- },
-);
-
test("local embeddings preserve an endpoint from a copied Windows environment", async () => {
const f = await fixture();
const finding = f.document.findings[0]!;