OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor!: prepare local deduplication in Node (#1840)
Michael D'Angelo1503a508778d
diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.tsindex 0993c9db8..e25974558 100644--- a/plugins/codex-security/mcp-app/helpers-main.ts+++ b/plugins/codex-security/mcp-app/helpers-main.ts@@ -23,7 +23,11 @@ import { validateTrackingSourceCommand } from "./src/helpers/validate-tracking-simport { deepReviewInputCommand } from "./src/helpers/deep-review-input";import { inventoryCommand } from "./src/helpers/generate-inventory";import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes";-import { escapeControls, stringifyJson } from "./src/helpers/json";+import {+ escapeControls,+ requireFiniteNumbers,+ stringifyJson,+} from "./src/helpers/json";import { decodeUtf8 } from "./src/helpers/utf8";// Importing the bundled parser does not invoke its CLI adapter.@@ -68,6 +72,8 @@ function runHelper(): void {const workbenchUsage: Record<string, string> = {"database-info":"Usage: database-info (reads a JSON absolute state-directory string from stdin)",+ "local-dedupe":+ "Usage: local-dedupe\nReads a JSON object from stdin with an absolute stateDirectory and a prepare or embed payload.","store-findings":"Usage: store-findings\nReads a JSON object from stdin with an absolute stateDirectory and payload.entries containing finding and embedding records; payload.repositoryId is optional.","list-stored-findings":@@ -145,6 +151,8 @@ function runHelper(): void {values["scan-id"] as string | undefined,);}+ // Preparation commits before response serialization, as other workbench writes do.+ if (command === "local-dedupe") requireFiniteNumbers(result);console.log(stringifyJson(result, 0).replace(/[\p{Cc}\p{Cf}]/gu, (character) =>characterdiff --git a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.tsindex 7c199d010..3bff94588 100644--- a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts+++ b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts@@ -11,13 +11,8 @@ import { parseArgs } from "node:util";import { mkdir, readFile, writeFile } from "./helper-files";import { expandHome, windowsFiles, windowsJoin } from "./resolve-security-md";import { decodePosixBytes, encodePosixPath } from "./posix-path";-import { stringifyJson } from "./json";-import {- compare,- print,- reportCommandError,- ArgumentError,-} from "./rank-worklists";+import { compareCodePoints, stringifyJson } from "./json";+import { print, reportCommandError, ArgumentError } from "./rank-worklists";import { decodeUtf8 } from "./utf8";import {ancestors,@@ -432,7 +427,7 @@ export async function generateInventory(return { count: unique.length, output };}const rows = [...rankRows.values()]- .sort((left, right) => compare(left.path, right.path))+ .sort((left, right) => compareCodePoints(left.path, right.path)).map((row) =>Buffer.from(stringifyJson(row, 0).replace(diff --git a/plugins/codex-security/mcp-app/src/helpers/json.ts b/plugins/codex-security/mcp-app/src/helpers/json.tsindex c39898d1e..573edc3fc 100644--- a/plugins/codex-security/mcp-app/src/helpers/json.ts+++ b/plugins/codex-security/mcp-app/src/helpers/json.ts@@ -1,5 +1,14 @@export { isRecord as object } from "../record.ts";+export function compareCodePoints(left: string, right: string): number {+ const a = Array.from(left, (character) => character.codePointAt(0)!);+ const b = Array.from(right, (character) => character.codePointAt(0)!);+ for (let index = 0; index < Math.min(a.length, b.length); index++) {+ if (a[index] !== b[index]) return a[index]! - b[index]!;+ }+ return a.length - b.length;+}+export function requireFiniteNumbers(value: unknown): void {if (typeof value === "number" && !Number.isFinite(value))throw new TypeError(diff --git a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.tsindex a14f0ee2c..7f3a403d0 100644--- a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts+++ b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts@@ -8,7 +8,7 @@ import {writeFile,} from "./helper-files";import { encodePosixPath } from "./posix-path";-import { formatDiagnostic, object, parseJson } from "./json";+import { compareCodePoints, formatDiagnostic, object, parseJson } from "./json";import { expandHome } from "./resolve-security-md";export interface RankRow {@@ -17,15 +17,6 @@ export interface RankRow {preview?: string;}const blank = (value: string) => value.trim().length === 0;-export function compare(left: string, right: string): number {- const a = Array.from(left, (character) => character.codePointAt(0)!);- const b = Array.from(right, (character) => character.codePointAt(0)!);- for (let index = 0; index < Math.min(a.length, b.length); index++) {- if (a[index] !== b[index]) return a[index]! - b[index]!;- }- return a.length - b.length;-}-function missingNames(expected: Iterable<string>,actual: Iterable<string>,@@ -33,7 +24,7 @@ function missingNames(const present = new Set(actual);return Array.from(expected).filter((name) => !present.has(name))- .sort(compare);+ .sort(compareCodePoints);}export function loadRankRows(path: string): RankRow[] {@@ -83,7 +74,7 @@ export function requireUniquePaths(rows: RankRow[], label: string): void {}if (duplicates.size)throw new Error(- `${label} contains duplicate paths: ${formatDiagnostic([...duplicates].sort(compare))}`,+ `${label} contains duplicate paths: ${formatDiagnostic([...duplicates].sort(compareCodePoints))}`,);}diff --git a/plugins/codex-security/mcp-app/src/workbench/commands.ts b/plugins/codex-security/mcp-app/src/workbench/commands.tsindex c91b9704f..c86a732b6 100644--- a/plugins/codex-security/mcp-app/src/workbench/commands.ts+++ b/plugins/codex-security/mcp-app/src/workbench/commands.ts@@ -1,5 +1,6 @@import { parseJson } from "../helpers/json";import { openWorkbenchDatabase, workbenchDatabasePath } from "./database";+import { localDedupe, type LocalDedupeRequest } from "./local-dedupe";import {listStoredFindings,storeFindings,@@ -88,6 +89,7 @@ export async function workbenchCommand(workbenchDatabasePath(stateDirectory),{deferred:+ command !== "local-dedupe" &&command !== "store-findings" &&command !== "store-external-findings" &&command !== "store-dedupe-groups" &&@@ -95,6 +97,12 @@ export async function workbenchCommand(},);try {+ if (command === "local-dedupe")+ return localDedupe(+ database,+ payload as LocalDedupeRequest,+ new Date().toISOString(),+ );if (command === "store-external-findings")return storeExternalFindings(database,diff --git a/plugins/codex-security/mcp-app/src/workbench/local-dedupe.ts b/plugins/codex-security/mcp-app/src/workbench/local-dedupe.tsnew file mode 100644index 000000000..d18c41a00--- /dev/null+++ b/plugins/codex-security/mcp-app/src/workbench/local-dedupe.ts@@ -0,0 +1,397 @@+import { createHash } from "node:crypto";+import type { DatabaseSync, SQLOutputValue } from "node:sqlite";+import { resolvedPathText } from "../helpers/helper-files";+import {+ compareCodePoints,+ parseJson,+ requireFiniteNumbers,+ stringifyJson,+} from "../helpers/json";+import { decodeUtf8 } from "../helpers/utf8";+import { requireSqliteText } from "./database";+import type { Finding } from "./findings";+import { stableTargetId } from "./migrations";+import { transaction } from "./transaction";++interface Preparation {+ action: "prepare";+ space: string;+ repositoryPath: string;+ anchorRepositoryId: string;+ repositoryId?: string;+ findings: readonly Finding[];+}++interface EmbeddingBatch {+ action: "embed";+ space: string;+ model: string;+ dimensions: number | bigint | boolean;+ entries: readonly {+ findingId: string;+ cacheKey: string;+ embedding: {+ model: string;+ vector: readonly (number | bigint | boolean)[];+ };+ }[];+}++export type LocalDedupeRequest = Preparation | EmbeddingBatch;++class LocalDedupeError extends Error {}++// Historical SQLite rows can contain BLOBs despite their TEXT affinity. Decode+// TEXT explicitly so Node does not silently replace invalid text.+function column(expression: string, name: string): string {+ return `CASE WHEN typeof(${expression}) = 'text' THEN CAST(${expression} AS BLOB)+ ELSE ${expression} END AS ${name}, typeof(${expression}) AS ${name}_type`;+}++function databaseEncoding(database: DatabaseSync): string {+ // CAST(TEXT AS BLOB) returns bytes in SQLite's database encoding.+ return database.prepare("PRAGMA encoding").get()!.encoding as string;+}++function columnReader(encoding: string) {+ const decoder =+ encoding === "UTF-8"+ ? undefined+ : new TextDecoder(encoding, { fatal: true, ignoreBOM: true });+ return (+ row: Record<string, SQLOutputValue>,+ name: string,+ ): SQLOutputValue => {+ const stored = row[name]!;+ if (row[`${name}_type`] !== "text") return stored;+ const bytes = stored as Uint8Array;+ return decoder ? decoder.decode(bytes) : decodeUtf8(Buffer.from(bytes));+ };+}++function cacheKey(space: string, document: SQLOutputValue): string {+ if (space === undefined)+ throw new TypeError("Local deduplication requires an embedding space.");+ let text: string;+ if (document instanceof Uint8Array) {+ // Existing BLOB cache keys hash the bytes representation, not decoded JSON.+ const quote = document.includes(39) && !document.includes(34) ? 34 : 39;+ text = `b${String.fromCharCode(quote)}`;+ for (const byte of document) {+ if (byte === quote || byte === 92)+ text += `\\${String.fromCharCode(byte)}`;+ else if (byte === 9) text += "\\t";+ else if (byte === 10) text += "\\n";+ else if (byte === 13) text += "\\r";+ else if (byte < 32 || byte >= 127)+ text += `\\x${byte.toString(16).padStart(2, "0")}`;+ else text += String.fromCharCode(byte);+ }+ text += String.fromCharCode(quote);+ } else {+ text = document === null ? "None" : String(document);+ }+ requireSqliteText([space, text]);+ return createHash("sha256").update(`${space}\0${text}`).digest("hex");+}++function parseStoredJson(text: string): unknown {+ if (!text.includes("NaN") && !text.includes("Infinity"))+ return parseJson(text);++ // Python's json.loads accepted three extra number literals. Index bare values+ // so JSON.parse still validates the structure and decodes strings normally.+ const values: unknown[] = [];+ try {+ const indexed = text.replace(+ /"(?:[^"\\]|\\[\s\S])*"?|[^\s"[\]{},:]+/gu,+ (token) => {+ if (token.startsWith('"')) return token;+ const value =+ token === "NaN" || token === "Infinity" || token === "-Infinity"+ ? Number(token)+ : parseJson(token);+ return String(values.push(value) - 1);+ },+ );+ return JSON.parse(indexed, (_key, value: unknown) =>+ typeof value === "number" ? values[value] : value,+ );+ } catch (error) {+ // Report malformed JSON against the original document.+ if (error instanceof SyntaxError) return parseJson(text);+ throw error;+ }+}++function parseBody(document: SQLOutputValue): unknown {+ if (typeof document === "string") return parseStoredJson(document);+ if (!(document instanceof Uint8Array))+ throw new TypeError("Stored finding JSON must be text or bytes.");+ const bytes = Buffer.from(document);+ let text: string;+ try {+ // json.loads historically accepted UTF-8, UTF-16 and UTF-32 BLOB bodies.+ if (+ bytes.subarray(0, 4).equals(Buffer.from([0, 0, 0xfe, 0xff])) ||+ bytes.subarray(0, 4).equals(Buffer.from([0xff, 0xfe, 0, 0])) ||+ (bytes.length >= 4 && bytes[0] === 0 && bytes[1] === 0) ||+ (bytes.length >= 4 && bytes[1] === 0 && bytes[2] === 0 && bytes[3] === 0)+ ) {+ const little = bytes[0] !== 0;+ if (bytes.length % 4) throw new Error("Truncated UTF-32 finding JSON.");+ text = "";+ for (let offset = 0; offset < bytes.length; offset += 4)+ text += String.fromCodePoint(+ little ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset),+ );+ } else if (+ bytes.subarray(0, 2).equals(Buffer.from([0xfe, 0xff])) ||+ bytes.subarray(0, 2).equals(Buffer.from([0xff, 0xfe])) ||+ (bytes.length >= 2 && (bytes[0] === 0 || bytes[1] === 0))+ ) {+ if (bytes.length % 2) throw new Error("Truncated UTF-16 finding JSON.");+ const little = bytes[0] !== 0 && bytes[0] !== 0xfe;+ text = (little ? bytes : bytes.swap16()).toString("utf16le");+ } else {+ text = decodeUtf8(bytes);+ }+ } catch (error) {+ throw new LocalDedupeError((error as Error).message);+ }+ return parseStoredJson(text.startsWith("\ufeff") ? text.slice(1) : text);+}++function orderedFinding(value: unknown): unknown {+ if (Array.isArray(value)) return value.map(orderedFinding);+ if (value === null || typeof value !== "object") return value;+ // Python sorted response keys recursively; preserve the SDK embedding input.+ return Object.fromEntries(+ Object.entries(value)+ .sort(([left], [right]) => compareCodePoints(left, right))+ .map(([key, item]) => [key, orderedFinding(item)]),+ );+}++function prepare(+ database: DatabaseSync,+ payload: Preparation,+ timestamp: string,+) {+ const encoding = databaseEncoding(database);+ const value = columnReader(encoding);+ // A sealed scan selects finding IDs, not permission to select another corpus.+ const target = resolvedPathText(payload.repositoryPath, false);+ requireSqliteText([target]);+ const registered = database+ .prepare(+ `SELECT ${column("id", "id")} FROM security_targets WHERE current_path = ?`,+ )+ .get(target);+ const targetId =+ registered === undefined ? stableTargetId(target) : value(registered, "id");+ if (+ payload.anchorRepositoryId !== targetId ||+ (payload.repositoryId !== undefined && payload.repositoryId !== targetId)+ )+ throw new LocalDedupeError("target_mismatch");+ if (payload.findings.length === 0)+ return {+ cacheKeys: new Map<SQLOutputValue, string>(),+ findingsToEmbed: [],+ };++ const existing = database.prepare(+ `SELECT ${["fingerprint", "rule_id", "identity_anchor", "identity_instance", "details_json"].map((name) => column(name, name)).join(", ")}+ FROM findings WHERE id = ?`,+ );+ const insert = database.prepare(+ `INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, identity_instance,+ details_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)+ ON CONFLICT(id) DO UPDATE SET details_json = excluded.details_json,+ updated_at = excluded.updated_at`,+ );+ const link = database.prepare(+ "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",+ );+ for (const finding of payload.findings) {+ const identity = [+ finding.fingerprints.primary,+ finding.ruleId,+ finding.identity.anchor,+ finding.identity.instance ?? null,+ ] as const;+ requireSqliteText([finding.findingId]);+ const current = existing.get(finding.findingId);+ const values =+ current &&+ [+ "fingerprint",+ "rule_id",+ "identity_anchor",+ "identity_instance",+ "details_json",+ ].map((name) => value(current, name));+ if (+ values &&+ values.slice(0, 4).some((value, index) => value !== identity[index])+ )+ throw new LocalDedupeError("finding_conflict");+ // Historical input must never overwrite the current body or its cache key.+ if (values === undefined || values[4] === null) {+ requireSqliteText([...identity, timestamp]);+ try {+ requireFiniteNumbers(finding);+ } catch (error) {+ throw new LocalDedupeError((error as Error).message);+ }+ const body = stringifyJson(finding, 0);+ // SQLite replaces these code points when transcoding bound UTF-8 to UTF-16.+ const details =+ encoding === "UTF-8"+ ? body+ : body+ .replaceAll("\uFFFE", "\\ufffe")+ .replaceAll("\uFFFF", "\\uffff");+ insert.run(finding.findingId, ...identity, details, timestamp, timestamp);+ }+ link.run(payload.anchorRepositoryId, finding.findingId);+ }++ const rows = database.prepare(+ `SELECT ${column("findings.id", "id")}, ${column("findings.details_json", "details")},+ ${column("embeddings.cache_key", "cache_key")} FROM findings+ LEFT JOIN local_finding_embeddings AS embeddings ON findings.id = embeddings.finding_id+ ${+ payload.repositoryId === undefined+ ? ""+ : "WHERE EXISTS (SELECT 1 FROM finding_repositories WHERE finding_id = findings.id AND repository_id = ?)"+ }+ ORDER BY findings.created_at, findings.id`,+ );+ const cacheKeys = new Map<SQLOutputValue, string>();+ const findingsToEmbed: unknown[] = [];+ for (const row of rows.iterate(+ ...(payload.repositoryId === undefined ? [] : [payload.repositoryId]),+ )) {+ const [id, document, savedKey] = ["id", "details", "cache_key"].map(+ (name) => value(row, name),+ );+ if (document === null) throw new LocalDedupeError("finding_not_indexed");+ const key = cacheKey(payload.space, document!);+ cacheKeys.set(id!, key);+ // A cache hit does not parse or rewrite the stored document.+ if (key !== savedKey)+ findingsToEmbed.push(orderedFinding(parseBody(document!)));+ }+ return { cacheKeys, findingsToEmbed };+}++function validVector(+ vector: EmbeddingBatch["entries"][number]["embedding"]["vector"],+): boolean {+ let maximum = 0;+ const numbers: number[] = [];+ for (const value of vector) {+ if (!["number", "bigint", "boolean"].includes(typeof value)) return false;+ const number = Number(value);+ if (typeof value === "bigint" && !Number.isFinite(number))+ throw new RangeError("int too large to convert to float");+ if (!Number.isFinite(number)) return false;+ maximum = Math.max(maximum, Math.abs(number));+ numbers.push(number);+ }+ if (maximum === 0) return false;+ let sum = 0;+ let correction = 0;+ for (const number of numbers) {+ const scaled = number / maximum;+ const term = scaled * scaled - correction;+ const next = sum + term;+ correction = next - sum - term;+ sum = next;+ }+ if (maximum * Math.sqrt(sum) < Number.MAX_VALUE / 2) return true;+ // Near overflow, a rounded sqrt/product can reject a norm that rounds to+ // MAX_VALUE. Compare exact squares in units of 2^-2148 with the overflow midpoint.+ const boundary = ((1n << 2098n) - (1n << 2044n)) ** 2n;+ const bytes = Buffer.allocUnsafe(8);+ let squares = 0n;+ for (const number of numbers) {+ bytes.writeDoubleBE(number);+ const bits = bytes.readBigUInt64BE();+ const exponent = Number((bits >> 52n) & 0x7ffn);+ const significand =+ (bits & ((1n << 52n) - 1n)) | (exponent === 0 ? 0n : 1n << 52n);+ squares +=+ (significand * significand) <<+ BigInt(exponent === 0 ? 0 : 2 * (exponent - 1));+ if (squares >= boundary) return false;+ }+ return true;+}++function embed(database: DatabaseSync, payload: EmbeddingBatch) {+ const value = columnReader(databaseEncoding(database));+ const current = database.prepare(+ `SELECT ${column("details_json", "details")} FROM findings WHERE id = ?`,+ );+ const save = database.prepare(+ `INSERT INTO local_finding_embeddings (finding_id, model, vector_json, cache_key)+ VALUES (?, ?, ?, ?) ON CONFLICT(finding_id) DO UPDATE SET+ model = excluded.model, vector_json = excluded.vector_json, cache_key = excluded.cache_key`,+ );+ for (const entry of payload.entries) {+ requireSqliteText([entry.findingId]);+ const row = current.get(entry.findingId);+ if (+ row === undefined ||+ cacheKey(payload.space, value(row, "details")) !== entry.cacheKey+ )+ throw new LocalDedupeError("finding_changed");+ const { model, vector } = entry.embedding;+ if (+ model !== payload.model ||+ !["number", "bigint", "boolean"].includes(typeof payload.dimensions) ||+ vector.length !== Number(payload.dimensions) ||+ !validVector(vector)+ )+ throw new LocalDedupeError("embedding_failed");+ requireSqliteText([model, entry.cacheKey]);+ save.run(entry.findingId, model, stringifyJson(vector, 0), entry.cacheKey);+ }+ return {};+}++export function localDedupe(+ database: DatabaseSync,+ payload: LocalDedupeRequest,+ timestamp: string,+) {+ try {+ const result = transaction(database, "BEGIN IMMEDIATE", () => {+ if (payload.action === "prepare")+ return prepare(database, payload, timestamp);+ if (payload.action === "embed") return embed(database, payload);+ throw new LocalDedupeError("invalid_request");+ });+ if ("cacheKeys" in result && result.cacheKeys instanceof Map) {+ // Unsupported JSON object keys fail at the response boundary after commit.+ for (const key of result.cacheKeys.keys())+ if (key instanceof Uint8Array)+ throw new TypeError("Finding cache keys cannot contain bytes.");+ return { ...result, cacheKeys: Object.fromEntries(result.cacheKeys) };+ }+ return result;+ } catch (error) {+ if (error instanceof LocalDedupeError || error instanceof SyntaxError)+ return { error: error.message };+ if (+ error instanceof Error &&+ "errcode" in error &&+ (Number(error.errcode) & 0xff) === 19+ )+ return { error: "finding_conflict" };+ throw error;+ }+}diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.tsindex f619eb30b..d462c80b3 100644--- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts+++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts@@ -129,6 +129,15 @@ function repairStableTargets(database: DatabaseSync): boolean {return true;}+export function stableTargetId(path: string): string {+ return (+ "target_sha256_" ++ createHash("sha256")+ .update("local-workspace\0" + path)+ .digest("hex")+ );+}+function backfillTargets(database: DatabaseSync): void {const targets = database.prepare(@@ -145,11 +154,7 @@ function backfillTargets(database: DatabaseSync): void {const path = String(row.target_path);let id = lookup.get(path)?.id;if (id === undefined) {- id =- "target_sha256_" +- createHash("sha256")- .update("local-workspace\0" + path)- .digest("hex");+ id = stableTargetId(path);const timestamp = new Date().toISOString();insert.run(id, path, basename(path), timestamp, timestamp);}diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_local_dedupe.ts b/plugins/codex-security/mcp-app/tests/test_workbench_local_dedupe.tsnew file mode 100644index 000000000..2c67c43f8--- /dev/null+++ b/plugins/codex-security/mcp-app/tests/test_workbench_local_dedupe.ts@@ -0,0 +1,567 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { DatabaseSync } from "node:sqlite";+import { after, test, type TestContext } from "node:test";+import { importSource } from "./import-module.ts";+import { createTemporaryDirectories } from "./support/temporary-directories.ts";+import type * as Local from "../src/workbench/local-dedupe.ts";+import type * as Migrations from "../src/workbench/migrations.ts";+import type { Finding } from "../src/workbench/findings.ts";++const { localDedupe } = (await importSource("src/workbench/local-dedupe.ts", {+ define: {+ // Native bindings are installed beside the generated server bundle.+ "import.meta.url": JSON.stringify(+ new URL(+ "../../../../sdk/typescript/_bundled_plugin/mcp/server.mjs",+ import.meta.url,+ ).href,+ ),+ },+})) as typeof Local;+const { applyMigrations } = (await importSource(+ "src/workbench/migrations.ts",+)) as typeof Migrations;+const temporary = createTemporaryDirectories(true);+after(() => temporary.cleanup());+const timestamp = "2026-01-01T00:00:00Z";+const space = "synthetic-space";++async function fixture(+ t: TestContext,+ encoding: "UTF-8" | "UTF-16le" | "UTF-16be" = "UTF-8",+) {+ const root = await temporary.create("local-dedupe-");+ const database = new DatabaseSync(":memory:");+ t.after(() => database.close());+ database.exec(`PRAGMA encoding = '${encoding}'`);+ database.exec("PRAGMA foreign_keys = ON");+ applyMigrations(database);+ const targetId = `target_sha256_${createHash("sha256").update(`local-workspace\0${root}`).digest("hex")}`;+ const prepare = (+ findings: Finding[],+ overrides: Partial<+ Extract<Local.LocalDedupeRequest, { action: "prepare" }>+ > = {},+ ) =>+ localDedupe(+ database,+ {+ action: "prepare",+ space,+ repositoryPath: root,+ anchorRepositoryId: targetId,+ repositoryId: targetId,+ findings,+ ...overrides,+ },+ timestamp,+ );+ const embed = (+ entries: Extract<Local.LocalDedupeRequest, { action: "embed" }>["entries"],+ ) =>+ localDedupe(+ database,+ { action: "embed", space, model: "synthetic", dimensions: 2, entries },+ timestamp,+ );+ return { root, database, targetId, prepare, embed };+}++function finding(id: string): Finding {+ return {+ findingId: id,+ fingerprints: { primary: `fingerprint-${id}` },+ ruleId: "synthetic-rule",+ identity: { anchor: `anchor-${id}` },+ title: `Finding ${id} λ`,+ evidence: { diagnostic: "complete\0message", count: 9007199254740993n },+ };+}++function prepared(result: ReturnType<typeof localDedupe>) {+ assert.ok("cacheKeys" in result && "findingsToEmbed" in result);+ return result as {+ cacheKeys: Record<string, string>;+ findingsToEmbed: Finding[];+ };+}++function entry(result: ReturnType<typeof prepared>, id: string) {+ return {+ findingId: id,+ cacheKey: result.cacheKeys[id]!,+ embedding: { model: "synthetic", vector: [1, 0] },+ };+}++function cacheKey(body: string) {+ return createHash("sha256").update(`${space}\0${body}`).digest("hex");+}++function snapshot(database: DatabaseSync) {+ return [+ "findings",+ "finding_repositories",+ "local_finding_embeddings",+ "finding_embeddings",+ ].map((table) => database.prepare(`SELECT * FROM ${table} ORDER BY 1`).all());+}++test("preparation retains current body bytes, large integers and independent service embeddings", async (t) => {+ const { database, prepare, embed } = await fixture(t);+ const original = finding("__proto__");+ prepared(prepare([original]));+ const body =+ '{ "evidence":{"count":9007199254740993,"diagnostic":"complete\\u0000message"},' ++ '"title":"Current \\u03bb evidence","identity":{"anchor":"anchor-__proto__"},' ++ '"ruleId":"synthetic-rule","fingerprints":{"primary":"fingerprint-__proto__"},"findingId":"__proto__" }';+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = ?")+ .run(body, original.findingId);+ const first = prepared(prepare([original]));+ assert.equal(first.cacheKeys[original.findingId], cacheKey(body));+ assert.deepEqual(first.findingsToEmbed, [+ { ...original, title: "Current λ evidence" },+ ]);+ assert.deepEqual(Object.keys(first.findingsToEmbed[0]!), [+ "evidence",+ "findingId",+ "fingerprints",+ "identity",+ "ruleId",+ "title",+ ]);+ assert.deepEqual(embed([entry(first, original.findingId)]), {});+ database+ .prepare(+ "INSERT INTO finding_embeddings VALUES (?, 'synthetic', '[0, 1]', NULL)",+ )+ .run(original.findingId);+ assert.deepEqual(prepare([original]), {+ cacheKeys: first.cacheKeys,+ findingsToEmbed: [],+ });+ assert.equal(+ database.prepare("SELECT details_json FROM findings").get()!.details_json,+ body,+ );+ assert.equal(+ database.prepare("SELECT vector_json FROM finding_embeddings").get()!+ .vector_json,+ "[0, 1]",+ );+ const changedSpace = prepared(+ prepare([original], { space: "another-space" }),+ );+ assert.notEqual(changedSpace.cacheKeys[original.findingId], cacheKey(body));+ assert.deepEqual(changedSpace.findingsToEmbed, first.findingsToEmbed);+});++for (const encoding of ["UTF-8", "UTF-16le", "UTF-16be"] as const) {+ test(`${encoding} preserves distinct noncharacter keys in stored finding JSON`, async (t) => {+ const { database, prepare } = await fixture(t, encoding);+ const keys = { "\uFFFE": "first", "\uFFFF": "second" };+ const original = { ...finding("synthetic"), extensions: keys };+ const { findingsToEmbed } = prepared(prepare([original]));+ assert.deepEqual(findingsToEmbed, [original]);+ const { bytes } = database+ .prepare("SELECT CAST(details_json AS BLOB) AS bytes FROM findings")+ .get()!;+ const stored = new TextDecoder(encoding).decode(bytes as Uint8Array);+ assert.deepEqual(JSON.parse(stored).extensions, keys);+ if (encoding === "UTF-8") {+ for (const key of Object.keys(keys)) assert.ok(stored.includes(key));+ }+ });+}++for (const encoding of ["UTF-16le", "UTF-16be"] as const) {+ test(`${encoding} databases reuse the same text and cache keys across preparation and embedding`, async (t) => {+ const { database, prepare, embed } = await fixture(t, encoding);+ assert.equal(database.prepare("PRAGMA encoding").get()!.encoding, encoding);+ const original = finding("synthetic");+ const first = prepared(prepare([original]));+ assert.deepEqual(first.findingsToEmbed, [original]);+ const body = database.prepare("SELECT details_json FROM findings").get()!+ .details_json;+ assert.equal(first.cacheKeys.synthetic, cacheKey(String(body)));+ assert.deepEqual(embed([entry(first, "synthetic")]), {});+ assert.deepEqual(prepare([original]), {+ cacheKeys: first.cacheKeys,+ findingsToEmbed: [],+ });+ });+}++test("a missing embedding space rolls back when deriving a cache key", async (t) => {+ const { database, prepare } = await fixture(t);+ const before = snapshot(database);+ assert.deepEqual(prepare([], { space: undefined }), {+ cacheKeys: {},+ findingsToEmbed: [],+ });+ assert.throws(+ () => prepare([finding("synthetic")], { space: undefined }),+ /embedding space/i,+ );+ assert.deepEqual(snapshot(database), before);+ assert.equal(+ prepared(prepare([finding("synthetic")], { space: "undefined" }))+ .findingsToEmbed.length,+ 1,+ );+});++for (const [encoding, expectedKey] of [+ ["UTF-8", "41841dc6a8e60f506e90d97fb9599f6cd6ddba109c24bd6ca2593ca72fb8f646"],+ [+ "UTF-16LE",+ "23ba87b2a052c2b711dc909ad478346d1e253701cecf6f4b82b62d4cab9597c1",+ ],+ [+ "UTF-32LE",+ "c4e623bd52874215c18d910e333ad041eaef16e11ca5b307c1a0a51cb061ab3e",+ ],+] as const) {+ test(`${encoding} BLOB bodies retain Python cache identity and stored bytes`, async (t) => {+ const { database, prepare, embed } = await fixture(t);+ prepared(prepare([finding("old")]));+ const text = '{"findingId":"old","title":"BLOB body"}';+ const body =+ encoding === "UTF-8"+ ? Buffer.from(text)+ : encoding === "UTF-16LE"+ ? Buffer.concat([+ Buffer.from([255, 254]),+ Buffer.from(text, "utf16le"),+ ])+ : Buffer.from([+ 255,+ 254,+ 0,+ 0,+ ...Array.from(Buffer.from(text)).flatMap((byte) => [+ byte,+ 0,+ 0,+ 0,+ ]),+ ]);+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")+ .run(body);+ const first = prepared(prepare([finding("old")]));+ // Frozen Python SHA256 values use its bytes representation, not decoded JSON.+ assert.equal(first.cacheKeys.old, expectedKey);+ assert.notEqual(first.cacheKeys.old, cacheKey(text));+ assert.deepEqual(first.findingsToEmbed, [JSON.parse(text)]);+ assert.deepEqual(embed([entry(first, "old")]), {});+ assert.deepEqual(prepare([finding("old")]), {+ cacheKeys: first.cacheKeys,+ findingsToEmbed: [],+ });+ const row = database+ .prepare(+ "SELECT typeof(details_json) AS storage, details_json FROM findings",+ )+ .get()!;+ assert.equal(row.storage, "blob");+ assert.deepEqual(Buffer.from(row.details_json as Uint8Array), body);+ });+}++test("a malformed cached BLOB body is reused without decoding and permits later inserts", async (t) => {+ const { database, prepare } = await fixture(t);+ prepared(prepare([finding("old")]));+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")+ .run(Buffer.from("{"));+ const key = cacheKey("b'{'");+ database+ .prepare(+ "INSERT INTO local_finding_embeddings VALUES ('old', 'synthetic', '[1,0]', ?)",+ )+ .run(key);+ const result = prepared(prepare([finding("new")]));+ assert.equal(result.cacheKeys.old, key);+ assert.deepEqual(result.findingsToEmbed, [finding("new")]);+ assert.ok(database.prepare("SELECT id FROM findings WHERE id = 'new'").get());+});++test("malformed UTF-8 TEXT is rejected before committing new findings", async (t) => {+ const { database, prepare } = await fixture(t);+ prepared(prepare([finding("old")]));+ const body = Buffer.concat([+ Buffer.from('{"note":"'),+ Buffer.from([255]),+ Buffer.from('"}'),+ ]);+ database+ .prepare(+ "UPDATE findings SET details_json = CAST(? AS TEXT) WHERE id = 'old'",+ )+ .run(body);+ const before = snapshot(database);+ assert.throws(() => prepare([finding("new")]), /utf-8/iu);+ assert.deepEqual(snapshot(database), before);+ assert.equal(+ database.prepare("SELECT hex(details_json) AS bytes FROM findings").get()!+ .bytes,+ body.toString("hex").toUpperCase(),+ );+});++test("preparation indexes legacy missing bodies and invalidates changed local vectors", async (t) => {+ const { database, prepare, embed } = await fixture(t);+ const original = finding("legacy");+ database+ .prepare(+ `INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, created_at, updated_at)+ VALUES ('legacy', 'fingerprint-legacy', 'synthetic-rule', 'anchor-legacy', ?, ?)`,+ )+ .run(timestamp, timestamp);+ const first = prepared(prepare([original]));+ assert.deepEqual(first.findingsToEmbed, [original]);+ assert.deepEqual(embed([entry(first, "legacy")]), {});+ const body = String(+ database.prepare("SELECT details_json FROM findings").get()!.details_json,+ );+ assert.equal(first.cacheKeys.legacy, cacheKey(body));+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'legacy'")+ .run(body.replace("Finding legacy", "Updated legacy"));+ assert.equal(+ database+ .prepare("SELECT COUNT(*) AS total FROM local_finding_embeddings")+ .get()!.total,+ 0,+ );+ assert.equal(+ prepared(prepare([original])).findingsToEmbed[0]!.title,+ "Updated legacy λ",+ );+});++test("repository scope preserves creation order, cached keys and empty-input behavior", async (t) => {+ const { database, targetId, prepare, embed } = await fixture(t);+ const other = prepared(prepare([finding("other")]));+ const first = prepared(prepare([finding("first")]));+ prepared(prepare([finding("second")]));+ database+ .prepare("DELETE FROM finding_repositories WHERE finding_id = 'other'")+ .run();+ database+ .prepare(+ "INSERT INTO finding_repositories VALUES ('other-target', 'other')",+ )+ .run();+ database+ .prepare("UPDATE findings SET created_at = '2025' WHERE id = 'second'")+ .run();+ assert.deepEqual(embed([entry(first, "first")]), {});+ const scoped = prepared(prepare([finding("first")]));+ assert.deepEqual(Object.keys(scoped.cacheKeys), ["second", "first"]);+ assert.deepEqual(scoped.findingsToEmbed, [finding("second")]);+ const all = prepared(+ prepare([finding("first")], { repositoryId: undefined }),+ );+ assert.deepEqual(Object.keys(all.cacheKeys), ["second", "first", "other"]);+ assert.equal(all.cacheKeys.other, other.cacheKeys.other);+ assert.equal(all.cacheKeys.first, first.cacheKeys.first);+ assert.deepEqual(prepare([], { repositoryId: undefined }), {+ cacheKeys: {},+ findingsToEmbed: [],+ });+ assert.deepEqual(prepare([], { anchorRepositoryId: `${targetId}-wrong` }), {+ error: "target_mismatch",+ });+});++test("registered target identity is authoritative and a later identity conflict rolls back the batch", async (t) => {+ const { root, database, prepare } = await fixture(t);+ database+ .prepare(+ "INSERT INTO security_targets (id, current_path, display_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",+ )+ .run("registered-target", root, "Synthetic target", timestamp, timestamp);+ assert.deepEqual(prepare([finding("old")]), { error: "target_mismatch" });+ const scope = {+ anchorRepositoryId: "registered-target",+ repositoryId: "registered-target",+ };+ prepared(prepare([finding("old")], scope));+ database.prepare("DELETE FROM finding_repositories").run();+ const before = snapshot(database);+ const conflicting = finding("old");+ conflicting.identity.anchor = "different-anchor";+ assert.deepEqual(prepare([finding("new"), conflicting], scope), {+ error: "finding_conflict",+ });+ assert.deepEqual(snapshot(database), before);+});++for (const failure of [+ "stale",+ "model",+ "dimensions",+ "zero",+ "nonfinite",+ "overflow-norm",+] as const) {+ test(`a later ${failure} embedding rolls back earlier replacements`, async (t) => {+ const { database, prepare, embed } = await fixture(t);+ const first = prepared(prepare([finding("one"), finding("two")]));+ assert.deepEqual(embed([entry(first, "one")]), {});+ const before = snapshot(database);+ const replacement = entry(first, "one");+ replacement.embedding.vector = [0, 1];+ const invalid = entry(first, "two");+ if (failure === "stale") invalid.cacheKey = "previous-input";+ else if (failure === "model") invalid.embedding.model = "another-model";+ else+ invalid.embedding.vector = {+ dimensions: [1],+ zero: [0, 0],+ nonfinite: [Number.NaN, 0],+ "overflow-norm": [Number.MAX_VALUE, Number.MAX_VALUE],+ }[failure];+ assert.deepEqual(embed([replacement, invalid]), {+ error: failure === "stale" ? "finding_changed" : "embedding_failed",+ });+ assert.deepEqual(snapshot(database), before);+ });+}++test("a changed document rejects an in-flight embedding and retains the previous batch", async (t) => {+ const { database, prepare, embed } = await fixture(t);+ const first = prepared(prepare([finding("one"), finding("two")]));+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'two'")+ .run('{"findingId":"two","title":"Newer evidence"}');+ const before = snapshot(database);+ assert.deepEqual(embed([entry(first, "one"), entry(first, "two")]), {+ error: "finding_changed",+ });+ assert.deepEqual(snapshot(database), before);+});++test("finite integer, boolean and subnormal vector components keep their stored values", async (t) => {+ const { database, prepare, embed } = await fixture(t);+ const first = prepared(prepare([finding("one")]));+ const saved = entry(first, "one");+ for (const [vector, stored] of [+ [[true, 9007199254740993n], "[true,9007199254740993]"],+ [[Number.MIN_VALUE, 0], "[5e-324,0]"],+ [[Number.MAX_VALUE, 0], "[1.7976931348623157e+308,0]"],+ ] as const) {+ assert.deepEqual(+ embed([{ ...saved, embedding: { model: "synthetic", vector } }]),+ {},+ );+ assert.equal(+ database+ .prepare("SELECT vector_json FROM local_finding_embeddings")+ .get()!.vector_json,+ stored,+ );+ }+ const before = snapshot(database);+ assert.throws(+ () =>+ embed([+ {+ ...saved,+ embedding: { model: "synthetic", vector: [10n ** 400n, 0] },+ },+ ]),+ /int too large to convert to float/u,+ );+ assert.deepEqual(snapshot(database), before);+});++test("vector norm admits the largest finite result and rejects its overflowing neighbor", async (t) => {+ const { database, prepare, embed } = await fixture(t);+ const first = prepared(prepare([finding("one")]));+ const saved = entry(first, "one");+ const accepted = 1.2711610061536462e308;+ assert.deepEqual(+ embed([+ {+ ...saved,+ embedding: { model: "synthetic", vector: [accepted, accepted] },+ },+ ]),+ {},+ );+ const before = snapshot(database);+ const rejected = 1.2711610061536464e308;+ assert.deepEqual(+ embed([+ {+ ...saved,+ embedding: { model: "synthetic", vector: [rejected, rejected] },+ },+ ]),+ { error: "embedding_failed" },+ );+ assert.deepEqual(snapshot(database), before);+});++test("cache hits skip body parsing while malformed cache misses roll back inserted findings", async (t) => {+ const { database, prepare } = await fixture(t);+ prepared(prepare([finding("old")]));+ const body = '{"evidence":[NaN,]}';+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")+ .run(body);+ database+ .prepare(+ "INSERT INTO local_finding_embeddings VALUES ('old', 'synthetic', '[1, 0]', ?)",+ )+ .run(cacheKey(body));+ assert.deepEqual(prepare([finding("old")]), {+ cacheKeys: { old: cacheKey(body) },+ findingsToEmbed: [],+ });+ database.prepare("DELETE FROM local_finding_embeddings").run();+ const before = snapshot(database);+ const failed = prepare([finding("new")]);+ assert.ok("error" in failed);+ assert.match(String(failed.error), /JSON|position|property|end/iu);+ assert.deepEqual(snapshot(database), before);+});++test("legacy non-finite numbers reach the response after preparation commits", async (t) => {+ const { database, prepare } = await fixture(t);+ prepared(prepare([finding("old")]));+ database+ .prepare("UPDATE findings SET details_json = ? WHERE id = 'old'")+ .run(+ String.raw`{"findingId":"old","evidence":[1e400,NaN,Infinity,-Infinity,9007199254740993,{"NaN":"quoted \"Infinity\""}]}`,+ );+ const result = prepared(prepare([finding("new")]));+ assert.deepEqual(+ result.findingsToEmbed.find((item) => item.findingId === "old")!.evidence,+ [+ Infinity,+ NaN,+ Infinity,+ -Infinity,+ 9007199254740993n,+ { NaN: 'quoted "Infinity"' },+ ],+ );+ assert.ok(database.prepare("SELECT id FROM findings WHERE id = 'new'").get());+});++test("an incomplete corpus body rolls back newly indexed findings", async (t) => {+ const { database, prepare } = await fixture(t);+ prepared(prepare([finding("old")]));+ database.prepare("UPDATE findings SET details_json = NULL").run();+ const before = snapshot(database);+ assert.deepEqual(prepare([finding("new")]), {+ error: "finding_not_indexed",+ });+ assert.deepEqual(snapshot(database), before);+});diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.jsonindex 65ddc8422..d1b0c1650 100644--- a/plugins/codex-security/plugin-files.json+++ b/plugins/codex-security/plugin-files.json@@ -96,7 +96,6 @@"scripts/workbench_feedback.py","scripts/workbench_finding_index.py","scripts/workbench_finding_workflows.py",- "scripts/workbench_local_dedupe.py","scripts/workbench_native_indexes.py","scripts/workbench_progress.py","scripts/workbench_publication.py",diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.pyindex 08b7636a1..1c206b242 100644--- a/plugins/codex-security/scripts/workbench_cli.py+++ b/plugins/codex-security/scripts/workbench_cli.py@@ -401,7 +401,6 @@ def parse_args(description: str, *, inspection_only: bool = False) -> argparse.Nsubparsers.add_parser("database-info")subparsers.add_parser("finding-workflow")- subparsers.add_parser("local-dedupe")args = parser.parse_args()if getattr(args, "user_context_stdin", False):args.user_context = sys.stdin.buffer.read().decode("utf-8")diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.pyindex e8b36ad18..e86164bb0 100644--- a/plugins/codex-security/scripts/workbench_db.py+++ b/plugins/codex-security/scripts/workbench_db.py@@ -77,7 +77,6 @@ from workbench_constants import (from workbench_feedback import get_scan_feedbackfrom workbench_finding_index import index_findingsfrom workbench_finding_workflows import finding_workflow, register_workflow_scan-from workbench_local_dedupe import local_dedupefrom workbench_scan_start import (archive_scan,insert_running_scan,@@ -3261,8 +3260,6 @@ def main(*, before_archive: Callable[[], None] | None = None) -> None:)elif args.command == "database-info":result = {"databasePath": str(database_path())}- elif args.command == "local-dedupe":- result = local_dedupe(connection, json.load(sys.stdin), now())elif args.command == "finding-workflow":result = finding_workflow(connection, json.load(sys.stdin), now())else:diff --git a/plugins/codex-security/scripts/workbench_local_dedupe.py b/plugins/codex-security/scripts/workbench_local_dedupe.pydeleted file mode 100644index 9f0c7f322..000000000--- a/plugins/codex-security/scripts/workbench_local_dedupe.py+++ /dev/null@@ -1,136 +0,0 @@-"""Prepare and cache embeddings without publishing findings to an HTTP service."""--from __future__ import annotations--import argparse-import hashlib-import json-import math-import sqlite3-import sys-from pathlib import Path-from typing import Any--sys.path.insert(0, str(Path(__file__).resolve().parent))-from workbench_finding_index import upsert_finding-from workbench_target_state import stable_target_id---def cache_key(space: str, document: str) -> str:- return hashlib.sha256(f"{space}\0{document}".encode()).hexdigest()---def local_dedupe(- connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str-) -> dict[str, Any]:- action = payload["action"]- try:- with connection:- connection.execute("BEGIN IMMEDIATE")- if action == "prepare":- return prepare(connection, payload, timestamp)- if action == "embed":- for entry in payload["entries"]:- row = connection.execute(- "SELECT details_json FROM findings WHERE id = ?", (entry["findingId"],)- ).fetchone()- if (- row is None- or cache_key(payload["space"], row["details_json"]) != entry["cacheKey"]- ):- raise ValueError("finding_changed")- embedding = entry["embedding"]- vector = embedding["vector"]- if (- embedding["model"] != payload["model"]- or len(vector) != payload["dimensions"]- or not all(isinstance(v, (int, float)) and math.isfinite(v) for v in vector)- or not math.isfinite(math.hypot(*vector))- or math.hypot(*vector) == 0- ):- raise ValueError("embedding_failed")- connection.execute(- "INSERT INTO local_finding_embeddings (finding_id, model, vector_json, cache_key) "- "VALUES (?, ?, ?, ?) ON CONFLICT(finding_id) DO UPDATE SET "- "model = excluded.model, vector_json = excluded.vector_json, "- "cache_key = excluded.cache_key",- (- entry["findingId"],- embedding["model"],- json.dumps(vector, allow_nan=False),- entry["cacheKey"],- ),- )- return {}- raise ValueError("invalid_request")- except sqlite3.IntegrityError:- return {"error": "finding_conflict"}- except ValueError as exc:- return {"error": str(exc)}---def prepare(- connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str-) -> dict[str, Any]:- # Artifact seals establish consistency, not permission to select another local corpus.- target = Path(payload["repositoryPath"]).resolve()- registered = connection.execute(- "SELECT id FROM security_targets WHERE current_path = ?", (str(target),)- ).fetchone()- target_id = registered["id"] if registered is not None else stable_target_id(target)- if (- payload["anchorRepositoryId"] != target_id- or payload.get("repositoryId", target_id) != target_id- ):- raise ValueError("target_mismatch")- if not payload["findings"]:- return {"cacheKeys": {}, "findingsToEmbed": []}- # A sealed historical scan selects logical IDs; never overwrite their current bodies.- for finding in payload["findings"]:- existing = connection.execute(- "SELECT fingerprint, rule_id, identity_anchor, identity_instance, details_json "- "FROM findings WHERE id = ?",- (finding["findingId"],),- ).fetchone()- identity = (- finding["fingerprints"]["primary"],- finding["ruleId"],- finding["identity"]["anchor"],- finding["identity"].get("instance"),- )- if existing is not None and tuple(existing)[:4] != identity:- raise sqlite3.IntegrityError("Conflicting finding identity")- if existing is None or existing["details_json"] is None:- upsert_finding(connection, finding, timestamp, payload["anchorRepositoryId"])- else:- connection.execute(- "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",- (payload["anchorRepositoryId"], finding["findingId"]),- )- repository_id = payload.get("repositoryId")- rows = connection.execute(- "SELECT findings.id, findings.details_json, embeddings.cache_key FROM findings "- "LEFT JOIN local_finding_embeddings AS embeddings ON findings.id = embeddings.finding_id "- + (- "WHERE EXISTS (SELECT 1 FROM finding_repositories WHERE finding_id = findings.id "- "AND repository_id = ?) "- if repository_id is not None- else ""- )- + "ORDER BY findings.created_at, findings.id",- (repository_id,) if repository_id is not None else (),- )- cache_keys = {}- findings_to_embed = []- for row in rows:- if row["details_json"] is None:- raise ValueError("finding_not_indexed")- key = cache_key(payload["space"], row["details_json"])- cache_keys[row["id"]] = key- if key != row["cache_key"]:- findings_to_embed.append(json.loads(row["details_json"]))- return {"cacheKeys": cache_keys, "findingsToEmbed": findings_to_embed}---if __name__ == "__main__":- argparse.ArgumentParser(description=__doc__).parse_args()diff --git a/plugins/codex-security/tests/test_workbench_local_dedupe.py b/plugins/codex-security/tests/test_workbench_local_dedupe.pyindex 607031a9d..b09236365 100644--- a/plugins/codex-security/tests/test_workbench_local_dedupe.py+++ b/plugins/codex-security/tests/test_workbench_local_dedupe.py@@ -1,119 +1,11 @@from __future__ import annotations-import jsonimport sqlite3from contextlib import closing-from pathlib import Path--import pytestTIMESTAMP = "2026-01-01T00:00:00Z"-def finding(index: int) -> dict:- source = Path(__file__).resolve().parents[1] / "examples/legacy-completed-scan/findings.json"- result = json.loads(source.read_text())["findings"][0]- result["findingId"] = f"finding-{index}"- result["fingerprints"]["primary"] = f"fingerprint-{index}"- return result---def request(api, connection, action, **values):- return api["local_dedupe"](- connection,- {- "action": action,- "space": "synthetic-v1",- "model": "synthetic",- "dimensions": 2,- **values,- },- TIMESTAMP,- )---def prepare(api, connection, findings, **values):- repository_path = str(Path(__file__).resolve().parent)- with connection:- target_id = api["ensure_security_target"](connection, repository_path)- return request(- api,- connection,- "prepare",- findings=findings,- anchorRepositoryId=target_id,- repositoryId=target_id,- repositoryPath=repository_path,- **values,- )---def entry(prepared, index=0, **values):- finding_id = prepared["findingsToEmbed"][index]["findingId"]- return {- "findingId": finding_id,- "cacheKey": prepared["cacheKeys"][finding_id],- "embedding": {"model": "synthetic", "vector": [1, 0]},- **values,- }---def test_cache_reuse_survives_service_embedding_writes(workbench_api, workbench_db):- first = prepare(workbench_api, workbench_db, [finding(1)])- assert first["findingsToEmbed"] == [finding(1)]- assert request(workbench_api, workbench_db, "embed", entries=[entry(first)]) == {}- assert prepare(workbench_api, workbench_db, [finding(1)]) == {- "cacheKeys": first["cacheKeys"],- "findingsToEmbed": [],- }- assert prepare(workbench_api, workbench_db, [finding(1)], space="synthetic-v2")[- "findingsToEmbed"- ] == [finding(1)]- with workbench_db:- workbench_db.execute(- "INSERT INTO finding_embeddings (finding_id, model, vector_json) VALUES (?, ?, ?)",- ("finding-1", "synthetic", "[0, 1]"),- )- assert prepare(workbench_api, workbench_db, [finding(1)])["findingsToEmbed"] == []---def test_preparation_returns_all_cache_keys_but_only_uncached_bodies(workbench_api, workbench_db):- first = prepare(workbench_api, workbench_db, [finding(1)])- request(workbench_api, workbench_db, "embed", entries=[entry(first)])- mixed = prepare(workbench_api, workbench_db, [finding(1), finding(2)])- assert mixed["findingsToEmbed"] == [finding(2)]- assert set(mixed["cacheKeys"]) == {"finding-1", "finding-2"}- assert mixed["cacheKeys"]["finding-1"] == first["cacheKeys"]["finding-1"]- assert prepare(workbench_api, workbench_db, []) == {"cacheKeys": {}, "findingsToEmbed": []}--[email protected]("failure", ["stale", "dimensions", "zero", "nonfinite"])-def test_embedding_batch_rolls_back_on_invalid_or_stale_input(workbench_api, workbench_db, failure):- prepared = prepare(workbench_api, workbench_db, [finding(1), finding(2)])- second = entry(prepared, 1)- if failure == "stale":- second["cacheKey"] = "old-input"- else:- second["embedding"]["vector"] = {- "dimensions": [1],- "zero": [0, 0],- "nonfinite": [float("nan"), 0],- }[failure]- result = request(workbench_api, workbench_db, "embed", entries=[entry(prepared), second])- assert result == {"error": "finding_changed" if failure == "stale" else "embedding_failed"}- assert workbench_db.execute("SELECT COUNT(*) FROM local_finding_embeddings").fetchone()[0] == 0---def test_preparation_conflict_rolls_back_new_findings(workbench_api, workbench_db):- prepare(workbench_api, workbench_db, [finding(1)])- conflicting = finding(1)- conflicting["identity"]["anchor"] = "different-anchor"- assert prepare(workbench_api, workbench_db, [finding(2), conflicting]) == {- "error": "finding_conflict"- }- assert workbench_db.execute("SELECT COUNT(*) FROM findings").fetchone()[0] == 1--def test_embedding_cache_migration_preserves_existing_vectors(workbench_api, tmp_path):with closing(sqlite3.connect(tmp_path / "legacy.sqlite3")) as connection:connection.row_factory = sqlite3.Row@@ -171,18 +63,3 @@ def test_local_cache_migration_keeps_service_and_local_rows_separate(workbench_a"SELECT finding_id, cache_key FROM local_finding_embeddings")] == [("local", "cached-input")]---def test_finding_body_update_invalidates_local_search_and_pending_groups(- workbench_api, workbench_db-):- prepared = prepare(workbench_api, workbench_db, [finding(1)])- request(workbench_api, workbench_db, "embed", entries=[entry(prepared)])- changed = {**finding(1), "title": "Updated evidence from another scan"}- with workbench_db:- workbench_db.execute(- "UPDATE findings SET details_json = ? WHERE id = ?",- (json.dumps(changed, sort_keys=True), "finding-1"),- )- assert workbench_db.execute("SELECT COUNT(*) FROM local_finding_embeddings").fetchone()[0] == 0- assert prepare(workbench_api, workbench_db, [finding(1)])["findingsToEmbed"] == [changed]diff --git a/sdk/typescript/docs/findings-service.md b/sdk/typescript/docs/findings-service.mdindex 4d6d26416..fab1dc1a4 100644--- a/sdk/typescript/docs/findings-service.md+++ b/sdk/typescript/docs/findings-service.md@@ -399,13 +399,23 @@ and JSON on stdin. For example, `request.json` selects the existing database:Use an absolute Windows path on Windows. The retained helper commands acceptthese `payload` fields:-| Command | Payload |-| --------------------------- | ------------------------------------------------------------------------------------------------------- |-| `store-findings` | `entries` containing finding and embedding records; optional `repositoryId`. |-| `list-stored-findings` | Positive integer `limit` and non-negative integer `offset`. |-| `find-potential-duplicates` | `findingId` and `scope`: either `{ "repositoryId": "REPOSITORY_ID" }` or `{ "allRepositories": true }`. |-| `store-dedupe-groups` | `groups`, an array of finding-ID arrays. |-| `list-dedupe-groups` | `findingId`. |+| Command | Payload |+| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `store-findings` | `entries` containing finding and embedding records; optional `repositoryId`. |+| `list-stored-findings` | Positive integer `limit` and non-negative integer `offset`. |+| `find-potential-duplicates` | `findingId` and `scope`: either `{ "repositoryId": "REPOSITORY_ID" }` or `{ "allRepositories": true }`. |+| `store-dedupe-groups` | `groups`, an array of finding-ID arrays. |+| `list-dedupe-groups` | `findingId`. |+| `local-dedupe` | `action: "prepare"` with `findings`, `repositoryPath`, `anchorRepositoryId`, optional `repositoryId`, and `space`; or `action: "embed"` with `entries`, `space`, `model`, and `dimensions`. |++Local embedding preparation and cache writes now use the bundled Node helper+instead of `workbench_db.py local-dedupe`. Each embed entry contains `findingId`,+the preparation response's `cacheKey`, and an `embedding` with `model` and+`vector`. Preparation returns `cacheKeys` and `findingsToEmbed`; embedding returns+an empty object. These operations use Node even when `PYTHON` is configured.+Explicitly selected external plugins retain their Python workbench route.+Saved-scan lookup and export still require Python, so the complete saved-scan+deduplication workflow continues to need it.Retained scan commands such as `workbench_db.py list-findings` keep their existinginterface.diff --git a/sdk/typescript/src/deduplication/local.ts b/sdk/typescript/src/deduplication/local.tsindex ce7a44368..ff7df1e46 100644--- a/sdk/typescript/src/deduplication/local.ts+++ b/sdk/typescript/src/deduplication/local.ts@@ -8,7 +8,7 @@ import type {import { CodexSecurityError } from "../errors.js";import { workflowDigest } from "../finding-workflow.js";import {- resolveWorkbenchRuntime,+ bundledPluginRoot,workbenchEnvironment,runWorkbench,type WorkbenchCommandOptions,@@ -36,7 +36,7 @@ export interface FindingEmbeddingBinding {/** Direct workbench adapter. Importing it starts no HTTP or MCP server. */export class LocalDeduplication {- private options?: Promise<WorkbenchCommandOptions>;+ private options?: Promise<Omit<WorkbenchCommandOptions, "python">>;private cacheKeys: Record<string, string> = {};private readonly space: string;private readonly embedding: FindingEmbeddingBinding;@@ -137,14 +137,9 @@ export class LocalDeduplication {this.signal?.throwIfAborted();this.options ??= (async () => {const environment = workbenchEnvironment(this.environment);- const [python, pluginRoot] = await resolveWorkbenchRuntime({- environment,- protectedRoot: this.repositoryPath,- signal: this.signal,- });return {- python,- pluginRoot,+ pluginRoot: await bundledPluginRoot(),+ stateDirectory: environment.CODEX_SECURITY_STATE_DIR,environment,signal: this.signal,protectedRoot: [diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.tsindex a56e7030d..f1bf4d7ae 100644--- a/sdk/typescript/src/runtime.ts+++ b/sdk/typescript/src/runtime.ts@@ -1670,24 +1670,30 @@ export async function runWorkbench(): Promise<JsonObject> {const script = join(options.pluginRoot, "scripts", "workbench_db.py");const signal = options.signal;+ // Explicitly selected external plugins retain their own workbench protocol.+ const nativeLocalDedupe =+ args[0] === "local-dedupe" &&+ (await sameFile(options.pluginRoot, await bundledPluginRoot()));const run = async (arguments_: readonly string[],input?: string,archiveHandshake = false,): Promise<string> => {- const native = [- "database-info",- "store-findings",- "list-stored-findings",- "store-external-findings",- "list-external-findings",- "find-potential-duplicates",- "store-dedupe-groups",- "list-dedupe-groups",- "dashboard",- "severity-classification",- "read-severity-classification",- ].includes(arguments_[0] ?? "");+ const native =+ nativeLocalDedupe ||+ [+ "database-info",+ "store-findings",+ "list-stored-findings",+ "store-external-findings",+ "list-external-findings",+ "find-potential-duplicates",+ "store-dedupe-groups",+ "list-dedupe-groups",+ "dashboard",+ "severity-classification",+ "read-severity-classification",+ ].includes(arguments_[0] ?? "");const node =native && process.versions["bun"]? await resolveTrustedExecutable("node", options.environment, [@@ -1713,12 +1719,8 @@ export async function runWorkbench(? (options.stateDirectory ??codexSecurityStateDirectory(options.environment)): undefined;- // Workbench text fields can contain NUL or exceed OS command-line limits.- const framedArguments = !native;- let program = archiveHandshake ? ARCHIVE_REGISTRATION_PROGRAM : undefined;- if (framedArguments)- program =- WORKBENCH_ARGUMENTS_PROGRAM + (program ?? WORKBENCH_SCRIPT_PROGRAM);+ // Validate one JSON document before framing its unchanged source as payload.+ if (nativeLocalDedupe && input !== undefined) JSON.parse(input);const result = await runCodexCommand({ command },native@@ -1728,28 +1730,34 @@ export async function runWorkbench("-X","utf8","-B",- ...(program === undefined ? [] : ["-c", program]),+ "-c",+ WORKBENCH_ARGUMENTS_PROGRAM ++ (archiveHandshake+ ? ARCHIVE_REGISTRATION_PROGRAM+ : WORKBENCH_SCRIPT_PROGRAM),script,- ...(framedArguments ? [] : arguments_),],pluginHelperEnvironment(node?.environment ?? options.environment),// The SDK owns configuration normalization; the helper receives its resolved location.- native- ? JSON.stringify(- arguments_[0] === "database-info"- ? stateDirectory- : {- stateDirectory,- payload: input === undefined ? undefined : JSON.parse(input),- },- )- : input,+ nativeLocalDedupe+ ? `{"stateDirectory":${JSON.stringify(stateDirectory)},"payload":${input ?? "null"}}`+ : native+ ? JSON.stringify(+ arguments_[0] === "database-info"+ ? stateDirectory+ : {+ stateDirectory,+ payload:+ input === undefined ? undefined : JSON.parse(input),+ },+ )+ : input,signal,archiveHandshake,- // Match native argv's UTF-8 encoding for the private argument frame.- framedArguments- ? `${JSON.stringify(arguments_.map((argument) => argument.toWellFormed()))}\n`- : undefined,+ // Frame Python arguments on stdin to preserve NUL and long diagnostics.+ native+ ? undefined+ : `${JSON.stringify(arguments_.map((argument) => argument.toWellFormed()))}\n`,);if (!result.success) {throw new Error(diff --git a/sdk/typescript/tests-ts/local-dedupe-routing.test.ts b/sdk/typescript/tests-ts/local-dedupe-routing.test.tsnew file mode 100644index 000000000..b04bcd341--- /dev/null+++ b/sdk/typescript/tests-ts/local-dedupe-routing.test.ts@@ -0,0 +1,365 @@+import { spawnSync } from "node:child_process";+import { existsSync } from "node:fs";+import { mkdir, readFile, writeFile } from "node:fs/promises";+import { join } from "node:path";+import { afterEach, expect, mock, spyOn, test } from "bun:test";+import { Tiktoken } from "js-tiktoken/lite";+import cl100kBase from "js-tiktoken/ranks/cl100k_base";+import { LocalDeduplication } from "../src/deduplication/local.js";+import type { Finding, FindingsDocument } from "../src/models.js";+import { resolvePluginPython, runWorkbench } from "../src/runtime.js";+import {+ EMBEDDING_DIMENSIONS,+ EMBEDDING_MODEL,+} from "../src/server/embeddings.js";+import { PLUGIN_ROOT } from "./plugin-root.js";+import { sha256 } from "./support/finding-identity.js";+import { createTemporaryDirectories } from "./support/temporary-directories.js";++const directories = createTemporaryDirectories(true);+afterEach(directories.cleanup);++async function fixture() {+ const root = await directories.create("local-dedupe-routing-");+ const repository = join(root, "repository");+ await mkdir(repository);+ const repositoryId = `target_sha256_${sha256(`local-workspace\0${repository}`)}`;+ const document = JSON.parse(+ await readFile(+ join(PLUGIN_ROOT, "examples/completed-scan/findings.json"),+ "utf8",+ ),+ ) as FindingsDocument;+ const finding = document.findings[0]!;+ const stateDirectory = join(root, "state with spaces");+ const options = {+ pluginRoot: PLUGIN_ROOT,+ environment: process.env,+ python: join(root, "missing-python"),+ stateDirectory,+ protectedRoot: repository,+ };+ const prepare = {+ action: "prepare",+ repositoryPath: repository,+ anchorRepositoryId: repositoryId,+ repositoryId,+ space: "synthetic-embedding-space",+ model: "synthetic-embedding",+ dimensions: 2,+ findings: [finding],+ };+ return { options, prepare, finding, stateDirectory };+}++test.skipIf(process.platform === "win32")(+ "uses the configured state path when resolution exposes a trailing filename space",+ async () => {+ const { options, prepare, stateDirectory } = await fixture();+ const selected = `${stateDirectory} `;+ const local = new LocalDeduplication(+ { ...options.environment, CODEX_SECURITY_STATE_DIR: `${selected}/.` },+ { repositoryId: prepare.repositoryId },+ prepare.repositoryPath,+ );+ await local.prepare([], prepare.repositoryId);+ expect(existsSync(join(selected, "workbench.sqlite3"))).toBe(true);+ expect(existsSync(join(stateDirectory, "workbench.sqlite3"))).toBe(false);+ },+);++test("bundled local dedupe retains raw integer bytes and SDK number values without Python", async () => {+ const { options, prepare, finding } = await fixture();+ finding.extensions = {+ integer: "9007199254740993",+ text: "synthetic evidence \0 λ",+ };+ const input = JSON.stringify(prepare).replace(+ '"integer":"9007199254740993"',+ '"integer":9007199254740993',+ );+ const prepared = await runWorkbench(options, ["local-dedupe"], input);+ expect(prepared["findingsToEmbed"] as unknown).toEqual([+ {+ ...finding,+ extensions: { ...finding.extensions, integer: 9007199254740992 },+ },+ ]);+ const cacheKeys = prepared["cacheKeys"] as Record<string, string>;+ expect(Object.keys(cacheKeys)).toEqual([finding.findingId]);+ const stored = spawnSync(+ "node",+ [+ "-e",+ `const { DatabaseSync } = require("node:sqlite");+const database = new DatabaseSync(process.argv[1]);+console.log(database.prepare("SELECT details_json FROM findings WHERE id = ?").get(process.argv[2]).details_json);+database.close();`,+ join(options.stateDirectory, "workbench.sqlite3"),+ finding.findingId,+ ],+ { encoding: "utf8" },+ );+ expect(stored.status, stored.stderr).toBe(0);+ const storedBody = stored.stdout.trim();+ expect(storedBody).toMatch(/"integer":\s*9007199254740993/u);+ expect(cacheKeys[finding.findingId]).toBe(+ sha256(`${prepare.space}\0${storedBody}`),+ );++ expect(+ await runWorkbench(+ options,+ ["local-dedupe"],+ JSON.stringify({+ action: "embed",+ space: prepare.space,+ model: prepare.model,+ dimensions: prepare.dimensions,+ entries: [+ {+ findingId: finding.findingId,+ cacheKey: cacheKeys[finding.findingId],+ embedding: { model: prepare.model, vector: [1, 0] },+ },+ ],+ }),+ ),+ ).toEqual({});+ expect(await runWorkbench(options, ["local-dedupe"], input)).toEqual({+ cacheKeys,+ findingsToEmbed: [],+ });+ const largeFinding = {+ ...finding,+ extensions: { ...finding.extensions, integer: 9007199254740992 },+ } as Finding;+ const embed = mock(async (batch: readonly Finding[]) => {+ expect(batch).toEqual([largeFinding]);+ return batch.map(() => ({ model: prepare.model, vector: [1, 0] }));+ });+ const local = new LocalDeduplication(+ {+ ...options.environment,+ CODEX_SECURITY_STATE_DIR: options.stateDirectory,+ PYTHON: options.python,+ },+ { repositoryId: prepare.repositoryId },+ prepare.repositoryPath,+ undefined,+ undefined,+ {+ model: prepare.model,+ dimensions: prepare.dimensions,+ cacheNamespace: "synthetic-adapter",+ embedder: { embed },+ },+ );+ await local.prepare([largeFinding], prepare.repositoryId);+ await local.prepare([largeFinding], prepare.repositoryId);+ expect(embed).toHaveBeenCalledTimes(1);+});++test("default embeddings retain historical key order and SDK number values and reuse their cache", async () => {+ const { options, prepare, finding } = await fixture();+ await runWorkbench(options, ["local-dedupe"], JSON.stringify(prepare));+ const storedBody =+ `{"z":true,"findingId":${JSON.stringify(finding.findingId)},` ++ '"extensions":{"😀":"astral","z":{"z":2,"a":1},"10":"ten","2":"two","integer":9007199254740993,' ++ '"\\ue000":"BMP","a":[{"z":4,"a":3}]},"a":false}';+ const stored = spawnSync(+ "node",+ [+ "-e",+ `const { DatabaseSync } = require("node:sqlite");+const database = new DatabaseSync(process.argv[1]);+database.prepare("UPDATE findings SET details_json = ? WHERE id = ?").run(process.argv[3], process.argv[2]);+database.close();`,+ join(options.stateDirectory, "workbench.sqlite3"),+ finding.findingId,+ storedBody,+ ],+ { encoding: "utf8" },+ );+ expect(stored.status, stored.stderr).toBe(0);+ const expectedInput = JSON.stringify({+ a: false,+ extensions: {+ "2": "two",+ "10": "ten",+ a: [{ a: 3, z: 4 }],+ integer: 9007199254740992,+ z: { a: 1, z: 2 },+ "\ue000": "BMP",+ "😀": "astral",+ },+ findingId: finding.findingId,+ z: true,+ });+ const encoding = new Tiktoken(cl100kBase);+ const request = spyOn(globalThis, "fetch").mockResolvedValue(+ Response.json({+ model: EMBEDDING_MODEL,+ data: [+ {+ index: 0,+ embedding: [1, ...Array<number>(EMBEDDING_DIMENSIONS - 1).fill(0)],+ },+ ],+ }),+ );+ try {+ const local = new LocalDeduplication(+ {+ ...options.environment,+ CODEX_SECURITY_STATE_DIR: options.stateDirectory,+ PYTHON: options.python,+ OPENAI_API_KEY: "synthetic-key",+ },+ { repositoryId: prepare.repositoryId },+ prepare.repositoryPath,+ );+ await local.prepare([finding], prepare.repositoryId);+ expect(request).toHaveBeenCalledTimes(1);+ const body = JSON.parse(String(request.mock.calls[0]![1]!.body)) as {+ input: number[][];+ };+ expect(body.input).toHaveLength(1);+ expect(body.input[0]).toEqual(encoding.encode(expectedInput, [], []));+ expect(encoding.decode(body.input[0]!)).toBe(expectedInput);+ await local.prepare([finding], prepare.repositoryId);+ expect(request).toHaveBeenCalledTimes(1);+ } finally {+ request.mockRestore();+ }+});++test("pre-aborted native local dedupe does not initialize its database", async () => {+ const { options, prepare, stateDirectory } = await fixture();+ await expect(+ runWorkbench(+ { ...options, signal: AbortSignal.abort() },+ ["local-dedupe"],+ JSON.stringify(prepare),+ ),+ ).rejects.toMatchObject({ name: "AbortError" });+ expect(existsSync(join(stateDirectory, "workbench.sqlite3"))).toBe(false);+});++test("malformed local dedupe input cannot replace the selected state directory", async () => {+ const { options, prepare, stateDirectory } = await fixture();+ const otherState = join(stateDirectory, "..", "alternate state");+ const input =+ JSON.stringify(prepare) + ',"stateDirectory":' + JSON.stringify(otherState);+ await expect(+ runWorkbench(options, ["local-dedupe"], input),+ ).rejects.toMatchObject({ cause: expect.any(SyntaxError) });+ expect(existsSync(stateDirectory)).toBe(false);+ expect(existsSync(otherState)).toBe(false);+});++test("explicit external plugins retain their Python local dedupe protocol", async () => {+ const { options, stateDirectory } = await fixture();+ const pluginRoot = join(stateDirectory, "external-plugin");+ await mkdir(join(pluginRoot, "scripts"), { recursive: true });+ await writeFile(+ join(pluginRoot, "scripts", "workbench_db.py"),+ `import json, sys+request = json.load(sys.stdin)+print(json.dumps({"action": request["action"], "arguments": sys.argv[1:]}))+`,+ );+ const python = await resolvePluginPython({+ environment: options.environment,+ protectedRoot: options.protectedRoot,+ });+ expect(+ await runWorkbench(+ { ...options, pluginRoot, python },+ ["local-dedupe"],+ '{"action":"embed","entries":[]}',+ ),+ ).toEqual({ action: "embed", arguments: ["local-dedupe"] });+});++for (const failure of [+ "1e400",+ "NaN",+ "Infinity",+ "-Infinity",+ "BLOB finding ID",+]) {+ test(`native local dedupe commits before reporting a stored ${failure} response error`, async () => {+ const { options, prepare, finding, stateDirectory } = await fixture();+ await runWorkbench(options, ["local-dedupe"], JSON.stringify(prepare));+ const database = join(stateDirectory, "workbench.sqlite3");+ function sql(script: string, ...args: string[]) {+ const result = spawnSync(+ "node",+ [+ "-e",+ `const { DatabaseSync } = require("node:sqlite");+const database = new DatabaseSync(process.argv[1]);+${script}+database.close();`,+ database,+ ...args,+ ],+ { encoding: "utf8" },+ );+ expect(result.status, result.stderr).toBe(0);+ return result.stdout;+ }+ if (failure === "BLOB finding ID") {+ sql(+ `+database.exec("DELETE FROM finding_repositories");+database.prepare("UPDATE findings SET id = CAST(id AS BLOB) WHERE id = ?")+ .run(process.argv[2]);+`,+ finding.findingId,+ );+ } else {+ sql(+ 'database.prepare("UPDATE findings SET details_json = ? WHERE id = ?").run(...process.argv.slice(2));',+ `{"evidence":[${failure}]}`,+ finding.findingId,+ );+ }+ const newId = `${finding.findingId}-new`;+ const payload = {+ ...prepare,+ repositoryId: undefined,+ findings: [+ {+ ...finding,+ findingId: newId,+ fingerprints: {+ ...finding.fingerprints,+ primary: `${finding.fingerprints.primary}-new`,+ },+ },+ ],+ };+ const result = spawnSync(+ "node",+ [join(PLUGIN_ROOT, "mcp", "helpers.mjs"), "local-dedupe"],+ {+ input: JSON.stringify({ stateDirectory, payload }),+ encoding: "utf8",+ },+ );+ expect(result.status).toBe(1);+ expect(result.stdout).toBe("");+ if (failure === "BLOB finding ID")+ expect(result.stderr).toMatch(/key|blob|bytes/iu);+ else expect(result.stderr).toContain("non-finite");+ expect(+ sql(+ `console.log(database.prepare("SELECT COUNT(*) AS total FROM findings f JOIN finding_repositories r ON r.finding_id = f.id WHERE f.id = ? AND r.repository_id = ?").get(...process.argv.slice(2)).total);`,+ newId,+ prepare.anchorRepositoryId,+ ).trim(),+ ).toBe("1");+ });+}diff --git a/sdk/typescript/tests-ts/local-dedupe.test.ts b/sdk/typescript/tests-ts/local-dedupe.test.tsindex 65c3fe0d4..35b9b74d4 100644--- a/sdk/typescript/tests-ts/local-dedupe.test.ts+++ b/sdk/typescript/tests-ts/local-dedupe.test.ts@@ -1,10 +1,7 @@import { setFindingIdentity, sha256 } from "./support/finding-identity.js";import { existsSync } from "node:fs";import { getEventListeners } from "node:events";-import {- setTimeout as delay,- setImmediate as nextTurn,-} from "node:timers/promises";+import { setImmediate as nextTurn } from "node:timers/promises";import { readFile, writeFile, mkdir, chmod, unlink } from "node:fs/promises";import { join } from "node:path";import { afterEach, expect, mock, spyOn, test } from "bun:test";@@ -43,7 +40,9 @@ const vector = [1, ...Array<number>(EMBEDDING_DIMENSIONS - 1).fill(0)];for (const mode of ["direct", "fresh workflow", "resumed workflow"]) {test.skipIf(process.platform === "win32")(- `directory dedupe rejects repository Python before probing it from another directory (${mode})`,+ mode === "direct"+ ? "direct directory dedupe uses native helpers without probing repository Python"+ : `directory dedupe rejects repository Python before probing it from another directory (${mode})`,async () => {const f = await fixture();const options = {@@ -66,19 +65,31 @@ for (const mode of ["direct", "fresh workflow", "resumed workflow"]) {);await chmod(python, 0o700);expect(process.cwd()).not.toBe(f.repository);- await expect(- deduplicateScanDirectoryInternal(f.scanDir, options, {- environment: {- ...f.environment,- PYTHON: python,- TEST_PYTHON_PROBE: marker,- },- runWorkbench: rejecting("Repository Python reached the workbench"),- reviewer: emptyNeighborhoodReviewer(),- }),- ).rejects.toThrow("PYTHON interpreter is unavailable or unusable");+ const result = deduplicateScanDirectoryInternal(f.scanDir, options, {+ environment: {+ ...f.environment,+ PYTHON: python,+ TEST_PYTHON_PROBE: marker,+ },+ ...(mode === "direct"+ ? {}+ : {+ runWorkbench: rejecting(+ "Repository Python reached the workbench",+ ),+ }),+ reviewer: emptyNeighborhoodReviewer(),+ });+ if (mode === "direct") {+ expect((await result).deduplicationStatus).toBe("completed");+ expect(f.embed).toHaveBeenCalledTimes(1);+ } else {+ await expect(result).rejects.toThrow(+ "PYTHON interpreter is unavailable or unusable",+ );+ expect(f.embed).not.toHaveBeenCalled();+ }expect(existsSync(marker)).toBe(false);- expect(f.embed).not.toHaveBeenCalled();},);}@@ -1232,43 +1243,6 @@ test("local workbench calls receive cancellation and cannot report success afterexpect(actions).toEqual(["prepare", "embed", "neighbors", "commit"]);});-test.skipIf(process.platform === "win32")(- "local preparation cancels an active Python probe",- async () => {- const f = await fixture();- const python = join(f.root, "slow-python");- const marker = join(f.root, "probe-started");- await writeFile(- python,- '#!/bin/sh\nprintf started > "$TEST_PYTHON_PROBE"\nexec sleep 10\n',- );- await chmod(python, 0o700);- const controller = new AbortController();- try {- const local = new LocalDeduplication(- { ...f.environment, PYTHON: python, TEST_PYTHON_PROBE: marker },- { repositoryId: f.targetId },- f.repository,- controller.signal,- undefined,- f.embedding,- );- const outcome = local- .prepare(f.document.findings, f.targetId)- .catch((error: unknown) => error);- const completion = outcome.then((error) => ({ error }));- while (!existsSync(marker)) {- const exited = await Promise.race([delay(10), completion]);- if (exited) throw exited.error;- }- controller.abort();- expect(await outcome).toMatchObject({ name: "AbortError" });- } finally {- controller.abort();- }- },-);-test("local embeddings preserve an endpoint from a copied Windows environment", async () => {const f = await fixture();const finding = f.document.findings[0]!;