Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(plugin)!: move findings and dashboard helpers to Node (#1408)

mldangelo-oai77ea91ea96b2
diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts
index 258a7ffc6..cc32f2751 100644
--- a/plugins/codex-security/mcp-app/helpers-main.ts
+++ b/plugins/codex-security/mcp-app/helpers-main.ts
@@ -9,7 +9,7 @@ import { deepReviewInputCommand } from "./src/helpers/deep-review-input";
import { rankShardsCommand } from "./src/helpers/rank-shards";
import { rankPoolCommand } from "./src/helpers/rank-pool";
import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes";
-import { escapeControls } from "./src/helpers/json";
+import { escapeControls, stringifyJson } from "./src/helpers/json";
import { decodeUtf8 } from "./src/helpers/utf8";
let commandLine = process.argv.slice(2);
@@ -36,6 +36,22 @@ if (commandLine[0] === "--helper") {
}
}
const [command, ...args] = commandLine;
+const workbenchUsage: Record<string, string> = {
+ "database-info":
+ "Usage: database-info (reads a JSON absolute state-directory string from stdin)",
+ "store-findings":
+ "Usage: store-findings\nReads a JSON object from stdin with an absolute stateDirectory and payload.entries containing finding and embedding records; payload.repositoryId is optional.",
+ "list-stored-findings":
+ "Usage: list-stored-findings\nReads a JSON object from stdin with an absolute stateDirectory, positive payload.limit and non-negative payload.offset.",
+ "find-potential-duplicates":
+ "Usage: find-potential-duplicates\nReads a JSON object from stdin with an absolute stateDirectory, payload.findingId and payload.scope containing either repositoryId or allRepositories: true.",
+ "store-dedupe-groups":
+ "Usage: store-dedupe-groups\nReads a JSON object from stdin with an absolute stateDirectory and payload.groups containing arrays of finding IDs.",
+ "list-dedupe-groups":
+ "Usage: list-dedupe-groups\nReads a JSON object from stdin with an absolute stateDirectory and payload.findingId.",
+ dashboard:
+ "Usage: dashboard\nReads a JSON object from stdin with an absolute stateDirectory and payload containing view (findings or groups), sort, limit and offset; direction, query, repository and id are optional.",
+};
if (command === "resolve-security-md") {
process.exitCode = resolveSecurityMdCommand(args, posixHome);
} else if (command === "normalize-candidates") {
@@ -61,23 +77,26 @@ if (command === "resolve-security-md") {
process.exitCode = rankPoolCommand(command, args, posixHome);
} else if (command === "bind-repo-scopes") {
process.exitCode = bindRepoScopesCommand(args, posixHome);
-} else if (command === "database-info") {
+} else if (Object.hasOwn(workbenchUsage, command)) {
void (async () => {
const { values } = parseArgs({
args,
options: { help: { type: "boolean", short: "h" } },
});
if (values.help) {
- console.log(
- "Usage: database-info (reads a JSON absolute state-directory string from stdin)",
- );
+ console.log(workbenchUsage[command]);
return;
}
- const { databaseInfo } = await import("./src/workbench/database");
+ let result: unknown;
+ if (command === "database-info") {
+ const { databaseInfo } = await import("./src/workbench/database");
+ result = await databaseInfo(JSON.parse(decodeUtf8(readFileSync(0))));
+ } else {
+ const { findingsCommand } = await import("./src/workbench/commands");
+ result = await findingsCommand(command, decodeUtf8(readFileSync(0)));
+ }
console.log(
- JSON.stringify(
- await databaseInfo(JSON.parse(decodeUtf8(readFileSync(0)))),
- ).replace(/[\p{Cc}\p{Cf}]/gu, (character) =>
+ stringifyJson(result, 0).replace(/[\p{Cc}\p{Cf}]/gu, (character) =>
character
.split("")
.map(
@@ -94,7 +113,7 @@ if (command === "resolve-security-md") {
});
} else {
console.error(
- "Usage: launch_codex_security_mcp[.cmd] --helper <resolve-security-md | normalize-candidates | validate-patch-risk-assessment | copy-deep-review-input | select-deep-review-input | make-rank-shards | validate-rank-shard | merge-rank-outputs | make-rank-pool-plan | validate-rank-worker | validate-rank-pool | bind-repo-scopes | database-info> [options]",
+ `Usage: launch_codex_security_mcp[.cmd] --helper <resolve-security-md | normalize-candidates | validate-patch-risk-assessment | copy-deep-review-input | select-deep-review-input | make-rank-shards | validate-rank-shard | merge-rank-outputs | make-rank-pool-plan | validate-rank-worker | validate-rank-pool | bind-repo-scopes | ${Object.keys(workbenchUsage).join(" | ")}> [options]`,
);
process.exitCode = 2;
}
diff --git a/plugins/codex-security/mcp-app/src/helpers/json.ts b/plugins/codex-security/mcp-app/src/helpers/json.ts
index 2e920ba73..06dae6ba3 100644
--- a/plugins/codex-security/mcp-app/src/helpers/json.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/json.ts
@@ -34,7 +34,7 @@ export function parseJson(source: string): unknown {
}
}
-export function stringifyJson(value: unknown): string {
+export function stringifyJson(value: unknown, space = 2): string {
return JSON.stringify(
value,
(_key, item: unknown) =>
@@ -43,6 +43,6 @@ export function stringifyJson(value: unknown): string {
String(item),
)
: item,
- 2,
+ space,
);
}
diff --git a/plugins/codex-security/mcp-app/src/workbench/commands.ts b/plugins/codex-security/mcp-app/src/workbench/commands.ts
new file mode 100644
index 000000000..2532cef79
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/workbench/commands.ts
@@ -0,0 +1,99 @@
+import { parseJson } from "../helpers/json";
+import { openWorkbenchDatabase, workbenchDatabasePath } from "./database";
+import {
+ listStoredFindings,
+ storeFindings,
+ type EmbeddedFinding,
+} from "./findings";
+import {
+ findPotentialDuplicates,
+ listDedupeGroups,
+ storeDedupeGroups,
+} from "./duplicates";
+
+import { dashboard, type DashboardQuery } from "./dashboard";
+
+export async function findingsCommand(
+ command: string,
+ input: string,
+): Promise<unknown> {
+ const request = parseJson(input) as {
+ stateDirectory: string;
+ payload: unknown;
+ };
+ const { stateDirectory, payload } = request;
+ const page = payload as { limit: number; offset: number };
+ if (
+ command === "list-stored-findings" &&
+ (!Number.isSafeInteger(page.limit) ||
+ page.limit <= 0 ||
+ !Number.isSafeInteger(page.offset) ||
+ page.offset < 0)
+ )
+ throw new Error(
+ "limit must be a positive integer and offset a non-negative integer.",
+ );
+ const selection = payload as {
+ findingId: string;
+ scope?: { repositoryId?: string; allRepositories?: true };
+ };
+ if (
+ (command === "find-potential-duplicates" ||
+ command === "list-dedupe-groups") &&
+ typeof selection?.findingId !== "string"
+ )
+ throw new Error("findingId must be a string.");
+ if (command === "find-potential-duplicates") {
+ const scope = selection.scope;
+ if (
+ !(
+ typeof scope?.repositoryId === "string" &&
+ scope.allRepositories === undefined
+ ) &&
+ !(scope?.allRepositories === true && scope.repositoryId === undefined)
+ )
+ throw new Error(
+ "scope must specify either repositoryId or allRepositories: true.",
+ );
+ }
+ const database = await openWorkbenchDatabase(
+ workbenchDatabasePath(stateDirectory),
+ {
+ deferred:
+ command !== "store-findings" && command !== "store-dedupe-groups",
+ },
+ );
+ try {
+ if (command === "store-findings") {
+ const { entries, repositoryId } = payload as {
+ entries: EmbeddedFinding[];
+ repositoryId?: string;
+ };
+ return storeFindings(
+ database,
+ entries,
+ new Date().toISOString(),
+ repositoryId,
+ );
+ }
+ if (command === "store-dedupe-groups")
+ return storeDedupeGroups(
+ database,
+ (payload as { groups: string[][] }).groups,
+ new Date().toISOString(),
+ );
+ if (command === "list-dedupe-groups")
+ return listDedupeGroups(database, selection.findingId);
+ if (command === "find-potential-duplicates")
+ return findPotentialDuplicates(
+ database,
+ selection.findingId,
+ selection.scope!.repositoryId,
+ );
+ if (command === "dashboard")
+ return dashboard(database, payload as DashboardQuery);
+ return listStoredFindings(database, page);
+ } finally {
+ database.close();
+ }
+}
diff --git a/plugins/codex-security/mcp-app/src/workbench/dashboard.ts b/plugins/codex-security/mcp-app/src/workbench/dashboard.ts
new file mode 100644
index 000000000..bfb5b089c
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/workbench/dashboard.ts
@@ -0,0 +1,185 @@
+import type { DatabaseSync, SQLOutputValue } from "node:sqlite";
+import { parseJson } from "../helpers/json";
+import { requireSqliteText } from "./database";
+import { listDedupeGroups } from "./duplicates";
+import { transaction } from "./transaction";
+
+const records = {
+ findings: `
+ SELECT findings.id, json_extract(details_json, '$.title') AS title,
+ COALESCE(repositories.ids, '[]') AS repositoryIds,
+ json_extract(details_json, '$.severity.level') AS severity,
+ findings.created_at AS createdAt, findings.updated_at AS updatedAt
+ FROM findings LEFT JOIN (
+ SELECT finding_id, json_group_array(repository_id) AS ids
+ FROM finding_repositories GROUP BY finding_id
+ ) AS repositories ON repositories.finding_id = findings.id
+ WHERE details_json IS NOT NULL`,
+ groups: `
+ SELECT groups.id, groups.id AS title,
+ (SELECT json_group_array(DISTINCT repository_id)
+ FROM finding_dedupe_group_members AS members
+ JOIN finding_repositories ON finding_repositories.finding_id = members.finding_id
+ WHERE members.group_id = groups.id) AS repositoryIds,
+ groups.created_at AS createdAt, groups.created_at AS updatedAt,
+ (SELECT COUNT(*) FROM finding_dedupe_group_members WHERE group_id = groups.id) AS memberCount
+ FROM finding_dedupe_groups AS groups`,
+};
+
+const sorts = {
+ activity: "records.updatedAt",
+ newest: "records.createdAt",
+ title: "dashboard_lower(json_quote(records.title))",
+ repository: "repository_label(records.repositoryIds)",
+ severity:
+ "CASE records.severity WHEN 'informational' THEN 0 WHEN 'low' THEN 1 " +
+ "WHEN 'medium' THEN 2 WHEN 'high' THEN 3 WHEN 'critical' THEN 4 END",
+ members: "records.memberCount",
+};
+
+export interface DashboardQuery {
+ view: keyof typeof records;
+ sort: keyof typeof sorts;
+ direction?: "asc" | "desc";
+ limit: number;
+ offset: number;
+ query?: string;
+ repository?: string;
+ id?: string;
+}
+
+function compare(left: string, right: string) {
+ return left < right ? -1 : left > right ? 1 : 0;
+}
+
+function repositoryIds(value: string[]): string[] {
+ return value.sort(
+ (left, right) =>
+ compare(left.toLowerCase(), right.toLowerCase()) || compare(left, right),
+ );
+}
+
+function item(
+ row: Record<string, SQLOutputValue>,
+): Record<string, SQLOutputValue | string[]> {
+ const value = JSON.parse(row.item as string);
+ return {
+ ...value,
+ repositoryIds: repositoryIds(value.repositoryIds),
+ };
+}
+
+function detail(
+ database: DatabaseSync,
+ view: DashboardQuery["view"],
+ selected: Record<string, SQLOutputValue>,
+) {
+ const selectedItem = item(selected);
+ const id = selectedItem.id as string;
+ return view === "findings"
+ ? {
+ item: selectedItem,
+ finding: parseJson(
+ database
+ .prepare("SELECT details_json FROM findings WHERE id = ?")
+ .get(id)!.details_json as string,
+ ),
+ groups: listDedupeGroups(database, id).groups,
+ }
+ : {
+ item: selectedItem,
+ group: JSON.parse(
+ database
+ .prepare(
+ `SELECT json_object('groupId', ?, 'createdAt', ?, 'findingIds', json_group_array(finding_id)) AS value
+ FROM (SELECT finding_id FROM finding_dedupe_group_members WHERE group_id = ? ORDER BY finding_id)`,
+ )
+ .get(id, selectedItem.createdAt as string, id)!.value as string,
+ ),
+ };
+}
+
+/** Read one snapshot without loading artifacts or modifying stored data. */
+export function dashboard(database: DatabaseSync, query: DashboardQuery) {
+ requireSqliteText([query.query, query.repository, query.id]);
+ // JSON preserves text across Node 22 SQLite result and callback boundaries.
+ database.function("dashboard_lower", { deterministic: true }, (value) =>
+ (JSON.parse(value as string) as string).toLowerCase(),
+ );
+ database.function("dashboard_upper", { deterministic: true }, (value) =>
+ (JSON.parse(value as string) as string).toUpperCase(),
+ );
+ database.function("repository_label", { deterministic: true }, (value) =>
+ repositoryIds(JSON.parse(value as string))
+ .join(", ")
+ .toLowerCase(),
+ );
+ const clauses: string[] = [];
+ const values: string[] = [];
+ if (query.query) {
+ const columns = ["id", "title", "repositoryIds"];
+ clauses.push(
+ `(${columns.map((column) => `instr(dashboard_upper(json_quote(COALESCE(records.${column}, ''))), ?) > 0`).join(" OR ")})`,
+ );
+ values.push(...columns.map(() => query.query!.toUpperCase()));
+ }
+ if (query.repository) {
+ clauses.push(
+ "EXISTS (SELECT 1 FROM json_each(records.repositoryIds) WHERE value = ?)",
+ );
+ values.push(query.repository);
+ }
+ const where = clauses.length ? ` WHERE ${clauses.join(" AND ")}` : "";
+ const direction = { asc: "ASC", desc: "DESC" }[query.direction ?? "desc"];
+ let order = `${sorts[query.sort]} ${direction}`;
+ if (query.view === "findings" && query.sort === "activity")
+ order += `, ${sorts.severity} DESC`;
+ order += ", records.id";
+ const source = records[query.view];
+ const field = query.view === "findings" ? "severity" : "memberCount";
+ const projection = `json_object('id', records.id, 'title', records.title,
+ 'repositoryIds', json(repositoryIds), 'createdAt', createdAt, 'updatedAt', updatedAt,
+ '${field}', ${field}) AS item`;
+ return transaction(database, "BEGIN", () => {
+ const repositories = database
+ .prepare(
+ `SELECT json_object('id', repository_id, 'label', repository_id) AS value
+ FROM (SELECT DISTINCT repository_id FROM finding_repositories ORDER BY repository_id)`,
+ )
+ .all()
+ .map((row) => JSON.parse(row.value as string));
+ const total = database
+ .prepare(`SELECT COUNT(*) AS count FROM (${source}) AS records ${where}`)
+ .get(...values)!.count as number;
+ const rows = database
+ .prepare(
+ `SELECT ${projection} FROM (${source}) AS records ${where} ORDER BY ${order} LIMIT ? OFFSET ?`,
+ )
+ .all(...values, query.limit, query.offset);
+ const selected = query.id
+ ? database
+ .prepare(
+ `SELECT ${projection} FROM (${source}) AS records WHERE records.id = ?`,
+ )
+ .get(query.id)
+ : undefined;
+ const nextOffset = query.offset + rows.length;
+ return {
+ overview: {
+ ...database
+ .prepare(
+ `SELECT (SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL) AS findings,
+ (SELECT COUNT(*) FROM finding_dedupe_groups) AS groups`,
+ )
+ .get()!,
+ },
+ repositories,
+ items: rows.map(item),
+ total,
+ limit: query.limit,
+ offset: query.offset,
+ nextOffset: nextOffset < total ? nextOffset : null,
+ detail: selected ? detail(database, query.view, selected) : null,
+ };
+ });
+}
diff --git a/plugins/codex-security/mcp-app/src/workbench/database.ts b/plugins/codex-security/mcp-app/src/workbench/database.ts
index 90cc7c584..73c5cde24 100644
--- a/plugins/codex-security/mcp-app/src/workbench/database.ts
+++ b/plugins/codex-security/mcp-app/src/workbench/database.ts
@@ -14,6 +14,13 @@ import { decodePosixBytes, encodePosixPath } from "../helpers/posix-path";
import { windowsBinding } from "../native";
import { widePath, windowsFileSystem } from "../../../native/windows-files.mjs";
+export function requireSqliteText(
+ values: readonly (string | null | undefined)[],
+): void {
+ if (values.some((value) => value != null && !value.isWellFormed()))
+ throw new TypeError("SQLite text keys must contain valid Unicode.");
+}
+
function createStateDirectory(path: string): void {
if (process.platform !== "win32") path = path.replace(/\/+$/u, "") || "/";
const nativePath =
@@ -69,9 +76,7 @@ export async function openWorkbenchDatabase(
}
}
-export async function databaseInfo(
- state: string,
-): Promise<{ databasePath: string }> {
+export function workbenchDatabasePath(state: string): string {
if (
typeof state !== "string" ||
!isAbsolute(state) ||
@@ -81,11 +86,16 @@ export async function databaseInfo(
"database-info requires an absolute Unicode state-directory string.",
);
}
+ return `${state}${sep}workbench.sqlite3`;
+}
+
+export async function databaseInfo(
+ state: string,
+): Promise<{ databasePath: string }> {
// Keep an ASCII alias usable even when its destination has raw POSIX bytes.
- const database = await openWorkbenchDatabase(
- `${state}${sep}workbench.sqlite3`,
- { deferred: true },
- );
+ const database = await openWorkbenchDatabase(workbenchDatabasePath(state), {
+ deferred: true,
+ });
database.close();
const canonicalState =
process.platform === "win32"
diff --git a/plugins/codex-security/mcp-app/src/workbench/duplicates.ts b/plugins/codex-security/mcp-app/src/workbench/duplicates.ts
new file mode 100644
index 000000000..3072745e8
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/workbench/duplicates.ts
@@ -0,0 +1,161 @@
+import { createHash } from "node:crypto";
+import type { DatabaseSync } from "node:sqlite";
+import { parseJson } from "../helpers/json";
+import { transaction } from "./transaction";
+import { requireSqliteText } from "./database";
+
+export interface DedupeGroup {
+ groupId: string;
+ findingIds: string[];
+ createdAt: string;
+}
+
+function normalizedVector(vector: number[]): number[] {
+ const maximum = vector.reduce(
+ (maximum, value) => Math.max(maximum, Math.abs(value)),
+ 0,
+ );
+ // Binary scaling keeps ratios exact; 1023 is the largest finite Number exponent.
+ const scale = 2 ** Math.min(Math.floor(Math.log2(maximum)), 1023);
+ const norm =
+ scale *
+ Math.sqrt(vector.reduce((sum, value) => sum + (value / scale) ** 2, 0));
+ if (!vector.every(Number.isFinite) || norm === 0 || !Number.isFinite(norm))
+ throw new RangeError("A stored embedding cannot be compared.");
+ return vector.map((value) => value / norm);
+}
+
+export function findPotentialDuplicates(
+ database: DatabaseSync,
+ findingId: string,
+ repositoryId?: string,
+) {
+ return transaction(database, "BEGIN", () => {
+ requireSqliteText([findingId, repositoryId]);
+ const source =
+ repositoryId === undefined
+ ? "finding_embeddings AS embeddings"
+ : "finding_repositories AS repositories JOIN finding_embeddings AS embeddings ON embeddings.finding_id = repositories.finding_id";
+ const predicate =
+ repositoryId === undefined ? "" : "repositories.repository_id = ? AND ";
+ const scope = repositoryId === undefined ? [] : [repositoryId];
+ const anchor = database
+ .prepare(
+ `SELECT embeddings.vector_json FROM ${source}
+ WHERE ${predicate}embeddings.finding_id = ?`,
+ )
+ .get(...scope, findingId);
+ if (!anchor) return { error: "finding_not_indexed" as const };
+
+ const rows = database.prepare(
+ `SELECT json_quote(embeddings.finding_id) AS finding_id_json, embeddings.vector_json FROM ${source}
+ JOIN findings ON findings.id = embeddings.finding_id
+ WHERE ${predicate}embeddings.model = (SELECT model FROM finding_embeddings WHERE finding_id = ?)
+ AND embeddings.finding_id != ?
+ ORDER BY findings.created_at, findings.id`,
+ );
+ const ranked: { id: string; similarity: number }[] = [];
+ try {
+ const vector = normalizedVector(JSON.parse(anchor.vector_json as string));
+ for (const row of rows.iterate(...scope, findingId, findingId)) {
+ const candidate: number[] = JSON.parse(row.vector_json as string);
+ if (candidate.length !== vector.length) continue;
+ const other = normalizedVector(candidate);
+ const similarity = vector.reduce(
+ (total, value, index) => total + value * other[index],
+ 0,
+ );
+ if (similarity >= 0.55)
+ ranked.push({
+ id: JSON.parse(row.finding_id_json as string),
+ similarity,
+ });
+ }
+ } catch (error) {
+ if (error instanceof SyntaxError || error instanceof RangeError)
+ return { error: "embedding_failed" as const };
+ throw error;
+ }
+ // Stable sorting keeps insertion-time / finding-ID order for ties.
+ ranked.sort((a, b) => b.similarity - a.similarity);
+ const selected = [findingId, ...ranked.slice(0, 50).map(({ id }) => id)];
+ const [finding, ...potentialDuplicates] = database
+ .prepare(
+ `SELECT findings.details_json FROM json_each(?) AS selected
+ JOIN findings ON findings.id = selected.value ORDER BY selected.key`,
+ )
+ .all(JSON.stringify(selected))
+ .map((row) => parseJson(row.details_json as string));
+ return { finding, potentialDuplicates };
+ });
+}
+
+export function storeDedupeGroups(
+ database: DatabaseSync,
+ groups: readonly (readonly string[])[],
+ timestamp: string,
+) {
+ try {
+ return transaction(database, "BEGIN IMMEDIATE", () => {
+ const insertGroup = database.prepare(
+ "INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?) ON CONFLICT(id) DO NOTHING",
+ );
+ const insertMember = database.prepare(
+ "INSERT INTO finding_dedupe_group_members (group_id, finding_id) VALUES (?, ?) ON CONFLICT(group_id, finding_id) DO NOTHING",
+ );
+ const created = database.prepare(
+ "SELECT created_at FROM finding_dedupe_groups WHERE id = ?",
+ );
+ const stored = new Map<string, DedupeGroup>();
+ for (const group of groups) {
+ requireSqliteText(group);
+ // Durable group IDs use code-point sorting and ASCII-escaped JSON.
+ const findingIds = [...new Set(group)].sort((left, right) =>
+ Buffer.compare(Buffer.from(left), Buffer.from(right)),
+ );
+ const encoded = JSON.stringify(findingIds).replace(
+ /[\u007f-\uffff]/g,
+ (character) =>
+ `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`,
+ );
+ const groupId =
+ "fdg_" + createHash("sha256").update(encoded).digest("hex");
+ insertGroup.run(groupId, timestamp);
+ for (const findingId of findingIds)
+ insertMember.run(groupId, findingId);
+ stored.set(groupId, {
+ groupId,
+ findingIds,
+ createdAt: created.get(groupId)!.created_at as string,
+ });
+ }
+ return { groups: [...stored.values()] };
+ });
+ } catch (error) {
+ if (
+ error instanceof Error &&
+ "errcode" in error &&
+ (Number(error.errcode) & 0xff) === 19
+ )
+ return { error: "finding_conflict" as const };
+ throw error;
+ }
+}
+
+export function listDedupeGroups(database: DatabaseSync, findingId: string) {
+ requireSqliteText([findingId]);
+ const rows = database.prepare(`
+ SELECT json_object('groupId', groups.id, 'createdAt', groups.created_at,
+ 'findingIds', json_group_array(members.finding_id ORDER BY members.finding_id)) AS document
+ FROM finding_dedupe_group_members AS matched
+ JOIN finding_dedupe_groups AS groups ON groups.id = matched.group_id
+ JOIN finding_dedupe_group_members AS members ON members.group_id = groups.id
+ WHERE matched.finding_id = ?
+ GROUP BY groups.id ORDER BY groups.created_at, groups.id
+ `);
+ return {
+ groups: rows
+ .all(findingId)
+ .map((row) => JSON.parse(row.document as string) as DedupeGroup),
+ };
+}
diff --git a/plugins/codex-security/mcp-app/src/workbench/findings.ts b/plugins/codex-security/mcp-app/src/workbench/findings.ts
new file mode 100644
index 000000000..ccc2398b2
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/workbench/findings.ts
@@ -0,0 +1,133 @@
+import type { DatabaseSync } from "node:sqlite";
+import { parseJson, stringifyJson } from "../helpers/json";
+import { requireSqliteText } from "./database";
+import { transaction } from "./transaction";
+
+export interface Finding {
+ findingId: string;
+ fingerprints: { primary: string };
+ ruleId: string;
+ identity: { anchor: string; instance?: string };
+ [key: string]: unknown;
+}
+
+export interface EmbeddedFinding {
+ finding: Finding;
+ embedding: { model: string; vector: number[] };
+}
+
+class FindingConflict extends Error {}
+
+function requireFiniteNumbers(value: unknown): void {
+ if (typeof value === "number" && !Number.isFinite(value))
+ throw new TypeError(
+ "Stored finding JSON cannot contain non-finite numbers.",
+ );
+ if (value !== null && typeof value === "object")
+ for (const item of Object.values(value)) requireFiniteNumbers(item);
+}
+
+export function storeFindings(
+ database: DatabaseSync,
+ entries: readonly EmbeddedFinding[],
+ timestamp: string,
+ repositoryId?: string,
+): { findingIds: string[] } | { error: "finding_conflict" } {
+ try {
+ return transaction(database, "BEGIN IMMEDIATE", () => {
+ requireSqliteText([timestamp, repositoryId]);
+ const upsert = database.prepare(
+ `INSERT INTO findings (id, fingerprint, rule_id, identity_anchor, identity_instance,
+ details_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO UPDATE SET details_json = excluded.details_json,
+ updated_at = excluded.updated_at
+ WHERE findings.fingerprint = excluded.fingerprint
+ AND findings.rule_id = excluded.rule_id
+ AND findings.identity_anchor = excluded.identity_anchor
+ AND findings.identity_instance IS excluded.identity_instance`,
+ );
+ const embedding = database.prepare(
+ `INSERT INTO finding_embeddings (finding_id, model, vector_json) VALUES (?, ?, ?)
+ ON CONFLICT(finding_id) DO UPDATE SET model = excluded.model, vector_json = excluded.vector_json`,
+ );
+ const repository = database.prepare(
+ "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
+ );
+ for (const entry of entries) {
+ const finding = entry.finding;
+ requireSqliteText([
+ finding.findingId,
+ finding.fingerprints.primary,
+ finding.ruleId,
+ finding.identity.anchor,
+ finding.identity.instance,
+ entry.embedding.model,
+ ]);
+ requireFiniteNumbers([finding, entry.embedding.vector]);
+ const { changes } = upsert.run(
+ finding.findingId,
+ finding.fingerprints.primary,
+ finding.ruleId,
+ finding.identity.anchor,
+ finding.identity.instance ?? null,
+ stringifyJson(finding, 0),
+ timestamp,
+ timestamp,
+ );
+ if (changes === 0)
+ throw new FindingConflict(
+ "The stored finding identity cannot be replaced.",
+ );
+ if (repositoryId !== undefined)
+ repository.run(repositoryId, finding.findingId);
+ embedding.run(
+ finding.findingId,
+ entry.embedding.model,
+ stringifyJson(entry.embedding.vector, 0),
+ );
+ }
+ return { findingIds: entries.map(({ finding }) => finding.findingId) };
+ });
+ } catch (error) {
+ if (
+ error instanceof FindingConflict ||
+ (error instanceof Error &&
+ "errcode" in error &&
+ (Number(error.errcode) & 0xff) === 19)
+ ) {
+ return { error: "finding_conflict" };
+ }
+ throw error;
+ }
+}
+
+export function listStoredFindings(
+ database: DatabaseSync,
+ { limit, offset }: { limit: number; offset: number },
+) {
+ return transaction(database, "BEGIN", () => {
+ const total = Number(
+ database
+ .prepare(
+ "SELECT COUNT(*) AS total FROM findings WHERE details_json IS NOT NULL",
+ )
+ .get()!.total,
+ );
+ const rows = database
+ .prepare(
+ `SELECT details_json FROM findings WHERE details_json IS NOT NULL
+ ORDER BY created_at, id LIMIT ? OFFSET ?`,
+ )
+ .all(limit, offset);
+ const nextOffset = offset + rows.length;
+ return {
+ findings: rows.map(
+ (row) => parseJson(String(row.details_json)) as Finding,
+ ),
+ limit,
+ offset,
+ total,
+ nextOffset: nextOffset < total ? nextOffset : null,
+ };
+ });
+}
diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts
index 1d6271550..0ea0324c3 100644
--- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts
+++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts
@@ -3,6 +3,7 @@ import { basename } from "node:path";
import type { DatabaseSync } from "node:sqlite";
import history from "../../../shared/workbench-migrations.json";
import { parseJson, stringifyJson } from "../helpers/json";
+import { transaction } from "./transaction";
export interface Migration {
version: number;
@@ -397,8 +398,7 @@ export function applyMigrations(
selected: readonly Migration[] = migrations,
immediate = false,
): void {
- database.exec(immediate ? "BEGIN IMMEDIATE" : "BEGIN");
- try {
+ transaction(database, immediate ? "BEGIN IMMEDIATE" : "BEGIN", () => {
database.exec(
"CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)",
);
@@ -449,13 +449,5 @@ export function applyMigrations(
addColumn(database, count);
}
if (backfill) backfillTargets(database);
- database.exec("COMMIT");
- } catch (error) {
- try {
- database.exec("ROLLBACK");
- } catch {
- // SQLite may have rolled back automatically after a storage failure.
- }
- throw error;
- }
+ });
}
diff --git a/plugins/codex-security/mcp-app/src/workbench/transaction.ts b/plugins/codex-security/mcp-app/src/workbench/transaction.ts
new file mode 100644
index 000000000..b1f6c0ac8
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/workbench/transaction.ts
@@ -0,0 +1,21 @@
+import type { DatabaseSync } from "node:sqlite";
+
+export function transaction<T>(
+ database: DatabaseSync,
+ begin: "BEGIN" | "BEGIN IMMEDIATE",
+ action: () => T,
+): T {
+ database.exec(begin);
+ try {
+ const result = action();
+ database.exec("COMMIT");
+ return result;
+ } catch (error) {
+ try {
+ database.exec("ROLLBACK");
+ } catch {
+ // SQLite can roll back automatically after a storage failure.
+ }
+ throw error;
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_dashboard.ts b/plugins/codex-security/mcp-app/tests/test_workbench_dashboard.ts
new file mode 100644
index 000000000..c10b26d4f
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_dashboard.ts
@@ -0,0 +1,230 @@
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { after, test, type TestContext } from "node:test";
+import { importSource } from "./import-module.ts";
+import { stringifyJson } from "../src/helpers/json.ts";
+import { createTemporaryDirectories } from "./support/temporary-directories.ts";
+import type * as Dashboard from "../src/workbench/dashboard.ts";
+import type * as Migrations from "../src/workbench/migrations.ts";
+
+const { dashboard } = (await importSource(
+ "src/workbench/dashboard.ts",
+)) as typeof Dashboard;
+const { applyMigrations } = (await importSource(
+ "src/workbench/migrations.ts",
+)) as typeof Migrations;
+const temporary = createTemporaryDirectories(true);
+after(() => temporary.cleanup());
+
+function database(t: TestContext, path = ":memory:") {
+ const db = new DatabaseSync(path);
+ t.after(() => db.close());
+ applyMigrations(db);
+ return db;
+}
+
+function insert(
+ db: DatabaseSync,
+ id: string,
+ repository: string,
+ title = `Évaluation ${id}`,
+) {
+ const document = {
+ title,
+ severity: { level: "high" },
+ extensions: { opaqueId: 9007199254740993n },
+ };
+ db.prepare(
+ `INSERT INTO findings
+ (id, fingerprint, rule_id, identity_anchor, created_at, updated_at, details_json)
+ VALUES (?, ?, 'rule', ?, 'created', 'updated', ?)`,
+ ).run(id, id, id, stringifyJson(document));
+ db.prepare("INSERT INTO finding_repositories VALUES (?, ?)").run(
+ repository,
+ id,
+ );
+ return document;
+}
+
+test("dashboard selects details independently of filters and pagination", (t) => {
+ const db = database(t);
+ insert(db, "a", "Alpha");
+ const selected = insert(db, "b", "beta", "ΟΣΑ");
+ insert(db, "c", "Alpha");
+ db.exec(`INSERT INTO finding_dedupe_groups VALUES ('group', 'created');
+ INSERT INTO finding_dedupe_group_members VALUES ('group', 'a'), ('group', 'b');`);
+ const before = db.prepare("SELECT total_changes() AS count").get();
+ const result = dashboard(db, {
+ view: "findings",
+ sort: "title",
+ direction: "asc",
+ query: "ÉVALUATION",
+ repository: "Alpha",
+ limit: 1,
+ offset: 1,
+ id: "b",
+ });
+ assert.deepEqual(
+ result.items.map((item) => item.id),
+ ["c"],
+ );
+ assert.equal(result.total, 2);
+ assert.equal(result.nextOffset, null);
+ assert.deepEqual(result.overview, { findings: 3, groups: 1 });
+ assert.deepEqual(result.repositories, [
+ { id: "Alpha", label: "Alpha" },
+ { id: "beta", label: "beta" },
+ ]);
+ assert.equal(result.detail?.item.id, "b");
+ assert.deepEqual(result.detail?.finding, selected);
+ assert.deepEqual(result.detail?.groups, [
+ { groupId: "group", createdAt: "created", findingIds: ["a", "b"] },
+ ]);
+ const groups = dashboard(db, {
+ view: "groups",
+ sort: "members",
+ repository: "Alpha",
+ limit: 1,
+ offset: 0,
+ id: "group",
+ });
+ assert.deepEqual(groups.items[0].repositoryIds, ["Alpha", "beta"]);
+ assert.equal(groups.items[0].memberCount, 2);
+ assert.deepEqual(groups.detail?.group, result.detail?.groups?.[0]);
+ for (const query of ["ΟΣ", "οσ", "ος"])
+ assert.deepEqual(
+ dashboard(db, {
+ view: "findings",
+ sort: "title",
+ query,
+ limit: 50,
+ offset: 0,
+ }).items.map((item) => item.id),
+ ["b"],
+ );
+ assert.deepEqual(db.prepare("SELECT total_changes() AS count").get(), before);
+});
+
+test("dashboard preserves NUL text through display, filters, search, and sorting", (t) => {
+ const db = database(t);
+ const first = "first\0id";
+ const second = "second\0id";
+ const selected = insert(db, first, "repository\0Zulu", "Title\0Zulu");
+ insert(db, second, "repository\0Alpha", "Title\0Alpha");
+ const query = {
+ view: "findings",
+ sort: "title",
+ direction: "asc",
+ limit: 50,
+ offset: 0,
+ } as const;
+ const result = dashboard(db, { ...query, id: first });
+ assert.deepEqual(
+ result.items.map((item) => item.id),
+ [second, first],
+ );
+ assert.deepEqual(
+ result.items.map((item) => item.title),
+ ["Title\0Alpha", "Title\0Zulu"],
+ );
+ assert.deepEqual(result.repositories, [
+ { id: "repository\0Alpha", label: "repository\0Alpha" },
+ { id: "repository\0Zulu", label: "repository\0Zulu" },
+ ]);
+ assert.equal(result.detail?.item.id, first);
+ assert.deepEqual(result.detail?.finding, selected);
+ assert.deepEqual(
+ dashboard(db, {
+ ...query,
+ repository: result.repositories[0].id,
+ }).items.map((item) => item.id),
+ [second],
+ );
+ assert.deepEqual(
+ dashboard(db, { ...query, query: "\0ALPHA" }).items.map((item) => item.id),
+ [second],
+ );
+ assert.deepEqual(
+ dashboard(db, { ...query, sort: "repository" }).items.map(
+ (item) => item.id,
+ ),
+ [second, first],
+ );
+ db.exec("INSERT INTO finding_dedupe_groups VALUES ('group', 'created')");
+ const member = db.prepare(
+ "INSERT INTO finding_dedupe_group_members VALUES ('group', ?)",
+ );
+ member.run(first);
+ member.run(second);
+ const groups = dashboard(db, {
+ ...query,
+ view: "groups",
+ query: "ALPHA",
+ id: "group",
+ });
+ assert.equal(groups.total, 1);
+ assert.deepEqual(groups.detail?.group?.findingIds, [first, second]);
+});
+
+test("dashboard rejects malformed Unicode query keys without aliasing stored values", (t) => {
+ const db = database(t);
+ const stored = insert(db, "record-\ufffd", "scope-\ufffd", "Title\ufffd");
+ const query = {
+ view: "findings",
+ sort: "title",
+ limit: 50,
+ offset: 0,
+ } as const;
+ assert.deepEqual(
+ dashboard(db, { ...query, repository: "scope-\ufffd", id: "record-\ufffd" })
+ .detail?.finding,
+ stored,
+ );
+ for (const filter of [
+ { query: "Title\ud800" },
+ { repository: "scope-\ud800" },
+ { id: "record-\ud800" },
+ ]) {
+ assert.throws(() => dashboard(db, { ...query, ...filter }));
+ }
+});
+
+test("dashboard reads its counts and rows from one WAL snapshot", async (t) => {
+ const path = join(
+ await temporary.create("workbench-dashboard-"),
+ "state.sqlite3",
+ );
+ const db = database(t, path);
+ db.exec("PRAGMA journal_mode = WAL");
+ insert(db, "before", "repository");
+ const writer = new DatabaseSync(path);
+ t.after(() => writer.close());
+ const prepare = db.prepare.bind(db);
+ let written = false;
+ t.mock.method(db, "prepare", (sql: string) => {
+ if (!written && sql.startsWith("SELECT COUNT(*) AS count FROM (")) {
+ written = true;
+ insert(writer, "after", "new-repository");
+ }
+ return prepare(sql);
+ });
+ const query = {
+ view: "findings",
+ sort: "newest",
+ limit: 50,
+ offset: 0,
+ } as const;
+ const snapshot = dashboard(db, query);
+ assert.equal(written, true);
+ assert.equal(snapshot.total, 1);
+ assert.equal(snapshot.overview.findings, 1);
+ assert.deepEqual(
+ snapshot.items.map((item) => item.id),
+ ["before"],
+ );
+ assert.deepEqual(snapshot.repositories, [
+ { id: "repository", label: "repository" },
+ ]);
+ assert.equal(dashboard(db, query).total, 2);
+});
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_duplicates.ts b/plugins/codex-security/mcp-app/tests/test_workbench_duplicates.ts
new file mode 100644
index 000000000..3d5851a72
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_duplicates.ts
@@ -0,0 +1,377 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { DatabaseSync } from "node:sqlite";
+import { after, test, type TestContext } from "node:test";
+import { importSource } from "./import-module.ts";
+import { createTemporaryDirectories } from "./support/temporary-directories.ts";
+import { stringifyJson } from "../src/helpers/json.ts";
+import type * as Duplicates from "../src/workbench/duplicates.ts";
+import type * as Migrations from "../src/workbench/migrations.ts";
+
+const { findPotentialDuplicates, storeDedupeGroups, listDedupeGroups } =
+ (await importSource("src/workbench/duplicates.ts")) as typeof Duplicates;
+const { applyMigrations } = (await importSource(
+ "src/workbench/migrations.ts",
+)) as typeof Migrations;
+
+const temporary = createTemporaryDirectories(true);
+after(() => temporary.cleanup());
+
+function open(t: TestContext, path = ":memory:") {
+ const database = new DatabaseSync(path);
+ t.after(() => database.close());
+ database.exec("PRAGMA foreign_keys = ON");
+ applyMigrations(database);
+ return database;
+}
+
+function finding(
+ database: DatabaseSync,
+ index: number | string,
+ vector = [1, 0],
+ repository = "synthetic-repository",
+ model = "synthetic-model",
+) {
+ const findingId =
+ typeof index === "number"
+ ? `csf_${String(index).padStart(24, "0")}`
+ : index;
+ const document = { findingId, extensions: { opaqueId: 9007199254740993n } };
+ database
+ .prepare(
+ `
+ INSERT INTO findings
+ (id, fingerprint, rule_id, identity_anchor, details_json, created_at, updated_at)
+ VALUES (?, ?, 'synthetic-rule', ?, ?, 'created', 'updated')
+ `,
+ )
+ .run(findingId, findingId, findingId, stringifyJson(document));
+ database
+ .prepare("INSERT INTO finding_embeddings VALUES (?, ?, ?)")
+ .run(findingId, model, JSON.stringify(vector));
+ database
+ .prepare("INSERT INTO finding_repositories VALUES (?, ?)")
+ .run(repository, findingId);
+ return document;
+}
+
+test("duplicate helper requires an explicit JSON scope before accessing state", async () => {
+ const stateDirectory = join(
+ await temporary.create("duplicate-scope-"),
+ "state",
+ );
+ const helper = fileURLToPath(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ),
+ );
+ const call = (scope: unknown) =>
+ spawnSync(process.execPath, [helper, "find-potential-duplicates"], {
+ input: JSON.stringify({
+ stateDirectory,
+ payload: { findingId: "missing", scope },
+ }),
+ encoding: "utf8",
+ });
+ for (const scope of [
+ undefined,
+ null,
+ {},
+ { allRepositories: false },
+ { repositoryId: 7 },
+ { repositoryId: "repository", allRepositories: true },
+ ]) {
+ const result = call(scope);
+ assert.equal(result.status, 1);
+ assert.ok(result.stderr.includes("scope must specify either"));
+ assert.equal(existsSync(stateDirectory), false);
+ }
+ for (const scope of [
+ { repositoryId: "repository" },
+ { allRepositories: true },
+ ]) {
+ const result = call(scope);
+ assert.equal(result.status, 0, result.stderr);
+ assert.deepEqual(JSON.parse(result.stdout), {
+ error: "finding_not_indexed",
+ });
+ }
+});
+
+test("duplicate retrieval filters before scoring and loads only the stable top 50 documents", (t) => {
+ const database = open(t);
+ const entries = Array.from({ length: 62 }, (_, index) =>
+ finding(database, index + 1),
+ );
+ // Out-of-scope vectors and documents beyond the top 50 need not be readable.
+ const foreign = finding(database, 100, [0, 0], "another-repository");
+ finding(database, 101, [0, 0], "synthetic-repository", "another-model");
+ finding(database, 102, [1, 0, 0]);
+ database.exec("DROP TRIGGER invalidate_finding_embedding");
+ for (const entry of entries.slice(51))
+ database
+ .prepare("UPDATE findings SET details_json = 'invalid JSON' WHERE id = ?")
+ .run(entry.findingId);
+ assert.deepEqual(
+ findPotentialDuplicates(
+ database,
+ entries[0].findingId,
+ "synthetic-repository",
+ ),
+ {
+ finding: entries[0],
+ potentialDuplicates: entries.slice(1, 51),
+ },
+ );
+ assert.deepEqual(
+ findPotentialDuplicates(
+ database,
+ foreign.findingId,
+ "synthetic-repository",
+ ),
+ {
+ error: "finding_not_indexed",
+ },
+ );
+ assert.deepEqual(findPotentialDuplicates(database, entries[0].findingId), {
+ error: "embedding_failed",
+ });
+});
+
+test("cosine scoring handles large and scaled vectors without argument spreading", (t) => {
+ const database = open(t);
+ const vector = Array<number>(150_000).fill(0);
+ vector[0] = 1e-300;
+ const anchor = finding(database, 1, vector);
+ vector[0] = 1e300;
+ const candidate = finding(database, 2, vector);
+ assert.deepEqual(findPotentialDuplicates(database, anchor.findingId), {
+ finding: anchor,
+ potentialDuplicates: [candidate],
+ });
+ const update = database.prepare(
+ "UPDATE finding_embeddings SET vector_json = ? WHERE finding_id = ?",
+ );
+ update.run(JSON.stringify([Number.MIN_VALUE, 0]), anchor.findingId);
+ update.run(JSON.stringify([Number.MAX_VALUE, 0]), candidate.findingId);
+ assert.deepEqual(findPotentialDuplicates(database, anchor.findingId), {
+ finding: anchor,
+ potentialDuplicates: [candidate],
+ });
+ update.run("[0,0]", anchor.findingId);
+ assert.deepEqual(findPotentialDuplicates(database, anchor.findingId), {
+ error: "embedding_failed",
+ });
+});
+
+test("cosine scoring includes the threshold and excludes scores below it", (t) => {
+ const database = open(t);
+ const anchor = finding(database, 1, [7, 0]);
+ const candidate = finding(database, 2, [0.55, Math.sqrt(1 - 0.55 ** 2)]);
+ finding(database, 3, [0.54, Math.sqrt(1 - 0.54 ** 2)]);
+ const strongest = finding(database, 4, [1, 0]);
+ finding(database, 5, [0.549e-161, Math.sqrt(1 - 0.549 ** 2) * 1e-161]);
+ for (const repository of [undefined, "synthetic-repository"])
+ assert.deepEqual(
+ findPotentialDuplicates(database, anchor.findingId, repository),
+ { finding: anchor, potentialDuplicates: [strongest, candidate] },
+ );
+});
+
+test("dedupe retries retain durable hashes, first timestamps, and overlapping groups", (t) => {
+ const database = open(t);
+ const a = finding(database, 1).findingId;
+ const b = finding(database, 2).findingId;
+ const c = finding(database, 3).findingId;
+ const original = storeDedupeGroups(
+ database,
+ [
+ [b, a],
+ [b, c],
+ ],
+ "first",
+ );
+ assert.ok("groups" in original);
+ assert.equal(
+ original.groups[0].groupId,
+ "fdg_e1bc8f7052a79c844323824d7f9e0707373dd9ed4391c477aa9694702baae619",
+ );
+ assert.deepEqual(
+ storeDedupeGroups(
+ database,
+ [
+ [a, b, a],
+ [c, b],
+ ],
+ "later",
+ ),
+ original,
+ );
+ assert.deepEqual(
+ listDedupeGroups(database, b).groups,
+ [...original.groups].sort((left, right) =>
+ left.groupId.localeCompare(right.groupId),
+ ),
+ );
+ assert.equal(
+ database.prepare("SELECT count(*) AS count FROM finding_embeddings").get()!
+ .count,
+ 3,
+ );
+});
+
+test("a missing member rolls back the entire dedupe batch and leaves the connection usable", (t) => {
+ const database = open(t);
+ const a = finding(database, 1).findingId;
+ const b = finding(database, 2).findingId;
+ assert.deepEqual(
+ storeDedupeGroups(
+ database,
+ [
+ [a, b],
+ [b, "missing"],
+ ],
+ "created",
+ ),
+ {
+ error: "finding_conflict",
+ },
+ );
+ assert.deepEqual(listDedupeGroups(database, b), { groups: [] });
+ assert.equal(
+ database
+ .prepare("SELECT count(*) AS count FROM finding_dedupe_groups")
+ .get()!.count,
+ 0,
+ );
+ assert.ok("groups" in storeDedupeGroups(database, [[a, b]], "retry"));
+});
+
+test("retries preserve persisted legacy Unicode group identities and timestamps", (t) => {
+ const database = open(t);
+ // Captured from the retired Python writer, including its persisted member order.
+ const group = {
+ groupId:
+ "fdg_6f25bc42f005cebe1de9d4efe6448acdb1d42b1882d58aed74476385dfaa9f3c",
+ findingIds: [
+ "finding-\u007f",
+ "finding-λ",
+ "finding-\ue000",
+ "finding-\u{10000}",
+ ],
+ createdAt: "first",
+ };
+ for (const id of group.findingIds) finding(database, id);
+ database
+ .prepare("INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?)")
+ .run(group.groupId, group.createdAt);
+ const member = database.prepare(
+ "INSERT INTO finding_dedupe_group_members (group_id, finding_id) VALUES (?, ?)",
+ );
+ for (const id of group.findingIds) member.run(group.groupId, id);
+ const original = { groups: [group] };
+ assert.deepEqual(
+ storeDedupeGroups(database, [[...group.findingIds].reverse()], "later"),
+ original,
+ );
+ assert.deepEqual(listDedupeGroups(database, group.findingIds[0]), original);
+ assert.equal(
+ database
+ .prepare("SELECT count(*) AS count FROM finding_dedupe_groups")
+ .get()!.count,
+ 1,
+ );
+});
+
+test("duplicate retrieval and group listing preserve NUL-bearing IDs and model scope", (t) => {
+ const database = open(t);
+ const anchor = finding(
+ database,
+ "finding\0anchor",
+ [1, 0],
+ "repository",
+ "model\0suffix",
+ );
+ const duplicate = finding(
+ database,
+ "finding\0duplicate",
+ [1, 0],
+ "repository",
+ "model\0suffix",
+ );
+ finding(database, "different-model", [1, 0], "repository", "model");
+ assert.deepEqual(
+ findPotentialDuplicates(database, anchor.findingId, "repository"),
+ {
+ finding: anchor,
+ potentialDuplicates: [duplicate],
+ },
+ );
+ const stored = storeDedupeGroups(
+ database,
+ [[duplicate.findingId, anchor.findingId]],
+ "created",
+ );
+ assert.deepEqual(listDedupeGroups(database, duplicate.findingId), stored);
+});
+
+test("unpaired surrogates cannot alias stored finding IDs or repository scopes", (t) => {
+ const database = open(t);
+ const replacement = finding(
+ database,
+ "finding-\ufffd",
+ [1, 0],
+ "repository-\ufffd",
+ );
+ const other = finding(database, "other", [1, 0], "repository-\ufffd");
+ for (const surrogate of ["\ud800", "\udfff"]) {
+ const malformedId = `finding-${surrogate}`;
+ assert.throws(
+ () => findPotentialDuplicates(database, malformedId),
+ TypeError,
+ );
+ assert.throws(
+ () =>
+ findPotentialDuplicates(
+ database,
+ replacement.findingId,
+ `repository-${surrogate}`,
+ ),
+ TypeError,
+ );
+ assert.throws(() => listDedupeGroups(database, malformedId), TypeError);
+ assert.throws(
+ () =>
+ storeDedupeGroups(
+ database,
+ [
+ [replacement.findingId, other.findingId],
+ [other.findingId, malformedId],
+ ],
+ "created",
+ ),
+ TypeError,
+ );
+ assert.equal(
+ database
+ .prepare("SELECT count(*) AS count FROM finding_dedupe_groups")
+ .get()!.count,
+ 0,
+ );
+ }
+ assert.deepEqual(
+ findPotentialDuplicates(
+ database,
+ replacement.findingId,
+ "repository-\ufffd",
+ ),
+ {
+ finding: replacement,
+ potentialDuplicates: [other],
+ },
+ );
+});
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts b/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts
new file mode 100644
index 000000000..f25f8fd25
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts
@@ -0,0 +1,258 @@
+import assert from "node:assert/strict";
+import { execFile, execFileSync, spawnSync } from "node:child_process";
+import { promisify } from "node:util";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { DatabaseSync } from "node:sqlite";
+import { after, test, type TestContext } from "node:test";
+import { importSource } from "./import-module.ts";
+import { createTemporaryDirectories } from "./support/temporary-directories.ts";
+import type * as Findings from "../src/workbench/findings.ts";
+import type * as Migrations from "../src/workbench/migrations.ts";
+
+const { storeFindings, listStoredFindings } = (await importSource(
+ "src/workbench/findings.ts",
+)) as typeof Findings;
+const { applyMigrations } = (await importSource(
+ "src/workbench/migrations.ts",
+)) as typeof Migrations;
+const { parseJson, stringifyJson } = await importSource("src/helpers/json.ts");
+const temporary = createTemporaryDirectories(true);
+after(() => temporary.cleanup());
+
+function open(t: TestContext, path = ":memory:") {
+ const database = new DatabaseSync(path);
+ t.after(() => database.close());
+ database.exec("PRAGMA foreign_keys = ON");
+ applyMigrations(database);
+ return database;
+}
+
+function entry(id: string): Findings.EmbeddedFinding {
+ return {
+ finding: {
+ findingId: id,
+ fingerprints: { primary: `fingerprint-${id}` },
+ ruleId: "synthetic-rule",
+ identity: { anchor: "synthetic-anchor" },
+ title: "Synthetic finding λ",
+ evidence: { message: "complete\0diagnostic", lines: [1, 2] },
+ },
+ embedding: { model: "synthetic-model", vector: [1, 0] },
+ };
+}
+
+test("finding helper help exits without reading stdin", async () => {
+ const helper = fileURLToPath(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ),
+ );
+ const commands = [
+ "store-findings",
+ "list-stored-findings",
+ "find-potential-duplicates",
+ "store-dedupe-groups",
+ "list-dedupe-groups",
+ "dashboard",
+ ];
+ for (const command of commands) {
+ // execFile leaves stdin open; help must exit without waiting for JSON.
+ const { stdout } = await promisify(execFile)(
+ process.execPath,
+ [helper, command, "--help"],
+ { timeout: 30_000 },
+ );
+ assert.ok(stdout.startsWith(`Usage: ${command}`));
+ assert.ok(stdout.includes("stateDirectory"));
+ }
+ const usage = spawnSync(process.execPath, [helper], { encoding: "utf8" });
+ assert.equal(usage.status, 2);
+ for (const command of commands) assert.ok(usage.stderr.includes(command));
+});
+
+test("import batches preserve identity, repository memberships and stable pages", (t) => {
+ const database = open(t);
+ const [a, b, c] = ["a", "b", "c"].map(entry);
+ a.finding.identity = {
+ anchor: "anchor\0suffix",
+ instance: "instance\0suffix",
+ };
+ assert.deepEqual(storeFindings(database, [b, a], "created", "repository-a"), {
+ findingIds: ["b", "a"],
+ });
+ assert.deepEqual(storeFindings(database, [a], "updated", "repository-b"), {
+ findingIds: ["a"],
+ });
+ assert.deepEqual(listStoredFindings(database, { limit: 1, offset: 0 }), {
+ findings: [a.finding],
+ limit: 1,
+ offset: 0,
+ total: 2,
+ nextOffset: 1,
+ });
+ assert.deepEqual(listStoredFindings(database, { limit: 1, offset: 1 }), {
+ findings: [b.finding],
+ limit: 1,
+ offset: 1,
+ total: 2,
+ nextOffset: null,
+ });
+ const conflict = structuredClone(a);
+ conflict.finding.identity.anchor = "anchor";
+ assert.deepEqual(
+ storeFindings(database, [c, conflict], "later", "repository-c"),
+ {
+ error: "finding_conflict",
+ },
+ );
+ assert.deepEqual(
+ listStoredFindings(database, { limit: 10, offset: 0 }).findings,
+ [a.finding, b.finding],
+ );
+ assert.deepEqual(
+ database
+ .prepare(
+ "SELECT repository_id FROM finding_repositories WHERE finding_id = 'a' ORDER BY repository_id",
+ )
+ .all()
+ .map((row) => row.repository_id),
+ ["repository-a", "repository-b"],
+ );
+ assert.equal(
+ database.prepare("SELECT COUNT(*) AS total FROM finding_embeddings").get()!
+ .total,
+ 2,
+ );
+ const duplicate = entry("duplicate");
+ duplicate.finding.fingerprints = a.finding.fingerprints;
+ assert.deepEqual(storeFindings(database, [c, duplicate], "later"), {
+ error: "finding_conflict",
+ });
+ assert.equal(listStoredFindings(database, { limit: 10, offset: 0 }).total, 2);
+});
+
+test("mixed writers retain unchanged Python embeddings and replace supplied Node embeddings", async (t) => {
+ const directory = await temporary.create("workbench-findings-");
+ const path = join(directory, "workbench.sqlite3");
+ const database = open(t, path);
+ const item = entry("finding");
+ item.finding.extensions = { opaqueId: 9007199254740993n };
+ item.finding.severity = { score: 10 };
+ storeFindings(database, [item], "initial");
+ const details = () =>
+ database
+ .prepare("SELECT details_json FROM findings WHERE id = 'finding'")
+ .get()!.details_json;
+ const embedding = () =>
+ database.prepare("SELECT model, vector_json FROM finding_embeddings").get();
+ const scripts = fileURLToPath(new URL("../../scripts/", import.meta.url));
+ const pythonUpsert = (finding: Findings.Finding | string) =>
+ execFileSync(
+ process.env.PYTHON ?? "python",
+ [
+ "-I",
+ "-X",
+ "utf8",
+ "-c",
+ `import json, sqlite3, sys
+sys.path.insert(0, sys.argv[1])
+from workbench_finding_index import upsert_finding
+with sqlite3.connect(sys.argv[2]) as db:
+ db.row_factory = sqlite3.Row
+ upsert_finding(db, json.load(sys.stdin), "python")`,
+ scripts,
+ path,
+ ],
+ {
+ input: typeof finding === "string" ? finding : stringifyJson(finding),
+ encoding: "utf8",
+ },
+ );
+ const original = details();
+ assert.match(String(original), /9007199254740993/u);
+ pythonUpsert(
+ stringifyJson(
+ Object.fromEntries(Object.entries(item.finding).reverse()),
+ ).replace('"score": 10', '"score": 10.0'),
+ );
+ assert.equal(details(), original);
+ assert.ok(embedding());
+ const changed = { ...item.finding, title: "Updated finding" };
+ pythonUpsert(changed);
+ assert.equal(embedding(), undefined);
+ item.embedding = { model: "replacement-model", vector: [0, 1] };
+ storeFindings(database, [{ ...item, finding: changed }], "node");
+ const stored = embedding()!;
+ assert.equal(stored.model, item.embedding.model);
+ assert.deepEqual(parseJson(String(stored.vector_json)), [0, 1]);
+ assert.deepEqual(
+ listStoredFindings(database, { limit: 10, offset: 0 }).findings,
+ [changed],
+ );
+ for (const [before, after] of [
+ [true, 1],
+ [1, true],
+ ]) {
+ storeFindings(
+ database,
+ [{ ...item, finding: { ...changed, extensions: { foo: before } } }],
+ "node",
+ );
+ pythonUpsert({ ...changed, extensions: { foo: after } });
+ assert.equal(JSON.parse(String(details())).extensions.foo, after);
+ assert.equal(embedding(), undefined);
+ }
+});
+
+test("imports reject lossy SQLite keys and non-finite JSON atomically", (t) => {
+ const database = open(t);
+ const original = entry("existing");
+ original.finding.identity = { anchor: "identity�", instance: "identity�" };
+ original.finding.extensions = {
+ opaqueId: 9007199254740993n,
+ opaqueText: "preserve\ud800",
+ };
+ storeFindings(database, [original], "initial", "scope�");
+ assert.deepEqual(
+ listStoredFindings(database, { limit: 10, offset: 0 }).findings,
+ [original.finding],
+ );
+ const snapshot = () =>
+ ["findings", "finding_embeddings", "finding_repositories"].map((table) =>
+ database.prepare(`SELECT * FROM ${table}`).all(),
+ );
+ const before = snapshot();
+ for (const location of [
+ "extension",
+ "vector",
+ "repository",
+ "anchor",
+ "instance",
+ ] as const) {
+ const updated = structuredClone(original);
+ updated.finding.title = "Updated title";
+ updated.embedding.vector = [0, 1];
+ const invalid = entry("invalid");
+ const overflow = parseJson("1e400") as number;
+ if (location === "extension")
+ invalid.finding.extensions = { evidence: [overflow] };
+ else if (location === "vector") invalid.embedding.vector = [overflow, 0];
+ else if (location !== "repository") {
+ invalid.finding = structuredClone(original.finding);
+ invalid.finding.identity[location] = "identity\ud800";
+ }
+ assert.throws(
+ () =>
+ storeFindings(
+ database,
+ [updated, entry("new"), invalid],
+ "later",
+ location === "repository" ? "scope\ud800" : "new-repository",
+ ),
+ /non-finite|Unicode/u,
+ );
+ assert.deepEqual(snapshot(), before);
+ }
+});
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index 36d0ede9f..235b70c16 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -88,8 +88,6 @@
"scripts/workbench_feedback.py",
"scripts/workbench_finding_index.py",
"scripts/workbench_finding_workflows.py",
- "scripts/workbench_dashboard.py",
- "scripts/workbench_findings.py",
"scripts/workbench_native_indexes.py",
"scripts/workbench_progress.py",
"scripts/workbench_publication.py",
diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py
index 9d094ad76..1d01c0d28 100644
--- a/plugins/codex-security/scripts/workbench_cli.py
+++ b/plugins/codex-security/scripts/workbench_cli.py
@@ -381,23 +381,10 @@ def parse_args(description: str) -> argparse.Namespace:
publication.add_argument("--input-file", required=True)
subparsers.add_parser("database-info")
- subparsers.add_parser("dashboard")
subparsers.add_parser("finding-workflow")
subparsers.add_parser("severity-classification")
severity = subparsers.add_parser("read-severity-classification")
severity.add_argument("--scan-id", required=True)
- subparsers.add_parser("store-findings")
- subparsers.add_parser("store-dedupe-groups")
- dedupe_groups = subparsers.add_parser("list-dedupe-groups")
- dedupe_groups.add_argument("--finding-id", required=True)
- potential_duplicates = subparsers.add_parser("find-potential-duplicates")
- potential_duplicates.add_argument("--finding-id", required=True)
- scope = potential_duplicates.add_mutually_exclusive_group(required=True)
- scope.add_argument("--repository-id")
- scope.add_argument("--all-repositories", action="store_true")
- stored_findings = subparsers.add_parser("list-stored-findings")
- stored_findings.add_argument("--limit", type=positive_int, required=True)
- stored_findings.add_argument("--offset", type=non_negative_int, required=True)
arguments = sys.argv[1:]
if "--user-context-stdin" in arguments:
if arguments.count("--user-context-stdin") != 1 or "--user-context" in arguments:
diff --git a/plugins/codex-security/scripts/workbench_dashboard.py b/plugins/codex-security/scripts/workbench_dashboard.py
deleted file mode 100644
index 50c13f333..000000000
--- a/plugins/codex-security/scripts/workbench_dashboard.py
+++ /dev/null
@@ -1,159 +0,0 @@
-"""Read-only dashboard projections for stored findings and duplicate groups."""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sqlite3
-import sys
-from pathlib import Path
-from typing import Any
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from workbench_findings import list_dedupe_groups
-
-FINDING_RECORDS = """
-SELECT findings.id, json_extract(details_json, '$.title') AS title,
- COALESCE(repositories.ids, '[]') AS repositoryIds,
- json_extract(details_json, '$.severity.level') AS severity,
- findings.created_at AS createdAt, findings.updated_at AS updatedAt
-FROM findings LEFT JOIN (
- SELECT finding_id, json_group_array(repository_id) AS ids
- FROM finding_repositories GROUP BY finding_id
-) AS repositories ON repositories.finding_id = findings.id
-WHERE details_json IS NOT NULL
-"""
-
-GROUP_RECORDS = """
-SELECT groups.id, groups.id AS title,
- (SELECT json_group_array(DISTINCT repository_id)
- FROM finding_dedupe_group_members AS members
- JOIN finding_repositories ON finding_repositories.finding_id = members.finding_id
- WHERE members.group_id = groups.id) AS repositoryIds,
- groups.created_at AS createdAt, groups.created_at AS updatedAt,
- (SELECT COUNT(*) FROM finding_dedupe_group_members WHERE group_id = groups.id) AS memberCount
-FROM finding_dedupe_groups AS groups
-"""
-
-RECORDS = {
- "findings": FINDING_RECORDS,
- "groups": GROUP_RECORDS,
-}
-
-SORT_COLUMNS = {
- "activity": "updatedAt",
- "newest": "createdAt",
- "title": "casefold(title)",
- "repository": "casefold(repository_label(repositoryIds))",
- "severity": "CASE severity WHEN 'informational' THEN 0 WHEN 'low' THEN 1 "
- "WHEN 'medium' THEN 2 WHEN 'high' THEN 3 WHEN 'critical' THEN 4 END",
- "members": "memberCount",
-}
-
-
-def repository_ids(value: str) -> list[str]:
- return sorted(json.loads(value), key=lambda name: (name.casefold(), name))
-
-
-def item(row: sqlite3.Row) -> dict[str, Any]:
- result = dict(row)
- result["repositoryIds"] = repository_ids(result["repositoryIds"])
- return result
-
-
-def detail(connection: sqlite3.Connection, view: str, selected: dict[str, Any]) -> dict[str, Any]:
- result: dict[str, Any] = {"item": selected}
- selected_id = selected["id"]
- if view == "findings":
- result["finding"] = json.loads(
- connection.execute(
- "SELECT details_json FROM findings WHERE id = ?",
- (selected_id,),
- ).fetchone()[0]
- )
- result["groups"] = list_dedupe_groups(connection, selected_id)["groups"]
- else:
- result["group"] = {
- "groupId": selected_id,
- "createdAt": selected["createdAt"],
- "findingIds": [
- r[0]
- for r in connection.execute(
- "SELECT finding_id FROM finding_dedupe_group_members WHERE group_id = ? ORDER BY finding_id",
- (selected_id,),
- )
- ],
- }
- return result
-
-
-def dashboard(connection: sqlite3.Connection, query: dict[str, Any]) -> dict[str, Any]:
- """One snapshot, no artifact reads, model calls, or writes."""
- view = query["view"]
- records = RECORDS[view]
- connection.create_function("casefold", 1, str.casefold, deterministic=True)
- connection.create_function(
- "repository_label", 1, lambda value: ", ".join(repository_ids(value)), deterministic=True
- )
- clauses: list[str] = []
- values: list[Any] = []
- if query.get("query"):
- columns = ["id", "title", "repositoryIds"]
- clauses.append(
- "("
- + " OR ".join(f"instr(casefold(COALESCE({c}, '')), casefold(?)) > 0" for c in columns)
- + ")"
- )
- values.extend([query["query"]] * len(columns))
- if query.get("repository"):
- clauses.append("EXISTS (SELECT 1 FROM json_each(repositoryIds) WHERE value = ?)")
- values.append(query["repository"])
- where = " WHERE " + " AND ".join(clauses) if clauses else ""
- direction = {"asc": "ASC", "desc": "DESC"}[query.get("direction", "desc")]
- order = f"{SORT_COLUMNS[query['sort']]} {direction}"
- if view == "findings" and query["sort"] == "activity":
- order += f", {SORT_COLUMNS['severity']} DESC"
- order += ", id"
- connection.execute("BEGIN")
- with connection:
- repositories = connection.execute("""
- SELECT DISTINCT repository_id AS id, repository_id AS label
- FROM finding_repositories ORDER BY repository_id
- """).fetchall()
- total = connection.execute(f"SELECT COUNT(*) FROM ({records}) {where}", values).fetchone()[
- 0
- ]
- rows = connection.execute(
- f"SELECT * FROM ({records}) {where} ORDER BY {order} LIMIT ? OFFSET ?",
- (*values, query["limit"], query["offset"]),
- ).fetchall()
- selected = (
- connection.execute(
- f"SELECT * FROM ({records}) WHERE id = ?",
- (query["id"],),
- ).fetchone()
- if query.get("id")
- else None
- )
- next_offset = query["offset"] + len(rows)
- return {
- "overview": {
- "findings": connection.execute(
- "SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL"
- ).fetchone()[0],
- "groups": connection.execute(
- "SELECT COUNT(*) FROM finding_dedupe_groups"
- ).fetchone()[0],
- },
- "repositories": [dict(row) for row in repositories],
- "items": [item(row) for row in rows],
- "total": total,
- "limit": query["limit"],
- "offset": query["offset"],
- "nextOffset": next_offset if next_offset < total else None,
- "detail": detail(connection, view, item(selected)) if selected is not None else None,
- }
-
-
-if __name__ == "__main__":
- argparse.ArgumentParser(description=__doc__).parse_args()
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index b614f4ab1..5968b470d 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -76,17 +76,9 @@ from workbench_constants import (
PATCH_PREVIEW_BYTES,
SQLITE_RETRY_ATTEMPTS,
)
-from workbench_dashboard import dashboard
from workbench_feedback import get_scan_feedback
from workbench_finding_index import index_findings
from workbench_finding_workflows import finding_workflow, register_workflow_scan
-from workbench_findings import (
- find_potential_duplicates,
- list_dedupe_groups,
- list_stored_findings,
- store_dedupe_groups,
- store_findings,
-)
from workbench_remediation import remediation_claim_is_active
from workbench_scan_start import (
archive_scan,
@@ -3443,21 +3435,6 @@ def main() -> None:
result = severity.checkpoint(connection, json.load(sys.stdin), now())
elif args.command == "finding-workflow":
result = finding_workflow(connection, json.load(sys.stdin), now())
- elif args.command == "dashboard":
- result = dashboard(connection, json.load(sys.stdin))
- elif args.command == "store-findings":
- payload = json.load(sys.stdin)
- result = store_findings(
- connection, payload["entries"], now(), payload.get("repositoryId")
- )
- elif args.command == "find-potential-duplicates":
- result = find_potential_duplicates(connection, args.finding_id, args.repository_id)
- elif args.command == "store-dedupe-groups":
- result = store_dedupe_groups(connection, json.load(sys.stdin)["groups"], now())
- elif args.command == "list-dedupe-groups":
- result = list_dedupe_groups(connection, args.finding_id)
- elif args.command == "list-stored-findings":
- result = list_stored_findings(connection, limit=args.limit, offset=args.offset)
else:
raise SystemExit(f"Unknown command: {args.command}")
print(json.dumps(result, allow_nan=False, sort_keys=True))
diff --git a/plugins/codex-security/scripts/workbench_finding_index.py b/plugins/codex-security/scripts/workbench_finding_index.py
index b898a65f4..0fecd9320 100644
--- a/plugins/codex-security/scripts/workbench_finding_index.py
+++ b/plugins/codex-security/scripts/workbench_finding_index.py
@@ -5,8 +5,15 @@ from __future__ import annotations
import argparse
import json
import sqlite3
+from decimal import Decimal
from typing import Any
+# Tag numbers so 10 and 10.0 compare equally without treating true as 1.
+_finding_json = json.JSONDecoder(
+ parse_int=lambda value: (Decimal(value),),
+ parse_float=lambda value: (Decimal(value),),
+)
+
def upsert_finding(
connection: sqlite3.Connection,
@@ -14,6 +21,17 @@ def upsert_finding(
timestamp: str,
repository_id: str | None = None,
) -> None:
+ current = connection.execute(
+ "SELECT details_json FROM findings WHERE id = ?", (finding["findingId"],)
+ ).fetchone()
+ details = json.dumps(finding, allow_nan=False, sort_keys=True)
+ # Preserve unchanged text without conflating JSON booleans and numbers.
+ if (
+ current is not None
+ and current["details_json"] is not None
+ and _finding_json.decode(current["details_json"]) == _finding_json.decode(details)
+ ):
+ details = current["details_json"]
connection.execute(
"""
INSERT INTO findings (
@@ -34,7 +52,7 @@ def upsert_finding(
finding["ruleId"],
finding["identity"]["anchor"],
finding["identity"].get("instance"),
- json.dumps(finding, allow_nan=False, sort_keys=True),
+ details,
timestamp,
timestamp,
),
diff --git a/plugins/codex-security/scripts/workbench_findings.py b/plugins/codex-security/scripts/workbench_findings.py
deleted file mode 100644
index 5446ea3b2..000000000
--- a/plugins/codex-security/scripts/workbench_findings.py
+++ /dev/null
@@ -1,221 +0,0 @@
-"""Storage operations for complete findings imported without a scan."""
-
-from __future__ import annotations
-
-import argparse
-import hashlib
-import json
-import math
-import sqlite3
-import sys
-from pathlib import Path
-from typing import Any
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from workbench_finding_index import upsert_finding
-
-
-def store_findings(
- connection: sqlite3.Connection,
- entries: list[dict[str, Any]],
- timestamp: str,
- repository_id: str | None = None,
-) -> dict[str, Any]:
- try:
- with connection:
- connection.execute("BEGIN IMMEDIATE")
- for entry in entries:
- finding = entry["finding"]
- embedding = entry["embedding"]
- identity = (
- finding["fingerprints"]["primary"],
- finding["ruleId"],
- finding["identity"]["anchor"],
- finding["identity"].get("instance"),
- )
- current = connection.execute(
- "SELECT fingerprint, rule_id, identity_anchor, identity_instance "
- "FROM findings WHERE id = ?",
- (finding["findingId"],),
- ).fetchone()
- if current is not None and tuple(current) != identity:
- raise sqlite3.IntegrityError("The stored finding identity cannot be replaced.")
- upsert_finding(connection, finding, timestamp, repository_id)
- connection.execute(
- """
- INSERT INTO finding_embeddings (finding_id, model, vector_json)
- VALUES (?, ?, ?)
- ON CONFLICT(finding_id) DO UPDATE SET
- model = excluded.model, vector_json = excluded.vector_json
- """,
- (
- finding["findingId"],
- embedding["model"],
- json.dumps(embedding["vector"], allow_nan=False),
- ),
- )
- except sqlite3.IntegrityError:
- return {"error": "finding_conflict"}
- return {"findingIds": [entry["finding"]["findingId"] for entry in entries]}
-
-
-def list_stored_findings(
- connection: sqlite3.Connection, *, limit: int, offset: int
-) -> dict[str, Any]:
- connection.execute("BEGIN")
- with connection:
- total = connection.execute(
- "SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL"
- ).fetchone()[0]
- rows = connection.execute(
- """
- SELECT details_json FROM findings WHERE details_json IS NOT NULL
- ORDER BY created_at, id LIMIT ? OFFSET ?
- """,
- (limit, offset),
- ).fetchall()
- next_offset = offset + len(rows)
- return {
- "findings": [json.loads(row["details_json"]) for row in rows],
- "limit": limit,
- "offset": offset,
- "total": total,
- "nextOffset": next_offset if next_offset < total else None,
- }
-
-
-def find_potential_duplicates(
- connection: sqlite3.Connection, finding_id: str, repository_id: str | None
-) -> dict[str, Any]:
- """Rank IDs and vectors in the requested scope before loading finding documents."""
- connection.execute("BEGIN")
- with connection:
- if repository_id is None:
- source = "finding_embeddings AS embeddings"
- predicate = ""
- scope_parameters: tuple[str, ...] = ()
- else:
- source = (
- "finding_repositories AS repositories JOIN finding_embeddings AS embeddings "
- "ON embeddings.finding_id = repositories.finding_id"
- )
- predicate = "repositories.repository_id = ? AND "
- scope_parameters = (repository_id,)
- anchor = connection.execute(
- f"SELECT embeddings.model, embeddings.vector_json FROM {source} "
- f"WHERE {predicate}embeddings.finding_id = ?",
- (*scope_parameters, finding_id),
- ).fetchone()
- if anchor is None:
- return {"error": "finding_not_indexed"}
- rows = connection.execute(
- f"SELECT embeddings.finding_id, embeddings.vector_json FROM {source} "
- "JOIN findings ON findings.id = embeddings.finding_id "
- f"WHERE {predicate}embeddings.model = ? AND embeddings.finding_id != ? "
- "ORDER BY findings.created_at, findings.id",
- (*scope_parameters, anchor["model"], finding_id),
- )
- ranked: list[tuple[str, float]] = []
- try:
- vector = normalized_vector(json.loads(anchor["vector_json"]))
- for row in rows:
- candidate = json.loads(row["vector_json"])
- if len(candidate) != len(vector):
- continue
- other = normalized_vector(candidate)
- similarity = sum(left * right for left, right in zip(vector, other))
- if similarity >= 0.55:
- ranked.append((row["finding_id"], similarity))
- except ValueError:
- return {"error": "embedding_failed"}
- # Stable sorting retains insertion-time / finding-ID order for ties.
- ranked.sort(key=lambda candidate: candidate[1], reverse=True)
- selected_ids = [finding_id, *(candidate[0] for candidate in ranked[:50])]
- documents = {
- row["id"]: json.loads(row["details_json"])
- for row in connection.execute(
- "SELECT id, details_json FROM findings WHERE id IN ("
- + ",".join("?" for _ in selected_ids)
- + ")",
- selected_ids,
- )
- }
- return {
- "finding": documents[finding_id],
- "potentialDuplicates": [documents[id] for id in selected_ids[1:]],
- }
-
-
-def store_dedupe_groups(
- connection: sqlite3.Connection, groups: list[list[str]], timestamp: str
-) -> dict[str, Any]:
- """Persist reviewed sets independently, including overlapping groups, in one transaction."""
- stored: dict[str, dict[str, Any]] = {}
- try:
- with connection:
- connection.execute("BEGIN IMMEDIATE")
- for group in groups:
- members = sorted(set(group))
- # Membership, not input order, identifies a group on retries.
- group_id = (
- "fdg_"
- + hashlib.sha256(
- json.dumps(members, ensure_ascii=True, separators=(",", ":")).encode(
- "utf-8"
- )
- ).hexdigest()
- )
- connection.execute(
- "INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?) "
- "ON CONFLICT(id) DO NOTHING",
- (group_id, timestamp),
- )
- connection.executemany(
- "INSERT INTO finding_dedupe_group_members (group_id, finding_id) VALUES (?, ?) "
- "ON CONFLICT(group_id, finding_id) DO NOTHING",
- ((group_id, finding_id) for finding_id in members),
- )
- created_at = connection.execute(
- "SELECT created_at FROM finding_dedupe_groups WHERE id = ?", (group_id,)
- ).fetchone()[0]
- stored[group_id] = {
- "groupId": group_id,
- "findingIds": members,
- "createdAt": created_at,
- }
- except sqlite3.IntegrityError:
- return {"error": "finding_conflict"}
- return {"groups": list(stored.values())}
-
-
-def list_dedupe_groups(connection: sqlite3.Connection, finding_id: str) -> dict[str, Any]:
- groups: dict[str, dict[str, Any]] = {}
- rows = connection.execute(
- """
- SELECT groups.id, groups.created_at, members.finding_id
- FROM finding_dedupe_group_members AS matched
- JOIN finding_dedupe_groups AS groups ON groups.id = matched.group_id
- JOIN finding_dedupe_group_members AS members ON members.group_id = groups.id
- WHERE matched.finding_id = ?
- ORDER BY groups.created_at, groups.id, members.finding_id
- """,
- (finding_id,),
- )
- for row in rows:
- group = groups.setdefault(
- row["id"],
- {"groupId": row["id"], "findingIds": [], "createdAt": row["created_at"]},
- )
- group["findingIds"].append(row["finding_id"])
- return {"groups": list(groups.values())}
-
-
-def normalized_vector(vector: list[float]) -> list[float]:
- norm = math.hypot(*vector)
- if norm == 0 or not math.isfinite(norm):
- raise ValueError("A stored embedding cannot be compared.")
- return [value / norm for value in vector]
-
-
-if __name__ == "__main__":
- argparse.ArgumentParser(description=__doc__).parse_args()
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index 8175a553b..79d88bf3e 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -30,7 +30,8 @@ Supported runtimes:
- Node.js 22.13.0+ within 22.x, or Node.js 24.x or 26.x, on macOS, Linux, or Windows.
- Python 3.10+ for scans, policy generation, exports, scan history, and saved
- findings. Python 3.10 also needs `tomli`.
+ findings. Python 3.10 also needs `tomli`. The findings server (`serve`) uses
+ Node’s built-in SQLite and does not require Python.
## Authentication
diff --git a/sdk/typescript/docs/findings-service.md b/sdk/typescript/docs/findings-service.md
index 74606c736..81120fa16 100644
--- a/sdk/typescript/docs/findings-service.md
+++ b/sdk/typescript/docs/findings-service.md
@@ -61,7 +61,6 @@ environment. For a service started outside Docker, export any of these settings:
| `CODEX_SECURITY_EMBEDDINGS_URL` | Full embeddings endpoint URL, including its path and any query parameters. Defaults to `https://api.openai.com/v1/embeddings` when unset or empty. |
| `CODEX_SECURITY_STATE_DIR` | State directory containing `workbench.sqlite3`; Compose uses `/state`. |
| `HOST` / `PORT` | Listen address and port; outside Compose, defaults are `127.0.0.1` and `3000`. |
-| `PYTHON` | Python interpreter used by the storage adapter. |
Exported values override Compose's `.env` values. The repository excludes `.env`
from Git and Docker builds. Startup, listing, duplicate-group operations, and
@@ -269,8 +268,53 @@ start scans, publication, or deduplication.
Overview counts cover the whole service, regardless of filters. The default
order is last update descending, then severity descending for findings, then ID
-ascending. Text sorts ignore case; severity and member counts use their natural
-order. The dashboard uses the same unauthenticated endpoint as the API.
+ascending. Search compares uppercase JavaScript strings; text sorts compare
+lowercase strings. Severity and member counts use their natural order. The
+dashboard uses the same unauthenticated endpoint as the API.
+
+### Migrating direct Python helper calls
+
+The Python `workbench_db.py` commands listed below have been retired. Direct
+helper callers can use the existing Node helpers from an installed plugin
+directory. For example:
+
+```bash
+scripts/launch_codex_security_mcp --helper dashboard < dashboard-request.json
+```
+
+On Windows, use `scripts\launch_codex_security_mcp.cmd --helper dashboard` with
+the same JSON on stdin. The request wraps the dashboard query in `payload` and
+specifies the absolute directory containing the existing `workbench.sqlite3`:
+
+```json
+{
+ "stateDirectory": "/absolute/path/to/state",
+ "payload": {
+ "view": "findings",
+ "sort": "activity",
+ "limit": 50,
+ "offset": 0
+ }
+}
+```
+
+Use an absolute Windows path on Windows. The SDK, service API, and
+`codex-security` CLI already use the Node implementation and need no changes.
+
+Use the same command name after `--helper`, with these fields in `payload`:
+
+| Command | Payload fields |
+| --------------------------- | ------------------------------------------------------------------------------------------------- |
+| `dashboard` | The dashboard query shown above; optional `direction`, `query`, `repository`, and `id`. |
+| `store-findings` | `entries` containing finding and embedding records; optional `repositoryId`. |
+| `list-stored-findings` | Positive integer `limit` and non-negative integer `offset`. |
+| `find-potential-duplicates` | `findingId` and `scope`: either `{"repositoryId":"REPOSITORY_ID"}` or `{"allRepositories":true}`. |
+| `store-dedupe-groups` | `groups`, an array of finding-ID arrays. |
+| `list-dedupe-groups` | `findingId`. |
+
+Pagination and finding/repository selectors move from Python command flags into
+these JSON fields. Retained scan commands such as `workbench_db.py list-findings`
+keep their existing interface.
## Deduplicate a scan
@@ -454,7 +498,7 @@ not support the migrated database.
## Run without Docker
-With the package's supported Node.js and Python versions installed:
+With the package's supported Node.js version installed (22.13 or later in a supported major):
```bash
npm install -g @openai/codex-security
@@ -463,8 +507,9 @@ CODEX_SECURITY_STATE_DIR="$HOME/.codex-security-findings" codex-security serve -
`--port` overrides `PORT`. The service does not load `.env`; export the embedding
key before importing findings. Without a state override, it shares the CLI's
-default state directory. `HOST`, `PORT`, `CODEX_SECURITY_STATE_DIR`, and `PYTHON`
-also work on Windows. Stop with Ctrl-C or SIGTERM.
+default state directory. `HOST`, `PORT`, and `CODEX_SECURITY_STATE_DIR` also
+work on Windows. The findings service uses Node’s built-in SQLite and does not
+require Python. Stop with Ctrl-C or SIGTERM.
For source builds, prepare the
[universal native payload](https://github.com/openai/codex-security/blob/main/plugins/codex-security/native/README.md#package-inputs),
diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts
index 18fc5b687..48abb6ee2 100644
--- a/sdk/typescript/src/runtime.ts
+++ b/sdk/typescript/src/runtime.ts
@@ -1564,7 +1564,15 @@ export async function runWorkbench(
arguments_: readonly string[],
input?: string,
): Promise<string> => {
- const native = arguments_[0] === "database-info";
+ const native = [
+ "database-info",
+ "store-findings",
+ "list-stored-findings",
+ "find-potential-duplicates",
+ "store-dedupe-groups",
+ "list-dedupe-groups",
+ "dashboard",
+ ].includes(arguments_[0] ?? "");
const node =
native && process.versions["bun"]
? await resolveTrustedExecutable(
@@ -1583,6 +1591,10 @@ export async function runWorkbench(
protectedRoot: options.protectedRoot,
signal: options.signal,
}));
+ const stateDirectory = native
+ ? (options.stateDirectory ??
+ codexSecurityStateDirectory(options.environment))
+ : undefined;
const result = await runCodexCommand(
{ command },
native
@@ -1592,8 +1604,12 @@ export async function runWorkbench(
// The SDK owns configuration normalization; the helper receives its resolved location.
native
? JSON.stringify(
- options.stateDirectory ??
- codexSecurityStateDirectory(options.environment),
+ arguments_[0] === "database-info"
+ ? stateDirectory
+ : {
+ stateDirectory,
+ payload: input === undefined ? undefined : JSON.parse(input),
+ },
)
: input,
options.signal,
@@ -3380,12 +3396,8 @@ function nullIfMissingFileError(error: unknown): null {
/** @internal */
export function workbenchEnvironment(environment: ProcessEnvironment) {
- const python = environmentValue(environment, "PYTHON");
return {
...environment,
- ...(python && isPythonPathCandidate(python)
- ? { PYTHON: resolve(expandHome(python, environment)) }
- : {}),
CODEX_SECURITY_STATE_DIR: codexSecurityStateDirectory(environment),
};
}
diff --git a/sdk/typescript/src/server/sqlite-store.ts b/sdk/typescript/src/server/sqlite-store.ts
index 84818a170..2fb1ccee8 100644
--- a/sdk/typescript/src/server/sqlite-store.ts
+++ b/sdk/typescript/src/server/sqlite-store.ts
@@ -24,94 +24,70 @@ export class SqliteFindingsStore implements FindingsStore {
constructor(private readonly environment: NodeJS.ProcessEnv = process.env) {}
async initialize(): Promise<void> {
- await this.run(["database-info"]);
+ await this.run("database-info");
}
- async dashboard(query: DashboardQuery): Promise<DashboardSnapshot> {
- return (await this.run(
- ["dashboard"],
- JSON.stringify(query),
- )) as unknown as DashboardSnapshot;
+ dashboard(query: DashboardQuery): Promise<DashboardSnapshot> {
+ return this.run("dashboard", query);
}
async insert(
entries: readonly EmbeddedFinding[],
repositoryId?: string,
): Promise<string[]> {
- const result = await this.run(
- ["store-findings"],
- JSON.stringify({ entries, repositoryId }),
- );
- if (result["error"] === "finding_conflict") {
- throw new FindingsError(
- "finding_conflict",
- "A finding identity conflicts with stored data.",
- );
- }
- return result["findingIds"] as string[];
+ const result = await this.run<{ findingIds: string[] }>("store-findings", {
+ entries,
+ repositoryId,
+ });
+ return result.findingIds;
}
- async list(page: { limit: number; offset: number }): Promise<FindingsPage> {
- return (await this.run([
- "list-stored-findings",
- "--limit",
- String(page.limit),
- "--offset",
- String(page.offset),
- ])) as unknown as FindingsPage;
+ list(page: { limit: number; offset: number }): Promise<FindingsPage> {
+ return this.run("list-stored-findings", page);
}
- async findPotentialDuplicates(
+ findPotentialDuplicates(
findingId: string,
scope: FindingSearchScope,
): Promise<FindingNeighborhood> {
- const result = await this.run([
- "find-potential-duplicates",
- `--finding-id=${findingId}`,
- ...(scope.allRepositories === true
- ? ["--all-repositories"]
- : [`--repository-id=${scope.repositoryId}`]),
- ]);
- if (result["error"] === "finding_not_indexed") {
- throw new FindingsError(
- "finding_not_indexed",
- "The finding has no current embedding in the requested scope. Import it with the matching repositoryId through POST /v1/bulk/findings before requesting potential duplicates.",
- );
- }
- if (result["error"] === "embedding_failed") {
- throw new FindingsError(
- "embedding_failed",
- "A stored embedding cannot be compared. Reimport the finding.",
- );
- }
- return result as unknown as FindingNeighborhood;
+ return this.run("find-potential-duplicates", { findingId, scope });
}
async storeDedupeGroups(
groups: readonly string[][],
): Promise<FindingDedupeGroup[]> {
- const result = await this.run(
- ["store-dedupe-groups"],
- JSON.stringify({ groups }),
+ const result = await this.run<{ groups: FindingDedupeGroup[] }>(
+ "store-dedupe-groups",
+ { groups },
);
- if (result["error"] === "finding_conflict") {
- throw new FindingsError(
- "finding_conflict",
- "Every dedupe group member must already exist in the findings database.",
- );
- }
- return result["groups"] as unknown as FindingDedupeGroup[];
+ return result.groups;
}
async listDedupeGroups(findingId: string): Promise<FindingDedupeGroup[]> {
- return (
- await this.run(["list-dedupe-groups", `--finding-id=${findingId}`])
- )["groups"] as unknown as FindingDedupeGroup[];
+ const result = await this.run<{ groups: FindingDedupeGroup[] }>(
+ "list-dedupe-groups",
+ { findingId },
+ );
+ return result.groups;
}
- private async run(args: string[], input?: string) {
+ private async run<T>(command: string, payload?: unknown): Promise<T> {
+ const input = JSON.stringify(payload);
const options = await (this.options ??= this.resolveOptions());
- return await runWorkbench(options, args, input);
+ const result = await runWorkbench(options, [command], input);
+ const messages = {
+ finding_conflict:
+ command === "store-findings"
+ ? "A finding identity conflicts with stored data."
+ : "Every dedupe group member must already exist in the findings database.",
+ finding_not_indexed:
+ "The finding has no current embedding in the requested scope. Import it with the matching repositoryId through POST /v1/bulk/findings before requesting potential duplicates.",
+ embedding_failed:
+ "A stored embedding cannot be compared. Reimport the finding.",
+ };
+ const error = result["error"] as keyof typeof messages | undefined;
+ if (error) throw new FindingsError(error, messages[error]);
+ return result as T;
}
private async resolveOptions(): Promise<
@@ -122,10 +98,7 @@ export class SqliteFindingsStore implements FindingsStore {
this.environment,
);
for (const [name, value] of Object.entries(environment)) {
- if (value === undefined) continue;
- if (name.toUpperCase() === "XDG_CACHE_HOME" && value.trim())
- environment[name] = resolve(value);
- if (name.toUpperCase() !== "PATH") continue;
+ if (name.toUpperCase() !== "PATH" || value === undefined) continue;
environment[name] = value
.split(delimiter)
.map((entry) => {
diff --git a/sdk/typescript/tests-ts/findings-server.test.ts b/sdk/typescript/tests-ts/findings-server.test.ts
index 92a06e622..9d8069583 100644
--- a/sdk/typescript/tests-ts/findings-server.test.ts
+++ b/sdk/typescript/tests-ts/findings-server.test.ts
@@ -10,10 +10,8 @@ import {
} from "node:fs/promises";
import type { Server } from "node:http";
import { tmpdir } from "node:os";
-import { basename, delimiter, dirname, join, relative } from "node:path";
-import { fileURLToPath, pathToFileURL } from "node:url";
+import { delimiter, dirname, join } from "node:path";
import { afterEach, expect, spyOn, test, mock } from "bun:test";
-import { build } from "esbuild";
import type { Finding, FindingsDocument } from "../src/models.js";
import type { FindingDedupeGroup } from "../src/finding-dedupe-groups.js";
import {
@@ -91,7 +89,13 @@ async function fixture() {
...process.env,
CODEX_SECURITY_STATE_DIR: join(directory, "state with spaces"),
};
- return { environment, store: new SqliteFindingsStore(environment) };
+ return {
+ environment,
+ store: new SqliteFindingsStore({
+ ...environment,
+ PYTHON: join(directory, "missing-python"),
+ }),
+ };
}
test("initializes the shared database concurrently without Python", async () => {
@@ -116,16 +120,24 @@ test("initializes the shared database concurrently without Python", async () =>
});
});
-test("missing explicit Python fails only when a Python operation runs", async () => {
- const { environment } = await fixture();
- const store = new SqliteFindingsStore({
- ...environment,
- PYTHON: join(environment.CODEX_SECURITY_STATE_DIR, "missing-python"),
- });
- await store.initialize();
- await expect(store.list({ limit: 50, offset: 0 })).rejects.toThrow(
- "The PYTHON interpreter is unavailable or unusable",
- );
+test("invalid findings pagination is rejected before database creation", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "findings-page-"));
+ directories.push(directory);
+ const stateDirectory = join(directory, "state");
+ for (const payload of [
+ { limit: 0, offset: 0 },
+ { limit: 1, offset: -1 },
+ ]) {
+ const result = await runCodexCommand(
+ { command: "node" },
+ [join(PLUGIN_ROOT, "mcp", "helpers.mjs"), "list-stored-findings"],
+ process.env,
+ JSON.stringify({ stateDirectory, payload }),
+ );
+ expect(result.success).toBe(false);
+ expect(result.stderr).toContain("limit must be a positive integer");
+ }
+ expect(await readdir(directory)).toEqual([]);
});
test.skipIf(process.platform === "win32")(
@@ -166,7 +178,8 @@ printf '%s\n' '{"databasePath":"shim"}'
const store = new SqliteFindingsStore();
await store.initialize();
process.chdir(${JSON.stringify(other)});
-await store.initialize();`,
+await store.initialize();
+await store.list({ limit: 1, offset: 0 });`,
],
{
cwd: repository,
@@ -189,7 +202,7 @@ await store.initialize();`,
);
test.skipIf(process.platform === "win32")(
- "database initialization and Python operations share the configured state directory",
+ "database initialization and findings operations share the configured state directory",
async () => {
const directory = await mkdtemp(join(tmpdir(), "database-info-location-"));
directories.push(directory);
@@ -228,6 +241,7 @@ console.log(JSON.stringify(await store.list({ limit: 50, offset: 0 })));`,
env: {
...process.env,
CODEX_SECURITY_STATE_DIR: "state",
+ PYTHON: join(directory, "missing-python"),
},
},
);
@@ -257,145 +271,6 @@ test("initialized stores retain their environment across findings operations", a
expect(await readdir(directory)).toEqual(["state with spaces"]);
});
-test.each(["explicit", "PATH", "Path", "XDG_CACHE_HOME", "xdg_cache_home"])(
- "findings stores retain relative Python paths before and after first use (%s)",
- async (configuration) => {
- const directory = await mkdtemp(join(tmpdir(), "findings-python-"));
- directories.push(directory);
- const first = join(directory, "first");
- const second = join(directory, "second", "nested");
- await Promise.all([mkdir(first), mkdir(second, { recursive: true })]);
- const python = await resolvePluginPython();
- const managedCache = configuration.toUpperCase() === "XDG_CACHE_HOME";
- const cacheDirectory = join(directory, "managed cache");
- if (managedCache) {
- const managedPython = join(
- cacheDirectory,
- "codex-runtimes/codex-primary-runtime/dependencies/python",
- );
- await mkdir(dirname(managedPython), { recursive: true });
- if (process.platform === "win32") {
- await symlink(dirname(python), managedPython, "junction");
- } else {
- await mkdir(join(managedPython, "bin"), { recursive: true });
- await symlink(python, join(managedPython, "bin", "python3"));
- }
- }
- let pythonDirectory = relative(first, dirname(python));
- if (
- process.platform !== "win32" &&
- ["PATH", "Path"].includes(configuration)
- ) {
- const link = join(directory, "python-link");
- await symlink(dirname(python), link, "dir");
- pythonDirectory = `${configuration === "PATH" ? link : relative(first, link)}/../${basename(dirname(python))}`;
- }
- const searchPath = [
- process.platform === "win32" ? `"${pythonDirectory}"` : pythonDirectory,
- "",
- dirname(Bun.which("node")!),
- ].join(delimiter);
- const storeModule = join(directory, "sqlite-store.cjs");
- await build({
- entryPoints: [
- fileURLToPath(
- new URL("../src/server/sqlite-store.ts", import.meta.url),
- ),
- ],
- outfile: storeModule,
- bundle: true,
- platform: "node",
- format: "cjs",
- define: {
- "import.meta.url": JSON.stringify(
- new URL("../src/runtime.ts", import.meta.url).href,
- ),
- },
- });
- const result = Bun.spawnSync(
- [
- Bun.which("node")!,
- "--input-type=module",
- "--eval",
- `const assert = await import("node:assert/strict");
-const { SqliteFindingsStore } = await import(${JSON.stringify(pathToFileURL(storeModule).href)});
-const environment = { ...process.env };
-if (${managedCache}) {
- for (const key of Object.keys(environment)) if (["PATH", "PYTHON", "XDG_CACHE_HOME"].includes(key.toUpperCase())) delete environment[key];
- environment.PATH = "";
- environment[${JSON.stringify(configuration)}] = ${JSON.stringify(relative(first, cacheDirectory))};
-} else if (${JSON.stringify(configuration)} !== "explicit") {
- for (const key of Object.keys(environment)) if (key.toUpperCase() === "PATH") delete environment[key];
- environment[${JSON.stringify(configuration)}] = ${JSON.stringify(searchPath)};
-}
-const original = ["PATH", "Path", "PYTHON", "XDG_CACHE_HOME", "xdg_cache_home", "CODEX_SECURITY_STATE_DIR"].map((key) => [key, environment[key], process.env[key]]);
-const store = new SqliteFindingsStore(environment);
-await store.initialize();
-process.chdir(${JSON.stringify(second)});
-await store.insert(${JSON.stringify([embedded(1)])});
-process.chdir(${JSON.stringify(first)});
-console.log(JSON.stringify(await store.list({ limit: 50, offset: 0 })));
-for (const [key, supplied, inherited] of original) {
- assert.equal(environment[key], supplied);
- assert.equal(process.env[key], inherited);
-}`,
- ],
- {
- cwd: first,
- env: {
- ...process.env,
- PYTHON:
- configuration === "explicit"
- ? relative(first, python)
- : basename(python),
- CODEX_SECURITY_STATE_DIR: join(directory, "state"),
- },
- },
- );
- expect(result.exitCode, result.stderr.toString()).toBe(0);
- expect(JSON.parse(result.stdout.toString()).findings).toEqual([finding(1)]);
- },
-);
-
-test.skipIf(process.platform === "win32")(
- "changing directories does not trust the original repository's Python shim",
- async () => {
- const directory = await mkdtemp(join(tmpdir(), "findings-python-shim-"));
- directories.push(directory);
- const repository = join(directory, "repository");
- const other = join(directory, "other");
- await Promise.all([mkdir(repository), mkdir(other)]);
- await writeFile(
- join(repository, "python"),
- '#!/bin/sh\nprintf invoked > "$SYNTHETIC_PYTHON_MARKER"\nprintf "codex-security-python-ok\\n"\n',
- { mode: 0o755 },
- );
- const result = Bun.spawnSync(
- [
- process.execPath,
- "--eval",
- `const assert = await import("node:assert/strict");
-const { SqliteFindingsStore } = await import(${JSON.stringify(new URL("../src/server/sqlite-store.ts", import.meta.url).href)});
-const store = new SqliteFindingsStore();
-await store.initialize();
-process.chdir(${JSON.stringify(other)});
-await assert.rejects(store.list({ limit: 50, offset: 0 }), /The PYTHON interpreter is unavailable or unusable/);`,
- ],
- {
- cwd: repository,
- env: {
- ...process.env,
- PYTHON: "./python",
- CODEX_SECURITY_STATE_DIR: join(directory, "state"),
- SYNTHETIC_PYTHON_MARKER: join(repository, "invoked"),
- },
- },
- );
- expect(result.exitCode, result.stderr.toString()).toBe(0);
- expect(await readdir(repository)).toEqual(["python"]);
- },
-);
-
test("escapes terminal controls in database helper diagnostics", async () => {
const directory = await mkdtemp(join(tmpdir(), "database-info-"));
directories.push(directory);
@@ -768,36 +643,6 @@ test("dashboard browses imported findings and overlapping groups without local r
environment,
"print(json.dumps(list(db.iterdump())))",
);
- expect(
- await database(
- environment,
- `from workbench_dashboard import dashboard
-allowed = {'findings', 'finding_repositories', 'finding_dedupe_groups', 'finding_dedupe_group_members'}
-def authorize(action, table, column, database, source):
- if action == sqlite3.SQLITE_READ and table not in allowed:
- return sqlite3.SQLITE_DENY
- return sqlite3.SQLITE_OK
-db.set_authorizer(authorize)
-queries = json.load(sys.stdin)
-print(json.dumps([dashboard(db, query)['total'] for query in queries]))`,
- [
- {
- view: "findings",
- limit: 50,
- offset: 0,
- sort: "activity",
- id: first.findingId,
- },
- {
- view: "groups",
- limit: 50,
- offset: 0,
- sort: "newest",
- id: groups[0]!.groupId,
- },
- ],
- ),
- ).toEqual([3, 2]);
const page = await dashboard(base, {
limit: "1",
@@ -1009,7 +854,7 @@ test("persists overlapping dedupe groups idempotently without changing findings
environment,
`print(json.dumps({
"memberships": db.execute("SELECT COUNT(*) FROM finding_dedupe_group_members").fetchone()[0],
- "embeddings": [list(row) for row in db.execute("SELECT finding_id, model, vector_json FROM finding_embeddings ORDER BY finding_id")]
+ "embeddings": [[row[0], row[1], json.loads(row[2])] for row in db.execute("SELECT finding_id, model, vector_json FROM finding_embeddings ORDER BY finding_id")]
}))`,
),
).toEqual({
@@ -1017,7 +862,7 @@ test("persists overlapping dedupe groups idempotently without changing findings
embeddings: entries.map((entry) => [
entry.finding.findingId,
"synthetic",
- "[1, 0]",
+ [1, 0],
]),
});
expect(await getGroups(base, "missing-finding")).toEqual([]);
@@ -1183,55 +1028,21 @@ test("retrieves complete potential duplicates without vectors or review calls",
);
});
-test("SQLite filters repository and embedding compatibility before exact cosine ranking", async () => {
- const { store, environment } = await fixture();
- await store.initialize();
- const anchor = embedded(1, [7, 0]);
- const boundary = embedded(2, [0.55, Math.sqrt(1 - 0.55 ** 2)]);
- const below = embedded(3, [0.54, Math.sqrt(1 - 0.54 ** 2)]);
- const otherModel = embedded(4, [1, 0], "other-model");
- const otherDimensions = embedded(5, [1, 0, 0]);
- const foreign = embedded(6);
- await store.insert(
- [anchor, below, otherModel, otherDimensions, boundary],
- "repository-a",
- );
- await store.insert([foreign], "repository-b");
+test("translates native duplicate retrieval failures without broadening scope", async () => {
+ const { store } = await fixture();
+ const anchor = embedded(1);
+ await store.insert([anchor], "repository-a");
+ await store.insert([embedded(2, [0, 0])], "repository-b");
expect(
await store.findPotentialDuplicates(anchor.finding.findingId, {
repositoryId: "repository-a",
}),
- ).toEqual({
- finding: anchor.finding,
- potentialDuplicates: [boundary.finding],
- });
- expect(
- await store.findPotentialDuplicates(anchor.finding.findingId, {
- allRepositories: true,
- }),
- ).toEqual({
- finding: anchor.finding,
- potentialDuplicates: [foreign.finding, boundary.finding],
- });
+ ).toEqual({ finding: anchor.finding, potentialDuplicates: [] });
await expect(
store.findPotentialDuplicates(anchor.finding.findingId, {
repositoryId: "repository-b",
}),
).rejects.toMatchObject({ code: "finding_not_indexed" });
- await database(
- environment,
- `with db:
- db.execute("UPDATE finding_embeddings SET vector_json = '[0,0]' WHERE finding_id = ?", (json.load(sys.stdin),))
-print("null")`,
- foreign.finding.findingId,
- );
- expect(
- (
- await store.findPotentialDuplicates(anchor.finding.findingId, {
- repositoryId: "repository-a",
- })
- ).potentialDuplicates,
- ).toEqual([boundary.finding]);
await expect(
store.findPotentialDuplicates(anchor.finding.findingId, {
allRepositories: true,
@@ -1239,53 +1050,6 @@ print("null")`,
).rejects.toMatchObject({ code: "embedding_failed" });
});
-test("SQLite reads only IDs and vectors before fetching the anchor and stable top 50 documents", async () => {
- const { store, environment } = await fixture();
- await store.initialize();
- const entries = Array.from({ length: 61 }, (_, index) => embedded(index + 1));
- await store.insert(entries, "repository-a");
- await store.insert([entries[1]!], "repository-b");
- const { result, queries } = (await database(
- environment,
- `from workbench_findings import find_potential_duplicates
-queries = []
-db.set_trace_callback(queries.append)
-result = find_potential_duplicates(db, json.load(sys.stdin), "repository-a")
-print(json.dumps({"result": result, "queries": queries}))`,
- entries[0]!.finding.findingId,
- )) as {
- result: { finding: Finding; potentialDuplicates: Finding[] };
- queries: string[];
- };
- expect(result).toEqual({
- finding: entries[0]!.finding,
- potentialDuplicates: entries.slice(1, 51).map((entry) => entry.finding),
- });
- const reads = queries.filter((query) => query.startsWith("SELECT"));
- expect(reads).toHaveLength(3);
- expect(reads[0]).toStartWith(
- "SELECT embeddings.model, embeddings.vector_json ",
- );
- expect(reads[1]).toStartWith(
- "SELECT embeddings.finding_id, embeddings.vector_json ",
- );
- expect(reads[1]).toContain("repositories.repository_id = 'repository-a'");
- expect(reads[2]).toStartWith(
- "SELECT id, details_json FROM findings WHERE id IN (",
- );
- const loadedIds = [...reads[2]!.matchAll(/csf_[0-9a-f]+/g)].map(([id]) => id);
- expect(loadedIds).toEqual(
- entries.slice(0, 51).map((entry) => entry.finding.findingId),
- );
- expect(
- (
- await store.findPotentialDuplicates(entries[0]!.finding.findingId, {
- allRepositories: true,
- })
- ).potentialDuplicates,
- ).toEqual(result.potentialDuplicates);
-});
-
test("imports persist repository associations and keep untagged findings in explicit all-repository scope", async () => {
const { store, environment } = await fixture();
const base = await start(store, {
diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts
index 2c26c617a..57e6fd798 100644
--- a/sdk/typescript/tests-ts/runtime.test.ts
+++ b/sdk/typescript/tests-ts/runtime.test.ts
@@ -6595,6 +6595,20 @@ describe("runtime directories and plugin Python boundary", () => {
}),
).rejects.toThrow(PluginPythonUnavailableError);
expect(existsSync(marker)).toBe(false);
+
+ await expect(
+ runWorkbench(
+ {
+ pluginRoot: PLUGIN_ROOT,
+ environment: { PATH: trustedBin, PYTHON: unsafePython },
+ protectedRoot: repository,
+ },
+ ["list-scans"],
+ ),
+ ).rejects.toMatchObject({
+ cause: expect.any(PluginPythonUnavailableError),
+ });
+ expect(existsSync(marker)).toBe(false);
},
);