Skip to content

Persistent project knowledge graph for coding agents. MCP server with semantic search, in-process embeddings, and web explorer.

test(db): add multi-process concurrency coverage

0xk3vina51b41d2c083
diff --git a/src/__tests__/db.concurrency.test.ts b/src/__tests__/db.concurrency.test.ts
new file mode 100644
index 0000000..4a800e4
--- /dev/null
+++ b/src/__tests__/db.concurrency.test.ts
@@ -0,0 +1,320 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { spawn } from "child_process";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import Database from "libsql";
+import { KnowledgeDB } from "../db.js";
+
+const TEST_TIMEOUT_MS = 30000;
+const FRESH_OPEN_WORKERS = [1, 2, 3, 4];
+const CONCURRENT_WRITERS = [1, 2, 3, 4];
+const repoRoot = path.resolve(import.meta.dirname, "../..");
+const workerScriptPath = path.resolve(
+ import.meta.dirname,
+ "helpers",
+ "db-worker.ts"
+);
+
+interface WorkerPayload {
+ mode: "fresh-open" | "writer" | "reader";
+ dbPath: string;
+ workerId: number;
+ startAt?: number;
+ startSignalPath?: string;
+ readerReadyPath?: string;
+ anchorId?: string;
+ iterations?: number;
+ sleepMs?: number;
+ preWriteSleepMs?: number;
+}
+
+interface WorkerResult {
+ code: number | null;
+ signal: NodeJS.Signals | null;
+ stdout: string;
+ stderr: string;
+ parsed: Record<string, unknown>;
+}
+
+const tmpDirs: string[] = [];
+
+afterEach(() => {
+ for (const tmpDir of tmpDirs.splice(0)) {
+ fs.rmSync(tmpDir, { recursive: true, force: true });
+ }
+});
+
+function makeTempDbPath(testName: string): string {
+ const tmpDir = fs.mkdtempSync(
+ path.join(os.tmpdir(), `megamemory-${testName}-`)
+ );
+ tmpDirs.push(tmpDir);
+ return path.join(tmpDir, "knowledge.db");
+}
+
+async function waitUntil(timestamp: number): Promise<void> {
+ const delay = timestamp - Date.now();
+ if (delay > 0) {
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ }
+}
+
+async function runWorkersWithSharedStart(
+ dbPath: string,
+ workers: WorkerPayload[],
+ startAt: number
+): Promise<WorkerResult[]> {
+ const startSignalPath = path.join(path.dirname(dbPath), "start.signal");
+ const readerReadyPath = path.join(path.dirname(dbPath), "reader.ready");
+ const workerPromises = workers.map((worker) =>
+ runWorker({
+ ...worker,
+ startAt,
+ startSignalPath,
+ readerReadyPath,
+ })
+ );
+
+ await waitUntil(startAt);
+ fs.writeFileSync(startSignalPath, `${startAt}\n`);
+
+ return Promise.all(workerPromises);
+}
+
+function runWorker(payload: WorkerPayload): Promise<WorkerResult> {
+ return new Promise((resolve, reject) => {
+ const child = spawn(
+ process.execPath,
+ ["--import", "tsx", workerScriptPath, JSON.stringify(payload)],
+ {
+ cwd: repoRoot,
+ env: process.env,
+ stdio: ["ignore", "pipe", "pipe"],
+ }
+ );
+
+ let stdout = "";
+ let stderr = "";
+
+ child.stdout.on("data", (chunk) => {
+ stdout += chunk.toString();
+ });
+
+ child.stderr.on("data", (chunk) => {
+ stderr += chunk.toString();
+ });
+
+ child.on("error", reject);
+
+ child.on("close", (code, signal) => {
+ if (code !== 0) {
+ reject(
+ new Error(
+ [
+ `Worker failed for payload ${JSON.stringify(payload)}`,
+ `exit code: ${code}, signal: ${signal}`,
+ `stdout:\n${stdout || "<empty>"}`,
+ `stderr:\n${stderr || "<empty>"}`,
+ ].join("\n\n")
+ )
+ );
+ return;
+ }
+
+ const trimmed = stdout.trim();
+ if (!trimmed) {
+ reject(
+ new Error(
+ `Worker produced no stdout for payload ${JSON.stringify(payload)}`
+ )
+ );
+ return;
+ }
+
+ const lastLine = trimmed.split(/\r?\n/).at(-1);
+ if (!lastLine) {
+ reject(
+ new Error(
+ `Worker stdout missing result line for payload ${JSON.stringify(payload)}`
+ )
+ );
+ return;
+ }
+
+ try {
+ resolve({
+ code,
+ signal,
+ stdout,
+ stderr,
+ parsed: JSON.parse(lastLine) as Record<string, unknown>,
+ });
+ } catch (error: unknown) {
+ reject(
+ new Error(
+ [
+ `Failed to parse worker stdout for payload ${JSON.stringify(payload)}`,
+ `stdout:\n${stdout}`,
+ `stderr:\n${stderr}`,
+ error instanceof Error ? error.message : String(error),
+ ].join("\n\n")
+ )
+ );
+ }
+ });
+ });
+}
+
+function runIntegrityCheck(dbPath: string): string {
+ const rawDb = new Database(dbPath);
+
+ try {
+ const row = rawDb
+ .prepare("PRAGMA integrity_check")
+ .get() as { integrity_check?: string } | undefined;
+ return row?.integrity_check ?? "";
+ } finally {
+ rawDb.close();
+ }
+}
+
+describe.sequential("KnowledgeDB multi-process concurrency", () => {
+ it(
+ "smokes concurrent fresh open and migration on a shared WAL database",
+ { timeout: TEST_TIMEOUT_MS },
+ async () => {
+ const dbPath = makeTempDbPath("fresh-open");
+ const startAt = Date.now() + 250;
+
+ const workerResults = await runWorkersWithSharedStart(
+ dbPath,
+ FRESH_OPEN_WORKERS.map((workerId) => ({
+ mode: "fresh-open",
+ dbPath,
+ workerId,
+ })),
+ startAt
+ );
+
+ expect(workerResults).toHaveLength(FRESH_OPEN_WORKERS.length);
+
+ const db = new KnowledgeDB(dbPath);
+ try {
+ const stats = db.getStats();
+ const nodeIds = db.getAllActiveNodes().map((node) => node.id).sort();
+
+ expect(stats).toEqual({
+ nodes: FRESH_OPEN_WORKERS.length,
+ edges: 0,
+ removed: 0,
+ });
+ expect(nodeIds).toEqual(
+ FRESH_OPEN_WORKERS.map((workerId) => `fresh-worker-${workerId}`).sort()
+ );
+ } finally {
+ db.close();
+ }
+
+ const reopened = new KnowledgeDB(dbPath);
+ try {
+ expect(reopened.getStats().nodes).toBe(FRESH_OPEN_WORKERS.length);
+ } finally {
+ reopened.close();
+ }
+
+ expect(runIntegrityCheck(dbPath)).toBe("ok");
+ }
+ );
+
+ it(
+ "smokes concurrent writers and a reader on a shared WAL database",
+ { timeout: TEST_TIMEOUT_MS },
+ async () => {
+ const dbPath = makeTempDbPath("writers-reader");
+ const anchorId = "anchor-node";
+ const seedDb = new KnowledgeDB(dbPath);
+
+ try {
+ seedDb.insertNode({
+ id: anchorId,
+ name: "Anchor Node",
+ kind: "module",
+ summary: "Anchor for concurrent edge creation",
+ why: null,
+ file_refs: null,
+ parent_id: null,
+ created_by_task: "db-concurrency-seed",
+ embedding: null,
+ });
+ } finally {
+ seedDb.close();
+ }
+
+ const startAt = Date.now() + 250;
+ const [readerResult, ...writerResults] = await runWorkersWithSharedStart(
+ dbPath,
+ [
+ {
+ mode: "reader",
+ dbPath,
+ workerId: 99,
+ anchorId,
+ iterations: 60,
+ sleepMs: 20,
+ },
+ ...CONCURRENT_WRITERS.map((workerId) => ({
+ mode: "writer",
+ dbPath,
+ workerId,
+ anchorId,
+ preWriteSleepMs: workerId * 60,
+ })),
+ ],
+ startAt
+ );
+
+ expect(readerResult.parsed.iterations).toBe(60);
+ expect(readerResult.parsed.firstNodes).toBe(1);
+ expect(readerResult.parsed.firstEdges).toBe(0);
+ expect(readerResult.parsed.lastNodes).toBe(1 + CONCURRENT_WRITERS.length);
+ expect(readerResult.parsed.lastEdges).toBe(CONCURRENT_WRITERS.length);
+ expect(readerResult.parsed.maxNodes).toBe(1 + CONCURRENT_WRITERS.length);
+ expect(readerResult.parsed.maxEdges).toBe(CONCURRENT_WRITERS.length);
+ expect(readerResult.parsed.growthSamples).toBeGreaterThan(0);
+ expect(readerResult.parsed.distinctStateCount).toBeGreaterThan(1);
+ expect(writerResults).toHaveLength(CONCURRENT_WRITERS.length);
+
+ const writerNodeIds = writerResults
+ .map((result) => result.parsed.nodeId)
+ .sort() as string[];
+ expect(writerNodeIds).toEqual(
+ CONCURRENT_WRITERS.map((workerId) => `writer-node-${workerId}`).sort()
+ );
+ for (const writerResult of writerResults) {
+ expect(writerResult.parsed.outgoingEdges).toBe(1);
+ }
+
+ const db = new KnowledgeDB(dbPath);
+ try {
+ const stats = db.getStats();
+ const incomingAnchorEdges = db.getIncomingEdges(anchorId);
+ const activeNodeIds = db.getAllActiveNodes().map((node) => node.id).sort();
+
+ expect(stats).toEqual({
+ nodes: 1 + CONCURRENT_WRITERS.length,
+ edges: CONCURRENT_WRITERS.length,
+ removed: 0,
+ });
+ expect(incomingAnchorEdges).toHaveLength(CONCURRENT_WRITERS.length);
+ expect(activeNodeIds).toEqual(
+ [anchorId, ...CONCURRENT_WRITERS.map((workerId) => `writer-node-${workerId}`)].sort()
+ );
+ } finally {
+ db.close();
+ }
+
+ expect(runIntegrityCheck(dbPath)).toBe("ok");
+ }
+ );
+});
diff --git a/src/__tests__/helpers/db-worker.ts b/src/__tests__/helpers/db-worker.ts
new file mode 100644
index 0000000..45317e8
--- /dev/null
+++ b/src/__tests__/helpers/db-worker.ts
@@ -0,0 +1,271 @@
+import fs from "fs";
+import { KnowledgeDB } from "../../db.js";
+
+type WorkerMode = "fresh-open" | "writer" | "reader";
+
+interface WorkerPayload {
+ mode: WorkerMode;
+ dbPath: string;
+ workerId: number;
+ startAt?: number;
+ startSignalPath?: string;
+ readerReadyPath?: string;
+ anchorId?: string;
+ iterations?: number;
+ sleepMs?: number;
+ preWriteSleepMs?: number;
+}
+
+const START_SIGNAL_TIMEOUT_MS = 5000;
+const OPEN_RETRY_DELAYS_MS = [25, 50, 100, 200, 400, 800];
+
+function sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitUntil(timestamp?: number): Promise<void> {
+ if (!timestamp) return;
+ const delay = timestamp - Date.now();
+ if (delay > 0) {
+ await sleep(delay);
+ }
+}
+
+async function waitForStart(payload: WorkerPayload): Promise<void> {
+ await waitUntil(payload.startAt);
+
+ if (!payload.startSignalPath) {
+ return;
+ }
+
+ const deadline = Date.now() + START_SIGNAL_TIMEOUT_MS;
+ while (Date.now() <= deadline) {
+ try {
+ if (fs.existsSync(payload.startSignalPath)) {
+ return;
+ }
+ } catch {
+ // Ignore transient filesystem errors while polling.
+ }
+
+ await sleep(10);
+ }
+
+ throw new Error(
+ `timed out waiting for start signal ${payload.startSignalPath}`
+ );
+}
+
+async function waitForSignalFile(
+ filePath: string | undefined,
+ label: string
+): Promise<void> {
+ if (!filePath) {
+ return;
+ }
+
+ const deadline = Date.now() + START_SIGNAL_TIMEOUT_MS;
+ while (Date.now() <= deadline) {
+ try {
+ if (fs.existsSync(filePath)) {
+ return;
+ }
+ } catch {
+ // Ignore transient filesystem errors while polling.
+ }
+
+ await sleep(10);
+ }
+
+ throw new Error(`timed out waiting for ${label} signal ${filePath}`);
+}
+
+function isLockedError(error: unknown): boolean {
+ const message = error instanceof Error ? error.message : String(error);
+ return (
+ message.includes("SQLITE_BUSY") ||
+ message.includes("database is locked") ||
+ message.includes("database schema is locked")
+ );
+}
+
+async function openDbWithRetry(dbPath: string): Promise<KnowledgeDB> {
+ let lastError: unknown;
+
+ for (const delayMs of [0, ...OPEN_RETRY_DELAYS_MS]) {
+ if (delayMs > 0) {
+ await sleep(delayMs);
+ }
+
+ try {
+ return new KnowledgeDB(dbPath);
+ } catch (error: unknown) {
+ if (!isLockedError(error)) {
+ throw error;
+ }
+
+ lastError = error;
+ }
+ }
+
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+
+async function run(payload: WorkerPayload): Promise<Record<string, unknown>> {
+ await waitForStart(payload);
+ const db = await openDbWithRetry(payload.dbPath);
+
+ try {
+ if (payload.mode === "fresh-open") {
+ const nodeId = `fresh-worker-${payload.workerId}`;
+ db.insertNode({
+ id: nodeId,
+ name: `Fresh Worker ${payload.workerId}`,
+ kind: "feature",
+ summary: `Fresh open worker ${payload.workerId}`,
+ why: null,
+ file_refs: null,
+ parent_id: null,
+ created_by_task: "db-concurrency-fresh-open",
+ embedding: null,
+ });
+
+ return { mode: payload.mode, workerId: payload.workerId, nodeId };
+ }
+
+ if (payload.mode === "writer") {
+ if (!payload.anchorId) {
+ throw new Error("writer mode requires anchorId");
+ }
+
+ await waitForSignalFile(payload.readerReadyPath, "reader-ready");
+
+ if ((payload.preWriteSleepMs ?? 0) > 0) {
+ await sleep(payload.preWriteSleepMs ?? 0);
+ }
+
+ const nodeId = `writer-node-${payload.workerId}`;
+ db.insertNodeAndEdges(
+ {
+ id: nodeId,
+ name: `Writer Node ${payload.workerId}`,
+ kind: "feature",
+ summary: `Concurrent writer ${payload.workerId}`,
+ why: null,
+ file_refs: JSON.stringify([
+ `src/__tests__/db.concurrency.test.ts#writer-${payload.workerId}`,
+ ]),
+ parent_id: null,
+ created_by_task: "db-concurrency-writer",
+ embedding: null,
+ },
+ [
+ {
+ to_id: payload.anchorId,
+ relation: "depends_on",
+ description: `writer ${payload.workerId} links to anchor`,
+ },
+ ]
+ );
+
+ return {
+ mode: payload.mode,
+ workerId: payload.workerId,
+ nodeId,
+ outgoingEdges: db.getOutgoingEdges(nodeId).length,
+ };
+ }
+
+ if (!payload.anchorId) {
+ throw new Error("reader mode requires anchorId");
+ }
+
+ const iterations = payload.iterations ?? 40;
+ const sleepMs = payload.sleepMs ?? 20;
+ let maxNodes = 0;
+ let maxEdges = 0;
+ let firstNodes: number | null = null;
+ let firstEdges: number | null = null;
+ let lastNodes = 0;
+ let lastEdges = 0;
+ let growthSamples = 0;
+ const observedStates = new Set<string>();
+
+ const recordSnapshot = (): void => {
+ const anchor = db.getNode(payload.anchorId!);
+ if (!anchor) {
+ throw new Error(`anchor node ${payload.anchorId} missing during read`);
+ }
+
+ const stats = db.getStats();
+ const nodes = db.getAllActiveNodes();
+ const edges = db.getAllEdges();
+ const incoming = db.getIncomingEdges(payload.anchorId!);
+ const outgoing = db.getOutgoingEdges(payload.anchorId!);
+
+ if (stats.nodes < 1) {
+ throw new Error("reader observed invalid empty active graph");
+ }
+
+ const nodeCount = Math.max(nodes.length, stats.nodes);
+ const edgeCount = Math.max(edges.length, incoming.length, outgoing.length);
+
+ if (firstNodes === null) {
+ firstNodes = nodeCount;
+ firstEdges = edgeCount;
+ } else if (nodeCount > lastNodes || edgeCount > lastEdges) {
+ growthSamples += 1;
+ }
+
+ lastNodes = nodeCount;
+ lastEdges = edgeCount;
+ maxNodes = Math.max(maxNodes, nodeCount);
+ maxEdges = Math.max(maxEdges, edgeCount);
+ observedStates.add(`${nodeCount}:${edgeCount}`);
+ };
+
+ recordSnapshot();
+
+ if (payload.readerReadyPath) {
+ fs.writeFileSync(payload.readerReadyPath, `${payload.workerId}\n`);
+ }
+
+ for (let i = 1; i < iterations; i++) {
+ await sleep(sleepMs);
+ recordSnapshot();
+ }
+
+ return {
+ mode: payload.mode,
+ workerId: payload.workerId,
+ iterations,
+ firstNodes,
+ firstEdges,
+ lastNodes,
+ lastEdges,
+ maxNodes,
+ maxEdges,
+ growthSamples,
+ distinctStateCount: observedStates.size,
+ };
+ } finally {
+ db.close();
+ }
+}
+
+async function main(): Promise<void> {
+ const rawPayload = process.argv[2];
+ if (!rawPayload) {
+ throw new Error("missing worker payload");
+ }
+
+ const payload = JSON.parse(rawPayload) as WorkerPayload;
+ const result = await run(payload);
+ process.stdout.write(`${JSON.stringify(result)}\n`);
+}
+
+main().catch((error: unknown) => {
+ const message = error instanceof Error ? error.stack ?? error.message : String(error);
+ process.stderr.write(`${message}\n`);
+ process.exit(1);
+});