Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

perf(remember): move the embedding call out of the global save lock (#1423)

Rohit Ghumarec2df01a62c64
diff --git a/src/functions/remember.ts b/src/functions/remember.ts
index 73f6e3c3..e3a7e8ea 100644
--- a/src/functions/remember.ts
+++ b/src/functions/remember.ts
@@ -6,7 +6,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js";
import { memoryToObservation } from "../state/memory-utils.js";
import { deleteAccessLog } from "./access-tracker.js";
import { recordAudit } from "./audit.js";
-import { getSearchIndex, isMemoryIndexReady, scheduleIndexSave, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSave } from "./search.js";
+import { getSearchIndex, getVectorIndex, isMemoryIndexReady, scheduleIndexSave, vectorIndexAddGuarded, vectorIndexRemove, flushIndexSave } from "./search.js";
import { getAgentId } from "../config.js";
import { logger } from "../logger.js";
@@ -68,7 +68,7 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
? data.project.trim()
: undefined;
- return withKeyedLock("mem:remember", async () => {
+ const { memory, supersededId, nearMatch } = await withKeyedLock("mem:remember", async () => {
// Candidate generation: query the BM25 index with the new content
// and Jaccard-compare only the top hits, instead of walking the
// full memory corpus on every save. The index receives every
@@ -204,43 +204,57 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
error: err instanceof Error ? err.message : String(err),
});
}
- await vectorIndexAddGuarded(
- memory.id,
- memory.sessionIds?.[0] ?? "memory",
- memory.title + " " + memory.content,
- { kind: "memory", logId: memory.id },
- );
+ return { memory, supersededId, nearMatch };
+ });
- if (supersededId) {
- await sdk.trigger({
- function_id: "mem::cascade-update",
- payload: {
- supersededMemoryId: supersededId,
- },
- action: TriggerAction.Void(),
- });
- }
+ await vectorIndexAddGuarded(
+ memory.id,
+ memory.sessionIds?.[0] ?? "memory",
+ memory.title + " " + memory.content,
+ { kind: "memory", logId: memory.id },
+ (embedding) =>
+ withKeyedLock("mem:remember", async () => {
+ const current = await kv.get<Memory>(KV.memories, memory.id);
+ if (!current || current.isLatest === false) return false;
+ getVectorIndex()?.add(
+ memory.id,
+ memory.sessionIds?.[0] ?? "memory",
+ embedding,
+ );
+ scheduleIndexSave();
+ return true;
+ }),
+ );
- logger.info("Memory saved", {
- memId: memory.id,
- type: memory.type,
- project: memory.project,
+ if (supersededId) {
+ await sdk.trigger({
+ function_id: "mem::cascade-update",
+ payload: {
+ supersededMemoryId: supersededId,
+ },
+ action: TriggerAction.Void(),
});
- // similarTo is advisory only: a close-but-not-superseding match
- // the caller may want to consolidate via memory_update/forget.
- return {
- success: true,
- memory,
- ...(nearMatch && !supersededId
- ? {
- similarTo: {
- ...nearMatch,
- similarity: Math.round(nearMatch.similarity * 100) / 100,
- },
- }
- : {}),
- };
+ }
+
+ logger.info("Memory saved", {
+ memId: memory.id,
+ type: memory.type,
+ project: memory.project,
});
+ // similarTo is advisory only: a close-but-not-superseding match
+ // the caller may want to consolidate via memory_update/forget.
+ return {
+ success: true,
+ memory,
+ ...(nearMatch && !supersededId
+ ? {
+ similarTo: {
+ ...nearMatch,
+ similarity: Math.round(nearMatch.similarity * 100) / 100,
+ },
+ }
+ : {}),
+ };
},
);
diff --git a/src/functions/search.ts b/src/functions/search.ts
index 2d2f4147..66be533b 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -150,24 +150,14 @@ export async function vectorIndexAddGuarded(
sessionId: string,
text: string,
context: { kind: "memory" | "observation" | "synthetic"; logId: string },
+ commit?: (embedding: Float32Array) => Promise<boolean> | boolean,
): Promise<boolean> {
const vi = vectorIndex
const ep = currentEmbeddingProvider
if (!vi || !ep) return false
+ let embedding: Float32Array
try {
- const embedding = await ep.embed(clipEmbedInput(text))
- if (embedding.length !== ep.dimensions) {
- logger.warn("vector-index add: dimension mismatch — skipping", {
- kind: context.kind,
- id: context.logId,
- provider: ep.name,
- expected: ep.dimensions,
- received: embedding.length,
- })
- return false
- }
- vi.add(id, sessionId, embedding)
- return true
+ embedding = await ep.embed(clipEmbedInput(text))
} catch (err) {
logger.warn("vector-index add: embed failed — skipping", {
kind: context.kind,
@@ -177,6 +167,31 @@ export async function vectorIndexAddGuarded(
})
return false
}
+ if (embedding.length !== ep.dimensions) {
+ logger.warn("vector-index add: dimension mismatch — skipping", {
+ kind: context.kind,
+ id: context.logId,
+ provider: ep.name,
+ expected: ep.dimensions,
+ received: embedding.length,
+ })
+ return false
+ }
+ if (commit) {
+ try {
+ return await commit(embedding)
+ } catch (err) {
+ logger.warn("vector-index add: commit failed — skipping", {
+ kind: context.kind,
+ id: context.logId,
+ provider: ep.name,
+ error: err instanceof Error ? err.message : String(err),
+ })
+ return false
+ }
+ }
+ vi.add(id, sessionId, embedding)
+ return true
}
// Batched variant: calls EmbeddingProvider.embedBatch ONCE for the whole
diff --git a/test/remember-lock-scope.test.ts b/test/remember-lock-scope.test.ts
new file mode 100644
index 00000000..750ec55f
--- /dev/null
+++ b/test/remember-lock-scope.test.ts
@@ -0,0 +1,217 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import { registerRememberFunction } from "../src/functions/remember.js";
+import {
+ setVectorIndex,
+ setEmbeddingProvider,
+ setIndexPersistence,
+ vectorIndexAddGuarded,
+} from "../src/functions/search.js";
+import { logger } from "../src/logger.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import { KV } from "../src/state/schema.js";
+import type { EmbeddingProvider, Memory } from "../src/types.js";
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> =>
+ (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ const entries = store.get(scope);
+ return entries ? (Array.from(entries.values()) as T[]) : [];
+ },
+ };
+}
+
+function mockSdk() {
+ const functions = new Map<string, Function>();
+ return {
+ registerFunction: (id: string, handler: Function) => {
+ functions.set(id, handler);
+ },
+ registerTrigger: () => {},
+ trigger: async (input: { function_id: string; payload: unknown }) => {
+ const fn = functions.get(input.function_id);
+ if (!fn) return null;
+ return fn(input.payload);
+ },
+ };
+}
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+type RememberResult = {
+ success: boolean;
+ memory: Memory;
+};
+
+describe("mem::remember lock scope", () => {
+ const EMBED_DELAY_MS = 60;
+ let vectorIndex: VectorIndex;
+ let embedCalls: Array<{ start: number; end: number }>;
+
+ beforeEach(() => {
+ vectorIndex = new VectorIndex();
+ setVectorIndex(vectorIndex);
+ embedCalls = [];
+ const slowEmbedder: EmbeddingProvider = {
+ name: "slow-test",
+ dimensions: 3,
+ embed: async (_text: string) => {
+ const start = Date.now();
+ await delay(EMBED_DELAY_MS);
+ embedCalls.push({ start, end: Date.now() });
+ return new Float32Array([0.1, 0.2, 0.3]);
+ },
+ embedBatch: async (texts: string[]) =>
+ texts.map(() => new Float32Array([0.1, 0.2, 0.3])),
+ };
+ setEmbeddingProvider(slowEmbedder);
+ });
+
+ afterEach(() => {
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ });
+
+ it("overlaps embedding time across concurrent saves instead of serializing behind the mem:remember lock", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ await Promise.all([
+ sdk.trigger({
+ function_id: "mem::remember",
+ payload: {
+ content: "alpha content about the deploy pipeline rollout gates",
+ type: "fact",
+ },
+ }),
+ sdk.trigger({
+ function_id: "mem::remember",
+ payload: {
+ content: "bravo content about a completely unrelated caching layer",
+ type: "fact",
+ },
+ }),
+ ]);
+
+ expect(embedCalls).toHaveLength(2);
+ const [a, b] = embedCalls;
+ expect(a.start).toBeLessThan(b.end);
+ expect(b.start).toBeLessThan(a.end);
+ });
+
+ it("still dedups two identical saves fired concurrently, even with a slow embedder", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const content =
+ "the deploy pipeline uses blue green rollout with health gates";
+ const [first, second] = (await Promise.all([
+ sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content, type: "architecture" },
+ }),
+ sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content, type: "architecture" },
+ }),
+ ])) as RememberResult[];
+
+ expect(first.memory.version).toBe(1);
+ expect(second.memory.version).toBe(2);
+ expect(second.memory.supersedes).toContain(first.memory.id);
+ expect(second.memory.parentId).toBe(first.memory.id);
+
+ const stored = await kv.get<Memory>(KV.memories, first.memory.id);
+ expect(stored?.isLatest).toBe(false);
+ });
+
+ it("keeps a superseded memory out of the vector index even when its embedding finishes after the second save removes it", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const content =
+ "the deploy pipeline uses blue green rollout with health gates";
+ const [first, second] = (await Promise.all([
+ sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content, type: "architecture" },
+ }),
+ sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content, type: "architecture" },
+ }),
+ ])) as RememberResult[];
+
+ const results = vectorIndex.search(new Float32Array([0.1, 0.2, 0.3]), 10);
+ const indexedIds = results.map((r) => r.obsId);
+ expect(indexedIds).not.toContain(first.memory.id);
+ expect(indexedIds).toContain(second.memory.id);
+ expect(vectorIndex.size).toBe(1);
+ });
+
+ it("schedules an index save after the deferred vector-index commit lands", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+ const scheduleSave = vi.fn();
+ setIndexPersistence({ scheduleSave, save: vi.fn(async () => {}) });
+
+ try {
+ await sdk.trigger({
+ function_id: "mem::remember",
+ payload: {
+ content: "schedule an index save after the vector commit lands",
+ type: "fact",
+ },
+ });
+
+ expect(scheduleSave.mock.calls.length).toBeGreaterThanOrEqual(2);
+ } finally {
+ setIndexPersistence(null);
+ }
+ });
+
+ it("logs a vector-index commit failure distinctly from an embed failure", async () => {
+ const failingCommit = vi.fn(async () => {
+ throw new Error("kv unavailable");
+ });
+
+ const result = await vectorIndexAddGuarded(
+ "mem_commit_fail",
+ "memory",
+ "some memory text",
+ { kind: "memory", logId: "mem_commit_fail" },
+ failingCommit,
+ );
+
+ expect(result).toBe(false);
+ expect(logger.warn).toHaveBeenCalledWith(
+ "vector-index add: commit failed — skipping",
+ expect.objectContaining({ id: "mem_commit_fail" }),
+ );
+ expect(logger.warn).not.toHaveBeenCalledWith(
+ "vector-index add: embed failed — skipping",
+ expect.anything(),
+ );
+ });
+});