Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(search): BM25 unicode + vector index live-write — improved (closes #295, supersedes #296) (#327)

Rohit Ghumare24f6c5fb2475
diff --git a/src/functions/compress.ts b/src/functions/compress.ts
index ed1de787..af3973cb 100644
--- a/src/functions/compress.ts
+++ b/src/functions/compress.ts
@@ -15,7 +15,7 @@ import {
} from "../prompts/compression.js";
import { VISION_DESCRIPTION_PROMPT } from "../prompts/vision.js";
import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
-import { getSearchIndex } from "./search.js";
+import { getSearchIndex, vectorIndexAddGuarded } from "./search.js";
import { CompressOutputSchema } from "../eval/schemas.js";
import { validateOutput } from "../eval/validator.js";
import { scoreCompression } from "../eval/quality.js";
@@ -173,7 +173,23 @@ export function registerCompressFunction(
compressed,
);
- getSearchIndex().add(compressed);
+ try {
+ getSearchIndex().add(compressed);
+ } catch (err) {
+ logger.warn("Failed to index compressed observation into BM25", {
+ obsId: compressed.id,
+ sessionId: compressed.sessionId,
+ title: compressed.title,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ }
+
+ await vectorIndexAddGuarded(
+ compressed.id,
+ compressed.sessionId,
+ compressed.title + " " + (compressed.narrative || ""),
+ { kind: "observation", logId: compressed.id },
+ );
const streamResults = await Promise.allSettled([
sdk.trigger({
diff --git a/src/functions/migrate-vector-index.ts b/src/functions/migrate-vector-index.ts
new file mode 100644
index 00000000..058eab84
--- /dev/null
+++ b/src/functions/migrate-vector-index.ts
@@ -0,0 +1,152 @@
+import type { EmbeddingProvider, CompressedObservation, Memory } from "../types.js";
+import { VectorIndex } from "../state/vector-index.js";
+import { KV } from "../state/schema.js";
+import type { StateKV } from "../state/kv.js";
+import { logger } from "../logger.js";
+
+export interface MigrateVectorIndexResult {
+ success: boolean;
+ totalProcessed: number;
+ failed: number;
+ vectorSize: number;
+ failedSessions: string[];
+}
+
+// Validate one embedding's shape against the provider's declared dimensions
+// before pushing it into the index. Mirrors the symmetric guard in
+// search.ts::vectorIndexAddGuarded — without this, a misconfigured
+// provider returning the wrong-length Float32Array would silently corrupt
+// the rebuilt index (per #248).
+function isValidEmbedding(
+ embedding: Float32Array,
+ provider: EmbeddingProvider,
+ context: { kind: "memory" | "observation"; id: string },
+): boolean {
+ if (embedding.length !== provider.dimensions) {
+ logger.warn("migrateVectorIndex: dimension mismatch — skipping", {
+ kind: context.kind,
+ id: context.id,
+ provider: provider.name,
+ expected: provider.dimensions,
+ received: embedding.length,
+ });
+ return false;
+ }
+ return true;
+}
+
+// Rebuilds a fresh VectorIndex against `newProvider`, re-embedding every
+// memory and per-session observation in `kv`. Each phase (memories +
+// per-session observations) is isolated — a single session that throws
+// on kv.list or embedBatch increments `failed` and appends to
+// `failedSessions`, but the migration continues. Returns a structured
+// result the caller can inspect to decide whether to swap the index in.
+export async function migrateVectorIndex(
+ kv: StateKV,
+ newProvider: EmbeddingProvider,
+): Promise<MigrateVectorIndexResult> {
+ const newIndex = new VectorIndex();
+ let failed = 0;
+ let processed = 0;
+ const failedSessions: string[] = [];
+
+ // --- Memories phase ----------------------------------------------------
+ // textMems is declared outside the try so the catch can attribute the
+ // batch-level failure to the correct number of missed embeddings (the
+ // size of the batch we were about to embed), not a flat +1.
+ let textMems: Memory[] = [];
+ try {
+ const memories = await kv.list<Memory>(KV.memories);
+ textMems = memories.filter(
+ (m) => m.isLatest !== false && m.title && m.content && m.content.trim() !== "",
+ );
+ const texts = textMems.map((m) => m.title + " " + m.content);
+
+ if (texts.length > 0) {
+ const embeddings = await newProvider.embedBatch(texts);
+ for (let i = 0; i < textMems.length; i++) {
+ if (!isValidEmbedding(embeddings[i], newProvider, { kind: "memory", id: textMems[i].id })) {
+ failed++;
+ continue;
+ }
+ newIndex.add(
+ textMems[i].id,
+ textMems[i].sessionIds[0] ?? "memory",
+ embeddings[i],
+ );
+ processed++;
+ }
+ }
+ } catch (err) {
+ // If kv.list threw before textMems was populated, the batch size is
+ // unknown — count as +1 (something failed but we don't know what).
+ // If embedBatch threw, textMems.length is the real number of missed
+ // memories. Caller relying on `failed` for retry math needs this
+ // attribution to be accurate.
+ const missed = textMems.length > 0 ? textMems.length : 1;
+ logger.warn("migrateVectorIndex: failed to re-embed memories", {
+ missed,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ failed += missed;
+ }
+
+ // --- Observations phase (per-session isolation) ------------------------
+ // Without per-session try/catch, one bad session (kv.list throws,
+ // embedBatch rejects, etc.) would abort every later session and silently
+ // truncate the migration. Each session now has its own boundary; failures
+ // increment `failed`, append the session id to failedSessions, and the
+ // loop moves on.
+ let sessions: Array<{ id: string }>;
+ try {
+ sessions = await kv.list<{ id: string }>(KV.sessions);
+ } catch (err) {
+ logger.warn("migrateVectorIndex: failed to list sessions", {
+ error: err instanceof Error ? err.message : String(err),
+ });
+ failed++;
+ // Distinguish a list-sessions failure (catastrophic: no sessions
+ // could be enumerated) from a per-session failure (one specific id
+ // threw). Without the marker the caller sees failed=N + an empty
+ // failedSessions list and can't tell apart "0 sessions, all OK"
+ // from "kv.list itself blew up".
+ failedSessions.push("<sessions-list-failed>");
+ return { success: false, totalProcessed: processed, failed, vectorSize: newIndex.size, failedSessions };
+ }
+
+ for (const session of sessions) {
+ try {
+ const observations = await kv.list<CompressedObservation>(
+ KV.observations(session.id),
+ );
+ const textObs = observations.filter((o) => o.title);
+ const texts = textObs.map((o) => o.title + " " + (o.narrative || ""));
+ if (texts.length === 0) continue;
+
+ const embeddings = await newProvider.embedBatch(texts);
+ for (let i = 0; i < textObs.length; i++) {
+ if (!isValidEmbedding(embeddings[i], newProvider, { kind: "observation", id: textObs[i].id })) {
+ failed++;
+ continue;
+ }
+ newIndex.add(textObs[i].id, textObs[i].sessionId, embeddings[i]);
+ processed++;
+ }
+ } catch (err) {
+ logger.warn("migrateVectorIndex: failed to re-embed session", {
+ sessionId: session.id,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ failed++;
+ failedSessions.push(session.id);
+ }
+ }
+
+ return {
+ success: failed === 0,
+ totalProcessed: processed,
+ failed,
+ vectorSize: newIndex.size,
+ failedSessions,
+ };
+}
diff --git a/src/functions/observe.ts b/src/functions/observe.ts
index 951bf5f5..57e2f7df 100644
--- a/src/functions/observe.ts
+++ b/src/functions/observe.ts
@@ -7,7 +7,7 @@ import { DedupMap } from "./dedup.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { isAutoCompressEnabled } from "../config.js";
import { buildSyntheticCompression } from "./compress-synthetic.js";
-import { getSearchIndex } from "./search.js";
+import { getSearchIndex, vectorIndexAddGuarded } from "./search.js";
import { logger } from "../logger.js";
export function extractImage(d: unknown): string | undefined {
@@ -238,6 +238,12 @@ export function registerObserveFunction(
synthetic,
);
getSearchIndex().add(synthetic);
+ await vectorIndexAddGuarded(
+ synthetic.id,
+ synthetic.sessionId,
+ synthetic.title + " " + (synthetic.narrative || ""),
+ { kind: "synthetic", logId: synthetic.id },
+ );
await sdk.trigger({
function_id: "stream::set",
payload: {
diff --git a/src/functions/remember.ts b/src/functions/remember.ts
index ba9d5409..8d667705 100644
--- a/src/functions/remember.ts
+++ b/src/functions/remember.ts
@@ -6,7 +6,7 @@ import { withKeyedLock } from "../state/keyed-mutex.js";
import { memoryToObservation } from "../state/memory-utils.js";
import { deleteAccessLog } from "./access-tracker.js";
import { recordAudit } from "./audit.js";
-import { getSearchIndex } from "./search.js";
+import { getSearchIndex, vectorIndexAddGuarded } from "./search.js";
import { logger } from "../logger.js";
export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
@@ -112,6 +112,12 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
error: err instanceof Error ? err.message : String(err),
});
}
+ await vectorIndexAddGuarded(
+ memory.id,
+ memory.sessionIds[0] ?? "memory",
+ memory.title + " " + memory.content,
+ { kind: "memory", logId: memory.id },
+ );
if (supersededId) {
await sdk.trigger({
diff --git a/src/functions/search.ts b/src/functions/search.ts
index edfa522b..74af9ff1 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -3,21 +3,99 @@ import type { CompactSearchResult, CompressedObservation, Memory, SearchResult,
import { KV } from '../state/schema.js'
import { StateKV } from '../state/kv.js'
import { SearchIndex } from '../state/search-index.js'
+import { VectorIndex } from '../state/vector-index.js'
+import type { EmbeddingProvider } from '../types.js'
import { memoryToObservation } from '../state/memory-utils.js'
import { recordAccessBatch } from './access-tracker.js'
import { logger } from "../logger.js";
let index: SearchIndex | null = null
+let vectorIndex: VectorIndex | null = null
+let currentEmbeddingProvider: EmbeddingProvider | null = null
export function getSearchIndex(): SearchIndex {
if (!index) index = new SearchIndex()
return index
}
+export function setVectorIndex(idx: VectorIndex | null): void {
+ vectorIndex = idx
+}
+
+export function getVectorIndex(): VectorIndex | null {
+ return vectorIndex
+}
+
+export function setEmbeddingProvider(provider: EmbeddingProvider | null): void {
+ currentEmbeddingProvider = provider
+}
+
+export function getEmbeddingProvider(): EmbeddingProvider | null {
+ return currentEmbeddingProvider
+}
+
+// Hard cap on embedding input length. Most providers cap input around
+// 8k tokens (~32k chars at ~4 chars/token). Truncate defensively so a
+// huge memory.content can't 400 the embed call or blow context budget
+// on a single doc. 16k chars ≈ 4k tokens, safely under every provider.
+const EMBED_MAX_CHARS = 16_000
+
+export function clipEmbedInput(text: string): string {
+ if (text.length <= EMBED_MAX_CHARS) return text
+ return text.slice(0, EMBED_MAX_CHARS)
+}
+
+// Single guarded vector-index write. Returns true on success. Logs and
+// no-ops on:
+// - dimension mismatch (mis-configured provider would silently corrupt
+// the index per #248 otherwise — guarded at persistence load there;
+// this is the symmetric guard at the write site)
+// - embed throwing (network, rate limit, provider down)
+// Always soft-fails so a downed embedder doesn't break the upstream save.
+export async function vectorIndexAddGuarded(
+ id: string,
+ sessionId: string,
+ text: string,
+ context: { kind: "memory" | "observation" | "synthetic"; logId: string },
+): Promise<boolean> {
+ const vi = vectorIndex
+ const ep = currentEmbeddingProvider
+ if (!vi || !ep) return false
+ try {
+ const embedding = await ep.embed(clipEmbedInput(text))
+ if (embedding.length !== ep.dimensions) {
+ logger.warn("vector-index add: dimension mismatch — skipping", {
+ kind: context.kind,
+ id: context.logId,
+ provider: ep.name,
+ expected: ep.dimensions,
+ received: embedding.length,
+ })
+ return false
+ }
+ vi.add(id, sessionId, embedding)
+ return true
+ } catch (err) {
+ logger.warn("vector-index add: embed failed — skipping", {
+ kind: context.kind,
+ id: context.logId,
+ provider: ep.name,
+ error: err instanceof Error ? err.message : String(err),
+ })
+ return false
+ }
+}
+
export async function rebuildIndex(kv: StateKV): Promise<number> {
const idx = getSearchIndex()
idx.clear()
+ // BM25 clear above wipes stale doc entries; the vector index has the
+ // symmetric concern — memories/observations deleted between runs
+ // would leave orphan embeddings here forever. Clear both before the
+ // repopulation loops run, so BM25 and vector stay in sync.
+ vectorIndex?.clear()
+
let count = 0
// Memories live in their own KV scope outside per-session observation
@@ -30,6 +108,12 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
if (memory.isLatest === false) continue
if (!memory.title || !memory.content) continue
idx.add(memoryToObservation(memory))
+ await vectorIndexAddGuarded(
+ memory.id,
+ memory.sessionIds[0] ?? 'memory',
+ memory.title + ' ' + memory.content,
+ { kind: "memory", logId: memory.id },
+ )
count++
}
} catch (err) {
@@ -64,6 +148,12 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
for (const obs of observations) {
if (obs.title && obs.narrative) {
idx.add(obs)
+ await vectorIndexAddGuarded(
+ obs.id,
+ obs.sessionId,
+ obs.title + ' ' + obs.narrative,
+ { kind: "observation", logId: obs.id },
+ )
count++
}
}
diff --git a/src/index.ts b/src/index.ts
index 7a746620..fba3bd6f 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -34,6 +34,8 @@ import {
registerSearchFunction,
rebuildIndex,
getSearchIndex,
+ setVectorIndex,
+ setEmbeddingProvider,
} from "./functions/search.js";
import { registerContextFunction } from "./functions/context.js";
import { registerSummarizeFunction } from "./functions/summarize.js";
@@ -176,6 +178,9 @@ async function main() {
const vectorIndex = embeddingProvider ? new VectorIndex() : null;
+ setVectorIndex(vectorIndex);
+ setEmbeddingProvider(embeddingProvider);
+
const meterAccessor = hasGetMeter(sdk)
? (sdk.getMeter.bind(sdk) as (name: string) => unknown)
: undefined;
diff --git a/src/state/search-index.ts b/src/state/search-index.ts
index f534791d..d9892346 100644
--- a/src/state/search-index.ts
+++ b/src/state/search-index.ts
@@ -223,7 +223,7 @@ export class SearchIndex {
private tokenize(text: string): string[] {
return text
- .replace(/[^\w\s/.\-_]/g, " ")
+ .replace(/[^\p{L}\p{N}\s/.\\-_]/gu, " ")
.split(/\s+/)
.filter((t) => t.length > 1)
.map((t) => stem(t));
diff --git a/test/multimodal.test.ts b/test/multimodal.test.ts
index ffadd120..4015435f 100644
--- a/test/multimodal.test.ts
+++ b/test/multimodal.test.ts
@@ -15,6 +15,7 @@ vi.mock("../src/functions/search.js", () => ({
getSearchIndex: () => ({
add: vi.fn(),
}),
+ vectorIndexAddGuarded: vi.fn().mockResolvedValue(false),
}));
const mockTriggerVoid = vi.fn();
diff --git a/test/search-index.test.ts b/test/search-index.test.ts
index 0a99681b..a6354563 100644
--- a/test/search-index.test.ts
+++ b/test/search-index.test.ts
@@ -125,4 +125,32 @@ describe("SearchIndex", () => {
expect(results[0].obsId).toBe("obs_both");
expect(results[0].score).toBeGreaterThan(results[1].score);
});
+
+ it("indexes and finds non-ASCII (Greek) text", () => {
+ index.add(
+ makeObs({
+ id: "obs_greek",
+ title: "Προβολή μνήμης",
+ narrative: "Δοκιμάζουμε αναζήτηση σε ελληνικά",
+ concepts: ["δοκιμή", "μνήμη"],
+ }),
+ );
+ const results = index.search("μνήμη");
+ expect(results.length).toBe(1);
+ expect(results[0].obsId).toBe("obs_greek");
+ });
+
+ it("tokenizes mixed ASCII and non-ASCII (Greek) queries", () => {
+ index.add(
+ makeObs({
+ id: "obs_mixed",
+ title: "JWT middleware ρύθμιση",
+ narrative: "Configured JWT with ελληνικά σχόλια",
+ concepts: ["auth", "jwt", "ρύθμιση"],
+ }),
+ );
+ const results = index.search("JWT ρύθμιση");
+ expect(results.length).toBe(1);
+ expect(results[0].obsId).toBe("obs_mixed");
+ });
});
diff --git a/test/search.test.ts b/test/search.test.ts
index a7b1d954..7bc635ce 100644
--- a/test/search.test.ts
+++ b/test/search.test.ts
@@ -4,7 +4,8 @@ vi.mock("../src/logger.js", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
-import { registerSearchFunction, getSearchIndex, rebuildIndex } from "../src/functions/search.js";
+import { registerSearchFunction, getSearchIndex, rebuildIndex, setVectorIndex, setEmbeddingProvider, getVectorIndex } from "../src/functions/search.js";
+import { VectorIndex } from "../src/state/vector-index.js";
import { KV } from "../src/state/schema.js";
import type { CompressedObservation, Session } from "../src/types.js";
@@ -182,4 +183,26 @@ describe("mem::search", () => {
expect(hit).toBeDefined();
expect(hit?.title).toBe("Pineapple belongs on pizza");
});
+
+ it("rebuildIndex populates the vector index", async () => {
+ const mockEmbedder = {
+ name: "test",
+ dimensions: 3,
+ embed: async (_text: string) => new Float32Array([0.1, 0.2, 0.3]),
+ embedBatch: async (_texts: string[]) =>
+ _texts.map(() => new Float32Array([0.1, 0.2, 0.3])),
+ };
+ setEmbeddingProvider(mockEmbedder);
+ setVectorIndex(new VectorIndex());
+
+ await rebuildIndex(kv as never);
+
+ const vi = getVectorIndex();
+ expect(vi).not.toBeNull();
+ expect(vi!.size).toBeGreaterThan(0);
+
+ // Cleanup
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ });
});
diff --git a/test/vector-index-dimensions.test.ts b/test/vector-index-dimensions.test.ts
index 380ef3f7..140d7f8a 100644
--- a/test/vector-index-dimensions.test.ts
+++ b/test/vector-index-dimensions.test.ts
@@ -1,5 +1,12 @@
-import { describe, it, expect } from "vitest";
+import { describe, it, expect, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
import { VectorIndex } from "../src/state/vector-index.js";
+import { migrateVectorIndex } from "../src/functions/migrate-vector-index.js";
+import type { EmbeddingProvider } from "../src/types.js";
describe("VectorIndex.validateDimensions", () => {
it("reports no mismatches and an empty dim set on an empty index", () => {
@@ -40,3 +47,86 @@ describe("VectorIndex.validateDimensions", () => {
expect(Array.from(result.seenDimensions)).toEqual([384]);
});
});
+
+describe("migrateVectorIndex", () => {
+ const newProvider: EmbeddingProvider = {
+ name: "test-4d",
+ dimensions: 4,
+ embed: async (_text: string) => new Float32Array([0.1, 0.2, 0.3, 0.4]),
+ embedBatch: async (_texts: string[]) =>
+ _texts.map(() => new Float32Array([0.1, 0.2, 0.3, 0.4])),
+ };
+
+ function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> =>
+ (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (_scope: string, _key: string): Promise<void> => {},
+ list: async <T>(scope: string): Promise<T[]> => {
+ const entries = store.get(scope);
+ return entries ? (Array.from(entries.values()) as T[]) : [];
+ },
+ };
+ }
+
+ it("re-embeds observations with new provider dimensions", async () => {
+ const kv = mockKV();
+ await kv.set("mem:sessions", "ses_1", { id: "ses_1" });
+ await kv.set("mem:obs:ses_1", "obs_1", {
+ id: "obs_1",
+ sessionId: "ses_1",
+ timestamp: new Date().toISOString(),
+ type: "decision",
+ title: "migration test",
+ facts: ["test"],
+ narrative: "Testing migration",
+ concepts: ["test"],
+ files: [],
+ importance: 5,
+ });
+
+ const result = await migrateVectorIndex(kv as never, newProvider);
+ expect(result.success).toBe(true);
+ expect(result.totalProcessed).toBe(1);
+ expect(result.vectorSize).toBe(1);
+ expect(result.failed).toBe(0);
+ });
+
+ it("re-embeds memories with new provider dimensions", async () => {
+ const kv = mockKV();
+ await kv.set("mem:memories", "mem_1", {
+ id: "mem_1",
+ createdAt: new Date().toISOString(),
+ updatedAt: new Date().toISOString(),
+ type: "fact",
+ title: "migration memory",
+ content: "This memory will be re-embedded",
+ concepts: ["test"],
+ files: [],
+ sessionIds: ["ses_1"],
+ strength: 7,
+ version: 1,
+ isLatest: true,
+ });
+
+ const result = await migrateVectorIndex(kv as never, newProvider);
+ expect(result.totalProcessed).toBe(1);
+ expect(result.vectorSize).toBe(1);
+ });
+
+ it("handles empty KV gracefully", async () => {
+ const kv = mockKV();
+
+ const result = await migrateVectorIndex(kv as never, newProvider);
+ expect(result.success).toBe(true);
+ expect(result.totalProcessed).toBe(0);
+ expect(result.vectorSize).toBe(0);
+ expect(result.failed).toBe(0);
+ });
+});
diff --git a/test/vector-index-populate.test.ts b/test/vector-index-populate.test.ts
new file mode 100644
index 00000000..936232a7
--- /dev/null
+++ b/test/vector-index-populate.test.ts
@@ -0,0 +1,121 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+vi.mock("../src/state/keyed-mutex.js", () => ({
+ withKeyedLock: <T>(_key: string, fn: () => Promise<T>) => fn(),
+}));
+
+import { registerRememberFunction } from "../src/functions/remember.js";
+import { setVectorIndex, setEmbeddingProvider, getVectorIndex } from "../src/functions/search.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import type { EmbeddingProvider } from "../src/types.js";
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> =>
+ (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ const entries = store.get(scope);
+ return entries ? (Array.from(entries.values()) as T[]) : [];
+ },
+ };
+}
+
+function mockSdk() {
+ const functions = new Map<string, Function>();
+ return {
+ registerFunction: (id: string, handler: Function) => {
+ functions.set(id, handler);
+ },
+ registerTrigger: () => {},
+ trigger: async (input: { function_id: string; payload: unknown }) => {
+ const fn = functions.get(input.function_id);
+ if (!fn) throw new Error(`unknown fn ${input.function_id}`);
+ return fn(input.payload);
+ },
+ };
+}
+
+describe("vector index population on remember", () => {
+ const mockEmbedder: EmbeddingProvider = {
+ name: "test",
+ dimensions: 3,
+ embed: async (_text: string) => new Float32Array([0.1, 0.2, 0.3]),
+ embedBatch: async (_texts: string[]) =>
+ _texts.map(() => new Float32Array([0.1, 0.2, 0.3])),
+ };
+
+ let vectorIndex: VectorIndex;
+
+ beforeEach(() => {
+ vectorIndex = new VectorIndex();
+ setVectorIndex(vectorIndex);
+ setEmbeddingProvider(mockEmbedder);
+ });
+
+ afterEach(() => {
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ });
+
+ it("calls vectorIndex.add() when remember saves a memory", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const result = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content: "Test memory for vector indexing", type: "fact" },
+ });
+
+ expect((result as { success: boolean }).success).toBe(true);
+ expect(vectorIndex.size).toBe(1);
+ });
+
+ it("calls vectorIndex.add() with short content (0% similarity dedup)", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content: "First unique memory", type: "fact" },
+ });
+ await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content: "Second completely different memory", type: "fact" },
+ });
+
+ expect(vectorIndex.size).toBe(2);
+ });
+
+ it("handles missing embedder gracefully (vectorIndex stays null)", async () => {
+ // Override beforeEach setup: this case wants null state explicitly.
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+
+ const result = await sdk.trigger({
+ function_id: "mem::remember",
+ payload: { content: "This should work without vector index", type: "fact" },
+ });
+
+ expect((result as { success: boolean }).success).toBe(true);
+ expect(getVectorIndex()).toBeNull();
+ });
+});