#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
perf(index): store vectors in fixed buckets and rebuild BM25 from content (#1416)
Rohit Ghumarebcf4f0d00d1c
diff --git a/.env.example b/.env.exampleindex 4b9deb0b..60d201e5 100644--- a/.env.example+++ b/.env.example@@ -130,6 +130,9 @@# AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging.# AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental).# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately.+# AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed.+# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save.+# AGENTMEMORY_VECTOR_BACKFILL=all # Required to backfill an entire store's vectors at once (no prior vector save, or a stale legacy manifest); unset means that case waits for an explicit opt-in.# -----------------------------------------------------------------------------# 6. CLI / runtime knobsdiff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.mdindex 5fe07156..44e300d1 100644--- a/plugin/skills/agentmemory-config/REFERENCE.md+++ b/plugin/skills/agentmemory-config/REFERENCE.md@@ -3,7 +3,7 @@Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->-Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables:+Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 47 recognized variables:- `AGENTMEMORY_AGENT_SCOPE`- `AGENTMEMORY_ALLOW_AGENT_SDK`@@ -46,6 +46,9 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e- `AGENTMEMORY_TOOLS`- `AGENTMEMORY_URL`- `AGENTMEMORY_USE_DOCKER`+- `AGENTMEMORY_VECTOR_BACKFILL`+- `AGENTMEMORY_VECTOR_BACKFILL_MAX`+- `AGENTMEMORY_VECTOR_BUCKET_SIZE`- `AGENTMEMORY_VERBOSE`- `AGENTMEMORY_VIEWER_HOST`- `AGENTMEMORY_VIEWER_URL`diff --git a/src/config.ts b/src/config.tsindex ac82bd44..a7986f96 100644--- a/src/config.ts+++ b/src/config.ts@@ -477,6 +477,24 @@ export function getIndexSaveIntervalMs(): number {return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS;}+export const VECTOR_BUCKET_SIZE_DEFAULT = 500;++export function getVectorBucketSize(): number {+ const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKET_SIZE"], VECTOR_BUCKET_SIZE_DEFAULT);+ return raw > 0 ? raw : VECTOR_BUCKET_SIZE_DEFAULT;+}++export const VECTOR_BACKFILL_MAX_DEFAULT = 500;++export function getVectorBackfillMax(): number {+ const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL_MAX"], VECTOR_BACKFILL_MAX_DEFAULT);+ return raw > 0 ? raw : VECTOR_BACKFILL_MAX_DEFAULT;+}++export function isVectorBackfillAllEnabled(): boolean {+ return getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL"] === "all";+}+export function isStandaloneMcp(): boolean {return getMergedEnv()["STANDALONE_MCP"] === "true";}diff --git a/src/functions/search.ts b/src/functions/search.tsindex 8abce07f..2d2f4147 100644--- a/src/functions/search.ts+++ b/src/functions/search.ts@@ -9,7 +9,12 @@ import type { EmbeddingProvider } from '../types.js'import { memoryToObservation } from '../state/memory-utils.js'import { recordAccessBatch } from './access-tracker.js'import { logger } from "../logger.js";-import { getAgentId, isAgentScopeIsolated } from "../config.js";+import {+ getAgentId,+ getVectorBackfillMax,+ isAgentScopeIsolated,+ isVectorBackfillAllEnabled,+} from "../config.js";let index: SearchIndex | null = nulllet vectorIndex: VectorIndex | null = null@@ -45,6 +50,19 @@ export function isMemoryIndexReady(): boolean {return memoryIndexReady}+let bm25RebuildIncomplete = false+export function isBm25RebuildIncomplete(): boolean {+ return bm25RebuildIncomplete+}++let pendingVectorBackfill = 0+export function getPendingVectorBackfillCount(): number {+ return pendingVectorBackfill+}+export function setPendingVectorBackfillCount(n: number): void {+ pendingVectorBackfill = Math.max(0, n)+}+export function getSearchIndex(): SearchIndex {if (!index) index = new SearchIndex()return index@@ -399,6 +417,146 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {return indexed}+export type VectorBackfillJob = {+ id: string+ sessionId: string+ text: string+ context: { kind: "memory" | "observation" | "synthetic"; logId: string }+}++const SESSIONS_LIST_RETRY_ATTEMPTS = 3+const SESSIONS_LIST_RETRY_BASE_MS = 250++function delay(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms))+}++async function listSessionsWithRetry(kv: StateKV): Promise<Session[] | null> {+ let lastErr: unknown+ for (let attempt = 0; attempt < SESSIONS_LIST_RETRY_ATTEMPTS; attempt++) {+ try {+ return await kv.list<Session>(KV.sessions)+ } catch (err) {+ lastErr = err+ if (attempt < SESSIONS_LIST_RETRY_ATTEMPTS - 1) {+ await delay(SESSIONS_LIST_RETRY_BASE_MS * (attempt + 1))+ }+ }+ }+ logger.warn('rebuildKeywordIndex: failed to load sessions after retries', {+ attempts: SESSIONS_LIST_RETRY_ATTEMPTS,+ error: lastErr instanceof Error ? lastErr.message : String(lastErr),+ })+ return null+}++export async function rebuildKeywordIndex(+ kv: StateKV,+ vectorBackfillSince?: string | null,+): Promise<{ documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number }> {+ const idx = getSearchIndex()+ idx.clear()+ memoryIndexReady = false+ bm25RebuildIncomplete = false+ const vi = vectorIndex+ const backfillEligible = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined+ const wholeStoreBackfill = vectorBackfillSince === null+ const backfillGated = backfillEligible && wholeStoreBackfill && !isVectorBackfillAllEnabled()+ const backfillActive = backfillEligible && !backfillGated+ const cutoff = typeof vectorBackfillSince === 'string' ? Date.parse(vectorBackfillSince) : Number.NaN+ const backfillCap = backfillActive ? getVectorBackfillMax() : 0+ const vectorJobs: VectorBackfillJob[] = []+ let fullBackfillPending = 0+ const consider = (+ id: string,+ sessionId: string,+ text: string,+ timestamp: string | undefined,+ kind: "memory" | "observation",+ ): void => {+ if (!backfillEligible || vi?.has(id)) return+ if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return+ if (backfillGated) {+ fullBackfillPending+++ return+ }+ if (!backfillActive || vectorJobs.length >= backfillCap) return+ vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } })+ }++ let documents = 0+ let memoriesLoaded = false+ try {+ const memories = await kv.list<Memory>(KV.memories)+ memoriesLoaded = true+ for (const memory of memories) {+ if (memory.isLatest === false) continue+ if (!memory.title || !memory.content) continue+ idx.add(memoryToObservation(memory))+ consider(memory.id, memory.sessionIds?.[0] ?? 'memory', memory.title + ' ' + memory.content, memory.createdAt, "memory")+ documents+++ }+ } catch (err) {+ logger.warn('rebuildKeywordIndex: failed to load memories', {+ error: err instanceof Error ? err.message : String(err),+ })+ }++ const sessions = await listSessionsWithRetry(kv)+ if (sessions === null) {+ bm25RebuildIncomplete = true+ if (memoriesLoaded) memoryIndexReady = true+ return { documents, vectorJobs, fullBackfillPending }+ }+ const failedSessions: string[] = []+ for (let batch = 0; batch < sessions.length; batch += 10) {+ const chunk = sessions.slice(batch, batch + 10)+ const results = await Promise.all(+ chunk.map(async (s) => {+ try {+ return await kv.list<CompressedObservation>(KV.observations(s.id))+ } catch {+ failedSessions.push(s.id)+ return [] as CompressedObservation[]+ }+ })+ )+ for (const obs of results.flat()) {+ if (!obs.title || !obs.narrative) continue+ idx.add(obs)+ consider(obs.id, obs.sessionId, obs.title + ' ' + obs.narrative, obs.timestamp, "observation")+ documents+++ }+ }+ if (failedSessions.length > 0) {+ bm25RebuildIncomplete = true+ logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions })+ }+ if (memoriesLoaded) memoryIndexReady = true+ return { documents, vectorJobs, fullBackfillPending }+}++const BACKFILL_SAVE_EVERY_BATCHES = 10++export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {+ const batchSize = getRebuildEmbedBatchSize()+ let added = 0+ let batchesSinceSave = 0+ setPendingVectorBackfillCount(jobs.length)+ for (let offset = 0; offset < jobs.length; offset += batchSize) {+ const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))+ added += ok+ setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize))+ batchesSinceSave+++ if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) {+ await flushIndexSave()+ batchesSinceSave = 0+ }+ }+ if (added > 0) await flushIndexSave()+ return added+}+export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {sdk.registerFunction('mem::search',@@ -480,10 +638,10 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {// Share one rebuild across concurrent cold-start queries so they// don't each walk the whole corpus and saturate the pool.if (!rebuildPromise) {- rebuildPromise = rebuildIndex(kv)- .then((count) => {- logger.info('Search index rebuilt', { entries: count })- return count+ rebuildPromise = rebuildKeywordIndex(kv)+ .then((result) => {+ logger.info('Search index rebuilt', { entries: result.documents })+ return result.documents}).catch((err) => {logger.warn('Index rebuild failed', {diff --git a/src/functions/status.ts b/src/functions/status.tsindex 0b3ab531..c6577c44 100644--- a/src/functions/status.ts+++ b/src/functions/status.ts@@ -1,4 +1,4 @@-import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js";+import type { IndexLegStatus, IndexPersistenceStatus } from "../state/index-persistence.js";export type StatusLevel = "ok" | "info" | "warn" | "error";@@ -57,6 +57,8 @@ export interface StatusInputs {observationsIndexed: number;missingObservations: number | null;sessions: number | null;+ bm25Incomplete: boolean;+ pendingVectorBackfill: number;};graph: GraphStatsInput | null;graphExtractionEnabled: boolean;@@ -85,7 +87,6 @@ export interface StatusReport {const FAILURE_RATE_THRESHOLD = 0.2;const FAILURE_MIN_CALLS = 5;const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60;-const VECTOR_STRING_WARN_RATIO = 0.8;const LEVEL_RANK: Record<StatusLevel, number> = { ok: 0, info: 1, warn: 2, error: 3 };const FUNCTION_FIXES: Record<string, string> = {@@ -173,42 +174,54 @@ export function evaluateStatus(input: StatusInputs): StatusReport {});}+ if (input.index.bm25Incomplete) {+ problems.push({+ level: "error",+ code: "bm25-rebuild-incomplete",+ message: "The keyword index could not load every session at boot, so some observations are missing from search until the next successful rebuild.",+ fix: "Check the server log for the session listing failure. Restarting or the next cold-start search retries automatically.",+ });+ }++ if (input.index.pendingVectorBackfill > 0) {+ problems.push({+ level: "info",+ code: "index-vector-backfill-pending",+ message: `${input.index.pendingVectorBackfill} documents are waiting for a vector embedding.`,+ fix: "This runs in the background and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. If it is stuck at a nonzero count with no embedding provider errors, set AGENTMEMORY_VECTOR_BACKFILL=all to run a full backfill.",+ });+ }+const persistence = input.indexPersistence ?? null;- if (persistence) {- const legs: Array<[string, IndexLegStatus | null]> = [- ["BM25", persistence.bm25],- ["vector", persistence.vector],- ];- for (const [label, leg] of legs) {- if (!leg) continue;- if (leg.lastError) {- problems.push({- level: "error",- code: "index-save-failing",- message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`,- fix: "Check the server log for the failing state write. The next save retries automatically.",- });- }- const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined);- if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {- problems.push({- level: "warn",- code: "index-save-stale",- message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,- fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",- });- }+ const vectorLeg = persistence?.vector ?? null;+ if (persistence && vectorLeg) {+ if (vectorLeg.lastError) {+ problems.push({+ level: "error",+ code: "index-save-failing",+ message: `The vector index could not be saved: ${vectorLeg.lastError}. Search keeps working from memory, but vectors added since the last save are lost on restart.`,+ fix: "Check the server log for the failing state write. The next save retries automatically.",+ });}- const vectorChars = persistence.vector?.serializedChars ?? null;- if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) {+ const dirtyAge = secondsBetween(input.now, vectorLeg.dirtySince ?? undefined);+ if (!vectorLeg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {problems.push({level: "warn",- code: "vector-index-near-string-limit",- message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`,- fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.",+ code: "index-save-stale",+ message: `The vector index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,+ fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",});}}+ if (persistence?.vectorCountShortfall) {+ const { expected, loaded } = persistence.vectorCountShortfall;+ problems.push({+ level: "warn",+ code: "index-vector-count-shortfall",+ message: `Only ${loaded} of ${expected} vectors loaded from the last save; a bounded backfill re-embeds the rest.`,+ fix: "No action needed — missing vectors are queued for backfill up to the per-boot cap.",+ });+ }let graph: StatusReport["graph"] = null;if (input.graph) {@@ -300,16 +313,13 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string {function indexPersistenceRows(report: StatusReport): string {const persistence = report.indexPersistence;if (!persistence) return "";- let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25)));+ let rows = row("BM25 index", "rebuilt from stored content at boot");if (persistence.vector) {rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector)));- const chars = persistence.vector.serializedChars;- if (chars !== null) {- rows += row(- "Vector index size",- escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`),- );- }+ rows += row(+ "Vector storage",+ escapeHtml(`${persistence.buckets} buckets, ${persistence.pendingChanges} unsaved changes`),+ );}return rows;}@@ -396,6 +406,8 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off")${row("Observations indexed", escapeHtml(idx.observationsIndexed))}${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))}${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}+${row("BM25 rebuild", idx.bm25Incomplete ? '<span class="warn">incomplete</span>' : "complete")}+${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))}${indexPersistenceRows(report)}</table><h2>Knowledge graph</h2><table>diff --git a/src/index.ts b/src/index.tsindex ff465afe..c043a31f 100644--- a/src/index.ts+++ b/src/index.ts@@ -21,7 +21,6 @@ import {createImageEmbeddingProvider,} from "./providers/index.js";import { StateKV } from "./state/kv.js";-import { KV } from "./state/schema.js";import { VectorIndex } from "./state/vector-index.js";import { HybridSearch } from "./state/hybrid-search.js";import { IndexPersistence } from "./state/index-persistence.js";@@ -35,13 +34,14 @@ import { registerDiskSizeManager } from "./functions/disk-size-manager.js";import { registerCompressFunction } from "./functions/compress.js";import {registerSearchFunction,- rebuildIndex,- reconcileIndex,+ backfillVectors,+ rebuildKeywordIndex,getSearchIndex,setVectorIndex,setEmbeddingProvider,setIndexPersistence,setHybridRanker,+ setPendingVectorBackfillCount,} from "./functions/search.js";import { registerContextFunction } from "./functions/context.js";import { registerSummarizeFunction } from "./functions/summarize.js";@@ -395,23 +395,13 @@ async function main() {const healthMonitor = registerHealthMonitor(sdk, kv);- const indexPersistence = new IndexPersistence(kv, bm25Index, vectorIndex);- // Wire the persistence hook so delete paths can flush BM25/vector- // index mutations to disk. Without this, an in-memory remove can be- // lost across a hard process exit and the persisted snapshot- // restores the deleted entry at next boot.+ const indexPersistence = new IndexPersistence(kv, vectorIndex);setIndexPersistence(indexPersistence);const loaded = await indexPersistence.load().catch((err) => {- console.warn(`[agentmemory] Failed to load persisted index:`, err);+ console.warn(`[agentmemory] Failed to load persisted vector index:`, err);return null;});- if (loaded?.bm25 && loaded.bm25.size > 0) {- bm25Index.restoreFrom(loaded.bm25);- bootLog(- `Loaded persisted BM25 index (${bm25Index.size} docs)`,- );- }if (loaded?.vector && vectorIndex && loaded.vector.size > 0) {// Persisted vectors carry whatever dimension the provider had when// they were written. If the active provider declares a different@@ -436,6 +426,7 @@ async function main() {const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", ");const dropStale = isDropStaleIndexEnabled();if (dropStale) {+ for (const [obsId] of loaded.vector.entries()) vectorIndex.markRemoved(obsId);console.warn(`[agentmemory] Persisted vector index has ${mismatches.length} of ` +`${loaded.vector.size} vectors with the wrong dimension. Active ` +@@ -466,79 +457,42 @@ async function main() {}}- const needsRebuild = bm25Index.size === 0;-- if (needsRebuild) {- // Fire-and-forget. rebuildIndex iterates every observation across- // every session and AWAITS an embedding-provider call per record.- // On a large corpus + rate-limited embedding endpoint that can- // take HOURS; awaiting it here blocks every subsequent boot step- // (including startViewerServer below, leaving the viewer port- // unbound for the duration). The index lazily fills in over time- // and search degrades gracefully — partial coverage > no viewer- // for hours. Errors still surface via the inner .catch.- void rebuildIndex(kv)- .then((indexCount) => {- if (indexCount > 0) {- bootLog(`Search index rebuilt: ${indexCount} entries`);- indexPersistence.scheduleSave();- }- })- .catch((err) => {- console.warn(`[agentmemory] Failed to rebuild search index:`, err);- });- } else {- // Backfill memories into BM25 for users upgrading from <0.9.5: prior- // versions of mem::remember never indexed memories, so the persisted- // BM25 covers observations only and `memory_smart_search` returns- // empty for everything saved via memory_save (#257). Walk KV.memories- // and add the ones missing from the restored index. Idempotent on- // re-runs because SearchIndex.has() short-circuits already-indexed- // ids.- try {- const memories = await kv.list<import("./types.js").Memory>(KV.memories);- let backfilled = 0;- for (const memory of memories) {- if (memory.isLatest === false) continue;- if (!memory.title || !memory.content) continue;- if (bm25Index.has(memory.id)) continue;- bm25Index.add({- id: memory.id,- sessionId: memory.sessionIds?.[0] ?? "memory",- timestamp: memory.createdAt,- type: "decision",- title: memory.title,- facts: [memory.content],- narrative: memory.content,- concepts: memory.concepts,- files: memory.files,- importance: memory.strength,- });- backfilled++;- }- if (backfilled > 0) {- bootLog(- `Backfilled ${backfilled} memories into BM25 (legacy index gap)`,- );- indexPersistence.scheduleSave();- }- } catch (err) {- console.warn(- `[agentmemory] Failed to backfill memories into BM25:`,- err,+ const vectorCountShortfall =+ Boolean(loaded?.vector) &&+ loaded?.expectedCount !== undefined &&+ loaded.vector!.size < loaded.expectedCount;+ const vectorBackfillSince =+ !loaded || loaded.state === "unavailable"+ ? undefined+ : loaded.state === "none" || vectorCountShortfall+ ? null+ : loaded.savedAt;+ const keywordStart = Date.now();+ try {+ const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince);+ bootLog(+ `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`,+ );+ setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending);+ if (keyword.fullBackfillPending > 0) {+ bootLog(+ `Vector backfill needs ${keyword.fullBackfillPending} embeddings but a full backfill was not started ` ++ `(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`,);}- void reconcileIndex(kv)- .then((count) => {- if (count > 0) {- bootLog(- `Search index reconciled: ${count} observations missing from the persisted snapshot were re-indexed`,- );- }- })- .catch((err) => {- console.warn(`[agentmemory] Failed to reconcile search index:`, err);- });+ if (keyword.vectorJobs.length > 0) {+ bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`);+ void backfillVectors(keyword.vectorJobs)+ .then((count) => {+ setPendingVectorBackfillCount(keyword.fullBackfillPending);+ if (count > 0) bootLog(`Vector index backfilled: ${count} entries`);+ })+ .catch((err) => {+ console.warn(`[agentmemory] Failed to backfill vectors:`, err);+ });+ }+ } catch (err) {+ console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err);}// Ready / Endpoints lines are emitted via `bootLog` so they'rediff --git a/src/state/index-persistence.ts b/src/state/index-persistence.tsindex 28a24d5a..d046d0dd 100644--- a/src/state/index-persistence.ts+++ b/src/state/index-persistence.ts@@ -1,30 +1,21 @@-import { SearchIndex } from "./search-index.js";-import { VectorIndex } from "./vector-index.js";+import { VectorIndex, base64ToFloat32, float32ToBase64 } from "./vector-index.js";import type { StateKV } from "./kv.js";-import { KV, generateId } from "./schema.js";+import { KV } from "./schema.js";import { logger } from "../logger.js";import { safeAudit } from "../functions/audit.js";-import { getIndexSaveIntervalMs } from "../config.js";+import { getIndexSaveIntervalMs, getVectorBucketSize } from "../config.js";-export const V8_MAX_STRING_CHARS = 536_870_888;const FAILURE_LOG_THROTTLE_MS = 60_000;const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence";-const BM25_KEY = "data";-const BM25_MANIFEST_KEY = "data:manifest";-const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`;-const VECTOR_KEY = "vectors";-const VECTOR_MANIFEST_KEY = "vectors:manifest";-const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`;-const INDEX_SHARD_KEY = "data";-const DEFAULT_INDEX_SHARD_CHARS = 2_000_000;--// mem:audit exists to record structural deletions of user data — that is-// the policy stated at the top of src/functions/audit.ts. Index shard-// writes and manifest publishes remove no user rows, so they fall outside-// it, yet a single save() emits three of them: on a real store they-// reached 59876 of 84028 entries (71%), which is what makes the audit log-// slow to query and bloats startup. Off by default; set-// AGENTMEMORY_AUDIT_INDEX_PERSIST=1 when debugging index persistence.+const LEGACY_BM25_KEY = "data";+const LEGACY_BM25_MANIFEST_KEY = "data:manifest";+const LEGACY_VECTOR_KEY = "vectors";+const LEGACY_VECTOR_MANIFEST_KEY = "vectors:manifest";+const VECTOR_META_KEY = "vectors:meta";+const VECTOR_BUCKET_SCOPE_PREFIX = `${KV.bm25Index}:vec:`;+const WRITE_CONCURRENCY = 32;+const LOAD_CONCURRENCY = 8;+function auditIndexPersistEnabled(): boolean {const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;if (!raw) return false;@@ -39,45 +30,57 @@ type IndexShardManifest = {chars: number;};+type VectorMeta = {+ v: 3;+ bucketCount: number;+ savedAt: string;+ count: number;+};++type PersistedVector = {+ id: string;+ s: string;+ e: string;+};+type IndexPersistenceOptions = {- shardChars?: number;- createGeneration?: () => string;saveIntervalMs?: number;+ bucketSize?: number;now?: () => number;};-export type IndexLeg = "bm25" | "vector";-export interface IndexLegStatus {lastSavedAt: string | null;lastError: string | null;lastErrorAt: string | null;dirtySince: string | null;- serializedChars: number | null;+}++export interface VectorCountShortfall {+ expected: number;+ loaded: number;}export interface IndexPersistenceStatus {saveIntervalMs: number;saving: boolean;- bm25: IndexLegStatus;+ buckets: number;+ pendingChanges: number;vector: IndexLegStatus | null;+ vectorCountShortfall: VectorCountShortfall | null;}-function emptyLegStatus(): IndexLegStatus {- return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null };-}+export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable";-function shardChars(options: IndexPersistenceOptions): number {- const configured = options.shardChars;- if (typeof configured !== "number" || !Number.isFinite(configured)) {- return DEFAULT_INDEX_SHARD_CHARS;- }- const wholeChars = Math.floor(configured);- return wholeChars >= 1 ? wholeChars : DEFAULT_INDEX_SHARD_CHARS;+export interface VectorLoadResult {+ vector: VectorIndex | null;+ state: VectorLoadState;+ savedAt: string | null;+ expectedCount?: number;}-function createIndexGeneration(): string {- return generateId("idx");+export function vectorBucketScope(bucket: number): string {+ return `${VECTOR_BUCKET_SCOPE_PREFIX}${String(bucket).padStart(4, "0")}`;}function statePath(scope: string, key: string): string {@@ -88,6 +91,13 @@ function errorMessage(err: unknown): string {return err instanceof Error ? err.message : String(err);}+function generationTime(generation: string | undefined): string | null {+ const stamp = generation?.split("_")[1];+ if (!stamp) return null;+ const ms = parseInt(stamp, 36);+ return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : null;+}+function isValidShardDescriptor(shard: unknown,): shard is IndexShardManifest["shards"][number] {@@ -99,31 +109,55 @@ function isValidShardDescriptor(typeof candidate.key === "string" &&candidate.key.length > 0 &&Number.isInteger(candidate.chars) &&- candidate.chars >= 0+ (candidate.chars as number) >= 0);}+function isPersistedVector(row: unknown): row is PersistedVector {+ if (!row || typeof row !== "object") return false;+ const candidate = row as Partial<PersistedVector>;+ return typeof candidate.id === "string" && typeof candidate.s === "string" && typeof candidate.e === "string";+}++async function inBatches<T>(items: T[], size: number, run: (item: T) => Promise<void>): Promise<unknown[]> {+ const failures: unknown[] = [];+ for (let offset = 0; offset < items.length; offset += size) {+ const results = await Promise.allSettled(items.slice(offset, offset + size).map(run));+ for (const result of results) {+ if (result.status === "rejected") failures.push(result.reason);+ }+ }+ return failures;+}++function emptyLegStatus(): IndexLegStatus {+ return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null };+}+export class IndexPersistence {private timer: ReturnType<typeof setTimeout> | null = null;- private lastFailureLogAt = new Map<IndexLeg, number>();+ private lastFailureLogAt = 0;private running: Promise<void> | null = null;private queued: Promise<void> | null = null;private stopped = false;private lastSaveAt: number;private dirtyEpoch = 0;private markedDuringRunAt: number | null = null;- private legs: Record<IndexLeg, IndexLegStatus> = {- bm25: emptyLegStatus(),- vector: emptyLegStatus(),- };+ private metaWritten = false;+ private leg: IndexLegStatus = emptyLegStatus();private readonly saveIntervalMs: number;+ private readonly bucketSize: number;private readonly now: () => number;+ private bucketOfId: Map<string, number> = new Map();+ private bucketCounts: Map<number, number> = new Map();+ private highestBucket = 0;+ private hasOpenBucket = false;+ private vectorCountShortfall: VectorCountShortfall | null = null;constructor(private kv: StateKV,- private bm25: SearchIndex,private vector: VectorIndex | null,- private options: IndexPersistenceOptions = {},+ options: IndexPersistenceOptions = {},) {this.now = options.now ?? Date.now;const interval = options.saveIntervalMs;@@ -131,22 +165,25 @@ export class IndexPersistence {typeof interval === "number" && Number.isFinite(interval) && interval > 0? interval: getIndexSaveIntervalMs();+ const bucketSize = options.bucketSize;+ this.bucketSize =+ typeof bucketSize === "number" && Number.isInteger(bucketSize) && bucketSize > 0+ ? bucketSize+ : getVectorBucketSize();this.lastSaveAt = this.now();}scheduleSave(): void {- if (this.stopped) return;+ if (this.stopped || !this.vector) return;const now = this.now();this.dirtyEpoch++;if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now;- for (const leg of this.activeLegs()) {- if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString();- }+ if (this.leg.dirtySince === null) this.leg.dirtySince = new Date(now).toISOString();if (this.timer) return;const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now);this.timer = setTimeout(() => {this.timer = null;- this.save().catch((err) => this.logFailure("bm25", err));+ this.save().catch((err) => this.logFailure(err));}, delay);}@@ -168,34 +205,79 @@ export class IndexPersistence {return {saveIntervalMs: this.saveIntervalMs,saving: this.running !== null,- bm25: { ...this.legs.bm25 },- vector: this.vector ? { ...this.legs.vector } : null,+ buckets: this.bucketCountInUse(),+ pendingChanges: this.vector?.pendingChanges ?? 0,+ vector: this.vector ? { ...this.leg } : null,+ vectorCountShortfall: this.vectorCountShortfall,};}- async load(): Promise<{- bm25: SearchIndex | null;- vector: VectorIndex | null;- }> {- let bm25: SearchIndex | null = null;- let vector: VectorIndex | null = null;+ stop(): void {+ this.stopped = true;+ this.clearTimer();+ }++ async load(): Promise<VectorLoadResult> {+ if (!this.vector) {+ await this.removeLegacyBm25Snapshot();+ return { vector: null, state: "none", savedAt: null };+ }- const bm25Data = await this.loadBm25Data();- if (bm25Data && typeof bm25Data === "string") {- bm25 = SearchIndex.deserialize(bm25Data);+ let meta: VectorMeta | null;+ try {+ meta = await this.kv.get<VectorMeta>(KV.bm25Index, VECTOR_META_KEY);+ } catch (err) {+ logger.warn("index persistence: vector metadata read failed", { message: errorMessage(err) });+ return { vector: null, state: "unavailable", savedAt: null };}- const vecData = await this.loadVectorData();- if (vecData && typeof vecData === "string") {- vector = VectorIndex.deserialize(vecData);+ if (meta && meta.v === 3 && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0) {+ const loaded = await this.loadBuckets(meta.bucketCount);+ if (!loaded) return { vector: null, state: "unavailable", savedAt: null };+ this.metaWritten = true;+ await this.removeLegacyVectorSnapshotIfPresent();+ await this.removeLegacyBm25Snapshot();+ const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined;+ this.vectorCountShortfall =+ expectedCount !== undefined && loaded.size < expectedCount+ ? { expected: expectedCount, loaded: loaded.size }+ : null;+ if (this.vectorCountShortfall) {+ logger.warn("index persistence: loaded fewer vectors than the last save recorded", {+ ...this.vectorCountShortfall,+ });+ }+ return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null, expectedCount };}- return { bm25, vector };+ return this.migrateLegacy();}- stop(): void {- this.stopped = true;- this.clearTimer();+ private bucketCountInUse(): number {+ return this.hasOpenBucket ? this.highestBucket + 1 : 0;+ }++ private assignBucket(id: string): number {+ const existing = this.bucketOfId.get(id);+ if (existing !== undefined) return existing;+ if (!this.hasOpenBucket) {+ this.hasOpenBucket = true;+ this.highestBucket = 0;+ } else if ((this.bucketCounts.get(this.highestBucket) ?? 0) >= this.bucketSize) {+ this.highestBucket++;+ }+ this.bucketOfId.set(id, this.highestBucket);+ this.bucketCounts.set(this.highestBucket, (this.bucketCounts.get(this.highestBucket) ?? 0) + 1);+ return this.highestBucket;+ }++ private releaseBucket(id: string): void {+ const bucket = this.bucketOfId.get(id);+ if (bucket === undefined) return;+ this.bucketOfId.delete(id);+ const remaining = (this.bucketCounts.get(bucket) ?? 1) - 1;+ if (remaining > 0) this.bucketCounts.set(bucket, remaining);+ else this.bucketCounts.delete(bucket);}private clearTimer(): void {@@ -205,10 +287,6 @@ export class IndexPersistence {}}- private activeLegs(): IndexLeg[] {- return this.vector ? ["bm25", "vector"] : ["bm25"];- }-private startRun(): Promise<void> {const run = this.runSave().finally(() => {if (this.running === run) this.running = null;@@ -218,176 +296,199 @@ export class IndexPersistence {}private async runSave(): Promise<void> {- this.markedDuringRunAt = null;- this.lastSaveAt = this.now();- await this.saveLeg("bm25", async () => {- const serialized = this.bm25.serialize();- this.legs.bm25.serializedChars = serialized.length;- await this.saveBm25Index(serialized);- });const vector = this.vector;- if (vector) {- await this.saveLeg("vector", async () => {- const serialized = vector.serialize();- this.legs.vector.serializedChars = serialized.length;- await this.saveVectorIndex(serialized);- });- }- }-- private async saveLeg(leg: IndexLeg, run: () => Promise<void>): Promise<void> {- const status = this.legs[leg];+ if (!vector) return;const epoch = this.dirtyEpoch;+ this.markedDuringRunAt = null;+ this.lastSaveAt = this.now();try {- await run();- status.lastSavedAt = new Date(this.now()).toISOString();- status.lastError = null;- status.lastErrorAt = null;- if (this.dirtyEpoch === epoch) {- status.dirtySince = null;+ await this.writeChanges(vector);+ this.leg.lastSavedAt = new Date(this.now()).toISOString();+ this.leg.lastError = null;+ this.leg.lastErrorAt = null;+ if (this.dirtyEpoch === epoch || vector.pendingChanges === 0) {+ this.leg.dirtySince = null;} else if (this.markedDuringRunAt !== null) {- status.dirtySince = new Date(this.markedDuringRunAt).toISOString();+ this.leg.dirtySince = new Date(this.markedDuringRunAt).toISOString();}} catch (err) {- status.lastError = errorMessage(err);- status.lastErrorAt = new Date(this.now()).toISOString();- if (status.dirtySince === null) status.dirtySince = status.lastErrorAt;- this.logFailure(leg, err);+ this.leg.lastError = errorMessage(err);+ this.leg.lastErrorAt = new Date(this.now()).toISOString();+ if (this.leg.dirtySince === null) this.leg.dirtySince = this.leg.lastErrorAt;+ this.logFailure(err);}}- private logFailure(leg: IndexLeg, err: unknown): void {- const now = this.now();- if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return;- this.lastFailureLogAt.set(leg, now);- const code = (err as { code?: string })?.code;- const message = err instanceof Error ? err.message : String(err);- logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, {- code,- message,- hint:- code === "TIMEOUT"- ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save"- : undefined,+ private async writeChanges(vector: VectorIndex): Promise<void> {+ const changes = vector.takeChanges();+ if (changes.size === 0 && this.metaWritten) return;+ const failed = new Map<string, boolean>();+ const failures = await inBatches([...changes], WRITE_CONCURRENCY, async ([id, present]) => {+ const entry = present ? vector.get(id) : undefined;+ try {+ if (entry) {+ const bucket = this.assignBucket(id);+ await this.kv.set<PersistedVector>(vectorBucketScope(bucket), id, {+ id,+ s: entry.sessionId,+ e: float32ToBase64(entry.embedding),+ });+ } else {+ const bucket = this.bucketOfId.get(id);+ if (bucket !== undefined) {+ await this.kv.delete(vectorBucketScope(bucket), id);+ this.releaseBucket(id);+ }+ }+ } catch (err) {+ failed.set(id, present);+ throw err;+ }});+ if (failures.length > 0) {+ vector.returnChanges(failed);+ throw new Error(+ `${failures.length} of ${changes.size} vector writes failed: ${errorMessage(failures[0])}`,+ );+ }+ await this.kv.set<VectorMeta>(KV.bm25Index, VECTOR_META_KEY, {+ v: 3,+ bucketCount: this.bucketCountInUse(),+ savedAt: new Date(this.now()).toISOString(),+ count: vector.size,+ });+ this.metaWritten = true;}- private async saveBm25Index(serialized: string): Promise<void> {- await this.saveShardedIndex(- serialized,- BM25_MANIFEST_KEY,- BM25_KEY,- BM25_SHARD_SCOPE_PREFIX,- );+ private async loadBuckets(bucketCount: number): Promise<VectorIndex | null> {+ const loaded = new VectorIndex();+ const bucketOfId = new Map<string, number>();+ const bucketCounts = new Map<number, number>();+ const bucketIds = Array.from({ length: bucketCount }, (_, bucket) => bucket);+ const failures = await inBatches(bucketIds, LOAD_CONCURRENCY, async (bucket) => {+ const rows = await this.kv.list<unknown>(vectorBucketScope(bucket));+ let count = 0;+ for (const row of rows) {+ if (!isPersistedVector(row)) continue;+ try {+ loaded.loadPersisted(row.id, row.s, base64ToFloat32(row.e));+ } catch {+ continue;+ }+ bucketOfId.set(row.id, bucket);+ count++;+ }+ if (count > 0) bucketCounts.set(bucket, count);+ });+ if (failures.length > 0) {+ logger.warn("index persistence: vector bucket read failed", {+ failed: failures.length,+ message: errorMessage(failures[0]),+ });+ return null;+ }+ this.bucketOfId = bucketOfId;+ this.bucketCounts = bucketCounts;+ this.highestBucket = bucketCount > 0 ? bucketCount - 1 : 0;+ this.hasOpenBucket = bucketCount > 0;+ return loaded;}- private async saveVectorIndex(serialized: string): Promise<void> {- await this.saveShardedIndex(- serialized,- VECTOR_MANIFEST_KEY,- VECTOR_KEY,- VECTOR_SHARD_SCOPE_PREFIX,+ private async migrateLegacy(): Promise<VectorLoadResult> {+ const manifest = await this.readIndexValue<IndexShardManifest>(+ KV.bm25Index,+ LEGACY_VECTOR_MANIFEST_KEY,+ "vector",+ "manifest",);- }-- private async saveShardedIndex(- serialized: string,- manifestKey: string,- legacyKey: string,- scopePrefix: string,- ): Promise<void> {- const previous = await this.kv- .get<IndexShardManifest>(KV.bm25Index, manifestKey)- .catch(() => null);- const generation =- this.options.createGeneration?.() ?? createIndexGeneration();- const chunkChars = shardChars(this.options);- const shards: IndexShardManifest["shards"] = [];- const chunks: string[] = [];-- for (let offset = 0; offset < serialized.length; offset += chunkChars) {- const shardIndex = shards.length;- const scope = `${scopePrefix}${generation}:${String(shardIndex).padStart(- 5,- "0",- )}`;- const chunk = serialized.slice(offset, offset + chunkChars);- shards.push({ scope, key: INDEX_SHARD_KEY, chars: chunk.length });- chunks.push(chunk);+ const legacyKey = await this.readIndexValue<string>(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy");+ if (!manifest.ok || !legacyKey.ok) return { vector: null, state: "unavailable", savedAt: null };+ const legacyPresent = manifest.value != null || (typeof legacyKey.value === "string" && legacyKey.value.length > 0);+ if (!legacyPresent) {+ await this.removeLegacyBm25Snapshot();+ return { vector: null, state: "none", savedAt: null };}- const writeResults = await Promise.allSettled(- shards.map(async (shard, index) => {- const chunk = chunks[index] ?? "";- await this.kv.set(shard.scope, shard.key, chunk);- await this.auditIndexPersistence("shard_write", [- statePath(shard.scope, shard.key),- ], {- scope: shard.scope,- key: shard.key,- manifestKey,- generation,- chars: chunk.length,- });- }),- );- const failedWrite = writeResults.find(- (result): result is PromiseRejectedResult => result.status === "rejected",- );- if (failedWrite) {- await this.deleteShards(shards, "shard_write_rollback");- throw failedWrite.reason;+ const data = await this.loadShardedData(LEGACY_VECTOR_KEY, LEGACY_VECTOR_MANIFEST_KEY, "vector");+ if (typeof data !== "string") {+ logger.warn("index persistence: legacy vector snapshot is unreadable; keeping it and skipping migration");+ return { vector: null, state: "unavailable", savedAt: null };}+ const legacy = VectorIndex.deserialize(data);+ const savedAt =+ (manifest.value && typeof manifest.value === "object" ? generationTime(manifest.value.generation) : null) ??+ new Date(this.now()).toISOString();- const nextManifest: IndexShardManifest = {- v: 1,- generation,- shards,- chars: serialized.length,- };+ const staging = new VectorIndex();+ for (const [id, entry] of legacy.entries()) staging.add(id, entry.sessionId, entry.embedding);try {- await this.kv.set<IndexShardManifest>(- KV.bm25Index,- manifestKey,- nextManifest,- );- await this.auditIndexPersistence("manifest_publish", [- statePath(KV.bm25Index, manifestKey),- ], {- manifestKey,- generation,- chars: serialized.length,- shards: shards.length,- result: "committed",- });+ await this.writeChanges(staging);} catch (err) {- if (await this.isManifestPublished(manifestKey, nextManifest)) {- await this.auditIndexPersistence("manifest_publish", [- statePath(KV.bm25Index, manifestKey),- ], {- manifestKey,- generation,- chars: serialized.length,- shards: shards.length,- result: "committed_after_error",- error: errorMessage(err),- });- } else {- await this.deleteShards(shards, "manifest_publish_rollback");- }- throw err;+ logger.warn("index persistence: migrating the legacy vector snapshot failed; it stays in place and the next save retries", {+ message: errorMessage(err),+ });+ legacy.markAllChanged();+ return { vector: legacy, state: "unavailable", savedAt };}- await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup");- if (previous?.v === 1 && Array.isArray(previous.shards)) {- const currentShardIds = new Set(- shards.map((shard) => `${shard.scope}\0${shard.key}`),- );- for (const shard of previous.shards) {- if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue;- await this.deleteShards([shard], "previous_generation_cleanup");- }+ await this.removeLegacyVectorSnapshot(manifest.value);+ await this.removeLegacyBm25Snapshot();+ await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], {+ vectors: legacy.size,+ buckets: this.bucketCountInUse(),+ });+ logger.info("index persistence: migrated the vector index to bucketed storage", {+ vectors: legacy.size,+ buckets: this.bucketCountInUse(),+ });+ const migrated = new VectorIndex();+ for (const [id, entry] of legacy.entries()) migrated.loadPersisted(id, entry.sessionId, entry.embedding);+ return { vector: migrated, state: "migrated", savedAt };+ }++ private async removeLegacyVectorSnapshot(manifest: IndexShardManifest | null): Promise<void> {+ if (manifest && Array.isArray(manifest.shards)) {+ await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_vector_cleanup");+ }+ await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "legacy_vector_cleanup");+ await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_KEY, "legacy_vector_cleanup");+ }++ private async removeLegacyVectorSnapshotIfPresent(): Promise<void> {+ const manifest = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "vector", "manifest");+ const legacy = await this.readIndexValue<string>(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy");+ if (!manifest.ok || !legacy.ok) return;+ if (manifest.value == null && legacy.value == null) return;+ await this.removeLegacyVectorSnapshot(manifest.value ?? null);+ }++ private async removeLegacyBm25Snapshot(): Promise<void> {+ const manifestRead = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "bm25", "manifest");+ const legacyRead = await this.readIndexValue<string>(KV.bm25Index, LEGACY_BM25_KEY, "bm25", "legacy");+ if (!manifestRead.ok || !legacyRead.ok) return;+ const manifest = manifestRead.value ?? null;+ const legacy = legacyRead.value ?? null;+ if (manifest == null && legacy == null) return;+ if (manifest && Array.isArray(manifest.shards)) {+ await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_bm25_cleanup");}+ if (manifest != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "legacy_bm25_cleanup");+ if (legacy != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_KEY, "legacy_bm25_cleanup");+ }++ private logFailure(err: unknown): void {+ const now = this.now();+ if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return;+ this.lastFailureLogAt = now;+ const code = (err as { code?: string })?.code;+ logger.warn("index persistence: failed to save the vector index", {+ code,+ message: errorMessage(err),+ hint:+ code === "TIMEOUT"+ ? "iii-engine state::set timed out; unsaved vectors stay in memory and retry on the next save"+ : undefined,+ });}private async auditIndexPersistence(@@ -396,20 +497,13 @@ export class IndexPersistence {details: Record<string, unknown>,): Promise<void> {if (!auditIndexPersistEnabled()) return;- await safeAudit(- this.kv,- "index_persist",- INDEX_PERSISTENCE_FUNCTION_ID,- targetIds,- { action, ...details },- );+ await safeAudit(this.kv, "index_persist", INDEX_PERSISTENCE_FUNCTION_ID, targetIds, {+ action,+ ...details,+ });}- private async deleteKey(- scope: string,- key: string,- reason: string,- ): Promise<void> {+ private async deleteKey(scope: string, key: string, reason: string): Promise<void> {let result = "deleted";let error: string | undefined;try {@@ -427,81 +521,17 @@ export class IndexPersistence {});}- private async deleteShards(- shards: IndexShardManifest["shards"],- reason: string,- ): Promise<void> {- for (const shard of shards) {- await this.deleteKey(shard.scope, shard.key, reason);- }- }-- private async isManifestPublished(- manifestKey: string,- expected: IndexShardManifest,- ): Promise<boolean> {- const published = await this.kv- .get<IndexShardManifest>(KV.bm25Index, manifestKey)- .catch(() => null);- if (- published?.v !== 1 ||- published.generation !== expected.generation ||- published.chars !== expected.chars ||- !Array.isArray(published.shards) ||- published.shards.length !== expected.shards.length- ) {- return false;- }- return published.shards.every((shard, index) => {- const expectedShard = expected.shards[index];- if (!expectedShard) return false;- return (- shard.scope === expectedShard.scope &&- shard.key === expectedShard.key &&- shard.chars === expectedShard.chars- );- });- }-- private async loadBm25Data(): Promise<string | null> {- return this.loadShardedData(BM25_KEY, BM25_MANIFEST_KEY, "BM25");+ private async deleteShards(shards: IndexShardManifest["shards"], reason: string): Promise<void> {+ await inBatches(shards, WRITE_CONCURRENCY, (shard) => this.deleteKey(shard.scope, shard.key, reason));}- private async loadVectorData(): Promise<string | null> {- return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector");- }-- private async loadShardedData(- legacyKey: string,- manifestKey: string,- label: string,- ): Promise<string | null> {- const manifest = await this.readIndexValue<IndexShardManifest>(- KV.bm25Index,- manifestKey,- label,- "manifest",- );+ private async loadShardedData(legacyKey: string, manifestKey: string, label: string): Promise<string | null> {+ const manifest = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, manifestKey, label, "manifest");if (!manifest.ok) return null;- // #797: some iii-state adapters return `undefined` (not `null`) for- // a missing key. The previous `value !== null` check passed- // undefined through to loadManifestData, which then crashed on- // `manifest.v` with TypeError. Treat both null and undefined as- // "no manifest" and fall through to the legacy path. The shape- // check stays so a malformed-but-present row still fails closed.- if (- manifest.value != null &&- typeof manifest.value === "object"- ) {+ if (manifest.value != null && typeof manifest.value === "object") {return this.loadManifestData(manifest.value, label);}-- const legacy = await this.readIndexValue<string>(- KV.bm25Index,- legacyKey,- label,- "legacy",- );+ const legacy = await this.readIndexValue<string>(KV.bm25Index, legacyKey, label, "legacy");if (!legacy.ok) return null;if (legacy.value && typeof legacy.value === "string") return legacy.value;return null;@@ -525,10 +555,7 @@ export class IndexPersistence {}}- private async loadManifestData(- manifest: IndexShardManifest,- label: string,- ): Promise<string | null> {+ private async loadManifestData(manifest: IndexShardManifest, label: string): Promise<string | null> {if (manifest.v !== 1 ||!Array.isArray(manifest.shards) ||@@ -555,10 +582,7 @@ export class IndexPersistence {let chars = 0;for (const { shard, chunk } of loadedShards) {if (typeof chunk !== "string") {- logger.warn(`index persistence: ${label} shard missing`, {- scope: shard.scope,- key: shard.key,- });+ logger.warn(`index persistence: ${label} shard missing`, { scope: shard.scope, key: shard.key });return null;}if (chunk.length !== shard.chars) {@@ -574,10 +598,7 @@ export class IndexPersistence {chars += chunk.length;}if (chars !== manifest.chars) {- logger.warn(`index persistence: ${label} total length mismatch`, {- expected: manifest.chars,- actual: chars,- });+ logger.warn(`index persistence: ${label} total length mismatch`, { expected: manifest.chars, actual: chars });return null;}return chunks.join("");diff --git a/src/state/vector-index.ts b/src/state/vector-index.tsindex d4b8bda7..2425bef8 100644--- a/src/state/vector-index.ts+++ b/src/state/vector-index.ts@@ -5,13 +5,13 @@// pool. Same risk on the encode side if the input Float32Array is itself// a sliced view. Reported as a phantom "2048 dimensions on disk" crash// in #455 / #469 / #584 / #587.-function float32ToBase64(arr: Float32Array): string {+export function float32ToBase64(arr: Float32Array): string {return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString("base64",);}-function base64ToFloat32(b64: string): Float32Array {+export function base64ToFloat32(b64: string): Float32Array {const buf = Buffer.from(b64, "base64");return new Float32Array(buf.buffer,@@ -34,16 +34,59 @@ function cosineSimilarity(a: Float32Array, b: Float32Array): number {return denom === 0 ? 0 : dot / denom;}+export type VectorEntry = { embedding: Float32Array; sessionId: string };+export class VectorIndex {- private vectors: Map<string, { embedding: Float32Array; sessionId: string }> =- new Map();+ private vectors: Map<string, VectorEntry> = new Map();+ private changes: Map<string, boolean> = new Map();add(obsId: string, sessionId: string, embedding: Float32Array): void {this.vectors.set(obsId, { embedding, sessionId });+ this.changes.set(obsId, true);}remove(obsId: string): void {- this.vectors.delete(obsId);+ if (this.vectors.delete(obsId)) this.changes.set(obsId, false);+ }++ has(obsId: string): boolean {+ return this.vectors.has(obsId);+ }++ get(obsId: string): VectorEntry | undefined {+ return this.vectors.get(obsId);+ }++ entries(): IterableIterator<[string, VectorEntry]> {+ return this.vectors.entries();+ }++ loadPersisted(obsId: string, sessionId: string, embedding: Float32Array): void {+ this.vectors.set(obsId, { embedding, sessionId });+ }++ get pendingChanges(): number {+ return this.changes.size;+ }++ takeChanges(): Map<string, boolean> {+ const taken = this.changes;+ this.changes = new Map();+ return taken;+ }++ returnChanges(changes: Map<string, boolean>): void {+ for (const [obsId, present] of changes) {+ if (!this.changes.has(obsId)) this.changes.set(obsId, present);+ }+ }++ markRemoved(obsId: string): void {+ if (!this.vectors.has(obsId)) this.changes.set(obsId, false);+ }++ markAllChanged(): void {+ for (const obsId of this.vectors.keys()) this.changes.set(obsId, true);}search(@@ -104,6 +147,7 @@ export class VectorIndex {}clear(): void {+ for (const obsId of this.vectors.keys()) this.changes.set(obsId, false);this.vectors.clear();}@@ -119,6 +163,7 @@ export class VectorIndex {sessionId: entry.sessionId,});}+ this.changes = new Map(other.changes);}serialize(): string {diff --git a/src/triggers/api.ts b/src/triggers/api.tsindex 2fc2f328..337e34b5 100644--- a/src/triggers/api.ts+++ b/src/triggers/api.ts@@ -12,7 +12,14 @@ import type { ResilientProvider } from "../providers/resilient.js";import { III_PINNED_VERSION, VERSION } from "../version.js";import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js";import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js";-import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js";+import {+ findUnindexedObservations,+ getIndexPersistenceStatus,+ getPendingVectorBackfillCount,+ getSearchIndex,+ getVectorIndex,+ isBm25RebuildIncomplete,+} from "../functions/search.js";import { timingSafeCompare } from "../auth.js";import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js";import { renderViewerDocument } from "../viewer/document.js";@@ -370,6 +377,8 @@ export function registerApiTriggers(observationsIndexed,missingObservations: unindexed ? unindexed.missing.length : null,sessions: unindexed ? unindexed.sessions : null,+ bm25Incomplete: isBm25RebuildIncomplete(),+ pendingVectorBackfill: getPendingVectorBackfillCount(),},graph,graphExtractionEnabled: isGraphExtractionEnabled(),diff --git a/test/index-keyword-rebuild.test.ts b/test/index-keyword-rebuild.test.tsnew file mode 100644index 00000000..f0080675--- /dev/null+++ b/test/index-keyword-rebuild.test.ts@@ -0,0 +1,240 @@+import { describe, it, expect, afterEach } from "vitest";+import {+ backfillVectors,+ getPendingVectorBackfillCount,+ getSearchIndex,+ isBm25RebuildIncomplete,+ isMemoryIndexReady,+ rebuildKeywordIndex,+ setEmbeddingProvider,+ setIndexPersistence,+ setVectorIndex,+} from "../src/functions/search.js";+import { VectorIndex } from "../src/state/vector-index.js";+import type { CompressedObservation, EmbeddingProvider, Memory, Session } from "../src/types.js";++function mockKV() {+ const store = new Map<string, Map<string, unknown>>();+ return {+ store,+ list: async <T>(scope: string): Promise<T[]> => Array.from(store.get(scope)?.values() ?? []) as T[],+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, data);+ return data;+ },+ delete: async (scope: string, key: string): Promise<void> => {+ store.get(scope)?.delete(key);+ },+ };+}++const WORDS = ["auth", "cache", "index", "vector", "session", "memory", "graph", "retry", "queue", "schema", "deploy", "worker"];++function makeObs(id: string, sessionId: string, i: number, timestamp: string): CompressedObservation {+ const a = WORDS[i % WORDS.length];+ const b = WORDS[(i * 7) % WORDS.length];+ return {+ id,+ sessionId,+ timestamp,+ type: "file_edit",+ title: `${a} change ${i}`,+ facts: [`touched ${b} path`],+ narrative: `Updated the ${a} module so ${b} handling works for case ${i}`,+ concepts: [a, b],+ files: [`src/${a}/${b}.ts`],+ importance: 5,+ };+}++function seed(kv: ReturnType<typeof mockKV>, sessions: number, perSession: number, timestamp: (i: number) => string) {+ let n = 0;+ for (let s = 0; s < sessions; s++) {+ const sessionId = `ses_${s}`;+ kv.store.set("mem:sessions", kv.store.get("mem:sessions") ?? new Map());+ kv.store.get("mem:sessions")!.set(sessionId, {+ id: sessionId,+ project: "p",+ cwd: "/tmp",+ startedAt: timestamp(0),+ status: "completed",+ observationCount: perSession,+ } satisfies Session);+ const scope = `mem:obs:${sessionId}`;+ kv.store.set(scope, new Map());+ for (let o = 0; o < perSession; o++) {+ const id = `obs_${n}`;+ kv.store.get(scope)!.set(id, makeObs(id, sessionId, n, timestamp(n)));+ n++;+ }+ }+ const memories = new Map<string, Memory>();+ memories.set("mem_1", {+ id: "mem_1",+ createdAt: timestamp(0),+ updatedAt: timestamp(0),+ type: "fact",+ title: "Deploy uses blue green",+ content: "Deploys switch traffic after health checks pass",+ concepts: ["deploy"],+ files: [],+ sessionIds: ["ses_0"],+ strength: 7,+ version: 1,+ isLatest: true,+ } as Memory);+ memories.set("mem_old", {+ id: "mem_old",+ createdAt: timestamp(0),+ updatedAt: timestamp(0),+ type: "fact",+ title: "Superseded",+ content: "Old version",+ concepts: [],+ files: [],+ sessionIds: [],+ strength: 1,+ version: 1,+ isLatest: false,+ } as Memory);+ kv.store.set("mem:memories", memories);+ kv.store.get("mem:obs:ses_0")!.set("raw_1", { id: "raw_1", sessionId: "ses_0", timestamp: timestamp(0) });+ return n;+}++function stubProvider(): EmbeddingProvider {+ return {+ name: "stub",+ dimensions: 3,+ embed: async () => new Float32Array([0.1, 0.2, 0.3]),+ embedBatch: async (texts: string[]) => texts.map(() => new Float32Array([0.1, 0.2, 0.3])),+ } as EmbeddingProvider;+}++describe("rebuildKeywordIndex", () => {+ afterEach(() => {+ getSearchIndex().clear();+ setVectorIndex(null);+ setEmbeddingProvider(null);+ setIndexPersistence(null);+ });++ it("rebuilds BM25 from stored observations and latest memories, quickly", async () => {+ const kv = mockKV();+ const total = seed(kv, 100, 30, () => "2026-09-01T00:00:00.000Z");+ getSearchIndex().add(makeObs("ghost", "ses_gone", 0, "2026-09-01T00:00:00.000Z"));++ const started = performance.now();+ const result = await rebuildKeywordIndex(kv as never);+ const elapsed = performance.now() - started;++ expect(total).toBe(3000);+ expect(result.documents).toBe(3001);+ expect(result.vectorJobs).toEqual([]);+ const idx = getSearchIndex();+ expect(idx.size).toBe(3001);+ expect(idx.has("ghost")).toBe(false);+ expect(idx.has("mem_1")).toBe(true);+ expect(idx.has("mem_old")).toBe(false);+ expect(idx.has("raw_1")).toBe(false);+ expect(idx.search("blue green").map((r) => r.obsId)).toContain("mem_1");+ expect(isMemoryIndexReady()).toBe(true);+ expect(elapsed).toBeLessThan(10_000);+ });++ it("keeps the vector index untouched and reports an incomplete rebuild when the sessions listing fails", async () => {+ const kv = mockKV();+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");+ const vector = new VectorIndex();+ vector.add("obs_0", "ses_0", new Float32Array([0.1, 0.2, 0.3]));+ setVectorIndex(vector);+ setEmbeddingProvider(stubProvider());+ const failingKv = {+ ...kv,+ list: async <T>(scope: string): Promise<T[]> => {+ if (scope === "mem:sessions") throw new Error("engine unreachable");+ return kv.list<T>(scope);+ },+ };++ const result = await rebuildKeywordIndex(failingKv as never, null);++ expect(result.vectorJobs).toEqual([]);+ expect(vector.size).toBe(1);+ expect(vector.has("obs_0")).toBe(true);+ expect(isBm25RebuildIncomplete()).toBe(true);+ }, 10_000);++ it("queues embeddings only for documents newer than the last vector save", async () => {+ const kv = mockKV();+ seed(kv, 2, 5, (i) => (i < 5 ? "2026-09-01T00:00:00.000Z" : "2026-09-20T00:00:00.000Z"));+ const vector = new VectorIndex();+ vector.add("obs_6", "ses_1", new Float32Array([0.1, 0.2, 0.3]));+ setVectorIndex(vector);+ setEmbeddingProvider(stubProvider());++ const result = await rebuildKeywordIndex(kv as never, "2026-09-10T00:00:00.000Z");++ expect(result.vectorJobs.map((j) => j.id).sort()).toEqual(["obs_5", "obs_7", "obs_8", "obs_9"]);+ });++ it("withholds a whole-store backfill by default and reports it as pending, and queues none when storage was unavailable", async () => {+ const kv = mockKV();+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");+ setVectorIndex(new VectorIndex());+ setEmbeddingProvider(stubProvider());++ const fresh = await rebuildKeywordIndex(kv as never, null);+ expect(fresh.vectorJobs).toEqual([]);+ expect(fresh.fullBackfillPending).toBe(4);++ const unavailable = await rebuildKeywordIndex(kv as never, undefined);+ expect(unavailable.vectorJobs).toEqual([]);+ expect(unavailable.fullBackfillPending).toBe(0);+ });++ it("runs the whole-store backfill, capped, once AGENTMEMORY_VECTOR_BACKFILL=all is set", async () => {+ const kv = mockKV();+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");+ setVectorIndex(new VectorIndex());+ setEmbeddingProvider(stubProvider());+ process.env.AGENTMEMORY_VECTOR_BACKFILL = "all";+ try {+ const opted = await rebuildKeywordIndex(kv as never, null);+ expect(opted.vectorJobs.map((j) => j.id).sort()).toEqual(["mem_1", "obs_0", "obs_1", "obs_2"]);+ expect(opted.fullBackfillPending).toBe(0);++ process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "2";+ const capped = await rebuildKeywordIndex(kv as never, null);+ expect(capped.vectorJobs.length).toBe(2);+ } finally {+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL;+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX;+ }+ });++ it("backfills queued vectors in batches and saves progress", async () => {+ const kv = mockKV();+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");+ const vector = new VectorIndex();+ setVectorIndex(vector);+ setEmbeddingProvider(stubProvider());+ let saved = 0;+ setIndexPersistence({ scheduleSave: () => {}, save: async () => void saved++ });+ process.env.AGENTMEMORY_VECTOR_BACKFILL = "all";++ try {+ const { vectorJobs } = await rebuildKeywordIndex(kv as never, null);+ const added = await backfillVectors(vectorJobs);++ expect(added).toBe(4);+ expect(vector.size).toBe(4);+ expect(saved).toBe(1);+ expect(getPendingVectorBackfillCount()).toBe(0);+ } finally {+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL;+ }+ });+});diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.tsindex 54eff133..bc03834d 100644--- a/test/index-persistence-throttle.test.ts+++ b/test/index-persistence-throttle.test.ts@@ -1,18 +1,15 @@import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";-import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js";-import { SearchIndex } from "../src/state/search-index.js";+import { IndexPersistence } from "../src/state/index-persistence.js";import { VectorIndex } from "../src/state/vector-index.js";import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js";-import type { CompressedObservation } from "../src/types.js";-const BM25_SCOPE = "mem:index:bm25";+const META_KEY = "vectors:meta";function mockKV() {const store = new Map<string, Map<string, unknown>>();const sets: Array<{ scope: string; key: string }> = [];return {sets,- store,get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,set: async <T>(scope: string, key: string, data: T): Promise<T> => {sets.push({ scope, key });@@ -27,23 +24,12 @@ function mockKV() {};}-function obs(id: string, title: string): CompressedObservation {- return {- id,- sessionId: "ses_1",- timestamp: new Date().toISOString(),- type: "file_edit",- title,- facts: [],- narrative: `${title} narrative`,- concepts: [],- files: [],- importance: 5,- };+function metaSaves(kv: ReturnType<typeof mockKV>): number {+ return kv.sets.filter((s) => s.key === META_KEY).length;}-function manifestSaves(kv: ReturnType<typeof mockKV>, key: string): number {- return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length;+function touch(vector: VectorIndex, id: string): void {+ vector.add(id, "ses_1", new Float32Array([Math.random(), 0.2, 0.3]));}describe("IndexPersistence save throttling", () => {@@ -59,35 +45,40 @@ describe("IndexPersistence save throttling", () => {});it("saves at most once per interval however often changes are scheduled", async () => {- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });- for (let i = 0; i < 50; i++) persistence.scheduleSave();+ for (let i = 0; i < 50; i++) {+ touch(vector, `obs_${i}`);+ persistence.scheduleSave();+ }await vi.advanceTimersByTimeAsync(59_000);- expect(manifestSaves(kv, "data:manifest")).toBe(0);+ expect(metaSaves(kv)).toBe(0);await vi.advanceTimersByTimeAsync(1_000);- expect(manifestSaves(kv, "data:manifest")).toBe(1);+ expect(metaSaves(kv)).toBe(1);- for (let i = 0; i < 50; i++) persistence.scheduleSave();+ for (let i = 0; i < 50; i++) {+ touch(vector, `obs_more_${i}`);+ persistence.scheduleSave();+ }await vi.advanceTimersByTimeAsync(30_000);- expect(manifestSaves(kv, "data:manifest")).toBe(1);+ expect(metaSaves(kv)).toBe(1);await vi.advanceTimersByTimeAsync(30_000);- expect(manifestSaves(kv, "data:manifest")).toBe(2);+ expect(metaSaves(kv)).toBe(2);});it("an explicit save runs immediately and cancels the pending timer", async () => {- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });+ touch(vector, "obs_1");persistence.scheduleSave();await persistence.save();- expect(manifestSaves(kv, "data:manifest")).toBe(1);+ expect(metaSaves(kv)).toBe(1);await vi.advanceTimersByTimeAsync(120_000);- expect(manifestSaves(kv, "data:manifest")).toBe(1);+ expect(metaSaves(kv)).toBe(1);});it("never runs two saves at once and coalesces requests made during a save into one", async () => {@@ -108,11 +99,12 @@ describe("IndexPersistence save throttling", () => {return result;},};- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });+ const vector = new VectorIndex();+ touch(vector, "obs_1");+ const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });const first = persistence.save();+ touch(vector, "obs_2");const second = persistence.save();const third = persistence.save();expect(second).toBe(third);@@ -122,42 +114,12 @@ describe("IndexPersistence save throttling", () => {release();await Promise.all([first, second, third]);- expect(manifestSaves(kv, "data:manifest")).toBe(2);+ expect(metaSaves(kv)).toBe(2);expect(maxInFlight).toBe(1);expect(persistence.status().saving).toBe(false);});- it("a failing BM25 leg does not stop the vector leg from saving", async () => {- const failingKv = {- ...kv,- set: async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed");- return kv.set(scope, key, data);- },- };- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const vector = new VectorIndex();- vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));- const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 });-- persistence.scheduleSave();- await persistence.save();-- const status = persistence.status();- expect(status.bm25.lastError).toBe("bm25 manifest write failed");- expect(status.bm25.dirtySince).not.toBeNull();- expect(status.vector?.lastError).toBeNull();- expect(status.vector?.lastSavedAt).not.toBeNull();- expect(status.vector?.dirtySince).toBeNull();- expect(status.vector?.serializedChars).toBeGreaterThan(0);-- const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load();- expect(loaded.bm25).toBeNull();- expect(loaded.vector?.size).toBe(1);- });-- it("keeps a leg dirty when a change arrives while it is being saved", async () => {+ it("keeps the index dirty when a change arrives while it is being saved", async () => {let release: () => void = () => undefined;const gate = new Promise<void>((resolve) => {release = resolve;@@ -173,63 +135,31 @@ describe("IndexPersistence save throttling", () => {return kv.set(scope, key, data);},};- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });+ const vector = new VectorIndex();+ touch(vector, "obs_1");+ const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });const saving = persistence.save();+ touch(vector, "obs_2");persistence.scheduleSave();release();await saving;- expect(persistence.status().bm25.lastSavedAt).not.toBeNull();- expect(persistence.status().bm25.dirtySince).not.toBeNull();+ expect(persistence.status().vector?.lastSavedAt).not.toBeNull();+ expect(persistence.status().vector?.dirtySince).not.toBeNull();await persistence.save();- expect(persistence.status().bm25.dirtySince).toBeNull();- });-- it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => {- let release: () => void = () => undefined;- const gate = new Promise<void>((resolve) => {- release = resolve;- });- let heldBm25Manifest = false;- const slowKv = {- ...kv,- set: async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) {- heldBm25Manifest = true;- await gate;- }- return kv.set(scope, key, data);- },- };- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const vector = new VectorIndex();- vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));- const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 });-- const saving = persistence.save();- persistence.scheduleSave();- release();- await saving;-- const status = persistence.status();- expect(status.bm25.dirtySince).not.toBeNull();- expect(status.vector?.dirtySince).toBeNull();- expect(status.vector?.lastSavedAt).not.toBeNull();+ expect(persistence.status().vector?.dirtySince).toBeNull();});it("stop prevents later scheduled saves", async () => {- const bm25 = new SearchIndex();- bm25.add(obs("obs_1", "alpha"));- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 });+ const vector = new VectorIndex();+ touch(vector, "obs_1");+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 1_000, bucketSize: 16 });persistence.stop();persistence.scheduleSave();await vi.advanceTimersByTimeAsync(10_000);- expect(manifestSaves(kv, "data:manifest")).toBe(0);+ expect(metaSaves(kv)).toBe(0);});});@@ -246,7 +176,15 @@ function statusInputs(overrides: Partial<StatusInputs> = {}): StatusInputs {provider: "llm",embeddingProvider: "embeddings",flags: [],- index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 },+ index: {+ bm25Documents: 1,+ vectorDocuments: 1,+ observationsIndexed: 1,+ missingObservations: 0,+ sessions: 1,+ bm25Incomplete: false,+ pendingVectorBackfill: 0,+ },graph: null,graphExtractionEnabled: false,...overrides,@@ -258,78 +196,53 @@ const cleanLeg = {lastError: null,lastErrorAt: null,dirtySince: null,- serializedChars: 1000,};+function persistenceStatus(vector: typeof cleanLeg | null, pendingChanges = 0) {+ return { saveIntervalMs: 600_000, saving: false, buckets: 256, pendingChanges, vector, vectorCountShortfall: null };+}+describe("status reports index persistence", () => {- it("is quiet when both legs saved cleanly", () => {- const report = evaluateStatus(- statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }),- );+ it("is quiet when the vector index saved cleanly", () => {+ const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(cleanLeg) }));expect(report.problems).toEqual([]);- expect(report.indexPersistence?.vector?.serializedChars).toBe(1000);+ expect(report.indexPersistence?.buckets).toBe(256);});- it("reports a failing leg as an error", () => {+ it("reports a failing vector save as an error", () => {const report = evaluateStatus(statusInputs({- indexPersistence: {- saveIntervalMs: 600_000,- saving: false,- bm25: cleanLeg,- vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" },- },+ indexPersistence: persistenceStatus({+ ...cleanLeg,+ lastError: "3 of 10 vector writes failed: timed out",+ lastErrorAt: "2026-09-25T11:59:30.000Z",+ } as never),}),);expect(report.status).toBe("error");expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]);- expect(report.problems[0].message).toContain("vector");- });-- it("warns when the vector index passes 80% of the Node.js string limit", () => {- const report = evaluateStatus(- statusInputs({- indexPersistence: {- saveIntervalMs: 600_000,- saving: false,- bm25: cleanLeg,- vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) },- },- }),- );- expect(report.status).toBe("warn");- expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]);- expect(report.problems[0].message).toContain("85%");+ expect(report.problems[0].message).toContain("3 of 10 vector writes failed");});it("warns when unsaved changes are older than twice the save interval", () => {const report = evaluateStatus(- statusInputs({- indexPersistence: {- saveIntervalMs: 600_000,- saving: false,- bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" },- vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" },- },- }),+ statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" } as never, 4) }),);expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]);- expect(report.problems[0].message).toContain("BM25");});- it("shows save state and vector size on the status page", () => {+ it("does not report vector persistence when vector search is off", () => {+ const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(null) }));+ expect(report.problems).toEqual([]);+ });++ it("shows save state and bucket storage on the status page", () => {const report = evaluateStatus(- statusInputs({- indexPersistence: {- saveIntervalMs: 600_000,- saving: false,- bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" },- vector: { ...cleanLeg, serializedChars: 53_687_089 },- },- }),+ statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" } as never, 3) }),);const html = renderStatusHtml(report, "n");- expect(html).toContain("<th>BM25 save</th><td>saved 1m ago, unsaved changes pending</td>");- expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)");+ expect(html).toContain("<th>BM25 index</th><td>rebuilt from stored content at boot</td>");+ expect(html).toContain("<th>Vector save</th><td>saved 1m ago, unsaved changes pending</td>");+ expect(html).toContain("<th>Vector storage</th><td>256 buckets, 3 unsaved changes</td>");});});diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.tsindex 802084bd..568d1800 100644--- a/test/index-persistence.test.ts+++ b/test/index-persistence.test.ts@@ -1,740 +1,475 @@import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";-import { IndexPersistence } from "../src/state/index-persistence.js";+import { IndexPersistence, vectorBucketScope } from "../src/state/index-persistence.js";import { SearchIndex } from "../src/state/search-index.js";import { VectorIndex } from "../src/state/vector-index.js";import type { CompressedObservation } from "../src/types.js";-const BM25_SCOPE = "mem:index:bm25";-const BM25_LEGACY_KEY = "data";-const BM25_MANIFEST_KEY = "data:manifest";-const VECTOR_LEGACY_KEY = "vectors";-const VECTOR_MANIFEST_KEY = "vectors:manifest";--type TestIndexShardManifest = {- v: 1;- generation?: string;- shards: Array<{ scope: string; key: string; chars: number }>;- chars: number;-};+const INDEX_SCOPE = "mem:index:bm25";+const META_KEY = "vectors:meta";function mockKV() {const store = new Map<string, Map<string, unknown>>();+ const ops: Array<{ op: "set" | "delete" | "list"; scope: string; key?: string }> = [];return {- get: async <T>(scope: string, key: string): Promise<T | null> => {- return (store.get(scope)?.get(key) as T) ?? null;- },+ store,+ ops,+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ ops.push({ op: "set", scope, key });if (!store.has(scope)) store.set(scope, new Map());store.get(scope)!.set(key, data);return data;},delete: async (scope: string, key: string): Promise<void> => {+ ops.push({ op: "delete", scope, key });store.get(scope)?.delete(key);},list: async <T>(scope: string): Promise<T[]> => {- const entries = store.get(scope);- return entries ? (Array.from(entries.values()) as T[]) : [];+ ops.push({ op: "list", scope });+ return Array.from(store.get(scope)?.values() ?? []) as T[];},};}type MockKV = ReturnType<typeof mockKV>;-function makeObs(- overrides: Partial<CompressedObservation> = {},-): CompressedObservation {+function vec(values: number[]): Float32Array {+ return new Float32Array(values);+}++function vectorWith(entries: Array<[string, number[]]>): VectorIndex {+ const index = new VectorIndex();+ for (const [id, values] of entries) index.add(id, `ses_${id}`, vec(values));+ return index;+}++function bucketSets(kv: MockKV): Array<{ scope: string; key?: string }> {+ return kv.ops.filter((o) => o.op === "set" && o.scope.startsWith(`${INDEX_SCOPE}:vec:`));+}++function obs(id: string): CompressedObservation {return {- id: "obs_1",+ id,sessionId: "ses_1",timestamp: new Date().toISOString(),type: "file_edit",- title: "Edit auth middleware",- subtitle: "JWT validation",- facts: ["Added token check"],- narrative: "Modified the auth middleware to validate JWT tokens",- concepts: ["authentication", "jwt"],- files: ["src/middleware/auth.ts"],- importance: 7,- ...overrides,+ title: `title ${id}`,+ facts: [],+ narrative: `narrative ${id}`,+ concepts: [],+ files: [],+ importance: 5,};}-function makeBm25(id: string, title: string): SearchIndex {- const bm25 = new SearchIndex();- bm25.add(makeObs({ id, title, narrative: `${title} narrative` }));- return bm25;+async function writeLegacyVectorSnapshot(+ kv: MockKV,+ vector: VectorIndex,+ options: { generation?: string; monolithic?: boolean; shardChars?: number } = {},+): Promise<void> {+ const serialized = vector.serialize();+ if (options.monolithic) {+ await kv.set(INDEX_SCOPE, "vectors", serialized);+ return;+ }+ const generation = options.generation ?? "idx_mfabcd12_aaaaaaaaaaaa";+ const size = options.shardChars ?? 40;+ const shards: Array<{ scope: string; key: string; chars: number }> = [];+ for (let offset = 0, i = 0; offset < serialized.length; offset += size, i++) {+ const scope = `${INDEX_SCOPE}:vectors:${generation}:${String(i).padStart(5, "0")}`;+ const chunk = serialized.slice(offset, offset + size);+ await kv.set(scope, "data", chunk);+ shards.push({ scope, key: "data", chars: chunk.length });+ }+ await kv.set(INDEX_SCOPE, "vectors:manifest", { v: 1, generation, shards, chars: serialized.length });}-function makeVector(id = "obs_1"): VectorIndex {- const vector = new VectorIndex();- vector.add(id, "ses_1", new Float32Array([0.1, 0.2, 0.3]));- return vector;+async function writeLegacyBm25Snapshot(kv: MockKV): Promise<void> {+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_legacy"));+ const serialized = bm25.serialize();+ const scope = `${INDEX_SCOPE}:bm25:idx_mfabcd12_bbbbbbbbbbbb:00000`;+ await kv.set(scope, "data", serialized);+ await kv.set(INDEX_SCOPE, "data:manifest", {+ v: 1,+ generation: "idx_mfabcd12_bbbbbbbbbbbb",+ shards: [{ scope, key: "data", chars: serialized.length }],+ chars: serialized.length,+ });}-async function getBm25Manifest(kv: MockKV): Promise<TestIndexShardManifest> {- const manifest = await kv.get<TestIndexShardManifest>(- BM25_SCOPE,- BM25_MANIFEST_KEY,- );- expect(manifest).not.toBeNull();- return manifest!;+function expectSameVectors(actual: VectorIndex | null, expected: VectorIndex): void {+ expect(actual).not.toBeNull();+ expect(actual!.size).toBe(expected.size);+ for (const [id, entry] of expected.entries()) {+ const loaded = actual!.get(id);+ expect(loaded?.sessionId).toBe(entry.sessionId);+ expect(Array.from(loaded!.embedding)).toEqual(Array.from(entry.embedding));+ }}-describe("IndexPersistence", () => {- let kv: ReturnType<typeof mockKV>;+describe("IndexPersistence bucketed vector storage", () => {+ let kv: MockKV;beforeEach(() => {- vi.useFakeTimers();kv = mockKV();});- afterEach(() => {- vi.useRealTimers();- });-- it("saves and loads BM25 index round-trip", async () => {- const bm25 = new SearchIndex();- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));+ it("fills buckets in insertion order and rolls to a new one at the cap", async () => {+ const vector = vectorWith([+ ["obs_a", [0.1, 0.2, 0.3]],+ ["obs_b", [0.4, 0.5, 0.6]],+ ["obs_c", [0.7, 0.8, 0.9]],+ ]);+ await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save();++ expect(kv.store.get(vectorBucketScope(0))?.has("obs_a")).toBe(true);+ expect(kv.store.get(vectorBucketScope(0))?.has("obs_b")).toBe(true);+ expect(kv.store.get(vectorBucketScope(1))?.has("obs_c")).toBe(true);+ const meta = await kv.get<{ v: number; bucketCount: number; count: number }>(INDEX_SCOPE, META_KEY);+ expect(meta).toMatchObject({ v: 3, bucketCount: 2, count: 3 });++ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 }).load();+ expect(loaded.state).toBe("buckets");+ expectSameVectors(loaded.vector, vector);+ expect(loaded.vector!.pendingChanges).toBe(0);+ });++ it("writes only the bucket entry of a single added vector", async () => {+ const vector = vectorWith([+ ["obs_a", [0.1, 0.2, 0.3]],+ ["obs_b", [0.4, 0.5, 0.6]],+ ]);+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });+ await persistence.save();+ kv.ops.length = 0;- const persistence = new IndexPersistence(kv as never, bm25, null);+ vector.add("obs_new", "ses_new", vec([1, 0, 0]));await persistence.save();- const loaded = await persistence.load();- expect(loaded.bm25).not.toBeNull();- expect(loaded.bm25!.size).toBe(1);- const results = loaded.bm25!.search("auth");- expect(results.length).toBe(1);+ const writes = bucketSets(kv);+ expect(writes).toEqual([{ op: "set", scope: vectorBucketScope(0), key: "obs_new" }]);});- it("saves BM25 index shards outside the BM25 metadata scope", async () => {- const bm25 = new SearchIndex();- bm25.add(- makeObs({- id: "obs_1",- title: "auth handler ".repeat(40),- narrative: "JWT middleware validation ".repeat(40),- }),- );-- const persistence = new IndexPersistence(kv as never, bm25, null, {- shardChars: 80,- createGeneration: () => "gen_bm25",- });+ it("does not write when nothing changed", async () => {+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });await persistence.save();+ kv.ops.length = 0;- const manifest = await getBm25Manifest(kv);- expect(manifest.generation).toBe("gen_bm25");- expect(manifest.shards.length).toBeGreaterThan(1);- expect(manifest.shards[0].scope).toContain(":gen_bm25:");- await expect(kv.get(BM25_SCOPE, BM25_LEGACY_KEY)).resolves.toBeNull();- await expect(- kv.get(manifest.shards[0].scope, manifest.shards[0].key),- ).resolves.toEqual(expect.any(String));+ await persistence.save();- const loaded = await persistence.load();- expect(loaded.bm25).not.toBeNull();- expect(loaded.bm25!.search("auth").length).toBe(1);+ expect(kv.ops.filter((o) => o.op !== "list")).toEqual([]);});- it("loads legacy monolithic BM25 and vector snapshots", async () => {- const bm25 = makeBm25("obs_1", "legacy auth handler");- const vector = makeVector("obs_1");- await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, bm25.serialize());- await kv.set(BM25_SCOPE, VECTOR_LEGACY_KEY, vector.serialize());-- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();-- expect(loaded.bm25).not.toBeNull();- expect(loaded.bm25!.search("legacy").length).toBe(1);- expect(loaded.vector).not.toBeNull();- expect(loaded.vector!.size).toBe(1);+ it("removes deleted vectors from their bucket", async () => {+ const vector = vectorWith([+ ["obs_a", [0.1, 0.2, 0.3]],+ ["obs_b", [0.4, 0.5, 0.6]],+ ]);+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });+ await persistence.save();++ vector.remove("obs_a");+ await persistence.save();++ expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(0), key: "obs_a" });+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expect(loaded.vector!.has("obs_a")).toBe(false);+ expect(loaded.vector!.has("obs_b")).toBe(true);});- it("fails closed instead of falling back when manifest reads fail", async () => {- const legacy = makeBm25("obs_legacy", "legacy stale snapshot");- await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, legacy.serialize());- const failingKv = {- ...kv,- get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {- if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {- throw new Error("manifest backend unavailable");- }- return kv.get(scope, key);- }),- };+ it("a delete touches only the bucket that owns the id, in a multi-bucket store", async () => {+ const vector = vectorWith(+ Array.from({ length: 5 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]),+ );+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 2 });+ await persistence.save();+ kv.ops.length = 0;- const loaded = await new IndexPersistence(- failingKv as never,- new SearchIndex(),- null,- ).load();+ vector.remove("obs_2");+ await persistence.save();- expect(loaded.bm25).toBeNull();+ const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));+ expect(touched).toEqual(new Set([vectorBucketScope(1)]));+ expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(1), key: "obs_2" });});- it("fails closed when legacy snapshot reads fail", async () => {- const failingKv = {- ...kv,- get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {- if (scope === BM25_SCOPE && key === BM25_LEGACY_KEY) {- throw new Error("legacy backend unavailable");- }- return kv.get(scope, key);- }),- };+ it("clearing the index deletes every persisted vector", async () => {+ const vector = vectorWith([+ ["obs_a", [0.1, 0.2, 0.3]],+ ["obs_b", [0.4, 0.5, 0.6]],+ ]);+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });+ await persistence.save();- const loaded = await new IndexPersistence(- failingKv as never,- new SearchIndex(),- null,- ).load();+ vector.clear();+ vector.add("obs_c", "ses_c", vec([0, 1, 0]));+ await persistence.save();- expect(loaded.bm25).toBeNull();+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expect([...loaded.vector!.entries()].map(([id]) => id)).toEqual(["obs_c"]);});- it("loads sharded manifests that omit optional generation metadata", async () => {- const bm25 = makeBm25("obs_1", "deterministic shard auth");- const serialized = bm25.serialize();- const chunks = [serialized.slice(0, 50), serialized.slice(50)];- await kv.set("mem:index:bm25:bm25:00000", "data", chunks[0]);- await kv.set("mem:index:bm25:bm25:00001", "data", chunks[1]);- await kv.set<TestIndexShardManifest>(BM25_SCOPE, BM25_MANIFEST_KEY, {- v: 1,- chars: serialized.length,- shards: [- {- scope: "mem:index:bm25:bm25:00000",- key: "data",- chars: chunks[0].length,- },- {- scope: "mem:index:bm25:bm25:00001",- key: "data",- chars: chunks[1].length,- },- ],- });-- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();-- expect(loaded.bm25).not.toBeNull();- expect(loaded.bm25!.search("deterministic").length).toBe(1);- });+ it("keeps failed writes pending and retries them on the next save", async () => {+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ let fail = true;+ const flakyKv = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("state::set timed out");+ return kv.set(scope, key, data);+ },+ };+ const persistence = new IndexPersistence(flakyKv as never, vector, { bucketSize: 16 });- it("saves and loads vector index round-trip", async () => {- const bm25 = new SearchIndex();- const vector = makeVector();+ await expect(persistence.save()).resolves.toBeUndefined();+ const failed = persistence.status();+ expect(failed.vector?.lastError).toBe("1 of 1 vector writes failed: state::set timed out");+ expect(failed.pendingChanges).toBe(1);+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();- const persistence = new IndexPersistence(kv as never, bm25, vector);+ fail = false;await persistence.save();-- const loaded = await persistence.load();- expect(loaded.vector).not.toBeNull();- expect(loaded.vector!.size).toBe(1);+ expect(persistence.status().vector?.lastError).toBeNull();+ expect(persistence.status().pendingChanges).toBe(0);+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expectSameVectors(loaded.vector, vector);});- it("saves vector index shards outside the BM25 scope", async () => {- const bm25 = new SearchIndex();+ it("a save after many vectors added over several waves touches only the currently open bucket", async () => {const vector = new VectorIndex();- vector.add(- "obs_1",- "ses_1",- new Float32Array(Array.from({ length: 32 }, (_, i) => i)),- );+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 100 });- const persistence = new IndexPersistence(kv as never, bm25, vector, {- shardChars: 40,- createGeneration: () => "gen_vector",- });+ for (let i = 0; i < 950; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0]));await persistence.save();+ expect(kv.store.get(vectorBucketScope(9))?.size).toBe(50);+ for (let bucket = 0; bucket <= 8; bucket++) expect(kv.store.get(vectorBucketScope(bucket))?.size).toBe(100);+ kv.ops.length = 0;- const manifest = await kv.get<TestIndexShardManifest>(- BM25_SCOPE,- VECTOR_MANIFEST_KEY,- );- expect(manifest).not.toBeNull();- expect(manifest!.generation).toBe("gen_vector");- expect(manifest!.shards.length).toBeGreaterThan(1);- expect(manifest!.shards[0].scope).toContain(":gen_vector:");- await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull();- await expect(- kv.get(manifest!.shards[0].scope, manifest!.shards[0].key),- ).resolves.toEqual(expect.any(String));+ for (let i = 950; i < 1000; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0]));+ await persistence.save();+ let touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));+ expect(touched).toEqual(new Set([vectorBucketScope(9)]));+ expect(kv.store.get(vectorBucketScope(9))?.size).toBe(100);+ kv.ops.length = 0;- const loaded = await persistence.load();- expect(loaded.vector).not.toBeNull();- expect(loaded.vector!.size).toBe(1);+ vector.add("obs_1000", "ses_1", vec([1000, 0, 0]));+ await persistence.save();+ touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));+ expect(touched).toEqual(new Set([vectorBucketScope(10)]));});- it("persists empty vector snapshots so cleared vectors do not reload", async () => {- const previousBm25 = makeBm25("obs_old", "alpha previous snapshot");- const previousVector = makeVector("obs_old");- await new IndexPersistence(kv as never, previousBm25, previousVector, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();-- const nextBm25 = makeBm25("obs_new", "bravo new snapshot");- const emptyVector = new VectorIndex();- await new IndexPersistence(kv as never, nextBm25, emptyVector, {- shardChars: 80,- createGeneration: () => "gen_empty",- }).save();-- const vectorManifest = await kv.get<TestIndexShardManifest>(- BM25_SCOPE,- VECTOR_MANIFEST_KEY,+ it("a smaller bucket size configured later opens a new bucket instead of moving existing data", async () => {+ const vector = vectorWith(+ Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]),);- expect(vectorManifest).not.toBeNull();- expect(vectorManifest!.generation).toBe("gen_empty");- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("bravo").length).toBe(1);- expect(loaded.vector).not.toBeNull();- expect(loaded.vector!.size).toBe(0);- });+ await new IndexPersistence(kv as never, vector, { bucketSize: 4 }).save();+ for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true);- it("avoids one oversized state::set string payload for persisted indexes", async () => {- const maxStringPayloadChars = 80;- const bm25 = new SearchIndex();- bm25.add(- makeObs({- id: "obs_1",- title: "large persisted snapshot ".repeat(40),- narrative: "oversized state set reproduction ".repeat(40),- }),- );- const vector = new VectorIndex();- vector.add(- "obs_1",- "ses_1",- new Float32Array(Array.from({ length: 64 }, (_, i) => i / 10)),- );- const guardedKv = {- ...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (- typeof data === "string" &&- data.length > maxStringPayloadChars- ) {- throw new Error(`oversized state::set payload: ${scope}/${key}`);- }- return kv.set(scope, key, data);- }),- };+ const reloaded = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, reloaded, { bucketSize: 2 });+ const loaded = await persistence.load();+ reloaded.restoreFrom(loaded.vector!);+ kv.ops.length = 0;++ reloaded.add("obs_new", "ses_new", vec([9, 9, 9]));+ await persistence.save();- await new IndexPersistence(guardedKv as never, bm25, vector, {- shardChars: maxStringPayloadChars,- createGeneration: () => "gen_payload_limit",- }).save();-- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("oversized").length).toBe(1);- expect(loaded.vector!.size).toBe(1);+ const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));+ expect(touched).toEqual(new Set([vectorBucketScope(1)]));+ for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true);});- it("falls back to the default shard size for fractional values below one", async () => {- const bm25 = makeBm25("obs_fraction", "fractional shard config");- let newShardWrites = 0;- const guardedKv = {- ...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope.includes(":gen_fraction:")) {- newShardWrites += 1;- if (newShardWrites > 3) {- throw new Error("fractional shard size caused zero-width shards");- }- }- return kv.set(scope, key, data);- }),- };+ it("detects a partial bucket set against the saved count and reports it in status", async () => {+ const vector = vectorWith(+ Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]),+ );+ await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save();+ kv.store.get(vectorBucketScope(1))?.clear();- await new IndexPersistence(guardedKv as never, bm25, null, {- shardChars: 0.5,- createGeneration: () => "gen_fraction",- }).save();+ const persistence = new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 });+ const loaded = await persistence.load();- const manifest = await getBm25Manifest(kv);- expect(manifest.generation).toBe("gen_fraction");- expect(manifest.shards.length).toBe(1);- expect(newShardWrites).toBe(1);+ expect(loaded.state).toBe("buckets");+ expect(loaded.vector!.size).toBe(2);+ expect(loaded.expectedCount).toBe(4);+ expect(persistence.status().vectorCountShortfall).toEqual({ expected: 4, loaded: 2 });});- it("keeps the previous generation when a shard write fails before manifest commit", async () => {- const previous = makeBm25("obs_old", "alpha previous snapshot");- await new IndexPersistence(kv as never, previous, null, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();- const previousManifest = await getBm25Manifest(kv);-- let newShardWrites = 0;+ it("reports storage as unavailable when the metadata read fails", async () => {const failingKv = {...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope.includes(":gen_new:")) {- newShardWrites += 1;- if (newShardWrites === 2) throw new Error("shard write failed");- }- return kv.set(scope, key, data);- }),+ get: async () => {+ throw new Error("engine down");+ },};+ const loaded = await new IndexPersistence(failingKv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null });+ });- const next = makeBm25("obs_new", "bravo new snapshot");- await new IndexPersistence(failingKv as never, next, null, {- shardChars: 80,- createGeneration: () => "gen_new",- }).save();-- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(- previousManifest,- );- await expect(- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),- ).resolves.toBeNull();- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("alpha").length).toBe(1);- expect(loaded.bm25!.search("bravo").length).toBe(0);+ it("treats a missing store as empty, including adapters that return undefined", async () => {+ const undefinedKv = { ...kv, get: async () => undefined };+ const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expect(loaded).toEqual({ vector: null, state: "none", savedAt: null });});+});- it("keeps the previous generation when manifest set rejects before commit", async () => {- const previous = makeBm25("obs_old", "alpha previous snapshot");- await new IndexPersistence(kv as never, previous, null, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();- const previousManifest = await getBm25Manifest(kv);+describe("IndexPersistence migration from the single-string format", () => {+ let kv: MockKV;- const failingKv = {- ...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {- throw new Error("manifest write failed");- }- return kv.set(scope, key, data);- }),- };+ beforeEach(() => {+ kv = mockKV();+ });- const next = makeBm25("obs_new", "bravo new snapshot");- await new IndexPersistence(failingKv as never, next, null, {- shardChars: 80,- createGeneration: () => "gen_new",- }).save();+ it("migrates a sharded vector snapshot into buckets and removes the old shards", async () => {+ const legacy = vectorWith([+ ["obs_a", [0.1, 0.2, 0.3]],+ ["obs_b", [0.4, 0.5, 0.6]],+ ["obs_c", [0.7, 0.8, 0.9]],+ ]);+ await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" });+ await writeLegacyBm25Snapshot(kv);++ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();++ expect(loaded.state).toBe("migrated");+ expect(loaded.savedAt).toBe(new Date(parseInt("mfabcd12", 36)).toISOString());+ expectSameVectors(loaded.vector, legacy);+ expect(loaded.vector!.pendingChanges).toBe(0);+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull();+ expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull();+ for (const [scope, entries] of kv.store) {+ if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0);+ }- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(- previousManifest,- );- await expect(- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),- ).resolves.toBeNull();- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("alpha").length).toBe(1);- expect(loaded.bm25!.search("bravo").length).toBe(0);+ const reloaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expect(reloaded.state).toBe("buckets");+ expectSameVectors(reloaded.vector, legacy);});- it("keeps a generation loadable when manifest set commits before rejecting", async () => {- const previous = makeBm25("obs_old", "alpha previous snapshot");- await new IndexPersistence(kv as never, previous, null, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();+ it("removes the old BM25 shards when missing keys read back as undefined, as the engine SDK returns them", async () => {+ const baseGet = kv.get;+ kv.get = (async <T>(scope: string, key: string) => {+ const value = await baseGet<T>(scope, key);+ return value === null ? undefined : value;+ }) as typeof kv.get;+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_bbbbbbbbbbbb" });+ await writeLegacyBm25Snapshot(kv);- const failingKv = {- ...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {- await kv.set(scope, key, data);- throw new Error("manifest write timed out after commit");- }- return kv.set(scope, key, data);- }),- };+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();- const next = makeBm25("obs_new", "bravo new snapshot");- await new IndexPersistence(failingKv as never, next, null, {- shardChars: 80,- createGeneration: () => "gen_new",- }).save();-- const manifest = await getBm25Manifest(kv);- expect(manifest.generation).toBe("gen_new");- await expect(- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),- ).resolves.toEqual(expect.any(String));- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("bravo").length).toBe(1);+ expect(loaded.state).toBe("migrated");+ expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeUndefined();+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeUndefined();+ for (const [scope, entries] of kv.store) {+ if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0);+ }});- it("deletes a shard that committed before set rejected", async () => {- const previous = makeBm25("obs_old", "alpha previous snapshot");- await new IndexPersistence(kv as never, previous, null, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();- const previousManifest = await getBm25Manifest(kv);+ it("migrates a monolithic vector snapshot", async () => {+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ await writeLegacyVectorSnapshot(kv, legacy, { monolithic: true });- const failingKv = {- ...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope === "mem:index:bm25:bm25:gen_new:00000") {- await kv.set(scope, key, data);- throw new Error("state::set timed out after commit");- }- return kv.set(scope, key, data);- }),- };-- const next = makeBm25("obs_new", "bravo new snapshot");- await new IndexPersistence(failingKv as never, next, null, {- shardChars: 80,- createGeneration: () => "gen_new",- }).save();+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(- previousManifest,- );- await expect(- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),- ).resolves.toBeNull();- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("alpha").length).toBe(1);- expect(loaded.bm25!.search("bravo").length).toBe(0);+ expect(loaded.state).toBe("migrated");+ expectSameVectors(loaded.vector, legacy);+ expect(await kv.get(INDEX_SCOPE, "vectors")).toBeNull();});- it("loads the new generation even when old generation cleanup fails", async () => {- const previous = makeBm25("obs_old", "alpha previous snapshot");- await new IndexPersistence(kv as never, previous, null, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();+ it("leaves an unreadable legacy snapshot in place and writes nothing", async () => {+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ await writeLegacyVectorSnapshot(kv, legacy);+ const manifest = await kv.get<{ shards: Array<{ scope: string; key: string }> }>(INDEX_SCOPE, "vectors:manifest");+ await kv.delete(manifest!.shards[0].scope, manifest!.shards[0].key);+ kv.ops.length = 0;- const cleanupKv = {- ...kv,- delete: vi.fn(async () => {- throw new Error("cleanup failed");- }),- };- const next = makeBm25("obs_new", "bravo new snapshot");- await new IndexPersistence(cleanupKv as never, next, null, {- shardChars: 80,- createGeneration: () => "gen_new",- }).save();-- const manifest = await getBm25Manifest(kv);- expect(manifest.generation).toBe("gen_new");- expect(cleanupKv.delete).toHaveBeenCalled();- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("bravo").length).toBe(1);- expect(loaded.bm25!.search("alpha").length).toBe(0);- });+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();- it("keeps the previous vector generation when vector save fails after BM25 publish", async () => {- const previousBm25 = makeBm25("obs_old", "alpha previous snapshot");- const previousVector = makeVector("obs_old");- await new IndexPersistence(kv as never, previousBm25, previousVector, {- shardChars: 80,- createGeneration: () => "gen_old",- }).save();+ expect(loaded.state).toBe("unavailable");+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();+ expect(kv.ops.filter((o) => o.op === "set" || o.op === "delete")).toEqual([]);+ });- const failingKv = {+ it("keeps the legacy snapshot when migration writes fail and finishes on a later save", async () => {+ const legacy = vectorWith([+ ["obs_a", [0.1, 0.2, 0.3]],+ ["obs_b", [0.4, 0.5, 0.6]],+ ]);+ await writeLegacyVectorSnapshot(kv, legacy);+ let fail = true;+ const flakyKv = {...kv,- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {- if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) {- throw new Error("vector manifest write failed");- }+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("write failed");return kv.set(scope, key, data);- }),+ },};- const nextBm25 = makeBm25("obs_new", "bravo new snapshot");- const nextVector = new VectorIndex();- nextVector.add("obs_new", "ses_1", new Float32Array([0.4, 0.5, 0.6]));-- await new IndexPersistence(failingKv as never, nextBm25, nextVector, {- shardChars: 80,- createGeneration: () => "gen_new",- }).save();-- await expect(- kv.get("mem:index:bm25:vectors:gen_new:00000", "data"),- ).resolves.toBeNull();- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();- expect(loaded.bm25!.search("bravo").length).toBe(1);- expect(loaded.vector!.size).toBe(1);- expect(- loaded.vector!.search(new Float32Array([0.1, 0.2, 0.3]))[0]?.obsId,- ).toBe("obs_old");- });-- it("fails closed when a manifest shard is missing", async () => {- const bm25 = makeBm25("obs_1", "alpha sharded snapshot");- await new IndexPersistence(kv as never, bm25, null, {- shardChars: 80,- createGeneration: () => "gen_missing",- }).save();- const manifest = await getBm25Manifest(kv);- await kv.delete(manifest.shards[0].scope, manifest.shards[0].key);-- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();-- expect(loaded.bm25).toBeNull();- });-- it("fails closed when a manifest shard length mismatches", async () => {- const bm25 = makeBm25("obs_1", "alpha sharded snapshot");- await new IndexPersistence(kv as never, bm25, null, {- shardChars: 80,- createGeneration: () => "gen_mismatch",- }).save();- const manifest = await getBm25Manifest(kv);- const firstShard = manifest.shards[0];- const chunk = await kv.get<string>(firstShard.scope, firstShard.key);- await kv.set(firstShard.scope, firstShard.key, `${chunk}x`);-- const loaded = await new IndexPersistence(- kv as never,- new SearchIndex(),- null,- ).load();-- expect(loaded.bm25).toBeNull();- });- it("fails closed before reading invalid shard descriptors", async () => {- await kv.set<TestIndexShardManifest>(BM25_SCOPE, BM25_MANIFEST_KEY, {- v: 1,- chars: 10,- shards: [{ scope: "", key: "data", chars: 10 }],- });- const guardedKv = {- ...kv,- get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {- if (scope === "") {- throw new Error("invalid shard descriptor was read");- }- return kv.get(scope, key);- }),- };+ const live = new VectorIndex();+ const persistence = new IndexPersistence(flakyKv as never, live, { bucketSize: 16 });+ const loaded = await persistence.load();+ expect(loaded.state).toBe("unavailable");+ expectSameVectors(loaded.vector, legacy);+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();- const loaded = await new IndexPersistence(- guardedKv as never,- new SearchIndex(),- null,- ).load();+ live.restoreFrom(loaded.vector!);+ fail = false;+ await persistence.save();- expect(loaded.bm25).toBeNull();- expect(guardedKv.get).not.toHaveBeenCalledWith("", "data");+ const next = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();+ expect(next.state).toBe("buckets");+ expectSameVectors(next.vector, legacy);+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull();});- it("scheduleSave debounces multiple calls", async () => {- const bm25 = new SearchIndex();- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 });+ it("without vector search, removes only the old BM25 snapshot", async () => {+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ await writeLegacyVectorSnapshot(kv, legacy);+ await writeLegacyBm25Snapshot(kv);- persistence.scheduleSave();- persistence.scheduleSave();- persistence.scheduleSave();+ const loaded = await new IndexPersistence(kv as never, null, { bucketSize: 16 }).load();- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toBeNull();-- vi.advanceTimersByTime(5000);- await vi.runAllTimersAsync();-- const saved = await kv.get<string>(BM25_SCOPE, BM25_MANIFEST_KEY);- expect(saved).not.toBeNull();+ expect(loaded).toEqual({ vector: null, state: "none", savedAt: null });+ expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull();+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();});+});- it("stop clears the pending timer", async () => {- const bm25 = new SearchIndex();- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));- const persistence = new IndexPersistence(kv as never, bm25, null);-- persistence.scheduleSave();- persistence.stop();+describe("IndexPersistence scheduling", () => {+ let kv: MockKV;- vi.advanceTimersByTime(10000);- const saved = await kv.get<string>(BM25_SCOPE, BM25_MANIFEST_KEY);- expect(saved).toBeNull();+ beforeEach(() => {+ vi.useFakeTimers();+ kv = mockKV();});- it("returns null indexes when nothing has been saved", async () => {- const bm25 = new SearchIndex();- const persistence = new IndexPersistence(kv as never, bm25, null);-- const loaded = await persistence.load();- expect(loaded.bm25).toBeNull();- expect(loaded.vector).toBeNull();+ afterEach(() => {+ vi.useRealTimers();});- it("scheduled save swallows kv.set rejection without unhandledRejection (#204)", async () => {+ it("scheduled saves never raise unhandled rejections", async () => {const failingKv = {- ...mockKV(),+ ...kv,set: vi.fn(async () => {- const err = new Error(- "TIMEOUT: invocation timed out after 30000ms",- ) as Error & { code?: string; function_id?: string };+ const err = new Error("TIMEOUT: invocation timed out after 30000ms") as Error & { code?: string };err.code = "TIMEOUT";- err.function_id = "state::set";throw err;}),};- const bm25 = new SearchIndex();- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));- const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });-+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ const persistence = new IndexPersistence(failingKv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 });let unhandled = false;const onUnhandled = () => {unhandled = true;};process.on("unhandledRejection", onUnhandled);-try {persistence.scheduleSave();- vi.advanceTimersByTime(5000);- await vi.runAllTimersAsync();- // give microtasks a chance to flush- await Promise.resolve();+ await vi.advanceTimersByTimeAsync(5000);expect(failingKv.set).toHaveBeenCalled();expect(unhandled).toBe(false);} finally {@@ -742,143 +477,12 @@ describe("IndexPersistence", () => {}});- it("save() does not throw when kv.set rejects (#204)", async () => {- const failingKv = {- ...mockKV(),- set: vi.fn(async () => {- throw new Error("TIMEOUT");- }),- };- const bm25 = new SearchIndex();- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));- const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });-- await expect(persistence.save()).resolves.toBeUndefined();- });-- // #797: first run after upgrading to 0.9.25 crashed with- // 'TypeError: Cannot read properties of undefined (reading "v")'- // because some iii-state adapters return `undefined` (not `null`)- // for a missing key. The load path's `value !== null` check passed- // undefined to loadManifestData, which then read `undefined.v`.- it("load() returns null instead of crashing when kv.get returns undefined for the manifest (#797)", async () => {- const undefinedKv = {- ...mockKV(),- get: vi.fn(async () => undefined),- };- const persistence = new IndexPersistence(- undefinedKv as never,- new SearchIndex(),- null,- );-- const loaded = await persistence.load();- expect(loaded.bm25).toBeNull();- expect(loaded.vector).toBeNull();- });-- it("load() does not crash when a manifest row value is the wrong shape (#797)", async () => {- const wrongShapeKv = {- ...mockKV(),- get: vi.fn(async () => "not-a-manifest"),- };- const persistence = new IndexPersistence(- wrongShapeKv as never,- new SearchIndex(),- null,- );-- await expect(persistence.load()).resolves.toBeDefined();- });-});--describe("index_persist audit gating", () => {- let kv: ReturnType<typeof mockKV>;- let previousFlag: string | undefined;-- beforeEach(() => {- // AGENTMEMORY_* variables are documented as living in- // ~/.agentmemory/.env, so a developer running the suite on a- // configured machine can inherit this one. Clear it going in and put- // whatever was there back on the way out.- previousFlag = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;- delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;- vi.useFakeTimers();- kv = mockKV();- });-- afterEach(() => {- vi.useRealTimers();- if (previousFlag === undefined) {- delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;- } else {- process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previousFlag;- }- });-- async function indexPersistEntries(): Promise<Array<{ operation: string }>> {- const entries = await kv.list<{ operation: string }>("mem:audit");- return entries.filter((entry) => entry.operation === "index_persist");- }-- it("writes no index_persist audit entries by default", async () => {- const persistence = new IndexPersistence(- kv as never,- makeBm25("obs_1", "auth handler"),- null,- );-- await persistence.save();-- expect(await indexPersistEntries()).toEqual([]);- });-- it.each(["1", " 1 ", "true", "TRUE", " true "])(- "writes index_persist audit entries when set to %j",- async (value) => {- process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value;- const persistence = new IndexPersistence(- kv as never,- makeBm25("obs_1", "auth handler"),- null,- );-- await persistence.save();-- expect((await indexPersistEntries()).length).toBeGreaterThan(0);- },- );-- // Anything that is not an affirmative stays off. "0" and "false" are the- // ones an operator is likely to reach for to disable it, and they must- // not read as "present, therefore enabled".- it.each(["0", "false", "yes", "", " "])(- "keeps auditing off when set to %j",- async (value) => {- process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value;- const persistence = new IndexPersistence(- kv as never,- makeBm25("obs_1", "auth handler"),- null,- );-- await persistence.save();-- expect(await indexPersistEntries()).toEqual([]);- },- );-- it("still persists the index when auditing is off", async () => {- const persistence = new IndexPersistence(- kv as never,- makeBm25("obs_1", "auth handler"),- null,- );-- await persistence.save();-- const loaded = await persistence.load();- expect(loaded.bm25).not.toBeNull();- expect(loaded.bm25!.size).toBe(1);+ it("stop clears the pending timer", async () => {+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 });+ persistence.scheduleSave();+ persistence.stop();+ await vi.advanceTimersByTimeAsync(10_000);+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();});});diff --git a/test/search.test.ts b/test/search.test.tsindex 84b8134e..99b7f4d6 100644--- a/test/search.test.ts+++ b/test/search.test.ts@@ -209,4 +209,21 @@ describe("mem::search", () => {setVectorIndex(null);setEmbeddingProvider(null);});++ it("a cold-start search rebuilds BM25 without touching the persisted vector index", async () => {+ const vector = new VectorIndex();+ vector.add("obs_persisted", "ses_1", new Float32Array([0.1, 0.2, 0.3]));+ setVectorIndex(vector);+ getSearchIndex().clear();++ const result = (await sdk.trigger("mem::search", {+ query: "auth middleware",+ })) as { results: Array<{ observation: CompressedObservation }> };++ expect(result.results[0]?.observation.id).toBe("obs_a");+ expect(getVectorIndex()?.has("obs_persisted")).toBe(true);+ expect(getVectorIndex()?.size).toBe(1);++ setVectorIndex(null);+ });});diff --git a/test/status.test.ts b/test/status.test.tsindex 6ecfb50c..2160a65f 100644--- a/test/status.test.ts+++ b/test/status.test.ts@@ -27,6 +27,8 @@ function inputs(overrides: Partial<StatusInputs> = {}): StatusInputs {observationsIndexed: 110,missingObservations: 0,sessions: 95,+ bm25Incomplete: false,+ pendingVectorBackfill: 0,},graph: {totalNodes: 55,@@ -93,6 +95,28 @@ describe("evaluateStatus", () => {expect(report.status).toBe("info");});+ it("reports an incomplete BM25 rebuild as an error", () => {+ const report = evaluateStatus(inputs({ index: { ...inputs().index, bm25Incomplete: true } }));+ expect(report.status).toBe("error");+ expect(codes(report)).toContain("bm25-rebuild-incomplete");+ });++ it("notes a pending vector backfill without raising the overall status", () => {+ const report = evaluateStatus(inputs({ index: { ...inputs().index, pendingVectorBackfill: 42 } }));+ expect(report.status).toBe("info");+ const problem = report.problems.find((p) => p.code === "index-vector-backfill-pending");+ expect(problem?.message).toBe("42 documents are waiting for a vector embedding.");+ });++ it("reports a vector count shortfall from the last save as a warning", () => {+ const report = evaluateStatus(+ inputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, buckets: 3, pendingChanges: 0, vector: null, vectorCountShortfall: { expected: 100, loaded: 40 } } }),+ );+ expect(report.status).toBe("warn");+ const problem = report.problems.find((p) => p.code === "index-vector-count-shortfall");+ expect(problem?.message).toContain("40 of 100 vectors");+ });+it("checks snapshot presence when extraction is on, and snapshot age whenever a snapshot exists", () => {const noSnapshot = { totalNodes: 0, totalEdges: 0, fromSnapshot: false };expect(codes(evaluateStatus(inputs({ graph: noSnapshot })))).toEqual(["graph-no-snapshot"]);