Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

perf(index): store vectors in fixed buckets and rebuild BM25 from content (#1416)

Rohit Ghumarebcf4f0d00d1c
diff --git a/.env.example b/.env.example
index 4b9deb0b..60d201e5 100644
--- a/.env.example
+++ b/.env.example
@@ -130,6 +130,9 @@
# AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging.
# AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental).
# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately.
+# AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed.
+# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save.
+# AGENTMEMORY_VECTOR_BACKFILL=all # Required to backfill an entire store's vectors at once (no prior vector save, or a stale legacy manifest); unset means that case waits for an explicit opt-in.
# -----------------------------------------------------------------------------
# 6. CLI / runtime knobs
diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md
index 5fe07156..44e300d1 100644
--- a/plugin/skills/agentmemory-config/REFERENCE.md
+++ b/plugin/skills/agentmemory-config/REFERENCE.md
@@ -3,7 +3,7 @@
Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.
<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->
-Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables:
+Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 47 recognized variables:
- `AGENTMEMORY_AGENT_SCOPE`
- `AGENTMEMORY_ALLOW_AGENT_SDK`
@@ -46,6 +46,9 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e
- `AGENTMEMORY_TOOLS`
- `AGENTMEMORY_URL`
- `AGENTMEMORY_USE_DOCKER`
+- `AGENTMEMORY_VECTOR_BACKFILL`
+- `AGENTMEMORY_VECTOR_BACKFILL_MAX`
+- `AGENTMEMORY_VECTOR_BUCKET_SIZE`
- `AGENTMEMORY_VERBOSE`
- `AGENTMEMORY_VIEWER_HOST`
- `AGENTMEMORY_VIEWER_URL`
diff --git a/src/config.ts b/src/config.ts
index ac82bd44..a7986f96 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -477,6 +477,24 @@ export function getIndexSaveIntervalMs(): number {
return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS;
}
+export const VECTOR_BUCKET_SIZE_DEFAULT = 500;
+
+export function getVectorBucketSize(): number {
+ const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKET_SIZE"], VECTOR_BUCKET_SIZE_DEFAULT);
+ return raw > 0 ? raw : VECTOR_BUCKET_SIZE_DEFAULT;
+}
+
+export const VECTOR_BACKFILL_MAX_DEFAULT = 500;
+
+export function getVectorBackfillMax(): number {
+ const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL_MAX"], VECTOR_BACKFILL_MAX_DEFAULT);
+ return raw > 0 ? raw : VECTOR_BACKFILL_MAX_DEFAULT;
+}
+
+export function isVectorBackfillAllEnabled(): boolean {
+ return getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL"] === "all";
+}
+
export function isStandaloneMcp(): boolean {
return getMergedEnv()["STANDALONE_MCP"] === "true";
}
diff --git a/src/functions/search.ts b/src/functions/search.ts
index 8abce07f..2d2f4147 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -9,7 +9,12 @@ import type { EmbeddingProvider } from '../types.js'
import { memoryToObservation } from '../state/memory-utils.js'
import { recordAccessBatch } from './access-tracker.js'
import { logger } from "../logger.js";
-import { getAgentId, isAgentScopeIsolated } from "../config.js";
+import {
+ getAgentId,
+ getVectorBackfillMax,
+ isAgentScopeIsolated,
+ isVectorBackfillAllEnabled,
+} from "../config.js";
let index: SearchIndex | null = null
let vectorIndex: VectorIndex | null = null
@@ -45,6 +50,19 @@ export function isMemoryIndexReady(): boolean {
return memoryIndexReady
}
+let bm25RebuildIncomplete = false
+export function isBm25RebuildIncomplete(): boolean {
+ return bm25RebuildIncomplete
+}
+
+let pendingVectorBackfill = 0
+export function getPendingVectorBackfillCount(): number {
+ return pendingVectorBackfill
+}
+export function setPendingVectorBackfillCount(n: number): void {
+ pendingVectorBackfill = Math.max(0, n)
+}
+
export function getSearchIndex(): SearchIndex {
if (!index) index = new SearchIndex()
return index
@@ -399,6 +417,146 @@ export async function rebuildIndex(kv: StateKV): Promise<number> {
return indexed
}
+export type VectorBackfillJob = {
+ id: string
+ sessionId: string
+ text: string
+ context: { kind: "memory" | "observation" | "synthetic"; logId: string }
+}
+
+const SESSIONS_LIST_RETRY_ATTEMPTS = 3
+const SESSIONS_LIST_RETRY_BASE_MS = 250
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+async function listSessionsWithRetry(kv: StateKV): Promise<Session[] | null> {
+ let lastErr: unknown
+ for (let attempt = 0; attempt < SESSIONS_LIST_RETRY_ATTEMPTS; attempt++) {
+ try {
+ return await kv.list<Session>(KV.sessions)
+ } catch (err) {
+ lastErr = err
+ if (attempt < SESSIONS_LIST_RETRY_ATTEMPTS - 1) {
+ await delay(SESSIONS_LIST_RETRY_BASE_MS * (attempt + 1))
+ }
+ }
+ }
+ logger.warn('rebuildKeywordIndex: failed to load sessions after retries', {
+ attempts: SESSIONS_LIST_RETRY_ATTEMPTS,
+ error: lastErr instanceof Error ? lastErr.message : String(lastErr),
+ })
+ return null
+}
+
+export async function rebuildKeywordIndex(
+ kv: StateKV,
+ vectorBackfillSince?: string | null,
+): Promise<{ documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number }> {
+ const idx = getSearchIndex()
+ idx.clear()
+ memoryIndexReady = false
+ bm25RebuildIncomplete = false
+ const vi = vectorIndex
+ const backfillEligible = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined
+ const wholeStoreBackfill = vectorBackfillSince === null
+ const backfillGated = backfillEligible && wholeStoreBackfill && !isVectorBackfillAllEnabled()
+ const backfillActive = backfillEligible && !backfillGated
+ const cutoff = typeof vectorBackfillSince === 'string' ? Date.parse(vectorBackfillSince) : Number.NaN
+ const backfillCap = backfillActive ? getVectorBackfillMax() : 0
+ const vectorJobs: VectorBackfillJob[] = []
+ let fullBackfillPending = 0
+ const consider = (
+ id: string,
+ sessionId: string,
+ text: string,
+ timestamp: string | undefined,
+ kind: "memory" | "observation",
+ ): void => {
+ if (!backfillEligible || vi?.has(id)) return
+ if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return
+ if (backfillGated) {
+ fullBackfillPending++
+ return
+ }
+ if (!backfillActive || vectorJobs.length >= backfillCap) return
+ vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } })
+ }
+
+ let documents = 0
+ let memoriesLoaded = false
+ try {
+ const memories = await kv.list<Memory>(KV.memories)
+ memoriesLoaded = true
+ for (const memory of memories) {
+ if (memory.isLatest === false) continue
+ if (!memory.title || !memory.content) continue
+ idx.add(memoryToObservation(memory))
+ consider(memory.id, memory.sessionIds?.[0] ?? 'memory', memory.title + ' ' + memory.content, memory.createdAt, "memory")
+ documents++
+ }
+ } catch (err) {
+ logger.warn('rebuildKeywordIndex: failed to load memories', {
+ error: err instanceof Error ? err.message : String(err),
+ })
+ }
+
+ const sessions = await listSessionsWithRetry(kv)
+ if (sessions === null) {
+ bm25RebuildIncomplete = true
+ if (memoriesLoaded) memoryIndexReady = true
+ return { documents, vectorJobs, fullBackfillPending }
+ }
+ const failedSessions: string[] = []
+ for (let batch = 0; batch < sessions.length; batch += 10) {
+ const chunk = sessions.slice(batch, batch + 10)
+ const results = await Promise.all(
+ chunk.map(async (s) => {
+ try {
+ return await kv.list<CompressedObservation>(KV.observations(s.id))
+ } catch {
+ failedSessions.push(s.id)
+ return [] as CompressedObservation[]
+ }
+ })
+ )
+ for (const obs of results.flat()) {
+ if (!obs.title || !obs.narrative) continue
+ idx.add(obs)
+ consider(obs.id, obs.sessionId, obs.title + ' ' + obs.narrative, obs.timestamp, "observation")
+ documents++
+ }
+ }
+ if (failedSessions.length > 0) {
+ bm25RebuildIncomplete = true
+ logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions })
+ }
+ if (memoriesLoaded) memoryIndexReady = true
+ return { documents, vectorJobs, fullBackfillPending }
+}
+
+const BACKFILL_SAVE_EVERY_BATCHES = 10
+
+export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {
+ const batchSize = getRebuildEmbedBatchSize()
+ let added = 0
+ let batchesSinceSave = 0
+ setPendingVectorBackfillCount(jobs.length)
+ for (let offset = 0; offset < jobs.length; offset += batchSize) {
+ const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))
+ added += ok
+ setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize))
+ batchesSinceSave++
+ if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) {
+ await flushIndexSave()
+ batchesSinceSave = 0
+ }
+ }
+ if (added > 0) await flushIndexSave()
+ return added
+}
+
export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {
sdk.registerFunction(
'mem::search',
@@ -480,10 +638,10 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {
// Share one rebuild across concurrent cold-start queries so they
// don't each walk the whole corpus and saturate the pool.
if (!rebuildPromise) {
- rebuildPromise = rebuildIndex(kv)
- .then((count) => {
- logger.info('Search index rebuilt', { entries: count })
- return count
+ rebuildPromise = rebuildKeywordIndex(kv)
+ .then((result) => {
+ logger.info('Search index rebuilt', { entries: result.documents })
+ return result.documents
})
.catch((err) => {
logger.warn('Index rebuild failed', {
diff --git a/src/functions/status.ts b/src/functions/status.ts
index 0b3ab531..c6577c44 100644
--- a/src/functions/status.ts
+++ b/src/functions/status.ts
@@ -1,4 +1,4 @@
-import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js";
+import type { IndexLegStatus, IndexPersistenceStatus } from "../state/index-persistence.js";
export type StatusLevel = "ok" | "info" | "warn" | "error";
@@ -57,6 +57,8 @@ export interface StatusInputs {
observationsIndexed: number;
missingObservations: number | null;
sessions: number | null;
+ bm25Incomplete: boolean;
+ pendingVectorBackfill: number;
};
graph: GraphStatsInput | null;
graphExtractionEnabled: boolean;
@@ -85,7 +87,6 @@ export interface StatusReport {
const FAILURE_RATE_THRESHOLD = 0.2;
const FAILURE_MIN_CALLS = 5;
const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60;
-const VECTOR_STRING_WARN_RATIO = 0.8;
const LEVEL_RANK: Record<StatusLevel, number> = { ok: 0, info: 1, warn: 2, error: 3 };
const FUNCTION_FIXES: Record<string, string> = {
@@ -173,42 +174,54 @@ export function evaluateStatus(input: StatusInputs): StatusReport {
});
}
+ if (input.index.bm25Incomplete) {
+ problems.push({
+ level: "error",
+ code: "bm25-rebuild-incomplete",
+ message: "The keyword index could not load every session at boot, so some observations are missing from search until the next successful rebuild.",
+ fix: "Check the server log for the session listing failure. Restarting or the next cold-start search retries automatically.",
+ });
+ }
+
+ if (input.index.pendingVectorBackfill > 0) {
+ problems.push({
+ level: "info",
+ code: "index-vector-backfill-pending",
+ message: `${input.index.pendingVectorBackfill} documents are waiting for a vector embedding.`,
+ fix: "This runs in the background and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. If it is stuck at a nonzero count with no embedding provider errors, set AGENTMEMORY_VECTOR_BACKFILL=all to run a full backfill.",
+ });
+ }
+
const persistence = input.indexPersistence ?? null;
- if (persistence) {
- const legs: Array<[string, IndexLegStatus | null]> = [
- ["BM25", persistence.bm25],
- ["vector", persistence.vector],
- ];
- for (const [label, leg] of legs) {
- if (!leg) continue;
- if (leg.lastError) {
- problems.push({
- level: "error",
- code: "index-save-failing",
- message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`,
- fix: "Check the server log for the failing state write. The next save retries automatically.",
- });
- }
- const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined);
- if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {
- problems.push({
- level: "warn",
- code: "index-save-stale",
- message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,
- fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",
- });
- }
+ const vectorLeg = persistence?.vector ?? null;
+ if (persistence && vectorLeg) {
+ if (vectorLeg.lastError) {
+ problems.push({
+ level: "error",
+ code: "index-save-failing",
+ message: `The vector index could not be saved: ${vectorLeg.lastError}. Search keeps working from memory, but vectors added since the last save are lost on restart.`,
+ fix: "Check the server log for the failing state write. The next save retries automatically.",
+ });
}
- const vectorChars = persistence.vector?.serializedChars ?? null;
- if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) {
+ const dirtyAge = secondsBetween(input.now, vectorLeg.dirtySince ?? undefined);
+ if (!vectorLeg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {
problems.push({
level: "warn",
- code: "vector-index-near-string-limit",
- message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`,
- fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.",
+ code: "index-save-stale",
+ message: `The vector index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,
+ fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",
});
}
}
+ if (persistence?.vectorCountShortfall) {
+ const { expected, loaded } = persistence.vectorCountShortfall;
+ problems.push({
+ level: "warn",
+ code: "index-vector-count-shortfall",
+ message: `Only ${loaded} of ${expected} vectors loaded from the last save; a bounded backfill re-embeds the rest.`,
+ fix: "No action needed — missing vectors are queued for backfill up to the per-boot cap.",
+ });
+ }
let graph: StatusReport["graph"] = null;
if (input.graph) {
@@ -300,16 +313,13 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string {
function indexPersistenceRows(report: StatusReport): string {
const persistence = report.indexPersistence;
if (!persistence) return "";
- let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25)));
+ let rows = row("BM25 index", "rebuilt from stored content at boot");
if (persistence.vector) {
rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector)));
- const chars = persistence.vector.serializedChars;
- if (chars !== null) {
- rows += row(
- "Vector index size",
- escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`),
- );
- }
+ rows += row(
+ "Vector storage",
+ escapeHtml(`${persistence.buckets} buckets, ${persistence.pendingChanges} unsaved changes`),
+ );
}
return rows;
}
@@ -396,6 +406,8 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off")
${row("Observations indexed", escapeHtml(idx.observationsIndexed))}
${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))}
${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}
+${row("BM25 rebuild", idx.bm25Incomplete ? '<span class="warn">incomplete</span>' : "complete")}
+${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))}
${indexPersistenceRows(report)}
</table>
<h2>Knowledge graph</h2><table>
diff --git a/src/index.ts b/src/index.ts
index ff465afe..c043a31f 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -21,7 +21,6 @@ import {
createImageEmbeddingProvider,
} from "./providers/index.js";
import { StateKV } from "./state/kv.js";
-import { KV } from "./state/schema.js";
import { VectorIndex } from "./state/vector-index.js";
import { HybridSearch } from "./state/hybrid-search.js";
import { IndexPersistence } from "./state/index-persistence.js";
@@ -35,13 +34,14 @@ import { registerDiskSizeManager } from "./functions/disk-size-manager.js";
import { registerCompressFunction } from "./functions/compress.js";
import {
registerSearchFunction,
- rebuildIndex,
- reconcileIndex,
+ backfillVectors,
+ rebuildKeywordIndex,
getSearchIndex,
setVectorIndex,
setEmbeddingProvider,
setIndexPersistence,
setHybridRanker,
+ setPendingVectorBackfillCount,
} from "./functions/search.js";
import { registerContextFunction } from "./functions/context.js";
import { registerSummarizeFunction } from "./functions/summarize.js";
@@ -395,23 +395,13 @@ async function main() {
const healthMonitor = registerHealthMonitor(sdk, kv);
- const indexPersistence = new IndexPersistence(kv, bm25Index, vectorIndex);
- // Wire the persistence hook so delete paths can flush BM25/vector
- // index mutations to disk. Without this, an in-memory remove can be
- // lost across a hard process exit and the persisted snapshot
- // restores the deleted entry at next boot.
+ const indexPersistence = new IndexPersistence(kv, vectorIndex);
setIndexPersistence(indexPersistence);
const loaded = await indexPersistence.load().catch((err) => {
- console.warn(`[agentmemory] Failed to load persisted index:`, err);
+ console.warn(`[agentmemory] Failed to load persisted vector index:`, err);
return null;
});
- if (loaded?.bm25 && loaded.bm25.size > 0) {
- bm25Index.restoreFrom(loaded.bm25);
- bootLog(
- `Loaded persisted BM25 index (${bm25Index.size} docs)`,
- );
- }
if (loaded?.vector && vectorIndex && loaded.vector.size > 0) {
// Persisted vectors carry whatever dimension the provider had when
// they were written. If the active provider declares a different
@@ -436,6 +426,7 @@ async function main() {
const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", ");
const dropStale = isDropStaleIndexEnabled();
if (dropStale) {
+ for (const [obsId] of loaded.vector.entries()) vectorIndex.markRemoved(obsId);
console.warn(
`[agentmemory] Persisted vector index has ${mismatches.length} of ` +
`${loaded.vector.size} vectors with the wrong dimension. Active ` +
@@ -466,79 +457,42 @@ async function main() {
}
}
- const needsRebuild = bm25Index.size === 0;
-
- if (needsRebuild) {
- // Fire-and-forget. rebuildIndex iterates every observation across
- // every session and AWAITS an embedding-provider call per record.
- // On a large corpus + rate-limited embedding endpoint that can
- // take HOURS; awaiting it here blocks every subsequent boot step
- // (including startViewerServer below, leaving the viewer port
- // unbound for the duration). The index lazily fills in over time
- // and search degrades gracefully — partial coverage > no viewer
- // for hours. Errors still surface via the inner .catch.
- void rebuildIndex(kv)
- .then((indexCount) => {
- if (indexCount > 0) {
- bootLog(`Search index rebuilt: ${indexCount} entries`);
- indexPersistence.scheduleSave();
- }
- })
- .catch((err) => {
- console.warn(`[agentmemory] Failed to rebuild search index:`, err);
- });
- } else {
- // Backfill memories into BM25 for users upgrading from <0.9.5: prior
- // versions of mem::remember never indexed memories, so the persisted
- // BM25 covers observations only and `memory_smart_search` returns
- // empty for everything saved via memory_save (#257). Walk KV.memories
- // and add the ones missing from the restored index. Idempotent on
- // re-runs because SearchIndex.has() short-circuits already-indexed
- // ids.
- try {
- const memories = await kv.list<import("./types.js").Memory>(KV.memories);
- let backfilled = 0;
- for (const memory of memories) {
- if (memory.isLatest === false) continue;
- if (!memory.title || !memory.content) continue;
- if (bm25Index.has(memory.id)) continue;
- bm25Index.add({
- id: memory.id,
- sessionId: memory.sessionIds?.[0] ?? "memory",
- timestamp: memory.createdAt,
- type: "decision",
- title: memory.title,
- facts: [memory.content],
- narrative: memory.content,
- concepts: memory.concepts,
- files: memory.files,
- importance: memory.strength,
- });
- backfilled++;
- }
- if (backfilled > 0) {
- bootLog(
- `Backfilled ${backfilled} memories into BM25 (legacy index gap)`,
- );
- indexPersistence.scheduleSave();
- }
- } catch (err) {
- console.warn(
- `[agentmemory] Failed to backfill memories into BM25:`,
- err,
+ const vectorCountShortfall =
+ Boolean(loaded?.vector) &&
+ loaded?.expectedCount !== undefined &&
+ loaded.vector!.size < loaded.expectedCount;
+ const vectorBackfillSince =
+ !loaded || loaded.state === "unavailable"
+ ? undefined
+ : loaded.state === "none" || vectorCountShortfall
+ ? null
+ : loaded.savedAt;
+ const keywordStart = Date.now();
+ try {
+ const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince);
+ bootLog(
+ `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`,
+ );
+ setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending);
+ if (keyword.fullBackfillPending > 0) {
+ bootLog(
+ `Vector backfill needs ${keyword.fullBackfillPending} embeddings but a full backfill was not started ` +
+ `(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`,
);
}
- void reconcileIndex(kv)
- .then((count) => {
- if (count > 0) {
- bootLog(
- `Search index reconciled: ${count} observations missing from the persisted snapshot were re-indexed`,
- );
- }
- })
- .catch((err) => {
- console.warn(`[agentmemory] Failed to reconcile search index:`, err);
- });
+ if (keyword.vectorJobs.length > 0) {
+ bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`);
+ void backfillVectors(keyword.vectorJobs)
+ .then((count) => {
+ setPendingVectorBackfillCount(keyword.fullBackfillPending);
+ if (count > 0) bootLog(`Vector index backfilled: ${count} entries`);
+ })
+ .catch((err) => {
+ console.warn(`[agentmemory] Failed to backfill vectors:`, err);
+ });
+ }
+ } catch (err) {
+ console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err);
}
// Ready / Endpoints lines are emitted via `bootLog` so they're
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 28a24d5a..d046d0dd 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -1,30 +1,21 @@
-import { SearchIndex } from "./search-index.js";
-import { VectorIndex } from "./vector-index.js";
+import { VectorIndex, base64ToFloat32, float32ToBase64 } from "./vector-index.js";
import type { StateKV } from "./kv.js";
-import { KV, generateId } from "./schema.js";
+import { KV } from "./schema.js";
import { logger } from "../logger.js";
import { safeAudit } from "../functions/audit.js";
-import { getIndexSaveIntervalMs } from "../config.js";
+import { getIndexSaveIntervalMs, getVectorBucketSize } from "../config.js";
-export const V8_MAX_STRING_CHARS = 536_870_888;
const FAILURE_LOG_THROTTLE_MS = 60_000;
const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence";
-const BM25_KEY = "data";
-const BM25_MANIFEST_KEY = "data:manifest";
-const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`;
-const VECTOR_KEY = "vectors";
-const VECTOR_MANIFEST_KEY = "vectors:manifest";
-const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`;
-const INDEX_SHARD_KEY = "data";
-const DEFAULT_INDEX_SHARD_CHARS = 2_000_000;
-
-// mem:audit exists to record structural deletions of user data — that is
-// the policy stated at the top of src/functions/audit.ts. Index shard
-// writes and manifest publishes remove no user rows, so they fall outside
-// it, yet a single save() emits three of them: on a real store they
-// reached 59876 of 84028 entries (71%), which is what makes the audit log
-// slow to query and bloats startup. Off by default; set
-// AGENTMEMORY_AUDIT_INDEX_PERSIST=1 when debugging index persistence.
+const LEGACY_BM25_KEY = "data";
+const LEGACY_BM25_MANIFEST_KEY = "data:manifest";
+const LEGACY_VECTOR_KEY = "vectors";
+const LEGACY_VECTOR_MANIFEST_KEY = "vectors:manifest";
+const VECTOR_META_KEY = "vectors:meta";
+const VECTOR_BUCKET_SCOPE_PREFIX = `${KV.bm25Index}:vec:`;
+const WRITE_CONCURRENCY = 32;
+const LOAD_CONCURRENCY = 8;
+
function auditIndexPersistEnabled(): boolean {
const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
if (!raw) return false;
@@ -39,45 +30,57 @@ type IndexShardManifest = {
chars: number;
};
+type VectorMeta = {
+ v: 3;
+ bucketCount: number;
+ savedAt: string;
+ count: number;
+};
+
+type PersistedVector = {
+ id: string;
+ s: string;
+ e: string;
+};
+
type IndexPersistenceOptions = {
- shardChars?: number;
- createGeneration?: () => string;
saveIntervalMs?: number;
+ bucketSize?: number;
now?: () => number;
};
-export type IndexLeg = "bm25" | "vector";
-
export interface IndexLegStatus {
lastSavedAt: string | null;
lastError: string | null;
lastErrorAt: string | null;
dirtySince: string | null;
- serializedChars: number | null;
+}
+
+export interface VectorCountShortfall {
+ expected: number;
+ loaded: number;
}
export interface IndexPersistenceStatus {
saveIntervalMs: number;
saving: boolean;
- bm25: IndexLegStatus;
+ buckets: number;
+ pendingChanges: number;
vector: IndexLegStatus | null;
+ vectorCountShortfall: VectorCountShortfall | null;
}
-function emptyLegStatus(): IndexLegStatus {
- return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null };
-}
+export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable";
-function shardChars(options: IndexPersistenceOptions): number {
- const configured = options.shardChars;
- if (typeof configured !== "number" || !Number.isFinite(configured)) {
- return DEFAULT_INDEX_SHARD_CHARS;
- }
- const wholeChars = Math.floor(configured);
- return wholeChars >= 1 ? wholeChars : DEFAULT_INDEX_SHARD_CHARS;
+export interface VectorLoadResult {
+ vector: VectorIndex | null;
+ state: VectorLoadState;
+ savedAt: string | null;
+ expectedCount?: number;
}
-function createIndexGeneration(): string {
- return generateId("idx");
+export function vectorBucketScope(bucket: number): string {
+ return `${VECTOR_BUCKET_SCOPE_PREFIX}${String(bucket).padStart(4, "0")}`;
}
function statePath(scope: string, key: string): string {
@@ -88,6 +91,13 @@ function errorMessage(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
+function generationTime(generation: string | undefined): string | null {
+ const stamp = generation?.split("_")[1];
+ if (!stamp) return null;
+ const ms = parseInt(stamp, 36);
+ return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : null;
+}
+
function isValidShardDescriptor(
shard: unknown,
): shard is IndexShardManifest["shards"][number] {
@@ -99,31 +109,55 @@ function isValidShardDescriptor(
typeof candidate.key === "string" &&
candidate.key.length > 0 &&
Number.isInteger(candidate.chars) &&
- candidate.chars >= 0
+ (candidate.chars as number) >= 0
);
}
+function isPersistedVector(row: unknown): row is PersistedVector {
+ if (!row || typeof row !== "object") return false;
+ const candidate = row as Partial<PersistedVector>;
+ return typeof candidate.id === "string" && typeof candidate.s === "string" && typeof candidate.e === "string";
+}
+
+async function inBatches<T>(items: T[], size: number, run: (item: T) => Promise<void>): Promise<unknown[]> {
+ const failures: unknown[] = [];
+ for (let offset = 0; offset < items.length; offset += size) {
+ const results = await Promise.allSettled(items.slice(offset, offset + size).map(run));
+ for (const result of results) {
+ if (result.status === "rejected") failures.push(result.reason);
+ }
+ }
+ return failures;
+}
+
+function emptyLegStatus(): IndexLegStatus {
+ return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null };
+}
+
export class IndexPersistence {
private timer: ReturnType<typeof setTimeout> | null = null;
- private lastFailureLogAt = new Map<IndexLeg, number>();
+ private lastFailureLogAt = 0;
private running: Promise<void> | null = null;
private queued: Promise<void> | null = null;
private stopped = false;
private lastSaveAt: number;
private dirtyEpoch = 0;
private markedDuringRunAt: number | null = null;
- private legs: Record<IndexLeg, IndexLegStatus> = {
- bm25: emptyLegStatus(),
- vector: emptyLegStatus(),
- };
+ private metaWritten = false;
+ private leg: IndexLegStatus = emptyLegStatus();
private readonly saveIntervalMs: number;
+ private readonly bucketSize: number;
private readonly now: () => number;
+ private bucketOfId: Map<string, number> = new Map();
+ private bucketCounts: Map<number, number> = new Map();
+ private highestBucket = 0;
+ private hasOpenBucket = false;
+ private vectorCountShortfall: VectorCountShortfall | null = null;
constructor(
private kv: StateKV,
- private bm25: SearchIndex,
private vector: VectorIndex | null,
- private options: IndexPersistenceOptions = {},
+ options: IndexPersistenceOptions = {},
) {
this.now = options.now ?? Date.now;
const interval = options.saveIntervalMs;
@@ -131,22 +165,25 @@ export class IndexPersistence {
typeof interval === "number" && Number.isFinite(interval) && interval > 0
? interval
: getIndexSaveIntervalMs();
+ const bucketSize = options.bucketSize;
+ this.bucketSize =
+ typeof bucketSize === "number" && Number.isInteger(bucketSize) && bucketSize > 0
+ ? bucketSize
+ : getVectorBucketSize();
this.lastSaveAt = this.now();
}
scheduleSave(): void {
- if (this.stopped) return;
+ if (this.stopped || !this.vector) return;
const now = this.now();
this.dirtyEpoch++;
if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now;
- for (const leg of this.activeLegs()) {
- if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString();
- }
+ if (this.leg.dirtySince === null) this.leg.dirtySince = new Date(now).toISOString();
if (this.timer) return;
const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now);
this.timer = setTimeout(() => {
this.timer = null;
- this.save().catch((err) => this.logFailure("bm25", err));
+ this.save().catch((err) => this.logFailure(err));
}, delay);
}
@@ -168,34 +205,79 @@ export class IndexPersistence {
return {
saveIntervalMs: this.saveIntervalMs,
saving: this.running !== null,
- bm25: { ...this.legs.bm25 },
- vector: this.vector ? { ...this.legs.vector } : null,
+ buckets: this.bucketCountInUse(),
+ pendingChanges: this.vector?.pendingChanges ?? 0,
+ vector: this.vector ? { ...this.leg } : null,
+ vectorCountShortfall: this.vectorCountShortfall,
};
}
- async load(): Promise<{
- bm25: SearchIndex | null;
- vector: VectorIndex | null;
- }> {
- let bm25: SearchIndex | null = null;
- let vector: VectorIndex | null = null;
+ stop(): void {
+ this.stopped = true;
+ this.clearTimer();
+ }
+
+ async load(): Promise<VectorLoadResult> {
+ if (!this.vector) {
+ await this.removeLegacyBm25Snapshot();
+ return { vector: null, state: "none", savedAt: null };
+ }
- const bm25Data = await this.loadBm25Data();
- if (bm25Data && typeof bm25Data === "string") {
- bm25 = SearchIndex.deserialize(bm25Data);
+ let meta: VectorMeta | null;
+ try {
+ meta = await this.kv.get<VectorMeta>(KV.bm25Index, VECTOR_META_KEY);
+ } catch (err) {
+ logger.warn("index persistence: vector metadata read failed", { message: errorMessage(err) });
+ return { vector: null, state: "unavailable", savedAt: null };
}
- const vecData = await this.loadVectorData();
- if (vecData && typeof vecData === "string") {
- vector = VectorIndex.deserialize(vecData);
+ if (meta && meta.v === 3 && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0) {
+ const loaded = await this.loadBuckets(meta.bucketCount);
+ if (!loaded) return { vector: null, state: "unavailable", savedAt: null };
+ this.metaWritten = true;
+ await this.removeLegacyVectorSnapshotIfPresent();
+ await this.removeLegacyBm25Snapshot();
+ const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined;
+ this.vectorCountShortfall =
+ expectedCount !== undefined && loaded.size < expectedCount
+ ? { expected: expectedCount, loaded: loaded.size }
+ : null;
+ if (this.vectorCountShortfall) {
+ logger.warn("index persistence: loaded fewer vectors than the last save recorded", {
+ ...this.vectorCountShortfall,
+ });
+ }
+ return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null, expectedCount };
}
- return { bm25, vector };
+ return this.migrateLegacy();
}
- stop(): void {
- this.stopped = true;
- this.clearTimer();
+ private bucketCountInUse(): number {
+ return this.hasOpenBucket ? this.highestBucket + 1 : 0;
+ }
+
+ private assignBucket(id: string): number {
+ const existing = this.bucketOfId.get(id);
+ if (existing !== undefined) return existing;
+ if (!this.hasOpenBucket) {
+ this.hasOpenBucket = true;
+ this.highestBucket = 0;
+ } else if ((this.bucketCounts.get(this.highestBucket) ?? 0) >= this.bucketSize) {
+ this.highestBucket++;
+ }
+ this.bucketOfId.set(id, this.highestBucket);
+ this.bucketCounts.set(this.highestBucket, (this.bucketCounts.get(this.highestBucket) ?? 0) + 1);
+ return this.highestBucket;
+ }
+
+ private releaseBucket(id: string): void {
+ const bucket = this.bucketOfId.get(id);
+ if (bucket === undefined) return;
+ this.bucketOfId.delete(id);
+ const remaining = (this.bucketCounts.get(bucket) ?? 1) - 1;
+ if (remaining > 0) this.bucketCounts.set(bucket, remaining);
+ else this.bucketCounts.delete(bucket);
}
private clearTimer(): void {
@@ -205,10 +287,6 @@ export class IndexPersistence {
}
}
- private activeLegs(): IndexLeg[] {
- return this.vector ? ["bm25", "vector"] : ["bm25"];
- }
-
private startRun(): Promise<void> {
const run = this.runSave().finally(() => {
if (this.running === run) this.running = null;
@@ -218,176 +296,199 @@ export class IndexPersistence {
}
private async runSave(): Promise<void> {
- this.markedDuringRunAt = null;
- this.lastSaveAt = this.now();
- await this.saveLeg("bm25", async () => {
- const serialized = this.bm25.serialize();
- this.legs.bm25.serializedChars = serialized.length;
- await this.saveBm25Index(serialized);
- });
const vector = this.vector;
- if (vector) {
- await this.saveLeg("vector", async () => {
- const serialized = vector.serialize();
- this.legs.vector.serializedChars = serialized.length;
- await this.saveVectorIndex(serialized);
- });
- }
- }
-
- private async saveLeg(leg: IndexLeg, run: () => Promise<void>): Promise<void> {
- const status = this.legs[leg];
+ if (!vector) return;
const epoch = this.dirtyEpoch;
+ this.markedDuringRunAt = null;
+ this.lastSaveAt = this.now();
try {
- await run();
- status.lastSavedAt = new Date(this.now()).toISOString();
- status.lastError = null;
- status.lastErrorAt = null;
- if (this.dirtyEpoch === epoch) {
- status.dirtySince = null;
+ await this.writeChanges(vector);
+ this.leg.lastSavedAt = new Date(this.now()).toISOString();
+ this.leg.lastError = null;
+ this.leg.lastErrorAt = null;
+ if (this.dirtyEpoch === epoch || vector.pendingChanges === 0) {
+ this.leg.dirtySince = null;
} else if (this.markedDuringRunAt !== null) {
- status.dirtySince = new Date(this.markedDuringRunAt).toISOString();
+ this.leg.dirtySince = new Date(this.markedDuringRunAt).toISOString();
}
} catch (err) {
- status.lastError = errorMessage(err);
- status.lastErrorAt = new Date(this.now()).toISOString();
- if (status.dirtySince === null) status.dirtySince = status.lastErrorAt;
- this.logFailure(leg, err);
+ this.leg.lastError = errorMessage(err);
+ this.leg.lastErrorAt = new Date(this.now()).toISOString();
+ if (this.leg.dirtySince === null) this.leg.dirtySince = this.leg.lastErrorAt;
+ this.logFailure(err);
}
}
- private logFailure(leg: IndexLeg, err: unknown): void {
- const now = this.now();
- if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return;
- this.lastFailureLogAt.set(leg, now);
- const code = (err as { code?: string })?.code;
- const message = err instanceof Error ? err.message : String(err);
- logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, {
- code,
- message,
- hint:
- code === "TIMEOUT"
- ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save"
- : undefined,
+ private async writeChanges(vector: VectorIndex): Promise<void> {
+ const changes = vector.takeChanges();
+ if (changes.size === 0 && this.metaWritten) return;
+ const failed = new Map<string, boolean>();
+ const failures = await inBatches([...changes], WRITE_CONCURRENCY, async ([id, present]) => {
+ const entry = present ? vector.get(id) : undefined;
+ try {
+ if (entry) {
+ const bucket = this.assignBucket(id);
+ await this.kv.set<PersistedVector>(vectorBucketScope(bucket), id, {
+ id,
+ s: entry.sessionId,
+ e: float32ToBase64(entry.embedding),
+ });
+ } else {
+ const bucket = this.bucketOfId.get(id);
+ if (bucket !== undefined) {
+ await this.kv.delete(vectorBucketScope(bucket), id);
+ this.releaseBucket(id);
+ }
+ }
+ } catch (err) {
+ failed.set(id, present);
+ throw err;
+ }
});
+ if (failures.length > 0) {
+ vector.returnChanges(failed);
+ throw new Error(
+ `${failures.length} of ${changes.size} vector writes failed: ${errorMessage(failures[0])}`,
+ );
+ }
+ await this.kv.set<VectorMeta>(KV.bm25Index, VECTOR_META_KEY, {
+ v: 3,
+ bucketCount: this.bucketCountInUse(),
+ savedAt: new Date(this.now()).toISOString(),
+ count: vector.size,
+ });
+ this.metaWritten = true;
}
- private async saveBm25Index(serialized: string): Promise<void> {
- await this.saveShardedIndex(
- serialized,
- BM25_MANIFEST_KEY,
- BM25_KEY,
- BM25_SHARD_SCOPE_PREFIX,
- );
+ private async loadBuckets(bucketCount: number): Promise<VectorIndex | null> {
+ const loaded = new VectorIndex();
+ const bucketOfId = new Map<string, number>();
+ const bucketCounts = new Map<number, number>();
+ const bucketIds = Array.from({ length: bucketCount }, (_, bucket) => bucket);
+ const failures = await inBatches(bucketIds, LOAD_CONCURRENCY, async (bucket) => {
+ const rows = await this.kv.list<unknown>(vectorBucketScope(bucket));
+ let count = 0;
+ for (const row of rows) {
+ if (!isPersistedVector(row)) continue;
+ try {
+ loaded.loadPersisted(row.id, row.s, base64ToFloat32(row.e));
+ } catch {
+ continue;
+ }
+ bucketOfId.set(row.id, bucket);
+ count++;
+ }
+ if (count > 0) bucketCounts.set(bucket, count);
+ });
+ if (failures.length > 0) {
+ logger.warn("index persistence: vector bucket read failed", {
+ failed: failures.length,
+ message: errorMessage(failures[0]),
+ });
+ return null;
+ }
+ this.bucketOfId = bucketOfId;
+ this.bucketCounts = bucketCounts;
+ this.highestBucket = bucketCount > 0 ? bucketCount - 1 : 0;
+ this.hasOpenBucket = bucketCount > 0;
+ return loaded;
}
- private async saveVectorIndex(serialized: string): Promise<void> {
- await this.saveShardedIndex(
- serialized,
- VECTOR_MANIFEST_KEY,
- VECTOR_KEY,
- VECTOR_SHARD_SCOPE_PREFIX,
+ private async migrateLegacy(): Promise<VectorLoadResult> {
+ const manifest = await this.readIndexValue<IndexShardManifest>(
+ KV.bm25Index,
+ LEGACY_VECTOR_MANIFEST_KEY,
+ "vector",
+ "manifest",
);
- }
-
- private async saveShardedIndex(
- serialized: string,
- manifestKey: string,
- legacyKey: string,
- scopePrefix: string,
- ): Promise<void> {
- const previous = await this.kv
- .get<IndexShardManifest>(KV.bm25Index, manifestKey)
- .catch(() => null);
- const generation =
- this.options.createGeneration?.() ?? createIndexGeneration();
- const chunkChars = shardChars(this.options);
- const shards: IndexShardManifest["shards"] = [];
- const chunks: string[] = [];
-
- for (let offset = 0; offset < serialized.length; offset += chunkChars) {
- const shardIndex = shards.length;
- const scope = `${scopePrefix}${generation}:${String(shardIndex).padStart(
- 5,
- "0",
- )}`;
- const chunk = serialized.slice(offset, offset + chunkChars);
- shards.push({ scope, key: INDEX_SHARD_KEY, chars: chunk.length });
- chunks.push(chunk);
+ const legacyKey = await this.readIndexValue<string>(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy");
+ if (!manifest.ok || !legacyKey.ok) return { vector: null, state: "unavailable", savedAt: null };
+ const legacyPresent = manifest.value != null || (typeof legacyKey.value === "string" && legacyKey.value.length > 0);
+ if (!legacyPresent) {
+ await this.removeLegacyBm25Snapshot();
+ return { vector: null, state: "none", savedAt: null };
}
- const writeResults = await Promise.allSettled(
- shards.map(async (shard, index) => {
- const chunk = chunks[index] ?? "";
- await this.kv.set(shard.scope, shard.key, chunk);
- await this.auditIndexPersistence("shard_write", [
- statePath(shard.scope, shard.key),
- ], {
- scope: shard.scope,
- key: shard.key,
- manifestKey,
- generation,
- chars: chunk.length,
- });
- }),
- );
- const failedWrite = writeResults.find(
- (result): result is PromiseRejectedResult => result.status === "rejected",
- );
- if (failedWrite) {
- await this.deleteShards(shards, "shard_write_rollback");
- throw failedWrite.reason;
+ const data = await this.loadShardedData(LEGACY_VECTOR_KEY, LEGACY_VECTOR_MANIFEST_KEY, "vector");
+ if (typeof data !== "string") {
+ logger.warn("index persistence: legacy vector snapshot is unreadable; keeping it and skipping migration");
+ return { vector: null, state: "unavailable", savedAt: null };
}
+ const legacy = VectorIndex.deserialize(data);
+ const savedAt =
+ (manifest.value && typeof manifest.value === "object" ? generationTime(manifest.value.generation) : null) ??
+ new Date(this.now()).toISOString();
- const nextManifest: IndexShardManifest = {
- v: 1,
- generation,
- shards,
- chars: serialized.length,
- };
+ const staging = new VectorIndex();
+ for (const [id, entry] of legacy.entries()) staging.add(id, entry.sessionId, entry.embedding);
try {
- await this.kv.set<IndexShardManifest>(
- KV.bm25Index,
- manifestKey,
- nextManifest,
- );
- await this.auditIndexPersistence("manifest_publish", [
- statePath(KV.bm25Index, manifestKey),
- ], {
- manifestKey,
- generation,
- chars: serialized.length,
- shards: shards.length,
- result: "committed",
- });
+ await this.writeChanges(staging);
} catch (err) {
- if (await this.isManifestPublished(manifestKey, nextManifest)) {
- await this.auditIndexPersistence("manifest_publish", [
- statePath(KV.bm25Index, manifestKey),
- ], {
- manifestKey,
- generation,
- chars: serialized.length,
- shards: shards.length,
- result: "committed_after_error",
- error: errorMessage(err),
- });
- } else {
- await this.deleteShards(shards, "manifest_publish_rollback");
- }
- throw err;
+ logger.warn("index persistence: migrating the legacy vector snapshot failed; it stays in place and the next save retries", {
+ message: errorMessage(err),
+ });
+ legacy.markAllChanged();
+ return { vector: legacy, state: "unavailable", savedAt };
}
- await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup");
- if (previous?.v === 1 && Array.isArray(previous.shards)) {
- const currentShardIds = new Set(
- shards.map((shard) => `${shard.scope}\0${shard.key}`),
- );
- for (const shard of previous.shards) {
- if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue;
- await this.deleteShards([shard], "previous_generation_cleanup");
- }
+ await this.removeLegacyVectorSnapshot(manifest.value);
+ await this.removeLegacyBm25Snapshot();
+ await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], {
+ vectors: legacy.size,
+ buckets: this.bucketCountInUse(),
+ });
+ logger.info("index persistence: migrated the vector index to bucketed storage", {
+ vectors: legacy.size,
+ buckets: this.bucketCountInUse(),
+ });
+ const migrated = new VectorIndex();
+ for (const [id, entry] of legacy.entries()) migrated.loadPersisted(id, entry.sessionId, entry.embedding);
+ return { vector: migrated, state: "migrated", savedAt };
+ }
+
+ private async removeLegacyVectorSnapshot(manifest: IndexShardManifest | null): Promise<void> {
+ if (manifest && Array.isArray(manifest.shards)) {
+ await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_vector_cleanup");
+ }
+ await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "legacy_vector_cleanup");
+ await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_KEY, "legacy_vector_cleanup");
+ }
+
+ private async removeLegacyVectorSnapshotIfPresent(): Promise<void> {
+ const manifest = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "vector", "manifest");
+ const legacy = await this.readIndexValue<string>(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy");
+ if (!manifest.ok || !legacy.ok) return;
+ if (manifest.value == null && legacy.value == null) return;
+ await this.removeLegacyVectorSnapshot(manifest.value ?? null);
+ }
+
+ private async removeLegacyBm25Snapshot(): Promise<void> {
+ const manifestRead = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "bm25", "manifest");
+ const legacyRead = await this.readIndexValue<string>(KV.bm25Index, LEGACY_BM25_KEY, "bm25", "legacy");
+ if (!manifestRead.ok || !legacyRead.ok) return;
+ const manifest = manifestRead.value ?? null;
+ const legacy = legacyRead.value ?? null;
+ if (manifest == null && legacy == null) return;
+ if (manifest && Array.isArray(manifest.shards)) {
+ await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_bm25_cleanup");
}
+ if (manifest != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "legacy_bm25_cleanup");
+ if (legacy != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_KEY, "legacy_bm25_cleanup");
+ }
+
+ private logFailure(err: unknown): void {
+ const now = this.now();
+ if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return;
+ this.lastFailureLogAt = now;
+ const code = (err as { code?: string })?.code;
+ logger.warn("index persistence: failed to save the vector index", {
+ code,
+ message: errorMessage(err),
+ hint:
+ code === "TIMEOUT"
+ ? "iii-engine state::set timed out; unsaved vectors stay in memory and retry on the next save"
+ : undefined,
+ });
}
private async auditIndexPersistence(
@@ -396,20 +497,13 @@ export class IndexPersistence {
details: Record<string, unknown>,
): Promise<void> {
if (!auditIndexPersistEnabled()) return;
- await safeAudit(
- this.kv,
- "index_persist",
- INDEX_PERSISTENCE_FUNCTION_ID,
- targetIds,
- { action, ...details },
- );
+ await safeAudit(this.kv, "index_persist", INDEX_PERSISTENCE_FUNCTION_ID, targetIds, {
+ action,
+ ...details,
+ });
}
- private async deleteKey(
- scope: string,
- key: string,
- reason: string,
- ): Promise<void> {
+ private async deleteKey(scope: string, key: string, reason: string): Promise<void> {
let result = "deleted";
let error: string | undefined;
try {
@@ -427,81 +521,17 @@ export class IndexPersistence {
});
}
- private async deleteShards(
- shards: IndexShardManifest["shards"],
- reason: string,
- ): Promise<void> {
- for (const shard of shards) {
- await this.deleteKey(shard.scope, shard.key, reason);
- }
- }
-
- private async isManifestPublished(
- manifestKey: string,
- expected: IndexShardManifest,
- ): Promise<boolean> {
- const published = await this.kv
- .get<IndexShardManifest>(KV.bm25Index, manifestKey)
- .catch(() => null);
- if (
- published?.v !== 1 ||
- published.generation !== expected.generation ||
- published.chars !== expected.chars ||
- !Array.isArray(published.shards) ||
- published.shards.length !== expected.shards.length
- ) {
- return false;
- }
- return published.shards.every((shard, index) => {
- const expectedShard = expected.shards[index];
- if (!expectedShard) return false;
- return (
- shard.scope === expectedShard.scope &&
- shard.key === expectedShard.key &&
- shard.chars === expectedShard.chars
- );
- });
- }
-
- private async loadBm25Data(): Promise<string | null> {
- return this.loadShardedData(BM25_KEY, BM25_MANIFEST_KEY, "BM25");
+ private async deleteShards(shards: IndexShardManifest["shards"], reason: string): Promise<void> {
+ await inBatches(shards, WRITE_CONCURRENCY, (shard) => this.deleteKey(shard.scope, shard.key, reason));
}
- private async loadVectorData(): Promise<string | null> {
- return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector");
- }
-
- private async loadShardedData(
- legacyKey: string,
- manifestKey: string,
- label: string,
- ): Promise<string | null> {
- const manifest = await this.readIndexValue<IndexShardManifest>(
- KV.bm25Index,
- manifestKey,
- label,
- "manifest",
- );
+ private async loadShardedData(legacyKey: string, manifestKey: string, label: string): Promise<string | null> {
+ const manifest = await this.readIndexValue<IndexShardManifest>(KV.bm25Index, manifestKey, label, "manifest");
if (!manifest.ok) return null;
- // #797: some iii-state adapters return `undefined` (not `null`) for
- // a missing key. The previous `value !== null` check passed
- // undefined through to loadManifestData, which then crashed on
- // `manifest.v` with TypeError. Treat both null and undefined as
- // "no manifest" and fall through to the legacy path. The shape
- // check stays so a malformed-but-present row still fails closed.
- if (
- manifest.value != null &&
- typeof manifest.value === "object"
- ) {
+ if (manifest.value != null && typeof manifest.value === "object") {
return this.loadManifestData(manifest.value, label);
}
-
- const legacy = await this.readIndexValue<string>(
- KV.bm25Index,
- legacyKey,
- label,
- "legacy",
- );
+ const legacy = await this.readIndexValue<string>(KV.bm25Index, legacyKey, label, "legacy");
if (!legacy.ok) return null;
if (legacy.value && typeof legacy.value === "string") return legacy.value;
return null;
@@ -525,10 +555,7 @@ export class IndexPersistence {
}
}
- private async loadManifestData(
- manifest: IndexShardManifest,
- label: string,
- ): Promise<string | null> {
+ private async loadManifestData(manifest: IndexShardManifest, label: string): Promise<string | null> {
if (
manifest.v !== 1 ||
!Array.isArray(manifest.shards) ||
@@ -555,10 +582,7 @@ export class IndexPersistence {
let chars = 0;
for (const { shard, chunk } of loadedShards) {
if (typeof chunk !== "string") {
- logger.warn(`index persistence: ${label} shard missing`, {
- scope: shard.scope,
- key: shard.key,
- });
+ logger.warn(`index persistence: ${label} shard missing`, { scope: shard.scope, key: shard.key });
return null;
}
if (chunk.length !== shard.chars) {
@@ -574,10 +598,7 @@ export class IndexPersistence {
chars += chunk.length;
}
if (chars !== manifest.chars) {
- logger.warn(`index persistence: ${label} total length mismatch`, {
- expected: manifest.chars,
- actual: chars,
- });
+ logger.warn(`index persistence: ${label} total length mismatch`, { expected: manifest.chars, actual: chars });
return null;
}
return chunks.join("");
diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts
index d4b8bda7..2425bef8 100644
--- a/src/state/vector-index.ts
+++ b/src/state/vector-index.ts
@@ -5,13 +5,13 @@
// pool. Same risk on the encode side if the input Float32Array is itself
// a sliced view. Reported as a phantom "2048 dimensions on disk" crash
// in #455 / #469 / #584 / #587.
-function float32ToBase64(arr: Float32Array): string {
+export function float32ToBase64(arr: Float32Array): string {
return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString(
"base64",
);
}
-function base64ToFloat32(b64: string): Float32Array {
+export function base64ToFloat32(b64: string): Float32Array {
const buf = Buffer.from(b64, "base64");
return new Float32Array(
buf.buffer,
@@ -34,16 +34,59 @@ function cosineSimilarity(a: Float32Array, b: Float32Array): number {
return denom === 0 ? 0 : dot / denom;
}
+export type VectorEntry = { embedding: Float32Array; sessionId: string };
+
export class VectorIndex {
- private vectors: Map<string, { embedding: Float32Array; sessionId: string }> =
- new Map();
+ private vectors: Map<string, VectorEntry> = new Map();
+ private changes: Map<string, boolean> = new Map();
add(obsId: string, sessionId: string, embedding: Float32Array): void {
this.vectors.set(obsId, { embedding, sessionId });
+ this.changes.set(obsId, true);
}
remove(obsId: string): void {
- this.vectors.delete(obsId);
+ if (this.vectors.delete(obsId)) this.changes.set(obsId, false);
+ }
+
+ has(obsId: string): boolean {
+ return this.vectors.has(obsId);
+ }
+
+ get(obsId: string): VectorEntry | undefined {
+ return this.vectors.get(obsId);
+ }
+
+ entries(): IterableIterator<[string, VectorEntry]> {
+ return this.vectors.entries();
+ }
+
+ loadPersisted(obsId: string, sessionId: string, embedding: Float32Array): void {
+ this.vectors.set(obsId, { embedding, sessionId });
+ }
+
+ get pendingChanges(): number {
+ return this.changes.size;
+ }
+
+ takeChanges(): Map<string, boolean> {
+ const taken = this.changes;
+ this.changes = new Map();
+ return taken;
+ }
+
+ returnChanges(changes: Map<string, boolean>): void {
+ for (const [obsId, present] of changes) {
+ if (!this.changes.has(obsId)) this.changes.set(obsId, present);
+ }
+ }
+
+ markRemoved(obsId: string): void {
+ if (!this.vectors.has(obsId)) this.changes.set(obsId, false);
+ }
+
+ markAllChanged(): void {
+ for (const obsId of this.vectors.keys()) this.changes.set(obsId, true);
}
search(
@@ -104,6 +147,7 @@ export class VectorIndex {
}
clear(): void {
+ for (const obsId of this.vectors.keys()) this.changes.set(obsId, false);
this.vectors.clear();
}
@@ -119,6 +163,7 @@ export class VectorIndex {
sessionId: entry.sessionId,
});
}
+ this.changes = new Map(other.changes);
}
serialize(): string {
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 2fc2f328..337e34b5 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -12,7 +12,14 @@ import type { ResilientProvider } from "../providers/resilient.js";
import { III_PINNED_VERSION, VERSION } from "../version.js";
import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js";
import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js";
-import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js";
+import {
+ findUnindexedObservations,
+ getIndexPersistenceStatus,
+ getPendingVectorBackfillCount,
+ getSearchIndex,
+ getVectorIndex,
+ isBm25RebuildIncomplete,
+} from "../functions/search.js";
import { timingSafeCompare } from "../auth.js";
import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js";
import { renderViewerDocument } from "../viewer/document.js";
@@ -370,6 +377,8 @@ export function registerApiTriggers(
observationsIndexed,
missingObservations: unindexed ? unindexed.missing.length : null,
sessions: unindexed ? unindexed.sessions : null,
+ bm25Incomplete: isBm25RebuildIncomplete(),
+ pendingVectorBackfill: getPendingVectorBackfillCount(),
},
graph,
graphExtractionEnabled: isGraphExtractionEnabled(),
diff --git a/test/index-keyword-rebuild.test.ts b/test/index-keyword-rebuild.test.ts
new file mode 100644
index 00000000..f0080675
--- /dev/null
+++ b/test/index-keyword-rebuild.test.ts
@@ -0,0 +1,240 @@
+import { describe, it, expect, afterEach } from "vitest";
+import {
+ backfillVectors,
+ getPendingVectorBackfillCount,
+ getSearchIndex,
+ isBm25RebuildIncomplete,
+ isMemoryIndexReady,
+ rebuildKeywordIndex,
+ setEmbeddingProvider,
+ setIndexPersistence,
+ setVectorIndex,
+} from "../src/functions/search.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import type { CompressedObservation, EmbeddingProvider, Memory, Session } from "../src/types.js";
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ store,
+ list: async <T>(scope: string): Promise<T[]> => Array.from(store.get(scope)?.values() ?? []) as T[],
+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ };
+}
+
+const WORDS = ["auth", "cache", "index", "vector", "session", "memory", "graph", "retry", "queue", "schema", "deploy", "worker"];
+
+function makeObs(id: string, sessionId: string, i: number, timestamp: string): CompressedObservation {
+ const a = WORDS[i % WORDS.length];
+ const b = WORDS[(i * 7) % WORDS.length];
+ return {
+ id,
+ sessionId,
+ timestamp,
+ type: "file_edit",
+ title: `${a} change ${i}`,
+ facts: [`touched ${b} path`],
+ narrative: `Updated the ${a} module so ${b} handling works for case ${i}`,
+ concepts: [a, b],
+ files: [`src/${a}/${b}.ts`],
+ importance: 5,
+ };
+}
+
+function seed(kv: ReturnType<typeof mockKV>, sessions: number, perSession: number, timestamp: (i: number) => string) {
+ let n = 0;
+ for (let s = 0; s < sessions; s++) {
+ const sessionId = `ses_${s}`;
+ kv.store.set("mem:sessions", kv.store.get("mem:sessions") ?? new Map());
+ kv.store.get("mem:sessions")!.set(sessionId, {
+ id: sessionId,
+ project: "p",
+ cwd: "/tmp",
+ startedAt: timestamp(0),
+ status: "completed",
+ observationCount: perSession,
+ } satisfies Session);
+ const scope = `mem:obs:${sessionId}`;
+ kv.store.set(scope, new Map());
+ for (let o = 0; o < perSession; o++) {
+ const id = `obs_${n}`;
+ kv.store.get(scope)!.set(id, makeObs(id, sessionId, n, timestamp(n)));
+ n++;
+ }
+ }
+ const memories = new Map<string, Memory>();
+ memories.set("mem_1", {
+ id: "mem_1",
+ createdAt: timestamp(0),
+ updatedAt: timestamp(0),
+ type: "fact",
+ title: "Deploy uses blue green",
+ content: "Deploys switch traffic after health checks pass",
+ concepts: ["deploy"],
+ files: [],
+ sessionIds: ["ses_0"],
+ strength: 7,
+ version: 1,
+ isLatest: true,
+ } as Memory);
+ memories.set("mem_old", {
+ id: "mem_old",
+ createdAt: timestamp(0),
+ updatedAt: timestamp(0),
+ type: "fact",
+ title: "Superseded",
+ content: "Old version",
+ concepts: [],
+ files: [],
+ sessionIds: [],
+ strength: 1,
+ version: 1,
+ isLatest: false,
+ } as Memory);
+ kv.store.set("mem:memories", memories);
+ kv.store.get("mem:obs:ses_0")!.set("raw_1", { id: "raw_1", sessionId: "ses_0", timestamp: timestamp(0) });
+ return n;
+}
+
+function stubProvider(): EmbeddingProvider {
+ return {
+ name: "stub",
+ dimensions: 3,
+ embed: async () => new Float32Array([0.1, 0.2, 0.3]),
+ embedBatch: async (texts: string[]) => texts.map(() => new Float32Array([0.1, 0.2, 0.3])),
+ } as EmbeddingProvider;
+}
+
+describe("rebuildKeywordIndex", () => {
+ afterEach(() => {
+ getSearchIndex().clear();
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ setIndexPersistence(null);
+ });
+
+ it("rebuilds BM25 from stored observations and latest memories, quickly", async () => {
+ const kv = mockKV();
+ const total = seed(kv, 100, 30, () => "2026-09-01T00:00:00.000Z");
+ getSearchIndex().add(makeObs("ghost", "ses_gone", 0, "2026-09-01T00:00:00.000Z"));
+
+ const started = performance.now();
+ const result = await rebuildKeywordIndex(kv as never);
+ const elapsed = performance.now() - started;
+
+ expect(total).toBe(3000);
+ expect(result.documents).toBe(3001);
+ expect(result.vectorJobs).toEqual([]);
+ const idx = getSearchIndex();
+ expect(idx.size).toBe(3001);
+ expect(idx.has("ghost")).toBe(false);
+ expect(idx.has("mem_1")).toBe(true);
+ expect(idx.has("mem_old")).toBe(false);
+ expect(idx.has("raw_1")).toBe(false);
+ expect(idx.search("blue green").map((r) => r.obsId)).toContain("mem_1");
+ expect(isMemoryIndexReady()).toBe(true);
+ expect(elapsed).toBeLessThan(10_000);
+ });
+
+ it("keeps the vector index untouched and reports an incomplete rebuild when the sessions listing fails", async () => {
+ const kv = mockKV();
+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");
+ const vector = new VectorIndex();
+ vector.add("obs_0", "ses_0", new Float32Array([0.1, 0.2, 0.3]));
+ setVectorIndex(vector);
+ setEmbeddingProvider(stubProvider());
+ const failingKv = {
+ ...kv,
+ list: async <T>(scope: string): Promise<T[]> => {
+ if (scope === "mem:sessions") throw new Error("engine unreachable");
+ return kv.list<T>(scope);
+ },
+ };
+
+ const result = await rebuildKeywordIndex(failingKv as never, null);
+
+ expect(result.vectorJobs).toEqual([]);
+ expect(vector.size).toBe(1);
+ expect(vector.has("obs_0")).toBe(true);
+ expect(isBm25RebuildIncomplete()).toBe(true);
+ }, 10_000);
+
+ it("queues embeddings only for documents newer than the last vector save", async () => {
+ const kv = mockKV();
+ seed(kv, 2, 5, (i) => (i < 5 ? "2026-09-01T00:00:00.000Z" : "2026-09-20T00:00:00.000Z"));
+ const vector = new VectorIndex();
+ vector.add("obs_6", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
+ setVectorIndex(vector);
+ setEmbeddingProvider(stubProvider());
+
+ const result = await rebuildKeywordIndex(kv as never, "2026-09-10T00:00:00.000Z");
+
+ expect(result.vectorJobs.map((j) => j.id).sort()).toEqual(["obs_5", "obs_7", "obs_8", "obs_9"]);
+ });
+
+ it("withholds a whole-store backfill by default and reports it as pending, and queues none when storage was unavailable", async () => {
+ const kv = mockKV();
+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");
+ setVectorIndex(new VectorIndex());
+ setEmbeddingProvider(stubProvider());
+
+ const fresh = await rebuildKeywordIndex(kv as never, null);
+ expect(fresh.vectorJobs).toEqual([]);
+ expect(fresh.fullBackfillPending).toBe(4);
+
+ const unavailable = await rebuildKeywordIndex(kv as never, undefined);
+ expect(unavailable.vectorJobs).toEqual([]);
+ expect(unavailable.fullBackfillPending).toBe(0);
+ });
+
+ it("runs the whole-store backfill, capped, once AGENTMEMORY_VECTOR_BACKFILL=all is set", async () => {
+ const kv = mockKV();
+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");
+ setVectorIndex(new VectorIndex());
+ setEmbeddingProvider(stubProvider());
+ process.env.AGENTMEMORY_VECTOR_BACKFILL = "all";
+ try {
+ const opted = await rebuildKeywordIndex(kv as never, null);
+ expect(opted.vectorJobs.map((j) => j.id).sort()).toEqual(["mem_1", "obs_0", "obs_1", "obs_2"]);
+ expect(opted.fullBackfillPending).toBe(0);
+
+ process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "2";
+ const capped = await rebuildKeywordIndex(kv as never, null);
+ expect(capped.vectorJobs.length).toBe(2);
+ } finally {
+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL;
+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX;
+ }
+ });
+
+ it("backfills queued vectors in batches and saves progress", async () => {
+ const kv = mockKV();
+ seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z");
+ const vector = new VectorIndex();
+ setVectorIndex(vector);
+ setEmbeddingProvider(stubProvider());
+ let saved = 0;
+ setIndexPersistence({ scheduleSave: () => {}, save: async () => void saved++ });
+ process.env.AGENTMEMORY_VECTOR_BACKFILL = "all";
+
+ try {
+ const { vectorJobs } = await rebuildKeywordIndex(kv as never, null);
+ const added = await backfillVectors(vectorJobs);
+
+ expect(added).toBe(4);
+ expect(vector.size).toBe(4);
+ expect(saved).toBe(1);
+ expect(getPendingVectorBackfillCount()).toBe(0);
+ } finally {
+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL;
+ }
+ });
+});
diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts
index 54eff133..bc03834d 100644
--- a/test/index-persistence-throttle.test.ts
+++ b/test/index-persistence-throttle.test.ts
@@ -1,18 +1,15 @@
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
-import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js";
-import { SearchIndex } from "../src/state/search-index.js";
+import { IndexPersistence } from "../src/state/index-persistence.js";
import { VectorIndex } from "../src/state/vector-index.js";
import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js";
-import type { CompressedObservation } from "../src/types.js";
-const BM25_SCOPE = "mem:index:bm25";
+const META_KEY = "vectors:meta";
function mockKV() {
const store = new Map<string, Map<string, unknown>>();
const sets: Array<{ scope: string; key: string }> = [];
return {
sets,
- store,
get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,
set: async <T>(scope: string, key: string, data: T): Promise<T> => {
sets.push({ scope, key });
@@ -27,23 +24,12 @@ function mockKV() {
};
}
-function obs(id: string, title: string): CompressedObservation {
- return {
- id,
- sessionId: "ses_1",
- timestamp: new Date().toISOString(),
- type: "file_edit",
- title,
- facts: [],
- narrative: `${title} narrative`,
- concepts: [],
- files: [],
- importance: 5,
- };
+function metaSaves(kv: ReturnType<typeof mockKV>): number {
+ return kv.sets.filter((s) => s.key === META_KEY).length;
}
-function manifestSaves(kv: ReturnType<typeof mockKV>, key: string): number {
- return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length;
+function touch(vector: VectorIndex, id: string): void {
+ vector.add(id, "ses_1", new Float32Array([Math.random(), 0.2, 0.3]));
}
describe("IndexPersistence save throttling", () => {
@@ -59,35 +45,40 @@ describe("IndexPersistence save throttling", () => {
});
it("saves at most once per interval however often changes are scheduled", async () => {
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });
- for (let i = 0; i < 50; i++) persistence.scheduleSave();
+ for (let i = 0; i < 50; i++) {
+ touch(vector, `obs_${i}`);
+ persistence.scheduleSave();
+ }
await vi.advanceTimersByTimeAsync(59_000);
- expect(manifestSaves(kv, "data:manifest")).toBe(0);
+ expect(metaSaves(kv)).toBe(0);
await vi.advanceTimersByTimeAsync(1_000);
- expect(manifestSaves(kv, "data:manifest")).toBe(1);
+ expect(metaSaves(kv)).toBe(1);
- for (let i = 0; i < 50; i++) persistence.scheduleSave();
+ for (let i = 0; i < 50; i++) {
+ touch(vector, `obs_more_${i}`);
+ persistence.scheduleSave();
+ }
await vi.advanceTimersByTimeAsync(30_000);
- expect(manifestSaves(kv, "data:manifest")).toBe(1);
+ expect(metaSaves(kv)).toBe(1);
await vi.advanceTimersByTimeAsync(30_000);
- expect(manifestSaves(kv, "data:manifest")).toBe(2);
+ expect(metaSaves(kv)).toBe(2);
});
it("an explicit save runs immediately and cancels the pending timer", async () => {
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });
+ touch(vector, "obs_1");
persistence.scheduleSave();
await persistence.save();
- expect(manifestSaves(kv, "data:manifest")).toBe(1);
+ expect(metaSaves(kv)).toBe(1);
await vi.advanceTimersByTimeAsync(120_000);
- expect(manifestSaves(kv, "data:manifest")).toBe(1);
+ expect(metaSaves(kv)).toBe(1);
});
it("never runs two saves at once and coalesces requests made during a save into one", async () => {
@@ -108,11 +99,12 @@ describe("IndexPersistence save throttling", () => {
return result;
},
};
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });
+ const vector = new VectorIndex();
+ touch(vector, "obs_1");
+ const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });
const first = persistence.save();
+ touch(vector, "obs_2");
const second = persistence.save();
const third = persistence.save();
expect(second).toBe(third);
@@ -122,42 +114,12 @@ describe("IndexPersistence save throttling", () => {
release();
await Promise.all([first, second, third]);
- expect(manifestSaves(kv, "data:manifest")).toBe(2);
+ expect(metaSaves(kv)).toBe(2);
expect(maxInFlight).toBe(1);
expect(persistence.status().saving).toBe(false);
});
- it("a failing BM25 leg does not stop the vector leg from saving", async () => {
- const failingKv = {
- ...kv,
- set: async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed");
- return kv.set(scope, key, data);
- },
- };
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const vector = new VectorIndex();
- vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
- const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 });
-
- persistence.scheduleSave();
- await persistence.save();
-
- const status = persistence.status();
- expect(status.bm25.lastError).toBe("bm25 manifest write failed");
- expect(status.bm25.dirtySince).not.toBeNull();
- expect(status.vector?.lastError).toBeNull();
- expect(status.vector?.lastSavedAt).not.toBeNull();
- expect(status.vector?.dirtySince).toBeNull();
- expect(status.vector?.serializedChars).toBeGreaterThan(0);
-
- const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load();
- expect(loaded.bm25).toBeNull();
- expect(loaded.vector?.size).toBe(1);
- });
-
- it("keeps a leg dirty when a change arrives while it is being saved", async () => {
+ it("keeps the index dirty when a change arrives while it is being saved", async () => {
let release: () => void = () => undefined;
const gate = new Promise<void>((resolve) => {
release = resolve;
@@ -173,63 +135,31 @@ describe("IndexPersistence save throttling", () => {
return kv.set(scope, key, data);
},
};
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });
+ const vector = new VectorIndex();
+ touch(vector, "obs_1");
+ const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });
const saving = persistence.save();
+ touch(vector, "obs_2");
persistence.scheduleSave();
release();
await saving;
- expect(persistence.status().bm25.lastSavedAt).not.toBeNull();
- expect(persistence.status().bm25.dirtySince).not.toBeNull();
+ expect(persistence.status().vector?.lastSavedAt).not.toBeNull();
+ expect(persistence.status().vector?.dirtySince).not.toBeNull();
await persistence.save();
- expect(persistence.status().bm25.dirtySince).toBeNull();
- });
-
- it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => {
- let release: () => void = () => undefined;
- const gate = new Promise<void>((resolve) => {
- release = resolve;
- });
- let heldBm25Manifest = false;
- const slowKv = {
- ...kv,
- set: async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) {
- heldBm25Manifest = true;
- await gate;
- }
- return kv.set(scope, key, data);
- },
- };
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const vector = new VectorIndex();
- vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
- const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 });
-
- const saving = persistence.save();
- persistence.scheduleSave();
- release();
- await saving;
-
- const status = persistence.status();
- expect(status.bm25.dirtySince).not.toBeNull();
- expect(status.vector?.dirtySince).toBeNull();
- expect(status.vector?.lastSavedAt).not.toBeNull();
+ expect(persistence.status().vector?.dirtySince).toBeNull();
});
it("stop prevents later scheduled saves", async () => {
- const bm25 = new SearchIndex();
- bm25.add(obs("obs_1", "alpha"));
- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 });
+ const vector = new VectorIndex();
+ touch(vector, "obs_1");
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 1_000, bucketSize: 16 });
persistence.stop();
persistence.scheduleSave();
await vi.advanceTimersByTimeAsync(10_000);
- expect(manifestSaves(kv, "data:manifest")).toBe(0);
+ expect(metaSaves(kv)).toBe(0);
});
});
@@ -246,7 +176,15 @@ function statusInputs(overrides: Partial<StatusInputs> = {}): StatusInputs {
provider: "llm",
embeddingProvider: "embeddings",
flags: [],
- index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 },
+ index: {
+ bm25Documents: 1,
+ vectorDocuments: 1,
+ observationsIndexed: 1,
+ missingObservations: 0,
+ sessions: 1,
+ bm25Incomplete: false,
+ pendingVectorBackfill: 0,
+ },
graph: null,
graphExtractionEnabled: false,
...overrides,
@@ -258,78 +196,53 @@ const cleanLeg = {
lastError: null,
lastErrorAt: null,
dirtySince: null,
- serializedChars: 1000,
};
+function persistenceStatus(vector: typeof cleanLeg | null, pendingChanges = 0) {
+ return { saveIntervalMs: 600_000, saving: false, buckets: 256, pendingChanges, vector, vectorCountShortfall: null };
+}
+
describe("status reports index persistence", () => {
- it("is quiet when both legs saved cleanly", () => {
- const report = evaluateStatus(
- statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }),
- );
+ it("is quiet when the vector index saved cleanly", () => {
+ const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(cleanLeg) }));
expect(report.problems).toEqual([]);
- expect(report.indexPersistence?.vector?.serializedChars).toBe(1000);
+ expect(report.indexPersistence?.buckets).toBe(256);
});
- it("reports a failing leg as an error", () => {
+ it("reports a failing vector save as an error", () => {
const report = evaluateStatus(
statusInputs({
- indexPersistence: {
- saveIntervalMs: 600_000,
- saving: false,
- bm25: cleanLeg,
- vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" },
- },
+ indexPersistence: persistenceStatus({
+ ...cleanLeg,
+ lastError: "3 of 10 vector writes failed: timed out",
+ lastErrorAt: "2026-09-25T11:59:30.000Z",
+ } as never),
}),
);
expect(report.status).toBe("error");
expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]);
- expect(report.problems[0].message).toContain("vector");
- });
-
- it("warns when the vector index passes 80% of the Node.js string limit", () => {
- const report = evaluateStatus(
- statusInputs({
- indexPersistence: {
- saveIntervalMs: 600_000,
- saving: false,
- bm25: cleanLeg,
- vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) },
- },
- }),
- );
- expect(report.status).toBe("warn");
- expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]);
- expect(report.problems[0].message).toContain("85%");
+ expect(report.problems[0].message).toContain("3 of 10 vector writes failed");
});
it("warns when unsaved changes are older than twice the save interval", () => {
const report = evaluateStatus(
- statusInputs({
- indexPersistence: {
- saveIntervalMs: 600_000,
- saving: false,
- bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" },
- vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" },
- },
- }),
+ statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" } as never, 4) }),
);
expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]);
- expect(report.problems[0].message).toContain("BM25");
});
- it("shows save state and vector size on the status page", () => {
+ it("does not report vector persistence when vector search is off", () => {
+ const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(null) }));
+ expect(report.problems).toEqual([]);
+ });
+
+ it("shows save state and bucket storage on the status page", () => {
const report = evaluateStatus(
- statusInputs({
- indexPersistence: {
- saveIntervalMs: 600_000,
- saving: false,
- bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" },
- vector: { ...cleanLeg, serializedChars: 53_687_089 },
- },
- }),
+ statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" } as never, 3) }),
);
const html = renderStatusHtml(report, "n");
- expect(html).toContain("<th>BM25 save</th><td>saved 1m ago, unsaved changes pending</td>");
- expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)");
+ expect(html).toContain("<th>BM25 index</th><td>rebuilt from stored content at boot</td>");
+ expect(html).toContain("<th>Vector save</th><td>saved 1m ago, unsaved changes pending</td>");
+ expect(html).toContain("<th>Vector storage</th><td>256 buckets, 3 unsaved changes</td>");
});
});
diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts
index 802084bd..568d1800 100644
--- a/test/index-persistence.test.ts
+++ b/test/index-persistence.test.ts
@@ -1,740 +1,475 @@
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
-import { IndexPersistence } from "../src/state/index-persistence.js";
+import { IndexPersistence, vectorBucketScope } from "../src/state/index-persistence.js";
import { SearchIndex } from "../src/state/search-index.js";
import { VectorIndex } from "../src/state/vector-index.js";
import type { CompressedObservation } from "../src/types.js";
-const BM25_SCOPE = "mem:index:bm25";
-const BM25_LEGACY_KEY = "data";
-const BM25_MANIFEST_KEY = "data:manifest";
-const VECTOR_LEGACY_KEY = "vectors";
-const VECTOR_MANIFEST_KEY = "vectors:manifest";
-
-type TestIndexShardManifest = {
- v: 1;
- generation?: string;
- shards: Array<{ scope: string; key: string; chars: number }>;
- chars: number;
-};
+const INDEX_SCOPE = "mem:index:bm25";
+const META_KEY = "vectors:meta";
function mockKV() {
const store = new Map<string, Map<string, unknown>>();
+ const ops: Array<{ op: "set" | "delete" | "list"; scope: string; key?: string }> = [];
return {
- get: async <T>(scope: string, key: string): Promise<T | null> => {
- return (store.get(scope)?.get(key) as T) ?? null;
- },
+ store,
+ ops,
+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,
set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ ops.push({ op: "set", scope, key });
if (!store.has(scope)) store.set(scope, new Map());
store.get(scope)!.set(key, data);
return data;
},
delete: async (scope: string, key: string): Promise<void> => {
+ ops.push({ op: "delete", scope, key });
store.get(scope)?.delete(key);
},
list: async <T>(scope: string): Promise<T[]> => {
- const entries = store.get(scope);
- return entries ? (Array.from(entries.values()) as T[]) : [];
+ ops.push({ op: "list", scope });
+ return Array.from(store.get(scope)?.values() ?? []) as T[];
},
};
}
type MockKV = ReturnType<typeof mockKV>;
-function makeObs(
- overrides: Partial<CompressedObservation> = {},
-): CompressedObservation {
+function vec(values: number[]): Float32Array {
+ return new Float32Array(values);
+}
+
+function vectorWith(entries: Array<[string, number[]]>): VectorIndex {
+ const index = new VectorIndex();
+ for (const [id, values] of entries) index.add(id, `ses_${id}`, vec(values));
+ return index;
+}
+
+function bucketSets(kv: MockKV): Array<{ scope: string; key?: string }> {
+ return kv.ops.filter((o) => o.op === "set" && o.scope.startsWith(`${INDEX_SCOPE}:vec:`));
+}
+
+function obs(id: string): CompressedObservation {
return {
- id: "obs_1",
+ id,
sessionId: "ses_1",
timestamp: new Date().toISOString(),
type: "file_edit",
- title: "Edit auth middleware",
- subtitle: "JWT validation",
- facts: ["Added token check"],
- narrative: "Modified the auth middleware to validate JWT tokens",
- concepts: ["authentication", "jwt"],
- files: ["src/middleware/auth.ts"],
- importance: 7,
- ...overrides,
+ title: `title ${id}`,
+ facts: [],
+ narrative: `narrative ${id}`,
+ concepts: [],
+ files: [],
+ importance: 5,
};
}
-function makeBm25(id: string, title: string): SearchIndex {
- const bm25 = new SearchIndex();
- bm25.add(makeObs({ id, title, narrative: `${title} narrative` }));
- return bm25;
+async function writeLegacyVectorSnapshot(
+ kv: MockKV,
+ vector: VectorIndex,
+ options: { generation?: string; monolithic?: boolean; shardChars?: number } = {},
+): Promise<void> {
+ const serialized = vector.serialize();
+ if (options.monolithic) {
+ await kv.set(INDEX_SCOPE, "vectors", serialized);
+ return;
+ }
+ const generation = options.generation ?? "idx_mfabcd12_aaaaaaaaaaaa";
+ const size = options.shardChars ?? 40;
+ const shards: Array<{ scope: string; key: string; chars: number }> = [];
+ for (let offset = 0, i = 0; offset < serialized.length; offset += size, i++) {
+ const scope = `${INDEX_SCOPE}:vectors:${generation}:${String(i).padStart(5, "0")}`;
+ const chunk = serialized.slice(offset, offset + size);
+ await kv.set(scope, "data", chunk);
+ shards.push({ scope, key: "data", chars: chunk.length });
+ }
+ await kv.set(INDEX_SCOPE, "vectors:manifest", { v: 1, generation, shards, chars: serialized.length });
}
-function makeVector(id = "obs_1"): VectorIndex {
- const vector = new VectorIndex();
- vector.add(id, "ses_1", new Float32Array([0.1, 0.2, 0.3]));
- return vector;
+async function writeLegacyBm25Snapshot(kv: MockKV): Promise<void> {
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_legacy"));
+ const serialized = bm25.serialize();
+ const scope = `${INDEX_SCOPE}:bm25:idx_mfabcd12_bbbbbbbbbbbb:00000`;
+ await kv.set(scope, "data", serialized);
+ await kv.set(INDEX_SCOPE, "data:manifest", {
+ v: 1,
+ generation: "idx_mfabcd12_bbbbbbbbbbbb",
+ shards: [{ scope, key: "data", chars: serialized.length }],
+ chars: serialized.length,
+ });
}
-async function getBm25Manifest(kv: MockKV): Promise<TestIndexShardManifest> {
- const manifest = await kv.get<TestIndexShardManifest>(
- BM25_SCOPE,
- BM25_MANIFEST_KEY,
- );
- expect(manifest).not.toBeNull();
- return manifest!;
+function expectSameVectors(actual: VectorIndex | null, expected: VectorIndex): void {
+ expect(actual).not.toBeNull();
+ expect(actual!.size).toBe(expected.size);
+ for (const [id, entry] of expected.entries()) {
+ const loaded = actual!.get(id);
+ expect(loaded?.sessionId).toBe(entry.sessionId);
+ expect(Array.from(loaded!.embedding)).toEqual(Array.from(entry.embedding));
+ }
}
-describe("IndexPersistence", () => {
- let kv: ReturnType<typeof mockKV>;
+describe("IndexPersistence bucketed vector storage", () => {
+ let kv: MockKV;
beforeEach(() => {
- vi.useFakeTimers();
kv = mockKV();
});
- afterEach(() => {
- vi.useRealTimers();
- });
-
- it("saves and loads BM25 index round-trip", async () => {
- const bm25 = new SearchIndex();
- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));
+ it("fills buckets in insertion order and rolls to a new one at the cap", async () => {
+ const vector = vectorWith([
+ ["obs_a", [0.1, 0.2, 0.3]],
+ ["obs_b", [0.4, 0.5, 0.6]],
+ ["obs_c", [0.7, 0.8, 0.9]],
+ ]);
+ await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save();
+
+ expect(kv.store.get(vectorBucketScope(0))?.has("obs_a")).toBe(true);
+ expect(kv.store.get(vectorBucketScope(0))?.has("obs_b")).toBe(true);
+ expect(kv.store.get(vectorBucketScope(1))?.has("obs_c")).toBe(true);
+ const meta = await kv.get<{ v: number; bucketCount: number; count: number }>(INDEX_SCOPE, META_KEY);
+ expect(meta).toMatchObject({ v: 3, bucketCount: 2, count: 3 });
+
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 }).load();
+ expect(loaded.state).toBe("buckets");
+ expectSameVectors(loaded.vector, vector);
+ expect(loaded.vector!.pendingChanges).toBe(0);
+ });
+
+ it("writes only the bucket entry of a single added vector", async () => {
+ const vector = vectorWith([
+ ["obs_a", [0.1, 0.2, 0.3]],
+ ["obs_b", [0.4, 0.5, 0.6]],
+ ]);
+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });
+ await persistence.save();
+ kv.ops.length = 0;
- const persistence = new IndexPersistence(kv as never, bm25, null);
+ vector.add("obs_new", "ses_new", vec([1, 0, 0]));
await persistence.save();
- const loaded = await persistence.load();
- expect(loaded.bm25).not.toBeNull();
- expect(loaded.bm25!.size).toBe(1);
- const results = loaded.bm25!.search("auth");
- expect(results.length).toBe(1);
+ const writes = bucketSets(kv);
+ expect(writes).toEqual([{ op: "set", scope: vectorBucketScope(0), key: "obs_new" }]);
});
- it("saves BM25 index shards outside the BM25 metadata scope", async () => {
- const bm25 = new SearchIndex();
- bm25.add(
- makeObs({
- id: "obs_1",
- title: "auth handler ".repeat(40),
- narrative: "JWT middleware validation ".repeat(40),
- }),
- );
-
- const persistence = new IndexPersistence(kv as never, bm25, null, {
- shardChars: 80,
- createGeneration: () => "gen_bm25",
- });
+ it("does not write when nothing changed", async () => {
+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });
await persistence.save();
+ kv.ops.length = 0;
- const manifest = await getBm25Manifest(kv);
- expect(manifest.generation).toBe("gen_bm25");
- expect(manifest.shards.length).toBeGreaterThan(1);
- expect(manifest.shards[0].scope).toContain(":gen_bm25:");
- await expect(kv.get(BM25_SCOPE, BM25_LEGACY_KEY)).resolves.toBeNull();
- await expect(
- kv.get(manifest.shards[0].scope, manifest.shards[0].key),
- ).resolves.toEqual(expect.any(String));
+ await persistence.save();
- const loaded = await persistence.load();
- expect(loaded.bm25).not.toBeNull();
- expect(loaded.bm25!.search("auth").length).toBe(1);
+ expect(kv.ops.filter((o) => o.op !== "list")).toEqual([]);
});
- it("loads legacy monolithic BM25 and vector snapshots", async () => {
- const bm25 = makeBm25("obs_1", "legacy auth handler");
- const vector = makeVector("obs_1");
- await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, bm25.serialize());
- await kv.set(BM25_SCOPE, VECTOR_LEGACY_KEY, vector.serialize());
-
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
-
- expect(loaded.bm25).not.toBeNull();
- expect(loaded.bm25!.search("legacy").length).toBe(1);
- expect(loaded.vector).not.toBeNull();
- expect(loaded.vector!.size).toBe(1);
+ it("removes deleted vectors from their bucket", async () => {
+ const vector = vectorWith([
+ ["obs_a", [0.1, 0.2, 0.3]],
+ ["obs_b", [0.4, 0.5, 0.6]],
+ ]);
+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });
+ await persistence.save();
+
+ vector.remove("obs_a");
+ await persistence.save();
+
+ expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(0), key: "obs_a" });
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expect(loaded.vector!.has("obs_a")).toBe(false);
+ expect(loaded.vector!.has("obs_b")).toBe(true);
});
- it("fails closed instead of falling back when manifest reads fail", async () => {
- const legacy = makeBm25("obs_legacy", "legacy stale snapshot");
- await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, legacy.serialize());
- const failingKv = {
- ...kv,
- get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {
- if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {
- throw new Error("manifest backend unavailable");
- }
- return kv.get(scope, key);
- }),
- };
+ it("a delete touches only the bucket that owns the id, in a multi-bucket store", async () => {
+ const vector = vectorWith(
+ Array.from({ length: 5 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]),
+ );
+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 2 });
+ await persistence.save();
+ kv.ops.length = 0;
- const loaded = await new IndexPersistence(
- failingKv as never,
- new SearchIndex(),
- null,
- ).load();
+ vector.remove("obs_2");
+ await persistence.save();
- expect(loaded.bm25).toBeNull();
+ const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));
+ expect(touched).toEqual(new Set([vectorBucketScope(1)]));
+ expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(1), key: "obs_2" });
});
- it("fails closed when legacy snapshot reads fail", async () => {
- const failingKv = {
- ...kv,
- get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {
- if (scope === BM25_SCOPE && key === BM25_LEGACY_KEY) {
- throw new Error("legacy backend unavailable");
- }
- return kv.get(scope, key);
- }),
- };
+ it("clearing the index deletes every persisted vector", async () => {
+ const vector = vectorWith([
+ ["obs_a", [0.1, 0.2, 0.3]],
+ ["obs_b", [0.4, 0.5, 0.6]],
+ ]);
+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 });
+ await persistence.save();
- const loaded = await new IndexPersistence(
- failingKv as never,
- new SearchIndex(),
- null,
- ).load();
+ vector.clear();
+ vector.add("obs_c", "ses_c", vec([0, 1, 0]));
+ await persistence.save();
- expect(loaded.bm25).toBeNull();
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expect([...loaded.vector!.entries()].map(([id]) => id)).toEqual(["obs_c"]);
});
- it("loads sharded manifests that omit optional generation metadata", async () => {
- const bm25 = makeBm25("obs_1", "deterministic shard auth");
- const serialized = bm25.serialize();
- const chunks = [serialized.slice(0, 50), serialized.slice(50)];
- await kv.set("mem:index:bm25:bm25:00000", "data", chunks[0]);
- await kv.set("mem:index:bm25:bm25:00001", "data", chunks[1]);
- await kv.set<TestIndexShardManifest>(BM25_SCOPE, BM25_MANIFEST_KEY, {
- v: 1,
- chars: serialized.length,
- shards: [
- {
- scope: "mem:index:bm25:bm25:00000",
- key: "data",
- chars: chunks[0].length,
- },
- {
- scope: "mem:index:bm25:bm25:00001",
- key: "data",
- chars: chunks[1].length,
- },
- ],
- });
-
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
-
- expect(loaded.bm25).not.toBeNull();
- expect(loaded.bm25!.search("deterministic").length).toBe(1);
- });
+ it("keeps failed writes pending and retries them on the next save", async () => {
+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ let fail = true;
+ const flakyKv = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("state::set timed out");
+ return kv.set(scope, key, data);
+ },
+ };
+ const persistence = new IndexPersistence(flakyKv as never, vector, { bucketSize: 16 });
- it("saves and loads vector index round-trip", async () => {
- const bm25 = new SearchIndex();
- const vector = makeVector();
+ await expect(persistence.save()).resolves.toBeUndefined();
+ const failed = persistence.status();
+ expect(failed.vector?.lastError).toBe("1 of 1 vector writes failed: state::set timed out");
+ expect(failed.pendingChanges).toBe(1);
+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();
- const persistence = new IndexPersistence(kv as never, bm25, vector);
+ fail = false;
await persistence.save();
-
- const loaded = await persistence.load();
- expect(loaded.vector).not.toBeNull();
- expect(loaded.vector!.size).toBe(1);
+ expect(persistence.status().vector?.lastError).toBeNull();
+ expect(persistence.status().pendingChanges).toBe(0);
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expectSameVectors(loaded.vector, vector);
});
- it("saves vector index shards outside the BM25 scope", async () => {
- const bm25 = new SearchIndex();
+ it("a save after many vectors added over several waves touches only the currently open bucket", async () => {
const vector = new VectorIndex();
- vector.add(
- "obs_1",
- "ses_1",
- new Float32Array(Array.from({ length: 32 }, (_, i) => i)),
- );
+ const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 100 });
- const persistence = new IndexPersistence(kv as never, bm25, vector, {
- shardChars: 40,
- createGeneration: () => "gen_vector",
- });
+ for (let i = 0; i < 950; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0]));
await persistence.save();
+ expect(kv.store.get(vectorBucketScope(9))?.size).toBe(50);
+ for (let bucket = 0; bucket <= 8; bucket++) expect(kv.store.get(vectorBucketScope(bucket))?.size).toBe(100);
+ kv.ops.length = 0;
- const manifest = await kv.get<TestIndexShardManifest>(
- BM25_SCOPE,
- VECTOR_MANIFEST_KEY,
- );
- expect(manifest).not.toBeNull();
- expect(manifest!.generation).toBe("gen_vector");
- expect(manifest!.shards.length).toBeGreaterThan(1);
- expect(manifest!.shards[0].scope).toContain(":gen_vector:");
- await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull();
- await expect(
- kv.get(manifest!.shards[0].scope, manifest!.shards[0].key),
- ).resolves.toEqual(expect.any(String));
+ for (let i = 950; i < 1000; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0]));
+ await persistence.save();
+ let touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));
+ expect(touched).toEqual(new Set([vectorBucketScope(9)]));
+ expect(kv.store.get(vectorBucketScope(9))?.size).toBe(100);
+ kv.ops.length = 0;
- const loaded = await persistence.load();
- expect(loaded.vector).not.toBeNull();
- expect(loaded.vector!.size).toBe(1);
+ vector.add("obs_1000", "ses_1", vec([1000, 0, 0]));
+ await persistence.save();
+ touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));
+ expect(touched).toEqual(new Set([vectorBucketScope(10)]));
});
- it("persists empty vector snapshots so cleared vectors do not reload", async () => {
- const previousBm25 = makeBm25("obs_old", "alpha previous snapshot");
- const previousVector = makeVector("obs_old");
- await new IndexPersistence(kv as never, previousBm25, previousVector, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
-
- const nextBm25 = makeBm25("obs_new", "bravo new snapshot");
- const emptyVector = new VectorIndex();
- await new IndexPersistence(kv as never, nextBm25, emptyVector, {
- shardChars: 80,
- createGeneration: () => "gen_empty",
- }).save();
-
- const vectorManifest = await kv.get<TestIndexShardManifest>(
- BM25_SCOPE,
- VECTOR_MANIFEST_KEY,
+ it("a smaller bucket size configured later opens a new bucket instead of moving existing data", async () => {
+ const vector = vectorWith(
+ Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]),
);
- expect(vectorManifest).not.toBeNull();
- expect(vectorManifest!.generation).toBe("gen_empty");
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("bravo").length).toBe(1);
- expect(loaded.vector).not.toBeNull();
- expect(loaded.vector!.size).toBe(0);
- });
+ await new IndexPersistence(kv as never, vector, { bucketSize: 4 }).save();
+ for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true);
- it("avoids one oversized state::set string payload for persisted indexes", async () => {
- const maxStringPayloadChars = 80;
- const bm25 = new SearchIndex();
- bm25.add(
- makeObs({
- id: "obs_1",
- title: "large persisted snapshot ".repeat(40),
- narrative: "oversized state set reproduction ".repeat(40),
- }),
- );
- const vector = new VectorIndex();
- vector.add(
- "obs_1",
- "ses_1",
- new Float32Array(Array.from({ length: 64 }, (_, i) => i / 10)),
- );
- const guardedKv = {
- ...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (
- typeof data === "string" &&
- data.length > maxStringPayloadChars
- ) {
- throw new Error(`oversized state::set payload: ${scope}/${key}`);
- }
- return kv.set(scope, key, data);
- }),
- };
+ const reloaded = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, reloaded, { bucketSize: 2 });
+ const loaded = await persistence.load();
+ reloaded.restoreFrom(loaded.vector!);
+ kv.ops.length = 0;
+
+ reloaded.add("obs_new", "ses_new", vec([9, 9, 9]));
+ await persistence.save();
- await new IndexPersistence(guardedKv as never, bm25, vector, {
- shardChars: maxStringPayloadChars,
- createGeneration: () => "gen_payload_limit",
- }).save();
-
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("oversized").length).toBe(1);
- expect(loaded.vector!.size).toBe(1);
+ const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope));
+ expect(touched).toEqual(new Set([vectorBucketScope(1)]));
+ for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true);
});
- it("falls back to the default shard size for fractional values below one", async () => {
- const bm25 = makeBm25("obs_fraction", "fractional shard config");
- let newShardWrites = 0;
- const guardedKv = {
- ...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope.includes(":gen_fraction:")) {
- newShardWrites += 1;
- if (newShardWrites > 3) {
- throw new Error("fractional shard size caused zero-width shards");
- }
- }
- return kv.set(scope, key, data);
- }),
- };
+ it("detects a partial bucket set against the saved count and reports it in status", async () => {
+ const vector = vectorWith(
+ Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]),
+ );
+ await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save();
+ kv.store.get(vectorBucketScope(1))?.clear();
- await new IndexPersistence(guardedKv as never, bm25, null, {
- shardChars: 0.5,
- createGeneration: () => "gen_fraction",
- }).save();
+ const persistence = new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 });
+ const loaded = await persistence.load();
- const manifest = await getBm25Manifest(kv);
- expect(manifest.generation).toBe("gen_fraction");
- expect(manifest.shards.length).toBe(1);
- expect(newShardWrites).toBe(1);
+ expect(loaded.state).toBe("buckets");
+ expect(loaded.vector!.size).toBe(2);
+ expect(loaded.expectedCount).toBe(4);
+ expect(persistence.status().vectorCountShortfall).toEqual({ expected: 4, loaded: 2 });
});
- it("keeps the previous generation when a shard write fails before manifest commit", async () => {
- const previous = makeBm25("obs_old", "alpha previous snapshot");
- await new IndexPersistence(kv as never, previous, null, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
- const previousManifest = await getBm25Manifest(kv);
-
- let newShardWrites = 0;
+ it("reports storage as unavailable when the metadata read fails", async () => {
const failingKv = {
...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope.includes(":gen_new:")) {
- newShardWrites += 1;
- if (newShardWrites === 2) throw new Error("shard write failed");
- }
- return kv.set(scope, key, data);
- }),
+ get: async () => {
+ throw new Error("engine down");
+ },
};
+ const loaded = await new IndexPersistence(failingKv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null });
+ });
- const next = makeBm25("obs_new", "bravo new snapshot");
- await new IndexPersistence(failingKv as never, next, null, {
- shardChars: 80,
- createGeneration: () => "gen_new",
- }).save();
-
- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
- previousManifest,
- );
- await expect(
- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
- ).resolves.toBeNull();
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("alpha").length).toBe(1);
- expect(loaded.bm25!.search("bravo").length).toBe(0);
+ it("treats a missing store as empty, including adapters that return undefined", async () => {
+ const undefinedKv = { ...kv, get: async () => undefined };
+ const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expect(loaded).toEqual({ vector: null, state: "none", savedAt: null });
});
+});
- it("keeps the previous generation when manifest set rejects before commit", async () => {
- const previous = makeBm25("obs_old", "alpha previous snapshot");
- await new IndexPersistence(kv as never, previous, null, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
- const previousManifest = await getBm25Manifest(kv);
+describe("IndexPersistence migration from the single-string format", () => {
+ let kv: MockKV;
- const failingKv = {
- ...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {
- throw new Error("manifest write failed");
- }
- return kv.set(scope, key, data);
- }),
- };
+ beforeEach(() => {
+ kv = mockKV();
+ });
- const next = makeBm25("obs_new", "bravo new snapshot");
- await new IndexPersistence(failingKv as never, next, null, {
- shardChars: 80,
- createGeneration: () => "gen_new",
- }).save();
+ it("migrates a sharded vector snapshot into buckets and removes the old shards", async () => {
+ const legacy = vectorWith([
+ ["obs_a", [0.1, 0.2, 0.3]],
+ ["obs_b", [0.4, 0.5, 0.6]],
+ ["obs_c", [0.7, 0.8, 0.9]],
+ ]);
+ await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" });
+ await writeLegacyBm25Snapshot(kv);
+
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
+
+ expect(loaded.state).toBe("migrated");
+ expect(loaded.savedAt).toBe(new Date(parseInt("mfabcd12", 36)).toISOString());
+ expectSameVectors(loaded.vector, legacy);
+ expect(loaded.vector!.pendingChanges).toBe(0);
+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull();
+ expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull();
+ for (const [scope, entries] of kv.store) {
+ if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0);
+ }
- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
- previousManifest,
- );
- await expect(
- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
- ).resolves.toBeNull();
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("alpha").length).toBe(1);
- expect(loaded.bm25!.search("bravo").length).toBe(0);
+ const reloaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expect(reloaded.state).toBe("buckets");
+ expectSameVectors(reloaded.vector, legacy);
});
- it("keeps a generation loadable when manifest set commits before rejecting", async () => {
- const previous = makeBm25("obs_old", "alpha previous snapshot");
- await new IndexPersistence(kv as never, previous, null, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
+ it("removes the old BM25 shards when missing keys read back as undefined, as the engine SDK returns them", async () => {
+ const baseGet = kv.get;
+ kv.get = (async <T>(scope: string, key: string) => {
+ const value = await baseGet<T>(scope, key);
+ return value === null ? undefined : value;
+ }) as typeof kv.get;
+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_bbbbbbbbbbbb" });
+ await writeLegacyBm25Snapshot(kv);
- const failingKv = {
- ...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {
- await kv.set(scope, key, data);
- throw new Error("manifest write timed out after commit");
- }
- return kv.set(scope, key, data);
- }),
- };
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
- const next = makeBm25("obs_new", "bravo new snapshot");
- await new IndexPersistence(failingKv as never, next, null, {
- shardChars: 80,
- createGeneration: () => "gen_new",
- }).save();
-
- const manifest = await getBm25Manifest(kv);
- expect(manifest.generation).toBe("gen_new");
- await expect(
- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
- ).resolves.toEqual(expect.any(String));
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("bravo").length).toBe(1);
+ expect(loaded.state).toBe("migrated");
+ expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeUndefined();
+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeUndefined();
+ for (const [scope, entries] of kv.store) {
+ if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0);
+ }
});
- it("deletes a shard that committed before set rejected", async () => {
- const previous = makeBm25("obs_old", "alpha previous snapshot");
- await new IndexPersistence(kv as never, previous, null, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
- const previousManifest = await getBm25Manifest(kv);
+ it("migrates a monolithic vector snapshot", async () => {
+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ await writeLegacyVectorSnapshot(kv, legacy, { monolithic: true });
- const failingKv = {
- ...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope === "mem:index:bm25:bm25:gen_new:00000") {
- await kv.set(scope, key, data);
- throw new Error("state::set timed out after commit");
- }
- return kv.set(scope, key, data);
- }),
- };
-
- const next = makeBm25("obs_new", "bravo new snapshot");
- await new IndexPersistence(failingKv as never, next, null, {
- shardChars: 80,
- createGeneration: () => "gen_new",
- }).save();
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
- previousManifest,
- );
- await expect(
- kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
- ).resolves.toBeNull();
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("alpha").length).toBe(1);
- expect(loaded.bm25!.search("bravo").length).toBe(0);
+ expect(loaded.state).toBe("migrated");
+ expectSameVectors(loaded.vector, legacy);
+ expect(await kv.get(INDEX_SCOPE, "vectors")).toBeNull();
});
- it("loads the new generation even when old generation cleanup fails", async () => {
- const previous = makeBm25("obs_old", "alpha previous snapshot");
- await new IndexPersistence(kv as never, previous, null, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
+ it("leaves an unreadable legacy snapshot in place and writes nothing", async () => {
+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ await writeLegacyVectorSnapshot(kv, legacy);
+ const manifest = await kv.get<{ shards: Array<{ scope: string; key: string }> }>(INDEX_SCOPE, "vectors:manifest");
+ await kv.delete(manifest!.shards[0].scope, manifest!.shards[0].key);
+ kv.ops.length = 0;
- const cleanupKv = {
- ...kv,
- delete: vi.fn(async () => {
- throw new Error("cleanup failed");
- }),
- };
- const next = makeBm25("obs_new", "bravo new snapshot");
- await new IndexPersistence(cleanupKv as never, next, null, {
- shardChars: 80,
- createGeneration: () => "gen_new",
- }).save();
-
- const manifest = await getBm25Manifest(kv);
- expect(manifest.generation).toBe("gen_new");
- expect(cleanupKv.delete).toHaveBeenCalled();
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("bravo").length).toBe(1);
- expect(loaded.bm25!.search("alpha").length).toBe(0);
- });
+ const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
- it("keeps the previous vector generation when vector save fails after BM25 publish", async () => {
- const previousBm25 = makeBm25("obs_old", "alpha previous snapshot");
- const previousVector = makeVector("obs_old");
- await new IndexPersistence(kv as never, previousBm25, previousVector, {
- shardChars: 80,
- createGeneration: () => "gen_old",
- }).save();
+ expect(loaded.state).toBe("unavailable");
+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();
+ expect(kv.ops.filter((o) => o.op === "set" || o.op === "delete")).toEqual([]);
+ });
- const failingKv = {
+ it("keeps the legacy snapshot when migration writes fail and finishes on a later save", async () => {
+ const legacy = vectorWith([
+ ["obs_a", [0.1, 0.2, 0.3]],
+ ["obs_b", [0.4, 0.5, 0.6]],
+ ]);
+ await writeLegacyVectorSnapshot(kv, legacy);
+ let fail = true;
+ const flakyKv = {
...kv,
- set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
- if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) {
- throw new Error("vector manifest write failed");
- }
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("write failed");
return kv.set(scope, key, data);
- }),
+ },
};
- const nextBm25 = makeBm25("obs_new", "bravo new snapshot");
- const nextVector = new VectorIndex();
- nextVector.add("obs_new", "ses_1", new Float32Array([0.4, 0.5, 0.6]));
-
- await new IndexPersistence(failingKv as never, nextBm25, nextVector, {
- shardChars: 80,
- createGeneration: () => "gen_new",
- }).save();
-
- await expect(
- kv.get("mem:index:bm25:vectors:gen_new:00000", "data"),
- ).resolves.toBeNull();
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
- expect(loaded.bm25!.search("bravo").length).toBe(1);
- expect(loaded.vector!.size).toBe(1);
- expect(
- loaded.vector!.search(new Float32Array([0.1, 0.2, 0.3]))[0]?.obsId,
- ).toBe("obs_old");
- });
-
- it("fails closed when a manifest shard is missing", async () => {
- const bm25 = makeBm25("obs_1", "alpha sharded snapshot");
- await new IndexPersistence(kv as never, bm25, null, {
- shardChars: 80,
- createGeneration: () => "gen_missing",
- }).save();
- const manifest = await getBm25Manifest(kv);
- await kv.delete(manifest.shards[0].scope, manifest.shards[0].key);
-
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
-
- expect(loaded.bm25).toBeNull();
- });
-
- it("fails closed when a manifest shard length mismatches", async () => {
- const bm25 = makeBm25("obs_1", "alpha sharded snapshot");
- await new IndexPersistence(kv as never, bm25, null, {
- shardChars: 80,
- createGeneration: () => "gen_mismatch",
- }).save();
- const manifest = await getBm25Manifest(kv);
- const firstShard = manifest.shards[0];
- const chunk = await kv.get<string>(firstShard.scope, firstShard.key);
- await kv.set(firstShard.scope, firstShard.key, `${chunk}x`);
-
- const loaded = await new IndexPersistence(
- kv as never,
- new SearchIndex(),
- null,
- ).load();
-
- expect(loaded.bm25).toBeNull();
- });
- it("fails closed before reading invalid shard descriptors", async () => {
- await kv.set<TestIndexShardManifest>(BM25_SCOPE, BM25_MANIFEST_KEY, {
- v: 1,
- chars: 10,
- shards: [{ scope: "", key: "data", chars: 10 }],
- });
- const guardedKv = {
- ...kv,
- get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {
- if (scope === "") {
- throw new Error("invalid shard descriptor was read");
- }
- return kv.get(scope, key);
- }),
- };
+ const live = new VectorIndex();
+ const persistence = new IndexPersistence(flakyKv as never, live, { bucketSize: 16 });
+ const loaded = await persistence.load();
+ expect(loaded.state).toBe("unavailable");
+ expectSameVectors(loaded.vector, legacy);
+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();
+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();
- const loaded = await new IndexPersistence(
- guardedKv as never,
- new SearchIndex(),
- null,
- ).load();
+ live.restoreFrom(loaded.vector!);
+ fail = false;
+ await persistence.save();
- expect(loaded.bm25).toBeNull();
- expect(guardedKv.get).not.toHaveBeenCalledWith("", "data");
+ const next = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load();
+ expect(next.state).toBe("buckets");
+ expectSameVectors(next.vector, legacy);
+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull();
});
- it("scheduleSave debounces multiple calls", async () => {
- const bm25 = new SearchIndex();
- const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 });
+ it("without vector search, removes only the old BM25 snapshot", async () => {
+ const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ await writeLegacyVectorSnapshot(kv, legacy);
+ await writeLegacyBm25Snapshot(kv);
- persistence.scheduleSave();
- persistence.scheduleSave();
- persistence.scheduleSave();
+ const loaded = await new IndexPersistence(kv as never, null, { bucketSize: 16 }).load();
- await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toBeNull();
-
- vi.advanceTimersByTime(5000);
- await vi.runAllTimersAsync();
-
- const saved = await kv.get<string>(BM25_SCOPE, BM25_MANIFEST_KEY);
- expect(saved).not.toBeNull();
+ expect(loaded).toEqual({ vector: null, state: "none", savedAt: null });
+ expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull();
+ expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull();
});
+});
- it("stop clears the pending timer", async () => {
- const bm25 = new SearchIndex();
- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));
- const persistence = new IndexPersistence(kv as never, bm25, null);
-
- persistence.scheduleSave();
- persistence.stop();
+describe("IndexPersistence scheduling", () => {
+ let kv: MockKV;
- vi.advanceTimersByTime(10000);
- const saved = await kv.get<string>(BM25_SCOPE, BM25_MANIFEST_KEY);
- expect(saved).toBeNull();
+ beforeEach(() => {
+ vi.useFakeTimers();
+ kv = mockKV();
});
- it("returns null indexes when nothing has been saved", async () => {
- const bm25 = new SearchIndex();
- const persistence = new IndexPersistence(kv as never, bm25, null);
-
- const loaded = await persistence.load();
- expect(loaded.bm25).toBeNull();
- expect(loaded.vector).toBeNull();
+ afterEach(() => {
+ vi.useRealTimers();
});
- it("scheduled save swallows kv.set rejection without unhandledRejection (#204)", async () => {
+ it("scheduled saves never raise unhandled rejections", async () => {
const failingKv = {
- ...mockKV(),
+ ...kv,
set: vi.fn(async () => {
- const err = new Error(
- "TIMEOUT: invocation timed out after 30000ms",
- ) as Error & { code?: string; function_id?: string };
+ const err = new Error("TIMEOUT: invocation timed out after 30000ms") as Error & { code?: string };
err.code = "TIMEOUT";
- err.function_id = "state::set";
throw err;
}),
};
- const bm25 = new SearchIndex();
- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));
- const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });
-
+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ const persistence = new IndexPersistence(failingKv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 });
let unhandled = false;
const onUnhandled = () => {
unhandled = true;
};
process.on("unhandledRejection", onUnhandled);
-
try {
persistence.scheduleSave();
- vi.advanceTimersByTime(5000);
- await vi.runAllTimersAsync();
- // give microtasks a chance to flush
- await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(5000);
expect(failingKv.set).toHaveBeenCalled();
expect(unhandled).toBe(false);
} finally {
@@ -742,143 +477,12 @@ describe("IndexPersistence", () => {
}
});
- it("save() does not throw when kv.set rejects (#204)", async () => {
- const failingKv = {
- ...mockKV(),
- set: vi.fn(async () => {
- throw new Error("TIMEOUT");
- }),
- };
- const bm25 = new SearchIndex();
- bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));
- const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });
-
- await expect(persistence.save()).resolves.toBeUndefined();
- });
-
- // #797: first run after upgrading to 0.9.25 crashed with
- // 'TypeError: Cannot read properties of undefined (reading "v")'
- // because some iii-state adapters return `undefined` (not `null`)
- // for a missing key. The load path's `value !== null` check passed
- // undefined to loadManifestData, which then read `undefined.v`.
- it("load() returns null instead of crashing when kv.get returns undefined for the manifest (#797)", async () => {
- const undefinedKv = {
- ...mockKV(),
- get: vi.fn(async () => undefined),
- };
- const persistence = new IndexPersistence(
- undefinedKv as never,
- new SearchIndex(),
- null,
- );
-
- const loaded = await persistence.load();
- expect(loaded.bm25).toBeNull();
- expect(loaded.vector).toBeNull();
- });
-
- it("load() does not crash when a manifest row value is the wrong shape (#797)", async () => {
- const wrongShapeKv = {
- ...mockKV(),
- get: vi.fn(async () => "not-a-manifest"),
- };
- const persistence = new IndexPersistence(
- wrongShapeKv as never,
- new SearchIndex(),
- null,
- );
-
- await expect(persistence.load()).resolves.toBeDefined();
- });
-});
-
-describe("index_persist audit gating", () => {
- let kv: ReturnType<typeof mockKV>;
- let previousFlag: string | undefined;
-
- beforeEach(() => {
- // AGENTMEMORY_* variables are documented as living in
- // ~/.agentmemory/.env, so a developer running the suite on a
- // configured machine can inherit this one. Clear it going in and put
- // whatever was there back on the way out.
- previousFlag = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
- delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
- vi.useFakeTimers();
- kv = mockKV();
- });
-
- afterEach(() => {
- vi.useRealTimers();
- if (previousFlag === undefined) {
- delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
- } else {
- process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previousFlag;
- }
- });
-
- async function indexPersistEntries(): Promise<Array<{ operation: string }>> {
- const entries = await kv.list<{ operation: string }>("mem:audit");
- return entries.filter((entry) => entry.operation === "index_persist");
- }
-
- it("writes no index_persist audit entries by default", async () => {
- const persistence = new IndexPersistence(
- kv as never,
- makeBm25("obs_1", "auth handler"),
- null,
- );
-
- await persistence.save();
-
- expect(await indexPersistEntries()).toEqual([]);
- });
-
- it.each(["1", " 1 ", "true", "TRUE", " true "])(
- "writes index_persist audit entries when set to %j",
- async (value) => {
- process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value;
- const persistence = new IndexPersistence(
- kv as never,
- makeBm25("obs_1", "auth handler"),
- null,
- );
-
- await persistence.save();
-
- expect((await indexPersistEntries()).length).toBeGreaterThan(0);
- },
- );
-
- // Anything that is not an affirmative stays off. "0" and "false" are the
- // ones an operator is likely to reach for to disable it, and they must
- // not read as "present, therefore enabled".
- it.each(["0", "false", "yes", "", " "])(
- "keeps auditing off when set to %j",
- async (value) => {
- process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value;
- const persistence = new IndexPersistence(
- kv as never,
- makeBm25("obs_1", "auth handler"),
- null,
- );
-
- await persistence.save();
-
- expect(await indexPersistEntries()).toEqual([]);
- },
- );
-
- it("still persists the index when auditing is off", async () => {
- const persistence = new IndexPersistence(
- kv as never,
- makeBm25("obs_1", "auth handler"),
- null,
- );
-
- await persistence.save();
-
- const loaded = await persistence.load();
- expect(loaded.bm25).not.toBeNull();
- expect(loaded.bm25!.size).toBe(1);
+ it("stop clears the pending timer", async () => {
+ const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]);
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 });
+ persistence.scheduleSave();
+ persistence.stop();
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();
});
});
diff --git a/test/search.test.ts b/test/search.test.ts
index 84b8134e..99b7f4d6 100644
--- a/test/search.test.ts
+++ b/test/search.test.ts
@@ -209,4 +209,21 @@ describe("mem::search", () => {
setVectorIndex(null);
setEmbeddingProvider(null);
});
+
+ it("a cold-start search rebuilds BM25 without touching the persisted vector index", async () => {
+ const vector = new VectorIndex();
+ vector.add("obs_persisted", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
+ setVectorIndex(vector);
+ getSearchIndex().clear();
+
+ const result = (await sdk.trigger("mem::search", {
+ query: "auth middleware",
+ })) as { results: Array<{ observation: CompressedObservation }> };
+
+ expect(result.results[0]?.observation.id).toBe("obs_a");
+ expect(getVectorIndex()?.has("obs_persisted")).toBe(true);
+ expect(getVectorIndex()?.size).toBe(1);
+
+ setVectorIndex(null);
+ });
});
diff --git a/test/status.test.ts b/test/status.test.ts
index 6ecfb50c..2160a65f 100644
--- a/test/status.test.ts
+++ b/test/status.test.ts
@@ -27,6 +27,8 @@ function inputs(overrides: Partial<StatusInputs> = {}): StatusInputs {
observationsIndexed: 110,
missingObservations: 0,
sessions: 95,
+ bm25Incomplete: false,
+ pendingVectorBackfill: 0,
},
graph: {
totalNodes: 55,
@@ -93,6 +95,28 @@ describe("evaluateStatus", () => {
expect(report.status).toBe("info");
});
+ it("reports an incomplete BM25 rebuild as an error", () => {
+ const report = evaluateStatus(inputs({ index: { ...inputs().index, bm25Incomplete: true } }));
+ expect(report.status).toBe("error");
+ expect(codes(report)).toContain("bm25-rebuild-incomplete");
+ });
+
+ it("notes a pending vector backfill without raising the overall status", () => {
+ const report = evaluateStatus(inputs({ index: { ...inputs().index, pendingVectorBackfill: 42 } }));
+ expect(report.status).toBe("info");
+ const problem = report.problems.find((p) => p.code === "index-vector-backfill-pending");
+ expect(problem?.message).toBe("42 documents are waiting for a vector embedding.");
+ });
+
+ it("reports a vector count shortfall from the last save as a warning", () => {
+ const report = evaluateStatus(
+ inputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, buckets: 3, pendingChanges: 0, vector: null, vectorCountShortfall: { expected: 100, loaded: 40 } } }),
+ );
+ expect(report.status).toBe("warn");
+ const problem = report.problems.find((p) => p.code === "index-vector-count-shortfall");
+ expect(problem?.message).toContain("40 of 100 vectors");
+ });
+
it("checks snapshot presence when extraction is on, and snapshot age whenever a snapshot exists", () => {
const noSnapshot = { totalNodes: 0, totalEdges: 0, fromSnapshot: false };
expect(codes(evaluateStatus(inputs({ graph: noSnapshot })))).toEqual(["graph-no-snapshot"]);