#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix(index): keep vectors across a force-kill with a pending log and a durable backlog
Rohit Ghumarecd4ca4eb5eae
diff --git a/src/functions/search.ts b/src/functions/search.tsindex 0130d0d7..8fe22e01 100644--- a/src/functions/search.ts+++ b/src/functions/search.ts@@ -70,7 +70,7 @@ export function isKeywordRebuildInProgress(): boolean {}let pendingVectorBackfill = 0-export type VectorBackfillState = "idle" | "running" | "waiting-for-opt-in"+export type VectorBackfillState = "idle" | "running" | "paused" | "waiting-for-opt-in"let vectorBackfillState: VectorBackfillState = "idle"export function getVectorBackfillState(): VectorBackfillState {return vectorBackfillState@@ -575,7 +575,7 @@ async function runKeywordRebuild(fullBackfillPending++return}- if (!backfillActive || vectorJobs.length >= backfillCap) return+ if (!backfillActive || (wholeStoreBackfill && vectorJobs.length >= backfillCap)) returnvectorJobs.push({ id, sessionId, text, context: { kind, logId: id } })}@@ -632,24 +632,64 @@ async function runKeywordRebuild(}const BACKFILL_SAVE_EVERY_BATCHES = 10+const VECTOR_BACKLOG_PAUSE_MS = 1000-export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {+async function embedBackfillJobs(+ jobs: VectorBackfillJob[],+ remainingAfter: number,+): Promise<{ ok: number; fail: number }> {const batchSize = getRebuildEmbedBatchSize()- let added = 0+ let ok = 0+ let fail = 0let batchesSinceSave = 0- setPendingVectorBackfillCount(jobs.length)+ setPendingVectorBackfillCount(remainingAfter + jobs.length)for (let offset = 0; offset < jobs.length; offset += batchSize) {- const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))- added += ok- setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize))+ const result = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))+ ok += result.ok+ fail += result.fail+ setPendingVectorBackfillCount(remainingAfter + jobs.length - Math.min(jobs.length, offset + batchSize))batchesSinceSave++if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) {await flushIndexSave()batchesSinceSave = 0}}- if (added > 0) await flushIndexSave()- return added+ if (ok > 0) await flushIndexSave()+ return { ok, fail }+}++export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {+ return (await embedBackfillJobs(jobs, 0)).ok+}++export type VectorBacklogResult = { added: number; failed: number; remaining: number; complete: boolean }++export async function backfillVectorBacklog(+ jobs: VectorBackfillJob[],+ options: { batchSize?: number; pauseMs?: number } = {},+): Promise<VectorBacklogResult> {+ const batchSize = options.batchSize && options.batchSize > 0 ? options.batchSize : getVectorBackfillMax()+ const pauseMs = options.pauseMs ?? VECTOR_BACKLOG_PAUSE_MS+ let added = 0+ let failed = 0+ for (let offset = 0; offset < jobs.length; offset += batchSize) {+ if (offset > 0 && pauseMs > 0) await delay(pauseMs)+ const after = Math.max(0, jobs.length - offset - batchSize)+ const batch = jobs.slice(offset, offset + batchSize).filter((job) => !vectorIndex?.has(job.id))+ if (batch.length === 0) {+ setPendingVectorBackfillCount(after)+ continue+ }+ const result = await embedBackfillJobs(batch, after)+ added += result.ok+ failed += result.fail+ if (result.ok === 0) {+ setPendingVectorBackfillCount(jobs.length - offset)+ return { added, failed, remaining: jobs.length - offset, complete: false }+ }+ }+ setPendingVectorBackfillCount(failed)+ return { added, failed, remaining: failed, complete: failed === 0 }}export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {diff --git a/src/functions/status.ts b/src/functions/status.tsindex 5e0b3537..9cc858eb 100644--- a/src/functions/status.ts+++ b/src/functions/status.ts@@ -205,12 +205,21 @@ function secondsBetween(later: Date, earlierIso: string | undefined): number | nreturn Math.max(0, Math.round((later.getTime() - earlier) / 1000));}+function describeBackfillState(state: VectorBackfillState): string {+ if (state === "waiting-for-opt-in") return "paused, waiting for opt-in";+ if (state === "paused") return "paused, retried on the next start";+ return state;+}+function vectorBackfillFix(state: VectorBackfillState | undefined): string {if (state === "waiting-for-opt-in") {return "Backfill is paused. Set AGENTMEMORY_VECTOR_BACKFILL=all and restart to opt in. This calls the embedding provider and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX.";}if (state === "running") {- return "Backfill is running in the background, capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. Check the server log for embedding provider errors if progress stops.";+ return "Backfill is running in the background in batches of AGENTMEMORY_VECTOR_BACKFILL_MAX with a short pause between batches. If agentmemory stops first, it continues on the next start. Check the server log for embedding provider errors if progress stops.";+ }+ if (state === "paused") {+ return "Backfill stopped because the embedding provider did not return vectors. Search still works by keyword. Check the embedding provider key and the server log; the remaining documents are retried on the next start.";}return "Backfill is not running. Check the server log and embedding provider configuration. A full backfill requires AGENTMEMORY_VECTOR_BACKFILL=all and a restart, and can consume embedding provider tokens.";}@@ -348,6 +357,14 @@ export function evaluateStatus(input: StatusInputs): StatusReport {});}}+ if (persistence?.pendingLogError) {+ problems.push({+ level: "warn",+ code: "index-pending-log-failing",+ message: `New vectors could not be written to the pending vector log: ${persistence.pendingLogError}. They are still kept by the next index save, but a crash before then means they are embedded again at the next start.`,+ fix: "Check the server log for the failing state write. Each new vector retries the log write.",+ });+ }if (persistence?.vectorCountShortfall) {const { expected, loaded } = persistence.vectorCountShortfall;problems.push({@@ -558,6 +575,9 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string {return leg.dirtySince ? `${saved}, unsaved changes pending` : saved;}+const PENDING_LOG_EXPLAINER =+ "Every new or removed vector is also written right away to this small log. After a crash or force-kill it is replayed at start, so vectors are not lost and nothing is embedded twice. Each index save empties it.";+function indexPersistenceRows(report: StatusReport): string {const persistence = report.indexPersistence;if (!persistence) return "";@@ -568,6 +588,13 @@ function indexPersistenceRows(report: StatusReport): string {"Vector storage",escapeHtml(`${plural(persistence.buckets, "bucket")}, ${plural(persistence.pendingChanges, "unsaved change")}`),);+ if (persistence.pendingLog !== undefined) {+ rows += row(+ "Pending vector log",+ escapeHtml(plural(persistence.pendingLog, "vector change")) ++ `<p class="note">${escapeHtml(PENDING_LOG_EXPLAINER)}</p>`,+ );+ }}return rows;}@@ -698,7 +725,7 @@ ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked")${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}${row("Keyword index rebuild", idx.bm25Incomplete ? '<span class="warn">incomplete</span>' : "complete")}${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))}-${idx.vectorBackfillState ? row("Vector backfill", escapeHtml(idx.vectorBackfillState === "waiting-for-opt-in" ? "paused, waiting for opt-in" : idx.vectorBackfillState)) : ""}+${idx.vectorBackfillState ? row("Vector backfill", escapeHtml(describeBackfillState(idx.vectorBackfillState))) : ""}${indexPersistenceRows(report)}</table><h2>Knowledge graph</h2><table>diff --git a/src/index.ts b/src/index.tsindex 3b711ec3..8b738d7b 100644--- a/src/index.ts+++ b/src/index.ts@@ -14,6 +14,7 @@ import {getConsolidationIntervalMs,isContextInjectionEnabled,isDropStaleIndexEnabled,+ getVectorBackfillMax,getAuditRetentionMonths,getStateBackend,isSessionSweepEnabled,@@ -42,6 +43,8 @@ import { registerCompressFunction } from "./functions/compress.js";import {registerSearchFunction,backfillVectors,+ backfillVectorBacklog,+ isBm25RebuildIncomplete,rebuildKeywordIndex,markKeywordRebuildPending,getSearchIndex,@@ -512,18 +515,37 @@ async function main() {}}+ if (vectorIndex && loaded && loaded.state !== "unavailable") {+ const replay = await indexPersistence.replayPendingLog(embeddingProvider?.dimensions ?? 0);+ if (replay.entries > 0) {+ bootLog(+ `Recovered ${replay.added} vectors and ${replay.removed} removals written after the last index save, without calling the embedding provider` ++ (replay.skipped > 0 ? ` (${replay.skipped} entries skipped)` : ""),+ );+ }+ }+const auditMigration = startAuditMigration(kv).catch(() => {});const vectorCountShortfall =Boolean(loaded?.vector) &&loaded?.expectedCount !== undefined &&loaded.vector!.size < loaded.expectedCount;- const vectorBackfillSince =+ let vectorBackfillSince =!loaded || loaded.state === "unavailable"? undefined: loaded.state === "none" || vectorCountShortfall? null: loaded.savedAt;+ const incrementalBackfill = typeof vectorBackfillSince === "string" && Boolean(vectorIndex && embeddingProvider);+ if (incrementalBackfill) {+ const since = vectorBackfillSince as string;+ const marker = await indexPersistence.readBackfillMarker();+ if (marker !== null && Date.parse(marker) < Date.parse(since)) vectorBackfillSince = marker;+ await indexPersistence.markBackfillSince(vectorBackfillSince as string).catch((err) => {+ bootWarn(`Could not save the vector backfill marker: ${err instanceof Error ? err.message : String(err)}`);+ });+ }const keywordStart = Date.now();try {const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince);@@ -538,7 +560,31 @@ async function main() {`(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`,);}- if (keyword.vectorJobs.length > 0) {+ const backlogScanComplete = !isBm25RebuildIncomplete();+ if (incrementalBackfill && backlogScanComplete && keyword.vectorJobs.length === 0) {+ await indexPersistence.clearBackfillMarker().catch(() => {});+ }+ if (incrementalBackfill && keyword.vectorJobs.length > 0) {+ setVectorBackfillState("running");+ bootLog(+ `Re-embedding ${keyword.vectorJobs.length} vectors missing since the last index save, ${getVectorBackfillMax()} per batch in the background`,+ );+ void backfillVectorBacklog(keyword.vectorJobs)+ .then(async (result) => {+ if (result.complete && backlogScanComplete) await indexPersistence.clearBackfillMarker().catch(() => {});+ setVectorBackfillState(result.complete ? "idle" : "paused");+ if (result.added > 0) bootLog(`Vector index backfilled: ${result.added} entries`);+ if (!result.complete) {+ bootWarn(+ `Vector backfill stopped with ${result.remaining} documents still missing a vector. They are retried on the next start.`,+ );+ }+ })+ .catch((err) => {+ setVectorBackfillState("paused");+ console.warn(`[agentmemory] Failed to backfill vectors:`, err);+ });+ } else if (keyword.vectorJobs.length > 0) {setVectorBackfillState("running");bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`);void backfillVectors(keyword.vectorJobs)diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.tsindex 7313c109..9f1cd810 100644--- a/src/state/index-persistence.ts+++ b/src/state/index-persistence.ts@@ -1,4 +1,4 @@-import { VectorIndex, base64ToFloat32, float32ToBase64 } from "./vector-index.js";+import { VectorIndex, base64ToFloat32, float32ToBase64, type VectorEntry } from "./vector-index.js";import type { StateKV } from "./kv.js";import { KV } from "./schema.js";import { logger } from "../logger.js";@@ -16,6 +16,11 @@ const VECTOR_BUCKET_SCOPE_PREFIX = `${KV.bm25Index}:vec:`;const WRITE_CONCURRENCY = 32;const LOAD_CONCURRENCY = 8;const FIRST_CHECKPOINT_DELAY_MS = 5_000;+const PENDING_CLEAR_KEY = "~clear";+const PENDING_LOG_SAVE_THRESHOLD = 500;+const EARLY_SAVE_MIN_GAP_MS = 5_000;+const BACKFILL_MARKER_KEY = "vectors:backfill";+const MEMORY_ID_PREFIX = "mem_";function auditIndexPersistEnabled(): boolean {const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;@@ -44,6 +49,29 @@ type PersistedVector = {e: string;};+type PendingVectorRow = {+ q: number;+ id?: string;+ s?: string;+ k?: "memory" | "observation";+ e?: string;+ t?: 1;+ c?: 1;+};++type BackfillMarker = {+ v: 1;+ since: string;+};++export interface PendingReplayResult {+ entries: number;+ added: number;+ removed: number;+ cleared: boolean;+ skipped: number;+}+type IndexPersistenceOptions = {saveIntervalMs?: number;bucketSize?: number;@@ -71,6 +99,8 @@ export interface IndexPersistenceStatus {pendingChanges: number;vector: IndexLegStatus | null;vectorCountShortfall: VectorCountShortfall | null;+ pendingLog?: number;+ pendingLogError?: string | null;}export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable";@@ -133,6 +163,15 @@ async function inBatches<T>(items: T[], size: number, run: (item: T) => Promise<return failures;}+function isPendingVectorRow(row: unknown): row is PendingVectorRow {+ if (!row || typeof row !== "object") return false;+ const candidate = row as PendingVectorRow;+ if (typeof candidate.q !== "number" || !Number.isFinite(candidate.q)) return false;+ if (candidate.c === 1) return true;+ if (typeof candidate.id !== "string" || candidate.id.length === 0) return false;+ return candidate.t === 1 || typeof candidate.e === "string";+}+function emptyLegStatus(): IndexLegStatus {return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null };}@@ -158,12 +197,20 @@ export class IndexPersistence {private highestBucket = 0;private hasOpenBucket = false;private vectorCountShortfall: VectorCountShortfall | null = null;+ private seq = 0;+ private pendingLog: Map<string, number> = new Map();+ private logChains: Map<string, Promise<void>> = new Map();+ private logSuppressed = false;+ private pendingLogUnknown = false;+ private pendingLogError: string | null = null;+ private earlySaveQueued = false;constructor(private kv: StateKV,private vector: VectorIndex | null,options: IndexPersistenceOptions = {},) {+ this.vector?.setChangeListener((id, entry) => this.logChange(id, entry));this.now = options.now ?? Date.now;const interval = options.saveIntervalMs;this.saveIntervalMs =@@ -211,9 +258,176 @@ export class IndexPersistence {pendingChanges: this.vector?.pendingChanges ?? 0,vector: this.vector ? { ...this.leg } : null,vectorCountShortfall: this.vectorCountShortfall,+ pendingLog: this.vector ? this.pendingLog.size : 0,+ pendingLogError: this.pendingLogError,};}+ async replayPendingLog(expectedDimensions = 0): Promise<PendingReplayResult> {+ const result: PendingReplayResult = { entries: 0, added: 0, removed: 0, cleared: false, skipped: 0 };+ const vector = this.vector;+ if (!vector) return result;+ let rows: unknown[];+ try {+ rows = await this.kv.list<unknown>(KV.vectorPendingLog);+ } catch (err) {+ this.pendingLogUnknown = true;+ this.pendingLogError = errorMessage(err);+ logger.warn("index persistence: could not read the pending vector log", { message: this.pendingLogError });+ return result;+ }+ const valid = (Array.isArray(rows) ? rows : []).filter(isPendingVectorRow).sort((a, b) => a.q - b.q);+ this.logSuppressed = true;+ try {+ for (const row of valid) {+ result.entries++;+ const key = row.c === 1 ? PENDING_CLEAR_KEY : row.id!;+ if ((this.pendingLog.get(key) ?? -Infinity) < row.q) this.pendingLog.set(key, row.q);+ if (row.q > this.seq) this.seq = row.q;+ if (row.c === 1) {+ vector.clear();+ result.cleared = true;+ continue;+ }+ const id = row.id!;+ if (row.t === 1) {+ if (vector.has(id)) {+ vector.remove(id);+ result.removed++;+ }+ continue;+ }+ let embedding: Float32Array;+ try {+ embedding = base64ToFloat32(row.e!);+ } catch {+ result.skipped++;+ continue;+ }+ if (embedding.length === 0 || (expectedDimensions > 0 && embedding.length !== expectedDimensions)) {+ result.skipped++;+ continue;+ }+ vector.add(id, typeof row.s === "string" ? row.s : "", embedding);+ result.added++;+ }+ } finally {+ this.logSuppressed = false;+ }+ if (result.added + result.removed > 0 || result.cleared) this.scheduleSave();+ return result;+ }++ async readBackfillMarker(): Promise<string | null> {+ if (!this.vector) return null;+ try {+ const marker = await this.kv.get<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY);+ return marker && typeof marker.since === "string" && !Number.isNaN(Date.parse(marker.since)) ? marker.since : null;+ } catch (err) {+ logger.warn("index persistence: vector backfill marker read failed", { message: errorMessage(err) });+ return null;+ }+ }++ async markBackfillSince(since: string): Promise<void> {+ if (!this.vector || Number.isNaN(Date.parse(since))) return;+ const current = await this.readBackfillMarker();+ if (current !== null && Date.parse(current) <= Date.parse(since)) return;+ await this.kv.set<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY, { v: 1, since });+ }++ async clearBackfillMarker(): Promise<void> {+ if (!this.vector) return;+ await this.kv.delete(KV.bm25Index, BACKFILL_MARKER_KEY);+ }++ async flushPendingLog(): Promise<void> {+ await Promise.all([...this.logChains.values()]);+ }++ private nextSeq(): number {+ this.seq = Math.max(this.seq + 1, this.now() * 1000);+ return this.seq;+ }++ private logChange(id: string | null, entry: VectorEntry | null): void {+ if (this.logSuppressed) return;+ const key = id ?? PENDING_CLEAR_KEY;+ const q = this.nextSeq();+ let row: PendingVectorRow;+ if (id === null) {+ row = { q, c: 1 };+ } else if (entry) {+ row = {+ q,+ id,+ s: entry.sessionId,+ k: id.startsWith(MEMORY_ID_PREFIX) ? "memory" : "observation",+ e: float32ToBase64(entry.embedding),+ };+ } else {+ row = { q, id, t: 1 };+ }+ this.pendingLog.set(key, q);+ this.enqueueLog(key, async () => {+ await this.kv.set<PendingVectorRow>(KV.vectorPendingLog, key, row);+ }).then(+ () => {+ this.pendingLogError = null;+ },+ (err) => {+ this.pendingLogError = errorMessage(err);+ this.logFailure(err);+ },+ );+ this.maybeEarlySave();+ }++ private enqueueLog(key: string, op: () => Promise<void>): Promise<void> {+ const run = (this.logChains.get(key) ?? Promise.resolve()).then(op);+ const tail = run.then(+ () => undefined,+ () => undefined,+ );+ this.logChains.set(key, tail);+ void tail.then(() => {+ if (this.logChains.get(key) === tail) this.logChains.delete(key);+ });+ return run;+ }++ private maybeEarlySave(): void {+ if (this.stopped || this.running || this.earlySaveQueued) return;+ if (this.pendingLog.size < PENDING_LOG_SAVE_THRESHOLD) return;+ if (this.now() - this.lastSaveAt < EARLY_SAVE_MIN_GAP_MS) return;+ this.earlySaveQueued = true;+ setTimeout(() => {+ this.earlySaveQueued = false;+ this.save().catch((err) => this.logFailure(err));+ }, 0);+ }++ private deletePendingKey(key: string, q: number | undefined): Promise<void> {+ return this.enqueueLog(key, async () => {+ if (this.pendingLog.get(key) !== q) return;+ await this.kv.delete(KV.vectorPendingLog, key);+ if (this.pendingLog.get(key) === q) this.pendingLog.delete(key);+ });+ }++ private async clearCoveredPendingLog(coveredSeq: number): Promise<void> {+ const clearSeq = this.pendingLog.get(PENDING_CLEAR_KEY);+ if (clearSeq !== undefined ? clearSeq <= coveredSeq : this.pendingLogUnknown) {+ await this.deletePendingKey(PENDING_CLEAR_KEY, clearSeq);+ }+ const covered = [...this.pendingLog].filter(([key, q]) => key !== PENDING_CLEAR_KEY && q <= coveredSeq);+ const failures = await inBatches(covered, WRITE_CONCURRENCY, ([key, q]) => this.deletePendingKey(key, q));+ if (failures.length > 0) {+ throw new Error(`${failures.length} of ${covered.length} pending vector log deletes failed: ${errorMessage(failures[0])}`);+ }+ this.pendingLogUnknown = false;+ }+stop(): void {this.stopped = true;this.clearTimer();@@ -321,8 +535,13 @@ export class IndexPersistence {this.markedDuringRunAt = null;this.lastSaveAt = this.now();this.firstSaveAttempted = true;+ const coveredSeq = this.seq;try {await this.writeChanges(vector);+ await this.clearCoveredPendingLog(coveredSeq).catch((err) => {+ this.pendingLogError = errorMessage(err);+ this.logFailure(err);+ });this.leg.lastSavedAt = new Date(this.now()).toISOString();this.leg.lastError = null;this.leg.lastErrorAt = null;diff --git a/src/state/schema.ts b/src/state/schema.tsindex 207be7df..c2b3925d 100644--- a/src/state/schema.ts+++ b/src/state/schema.ts@@ -11,6 +11,7 @@ export const KV = {health: "mem:health",embeddings: (obsId: string) => `mem:emb:${obsId}`,bm25Index: "mem:index:bm25",+ vectorPendingLog: "mem:index:vec-pending",relations: "mem:relations",profiles: "mem:profiles",claudeBridge: "mem:claude-bridge",diff --git a/src/state/vector-index.ts b/src/state/vector-index.tsindex 2425bef8..ddd79a4a 100644--- a/src/state/vector-index.ts+++ b/src/state/vector-index.ts@@ -36,17 +36,29 @@ function cosineSimilarity(a: Float32Array, b: Float32Array): number {export type VectorEntry = { embedding: Float32Array; sessionId: string };+export type VectorChangeListener = (obsId: string | null, entry: VectorEntry | null) => void;+export class VectorIndex {private vectors: Map<string, VectorEntry> = new Map();private changes: Map<string, boolean> = new Map();+ private listener: VectorChangeListener | null = null;++ setChangeListener(listener: VectorChangeListener | null): void {+ this.listener = listener;+ }add(obsId: string, sessionId: string, embedding: Float32Array): void {- this.vectors.set(obsId, { embedding, sessionId });+ const entry = { embedding, sessionId };+ this.vectors.set(obsId, entry);this.changes.set(obsId, true);+ this.listener?.(obsId, entry);}remove(obsId: string): void {- if (this.vectors.delete(obsId)) this.changes.set(obsId, false);+ if (this.vectors.delete(obsId)) {+ this.changes.set(obsId, false);+ this.listener?.(obsId, null);+ }}has(obsId: string): boolean {@@ -147,8 +159,10 @@ export class VectorIndex {}clear(): void {+ const hadVectors = this.vectors.size > 0;for (const obsId of this.vectors.keys()) this.changes.set(obsId, false);this.vectors.clear();+ if (hadVectors) this.listener?.(null, null);}restoreFrom(other: VectorIndex): void {diff --git a/src/viewer/index.html b/src/viewer/index.htmlindex 20a85c93..1e43ea3d 100644--- a/src/viewer/index.html+++ b/src/viewer/index.html@@ -7164,11 +7164,20 @@(b ? esc(fmtNum(b.observations) + ' observations + ' + fmtNum(b.memories) + ' memories + ' + fmtNum(b.lessons) + ' lessons. ') : '') + esc(BM25_EXPLAINER));html += hrow('Vector index', hasVectors ? esc(fmtNum(idx.vectorDocuments) + ' vectors') : 'off', 'vector',hasVectors ? 'Meaning-based search. Each vector is one embedded record.' : 'No embedding provider, so search is keyword only. Set an embedding provider key (for example OPENAI_API_KEY) and restart.');- html += hrow('Waiting for embeddings', esc(fmtNum(idx.pendingVectorBackfill || 0)), 'pendingBackfill', idx.pendingVectorBackfill ? 'Embedded in the background, capped per start by AGENTMEMORY_VECTOR_BACKFILL_MAX.' : '');+ var bfState = ri.vectorBackfillState;+ var bfNote = bfState === 'paused' ? 'Paused because the embedding provider did not return vectors. Retried on the next start.' :+ bfState === 'waiting-for-opt-in' ? 'Waiting for AGENTMEMORY_VECTOR_BACKFILL=all and a restart.' :+ 'Embedded in the background in batches of AGENTMEMORY_VECTOR_BACKFILL_MAX. If agentmemory stops first, it continues on the next start.';+ html += hrow('Waiting for embeddings', esc(fmtNum(idx.pendingVectorBackfill || 0) + (bfState && bfState !== 'idle' && idx.pendingVectorBackfill ? ' · ' + (bfState === 'waiting-for-opt-in' ? 'waiting for opt-in' : bfState) : '')), 'pendingBackfill', idx.pendingVectorBackfill ? esc(bfNote) : '');html += hrow('Missing from index', ri.missingObservations === null || ri.missingObservations === undefined ? 'not checked yet' : esc(fmtNum(ri.missingObservations)), 'missingFromIndex');html += hrow('Keyword rebuild', ri.bm25Incomplete ? '<span style="color:var(--accent);">incomplete</span>' : 'complete', null, 'Rebuilt from stored data at every start, so it is never saved to disk.');if (p) {html += hrow('Vector save', esc(legText(p.vector)), null, 'Saved at most every ' + esc(durationText(p.saveIntervalMs || 600000)) + ' (AGENTMEMORY_INDEX_SAVE_INTERVAL_MS). ' + esc(fmtNum(p.buckets) + ' bucket' + (p.buckets === 1 ? '' : 's') + ', ' + fmtNum(p.pendingChanges) + ' unsaved change' + (p.pendingChanges === 1 ? '' : 's') + (p.saving ? ', saving now' : '') + '.'));+ if (typeof p.pendingLog === 'number') {+ html += hrow('Pending vector log', esc(fmtNum(p.pendingLog) + ' vector change' + (p.pendingLog === 1 ? '' : 's')), null,+ p.pendingLogError ? '<span style="color:var(--accent);overflow-wrap:anywhere;">' + esc('Log writes failing: ' + p.pendingLogError) + '</span>' :+ 'New and removed vectors are written here right away and replayed after a crash, so nothing is embedded twice. Each save empties it.');+ }}html += hrow('Graph', esc(fmtNum(c.graphNodes) + ' nodes · ' + fmtNum(c.graphEdges) + ' edges'), 'graph', store.graph && store.graph.updatedAt ? 'Snapshot rebuilt ' + esc(timeAgo(store.graph.updatedAt)) + '.' : '');if (r && r.graphCompaction) html += graphCompactionRowHtml(r.graphCompaction);diff --git a/test/index-first-checkpoint.test.ts b/test/index-first-checkpoint.test.tsindex b0e02eaa..bbcef2ee 100644--- a/test/index-first-checkpoint.test.ts+++ b/test/index-first-checkpoint.test.ts@@ -14,7 +14,7 @@ function mockKV() {writes,get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,set: async <T>(scope: string, key: string, data: T): Promise<T> => {- writes.push({ scope, key });+ if (scope !== "mem:index:vec-pending") writes.push({ scope, key });if (!store.has(scope)) store.set(scope, new Map());store.get(scope)!.set(key, data);return data;diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.tsindex 9a28a668..62747017 100644--- a/test/index-persistence-throttle.test.ts+++ b/test/index-persistence-throttle.test.ts@@ -91,6 +91,7 @@ describe("IndexPersistence save throttling", () => {const slowKv = {...kv,set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === "mem:index:vec-pending") return kv.set(scope, key, data);inFlight++;maxInFlight = Math.max(maxInFlight, inFlight);if (gate) await gate;diff --git a/test/status.test.ts b/test/status.test.tsindex c3ed308e..4950f73e 100644--- a/test/status.test.ts+++ b/test/status.test.ts@@ -508,3 +508,32 @@ describe("markLlmFunctions", () => {expect(markLlmFunctions(metrics, "llm").some((m) => m.offWithoutLlm)).toBe(false);});});++describe("vector durability rows", () => {+ it("shows the pending vector log, a paused backlog and skipped repeats in plain language", () => {+ const report = evaluateStatus(+ inputs({+ index: { ...inputs().index, pendingVectorBackfill: 7, vectorBackfillState: "paused" },+ indexPersistence: {+ saveIntervalMs: 600000,+ saving: false,+ buckets: 2,+ pendingChanges: 3,+ vector: { lastSavedAt: "2026-09-24T11:59:00.000Z", dirtySince: null, lastError: null, lastErrorAt: null },+ vectorCountShortfall: null,+ pendingLog: 3,+ pendingLogError: "state write timed out",+ },+ observeDedup: { skippedSinceStart: 4, windowSeconds: 300 },+ }),+ );+ expect(codes(report)).toContain("index-pending-log-failing");+ const backlog = report.problems.find((p) => p.code === "index-vector-backfill-pending");+ expect(backlog?.fix).toContain("retried on the next start");+ expect(report.observeDedup).toEqual({ skippedSinceStart: 4, windowSeconds: 300 });+ const html = renderStatusHtml(report, "n");+ expect(html).toContain("3 vector changes");+ expect(html).toContain("paused, retried on the next start");+ expect(html).toContain("4 since start");+ });+});diff --git a/test/vector-pending-log.test.ts b/test/vector-pending-log.test.tsnew file mode 100644index 00000000..8bbf7ca5--- /dev/null+++ b/test/vector-pending-log.test.ts@@ -0,0 +1,343 @@+import { describe, it, expect, afterEach, vi } from "vitest";+import { IndexPersistence } from "../src/state/index-persistence.js";+import { VectorIndex } from "../src/state/vector-index.js";+import {+ backfillVectorBacklog,+ getPendingVectorBackfillCount,+ getSearchIndex,+ rebuildKeywordIndex,+ setEmbeddingProvider,+ setIndexPersistence,+ setVectorIndex,+} from "../src/functions/search.js";+import type { CompressedObservation, EmbeddingProvider, Session } from "../src/types.js";++const PENDING_SCOPE = "mem:index:vec-pending";+const INDEX_SCOPE = "mem:index:bm25";++function mockKV(options: { shuffle?: boolean } = {}) {+ const store = new Map<string, Map<string, unknown>>();+ const ops: Array<{ op: "set" | "delete" | "list"; scope: string; key?: string }> = [];+ return {+ store,+ ops,+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ ops.push({ op: "set", scope, key });+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, JSON.parse(JSON.stringify(data)));+ return data;+ },+ delete: async (scope: string, key: string): Promise<void> => {+ ops.push({ op: "delete", scope, key });+ store.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> => {+ ops.push({ op: "list", scope });+ const rows = Array.from(store.get(scope)?.values() ?? []);+ if (options.shuffle) rows.reverse();+ return rows as T[];+ },+ };+}++type MockKV = ReturnType<typeof mockKV>;++function v(seed: number): Float32Array {+ return new Float32Array([seed, seed + 0.5, seed + 1]);+}++function pendingRows(kv: MockKV): Map<string, Record<string, unknown>> {+ return (kv.store.get(PENDING_SCOPE) ?? new Map()) as Map<string, Record<string, unknown>>;+}++async function boot(kv: MockKV, dims = 3): Promise<{ vector: VectorIndex; persistence: IndexPersistence }> {+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000, bucketSize: 4 });+ const loaded = await persistence.load();+ if (loaded.vector && loaded.vector.size > 0) vector.restoreFrom(loaded.vector);+ await persistence.replayPendingLog(dims);+ return { vector, persistence };+}++describe("pending vector log", () => {+ afterEach(() => {+ vi.useRealTimers();+ getSearchIndex().clear();+ setVectorIndex(null);+ setEmbeddingProvider(null);+ setIndexPersistence(null);+ });++ it("writes a compact entry for every added vector and a tombstone for every removal", async () => {+ const kv = mockKV();+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000 });++ vector.add("obs_a", "ses_1", v(1));+ vector.add("mem_b", "ses_2", v(2));+ vector.remove("obs_a");+ await persistence.flushPendingLog();++ const rows = pendingRows(kv);+ expect(rows.size).toBe(2);+ expect(rows.get("obs_a")).toMatchObject({ id: "obs_a", t: 1 });+ expect(rows.get("mem_b")).toMatchObject({ id: "mem_b", s: "ses_2", k: "memory" });+ expect(typeof rows.get("mem_b")!.e).toBe("string");+ expect(rows.get("obs_a")!.q as number).toBeGreaterThan(rows.get("mem_b")!.q as number);+ expect(persistence.status().pendingLog).toBe(2);+ });++ it("does nothing when there is no vector index", async () => {+ const kv = mockKV();+ const persistence = new IndexPersistence(kv as never, null);+ const standalone = new VectorIndex();+ standalone.add("obs_a", "ses_1", v(1));+ await persistence.save();+ const replay = await persistence.replayPendingLog(3);++ expect(kv.store.get(PENDING_SCOPE)).toBeUndefined();+ expect(kv.ops.filter((o) => o.scope === PENDING_SCOPE)).toEqual([]);+ expect(replay.entries).toBe(0);+ expect(persistence.status().pendingLog).toBe(0);+ });++ it("replays adds and removals after a force-kill without any embedding calls", async () => {+ const kv = mockKV();+ const first = await boot(kv);+ first.vector.add("obs_1", "ses_1", v(1));+ first.vector.add("obs_2", "ses_1", v(2));+ await first.persistence.save();+ expect(pendingRows(kv).size).toBe(0);++ first.vector.add("obs_3", "ses_1", v(3));+ first.vector.add("obs_4", "ses_1", v(4));+ first.vector.remove("obs_1");+ await first.persistence.flushPendingLog();+ first.persistence.stop();++ const embed = vi.fn();+ const second = await boot(kv);+ expect(embed).not.toHaveBeenCalled();+ expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);+ expect(Array.from(second.vector.get("obs_3")!.embedding)).toEqual(Array.from(v(3)));+ expect(second.persistence.status().pendingLog).toBe(3);++ await second.persistence.save();+ expect(pendingRows(kv).size).toBe(0);+ const third = await boot(kv);+ expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);+ });++ it("skips logged vectors whose dimension does not match the active provider", async () => {+ const kv = mockKV();+ const first = await boot(kv);+ first.vector.add("obs_ok", "ses_1", v(1));+ first.vector.add("obs_wide", "ses_1", new Float32Array([1, 2, 3, 4]));+ await first.persistence.flushPendingLog();++ const vector = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, vector);+ await persistence.load();+ const replay = await persistence.replayPendingLog(3);+ expect(replay).toMatchObject({ added: 1, skipped: 1 });+ expect(vector.has("obs_wide")).toBe(false);+ });++ it("clears only the entries a snapshot covered", async () => {+ const kv = mockKV();+ let release: () => void = () => undefined;+ let gate: Promise<void> | null = new Promise<void>((resolve) => {+ release = resolve;+ });+ const slow = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope !== PENDING_SCOPE && gate) await gate;+ return kv.set(scope, key, data);+ },+ };+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(slow as never, vector, { saveIntervalMs: 600_000 });+ vector.add("obs_before", "ses_1", v(1));+ await persistence.flushPendingLog();++ const saving = persistence.save();+ vector.add("obs_during", "ses_1", v(2));+ await persistence.flushPendingLog();+ gate = null;+ release();+ await saving;++ expect([...pendingRows(kv).keys()]).toEqual(["obs_during"]);+ expect(persistence.status().pendingLog).toBe(1);+ });++ it("keeps every entry when the snapshot write fails", async () => {+ const kv = mockKV();+ const failing = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === INDEX_SCOPE) throw new Error("state write refused");+ return kv.set(scope, key, data);+ },+ };+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(failing as never, vector, { saveIntervalMs: 600_000 });+ vector.add("obs_1", "ses_1", v(1));+ vector.add("obs_2", "ses_1", v(2));+ await persistence.save();++ expect(persistence.status().vector?.lastError).toContain("state write refused");+ expect(pendingRows(kv).size).toBe(2);+ expect(kv.ops.filter((o) => o.op === "delete" && o.scope === PENDING_SCOPE)).toEqual([]);+ });++ it("orders replay by sequence, not by list order, including a full clear", async () => {+ const kv = mockKV({ shuffle: true });+ const first = await boot(kv);+ first.vector.add("obs_old", "ses_1", v(1));+ await first.persistence.save();+ first.vector.add("obs_gone", "ses_1", v(2));+ first.vector.clear();+ first.vector.add("obs_new", "ses_1", v(3));+ first.vector.add("obs_flip", "ses_1", v(4));+ first.vector.remove("obs_flip");+ first.vector.add("obs_flip", "ses_1", v(5));+ await first.persistence.flushPendingLog();++ const second = await boot(kv);+ expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);+ expect(Array.from(second.vector.get("obs_flip")!.embedding)).toEqual(Array.from(v(5)));++ await second.persistence.save();+ expect(pendingRows(kv).size).toBe(0);+ const third = await boot(kv);+ expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);+ });++ it("saves early once the log grows past its bound", async () => {+ vi.useFakeTimers();+ const kv = mockKV();+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000 });+ await vi.advanceTimersByTimeAsync(6_000);+ const metaSet = () => kv.ops.some((o) => o.op === "set" && o.key === "vectors:meta");+ for (let i = 0; i < 520; i++) vector.add(`obs_${i}`, "ses_1", v(i));+ expect(metaSet()).toBe(false);+ await vi.advanceTimersByTimeAsync(10);+ expect(persistence.status().saving || metaSet()).toBe(true);+ await persistence.save();+ await persistence.flushPendingLog();++ expect(kv.ops.some((o) => o.op === "set" && o.key === "vectors:meta")).toBe(true);+ expect(pendingRows(kv).size).toBe(0);+ });+});++describe("durable vector backlog", () => {+ afterEach(() => {+ getSearchIndex().clear();+ setVectorIndex(null);+ setEmbeddingProvider(null);+ setIndexPersistence(null);+ });++ function seed(kv: MockKV, count: number, timestamp: string): void {+ const sessionId = "ses_0";+ kv.store.set("mem:sessions", new Map([[sessionId, { id: sessionId, project: "p", cwd: "/tmp", startedAt: timestamp, status: "completed", observationCount: count } as Session]]));+ const scope = new Map<string, unknown>();+ for (let i = 0; i < count; i++) {+ const id = `obs_${String(i).padStart(3, "0")}`;+ scope.set(id, {+ id,+ sessionId,+ timestamp,+ type: "file_edit",+ title: `change ${i}`,+ facts: [],+ narrative: `narrative for change ${i}`,+ concepts: [],+ files: [],+ importance: 5,+ } satisfies CompressedObservation);+ }+ kv.store.set(`mem:obs:${sessionId}`, scope);+ }++ function provider(failAfterCalls: number | null, calls: { n: number }): EmbeddingProvider {+ return {+ name: "stub",+ dimensions: 3,+ embed: async () => new Float32Array([0.1, 0.2, 0.3]),+ embedBatch: async (texts: string[]) => {+ calls.n++;+ if (failAfterCalls !== null && calls.n > failAfterCalls) throw new Error("provider down");+ return texts.map(() => new Float32Array([0.1, 0.2, 0.3]));+ },+ } as EmbeddingProvider;+ }++ it("keeps the backfill marker until the backlog is done and resumes after a restart", async () => {+ process.env.REBUILD_EMBED_BATCH_SIZE = "5";+ try {+ const kv = mockKV();+ seed(kv, 12, "2026-09-02T00:00:00.000Z");++ const first = await boot(kv);+ setVectorIndex(first.vector);+ setIndexPersistence(first.persistence);+ const calls = { n: 0 };+ setEmbeddingProvider(provider(1, calls));+ await first.persistence.markBackfillSince("2026-09-01T00:00:00.000Z");+ const rebuilt = await rebuildKeywordIndex(kv as never, "2026-09-01T00:00:00.000Z");+ expect(rebuilt.vectorJobs).toHaveLength(12);++ const run = await backfillVectorBacklog(rebuilt.vectorJobs, { batchSize: 5, pauseMs: 0 });+ expect(run).toMatchObject({ added: 5, complete: false, remaining: 7 });+ expect(getPendingVectorBackfillCount()).toBe(7);+ expect(await first.persistence.readBackfillMarker()).toBe("2026-09-01T00:00:00.000Z");+ await first.persistence.flushPendingLog();+ first.persistence.stop();++ await first.persistence.markBackfillSince("2026-09-03T00:00:00.000Z");+ expect(await first.persistence.readBackfillMarker()).toBe("2026-09-01T00:00:00.000Z");++ getSearchIndex().clear();+ const second = await boot(kv);+ expect(second.vector.size).toBe(5);+ setVectorIndex(second.vector);+ setIndexPersistence(second.persistence);+ const resumedCalls = { n: 0 };+ setEmbeddingProvider(provider(null, resumedCalls));+ const since = await second.persistence.readBackfillMarker();+ const resumed = await rebuildKeywordIndex(kv as never, since);+ expect(resumed.vectorJobs).toHaveLength(7);++ const done = await backfillVectorBacklog(resumed.vectorJobs, { batchSize: 5, pauseMs: 0 });+ expect(done).toMatchObject({ added: 7, complete: true, remaining: 0 });+ expect(second.vector.size).toBe(12);+ expect(resumedCalls.n).toBe(2);+ await second.persistence.clearBackfillMarker();+ expect(await second.persistence.readBackfillMarker()).toBeNull();+ } finally {+ delete process.env.REBUILD_EMBED_BATCH_SIZE;+ }+ });++ it("does not cap incremental recovery at AGENTMEMORY_VECTOR_BACKFILL_MAX", async () => {+ process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "4";+ try {+ const kv = mockKV();+ seed(kv, 10, "2026-09-02T00:00:00.000Z");+ setVectorIndex(new VectorIndex());+ setEmbeddingProvider(provider(null, { n: 0 }));+ const rebuilt = await rebuildKeywordIndex(kv as never, "2026-09-01T00:00:00.000Z");+ expect(rebuilt.vectorJobs).toHaveLength(10);+ const run = await backfillVectorBacklog(rebuilt.vectorJobs, { pauseMs: 0 });+ expect(run).toMatchObject({ added: 10, complete: true });+ } finally {+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX;+ }+ });+});