Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(index): keep vectors across a force-kill with a pending log and a durable backlog

Rohit Ghumarecd4ca4eb5eae
diff --git a/src/functions/search.ts b/src/functions/search.ts
index 0130d0d7..8fe22e01 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -70,7 +70,7 @@ export function isKeywordRebuildInProgress(): boolean {
}
let pendingVectorBackfill = 0
-export type VectorBackfillState = "idle" | "running" | "waiting-for-opt-in"
+export type VectorBackfillState = "idle" | "running" | "paused" | "waiting-for-opt-in"
let vectorBackfillState: VectorBackfillState = "idle"
export function getVectorBackfillState(): VectorBackfillState {
return vectorBackfillState
@@ -575,7 +575,7 @@ async function runKeywordRebuild(
fullBackfillPending++
return
}
- if (!backfillActive || vectorJobs.length >= backfillCap) return
+ if (!backfillActive || (wholeStoreBackfill && vectorJobs.length >= backfillCap)) return
vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } })
}
@@ -632,24 +632,64 @@ async function runKeywordRebuild(
}
const BACKFILL_SAVE_EVERY_BATCHES = 10
+const VECTOR_BACKLOG_PAUSE_MS = 1000
-export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {
+async function embedBackfillJobs(
+ jobs: VectorBackfillJob[],
+ remainingAfter: number,
+): Promise<{ ok: number; fail: number }> {
const batchSize = getRebuildEmbedBatchSize()
- let added = 0
+ let ok = 0
+ let fail = 0
let batchesSinceSave = 0
- setPendingVectorBackfillCount(jobs.length)
+ setPendingVectorBackfillCount(remainingAfter + jobs.length)
for (let offset = 0; offset < jobs.length; offset += batchSize) {
- const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))
- added += ok
- setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize))
+ const result = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))
+ ok += result.ok
+ fail += result.fail
+ setPendingVectorBackfillCount(remainingAfter + jobs.length - Math.min(jobs.length, offset + batchSize))
batchesSinceSave++
if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) {
await flushIndexSave()
batchesSinceSave = 0
}
}
- if (added > 0) await flushIndexSave()
- return added
+ if (ok > 0) await flushIndexSave()
+ return { ok, fail }
+}
+
+export async function backfillVectors(jobs: VectorBackfillJob[]): Promise<number> {
+ return (await embedBackfillJobs(jobs, 0)).ok
+}
+
+export type VectorBacklogResult = { added: number; failed: number; remaining: number; complete: boolean }
+
+export async function backfillVectorBacklog(
+ jobs: VectorBackfillJob[],
+ options: { batchSize?: number; pauseMs?: number } = {},
+): Promise<VectorBacklogResult> {
+ const batchSize = options.batchSize && options.batchSize > 0 ? options.batchSize : getVectorBackfillMax()
+ const pauseMs = options.pauseMs ?? VECTOR_BACKLOG_PAUSE_MS
+ let added = 0
+ let failed = 0
+ for (let offset = 0; offset < jobs.length; offset += batchSize) {
+ if (offset > 0 && pauseMs > 0) await delay(pauseMs)
+ const after = Math.max(0, jobs.length - offset - batchSize)
+ const batch = jobs.slice(offset, offset + batchSize).filter((job) => !vectorIndex?.has(job.id))
+ if (batch.length === 0) {
+ setPendingVectorBackfillCount(after)
+ continue
+ }
+ const result = await embedBackfillJobs(batch, after)
+ added += result.ok
+ failed += result.fail
+ if (result.ok === 0) {
+ setPendingVectorBackfillCount(jobs.length - offset)
+ return { added, failed, remaining: jobs.length - offset, complete: false }
+ }
+ }
+ setPendingVectorBackfillCount(failed)
+ return { added, failed, remaining: failed, complete: failed === 0 }
}
export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {
diff --git a/src/functions/status.ts b/src/functions/status.ts
index 5e0b3537..9cc858eb 100644
--- a/src/functions/status.ts
+++ b/src/functions/status.ts
@@ -205,12 +205,21 @@ function secondsBetween(later: Date, earlierIso: string | undefined): number | n
return Math.max(0, Math.round((later.getTime() - earlier) / 1000));
}
+function describeBackfillState(state: VectorBackfillState): string {
+ if (state === "waiting-for-opt-in") return "paused, waiting for opt-in";
+ if (state === "paused") return "paused, retried on the next start";
+ return state;
+}
+
function vectorBackfillFix(state: VectorBackfillState | undefined): string {
if (state === "waiting-for-opt-in") {
return "Backfill is paused. Set AGENTMEMORY_VECTOR_BACKFILL=all and restart to opt in. This calls the embedding provider and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX.";
}
if (state === "running") {
- return "Backfill is running in the background, capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. Check the server log for embedding provider errors if progress stops.";
+ return "Backfill is running in the background in batches of AGENTMEMORY_VECTOR_BACKFILL_MAX with a short pause between batches. If agentmemory stops first, it continues on the next start. Check the server log for embedding provider errors if progress stops.";
+ }
+ if (state === "paused") {
+ return "Backfill stopped because the embedding provider did not return vectors. Search still works by keyword. Check the embedding provider key and the server log; the remaining documents are retried on the next start.";
}
return "Backfill is not running. Check the server log and embedding provider configuration. A full backfill requires AGENTMEMORY_VECTOR_BACKFILL=all and a restart, and can consume embedding provider tokens.";
}
@@ -348,6 +357,14 @@ export function evaluateStatus(input: StatusInputs): StatusReport {
});
}
}
+ if (persistence?.pendingLogError) {
+ problems.push({
+ level: "warn",
+ code: "index-pending-log-failing",
+ message: `New vectors could not be written to the pending vector log: ${persistence.pendingLogError}. They are still kept by the next index save, but a crash before then means they are embedded again at the next start.`,
+ fix: "Check the server log for the failing state write. Each new vector retries the log write.",
+ });
+ }
if (persistence?.vectorCountShortfall) {
const { expected, loaded } = persistence.vectorCountShortfall;
problems.push({
@@ -558,6 +575,9 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string {
return leg.dirtySince ? `${saved}, unsaved changes pending` : saved;
}
+const PENDING_LOG_EXPLAINER =
+ "Every new or removed vector is also written right away to this small log. After a crash or force-kill it is replayed at start, so vectors are not lost and nothing is embedded twice. Each index save empties it.";
+
function indexPersistenceRows(report: StatusReport): string {
const persistence = report.indexPersistence;
if (!persistence) return "";
@@ -568,6 +588,13 @@ function indexPersistenceRows(report: StatusReport): string {
"Vector storage",
escapeHtml(`${plural(persistence.buckets, "bucket")}, ${plural(persistence.pendingChanges, "unsaved change")}`),
);
+ if (persistence.pendingLog !== undefined) {
+ rows += row(
+ "Pending vector log",
+ escapeHtml(plural(persistence.pendingLog, "vector change")) +
+ `<p class="note">${escapeHtml(PENDING_LOG_EXPLAINER)}</p>`,
+ );
+ }
}
return rows;
}
@@ -698,7 +725,7 @@ ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked")
${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}
${row("Keyword index rebuild", idx.bm25Incomplete ? '<span class="warn">incomplete</span>' : "complete")}
${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))}
-${idx.vectorBackfillState ? row("Vector backfill", escapeHtml(idx.vectorBackfillState === "waiting-for-opt-in" ? "paused, waiting for opt-in" : idx.vectorBackfillState)) : ""}
+${idx.vectorBackfillState ? row("Vector backfill", escapeHtml(describeBackfillState(idx.vectorBackfillState))) : ""}
${indexPersistenceRows(report)}
</table>
<h2>Knowledge graph</h2><table>
diff --git a/src/index.ts b/src/index.ts
index 3b711ec3..8b738d7b 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -14,6 +14,7 @@ import {
getConsolidationIntervalMs,
isContextInjectionEnabled,
isDropStaleIndexEnabled,
+ getVectorBackfillMax,
getAuditRetentionMonths,
getStateBackend,
isSessionSweepEnabled,
@@ -42,6 +43,8 @@ import { registerCompressFunction } from "./functions/compress.js";
import {
registerSearchFunction,
backfillVectors,
+ backfillVectorBacklog,
+ isBm25RebuildIncomplete,
rebuildKeywordIndex,
markKeywordRebuildPending,
getSearchIndex,
@@ -512,18 +515,37 @@ async function main() {
}
}
+ if (vectorIndex && loaded && loaded.state !== "unavailable") {
+ const replay = await indexPersistence.replayPendingLog(embeddingProvider?.dimensions ?? 0);
+ if (replay.entries > 0) {
+ bootLog(
+ `Recovered ${replay.added} vectors and ${replay.removed} removals written after the last index save, without calling the embedding provider` +
+ (replay.skipped > 0 ? ` (${replay.skipped} entries skipped)` : ""),
+ );
+ }
+ }
+
const auditMigration = startAuditMigration(kv).catch(() => {});
const vectorCountShortfall =
Boolean(loaded?.vector) &&
loaded?.expectedCount !== undefined &&
loaded.vector!.size < loaded.expectedCount;
- const vectorBackfillSince =
+ let vectorBackfillSince =
!loaded || loaded.state === "unavailable"
? undefined
: loaded.state === "none" || vectorCountShortfall
? null
: loaded.savedAt;
+ const incrementalBackfill = typeof vectorBackfillSince === "string" && Boolean(vectorIndex && embeddingProvider);
+ if (incrementalBackfill) {
+ const since = vectorBackfillSince as string;
+ const marker = await indexPersistence.readBackfillMarker();
+ if (marker !== null && Date.parse(marker) < Date.parse(since)) vectorBackfillSince = marker;
+ await indexPersistence.markBackfillSince(vectorBackfillSince as string).catch((err) => {
+ bootWarn(`Could not save the vector backfill marker: ${err instanceof Error ? err.message : String(err)}`);
+ });
+ }
const keywordStart = Date.now();
try {
const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince);
@@ -538,7 +560,31 @@ async function main() {
`(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`,
);
}
- if (keyword.vectorJobs.length > 0) {
+ const backlogScanComplete = !isBm25RebuildIncomplete();
+ if (incrementalBackfill && backlogScanComplete && keyword.vectorJobs.length === 0) {
+ await indexPersistence.clearBackfillMarker().catch(() => {});
+ }
+ if (incrementalBackfill && keyword.vectorJobs.length > 0) {
+ setVectorBackfillState("running");
+ bootLog(
+ `Re-embedding ${keyword.vectorJobs.length} vectors missing since the last index save, ${getVectorBackfillMax()} per batch in the background`,
+ );
+ void backfillVectorBacklog(keyword.vectorJobs)
+ .then(async (result) => {
+ if (result.complete && backlogScanComplete) await indexPersistence.clearBackfillMarker().catch(() => {});
+ setVectorBackfillState(result.complete ? "idle" : "paused");
+ if (result.added > 0) bootLog(`Vector index backfilled: ${result.added} entries`);
+ if (!result.complete) {
+ bootWarn(
+ `Vector backfill stopped with ${result.remaining} documents still missing a vector. They are retried on the next start.`,
+ );
+ }
+ })
+ .catch((err) => {
+ setVectorBackfillState("paused");
+ console.warn(`[agentmemory] Failed to backfill vectors:`, err);
+ });
+ } else if (keyword.vectorJobs.length > 0) {
setVectorBackfillState("running");
bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`);
void backfillVectors(keyword.vectorJobs)
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 7313c109..9f1cd810 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -1,4 +1,4 @@
-import { VectorIndex, base64ToFloat32, float32ToBase64 } from "./vector-index.js";
+import { VectorIndex, base64ToFloat32, float32ToBase64, type VectorEntry } from "./vector-index.js";
import type { StateKV } from "./kv.js";
import { KV } from "./schema.js";
import { logger } from "../logger.js";
@@ -16,6 +16,11 @@ const VECTOR_BUCKET_SCOPE_PREFIX = `${KV.bm25Index}:vec:`;
const WRITE_CONCURRENCY = 32;
const LOAD_CONCURRENCY = 8;
const FIRST_CHECKPOINT_DELAY_MS = 5_000;
+const PENDING_CLEAR_KEY = "~clear";
+const PENDING_LOG_SAVE_THRESHOLD = 500;
+const EARLY_SAVE_MIN_GAP_MS = 5_000;
+const BACKFILL_MARKER_KEY = "vectors:backfill";
+const MEMORY_ID_PREFIX = "mem_";
function auditIndexPersistEnabled(): boolean {
const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST;
@@ -44,6 +49,29 @@ type PersistedVector = {
e: string;
};
+type PendingVectorRow = {
+ q: number;
+ id?: string;
+ s?: string;
+ k?: "memory" | "observation";
+ e?: string;
+ t?: 1;
+ c?: 1;
+};
+
+type BackfillMarker = {
+ v: 1;
+ since: string;
+};
+
+export interface PendingReplayResult {
+ entries: number;
+ added: number;
+ removed: number;
+ cleared: boolean;
+ skipped: number;
+}
+
type IndexPersistenceOptions = {
saveIntervalMs?: number;
bucketSize?: number;
@@ -71,6 +99,8 @@ export interface IndexPersistenceStatus {
pendingChanges: number;
vector: IndexLegStatus | null;
vectorCountShortfall: VectorCountShortfall | null;
+ pendingLog?: number;
+ pendingLogError?: string | null;
}
export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable";
@@ -133,6 +163,15 @@ async function inBatches<T>(items: T[], size: number, run: (item: T) => Promise<
return failures;
}
+function isPendingVectorRow(row: unknown): row is PendingVectorRow {
+ if (!row || typeof row !== "object") return false;
+ const candidate = row as PendingVectorRow;
+ if (typeof candidate.q !== "number" || !Number.isFinite(candidate.q)) return false;
+ if (candidate.c === 1) return true;
+ if (typeof candidate.id !== "string" || candidate.id.length === 0) return false;
+ return candidate.t === 1 || typeof candidate.e === "string";
+}
+
function emptyLegStatus(): IndexLegStatus {
return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null };
}
@@ -158,12 +197,20 @@ export class IndexPersistence {
private highestBucket = 0;
private hasOpenBucket = false;
private vectorCountShortfall: VectorCountShortfall | null = null;
+ private seq = 0;
+ private pendingLog: Map<string, number> = new Map();
+ private logChains: Map<string, Promise<void>> = new Map();
+ private logSuppressed = false;
+ private pendingLogUnknown = false;
+ private pendingLogError: string | null = null;
+ private earlySaveQueued = false;
constructor(
private kv: StateKV,
private vector: VectorIndex | null,
options: IndexPersistenceOptions = {},
) {
+ this.vector?.setChangeListener((id, entry) => this.logChange(id, entry));
this.now = options.now ?? Date.now;
const interval = options.saveIntervalMs;
this.saveIntervalMs =
@@ -211,9 +258,176 @@ export class IndexPersistence {
pendingChanges: this.vector?.pendingChanges ?? 0,
vector: this.vector ? { ...this.leg } : null,
vectorCountShortfall: this.vectorCountShortfall,
+ pendingLog: this.vector ? this.pendingLog.size : 0,
+ pendingLogError: this.pendingLogError,
};
}
+ async replayPendingLog(expectedDimensions = 0): Promise<PendingReplayResult> {
+ const result: PendingReplayResult = { entries: 0, added: 0, removed: 0, cleared: false, skipped: 0 };
+ const vector = this.vector;
+ if (!vector) return result;
+ let rows: unknown[];
+ try {
+ rows = await this.kv.list<unknown>(KV.vectorPendingLog);
+ } catch (err) {
+ this.pendingLogUnknown = true;
+ this.pendingLogError = errorMessage(err);
+ logger.warn("index persistence: could not read the pending vector log", { message: this.pendingLogError });
+ return result;
+ }
+ const valid = (Array.isArray(rows) ? rows : []).filter(isPendingVectorRow).sort((a, b) => a.q - b.q);
+ this.logSuppressed = true;
+ try {
+ for (const row of valid) {
+ result.entries++;
+ const key = row.c === 1 ? PENDING_CLEAR_KEY : row.id!;
+ if ((this.pendingLog.get(key) ?? -Infinity) < row.q) this.pendingLog.set(key, row.q);
+ if (row.q > this.seq) this.seq = row.q;
+ if (row.c === 1) {
+ vector.clear();
+ result.cleared = true;
+ continue;
+ }
+ const id = row.id!;
+ if (row.t === 1) {
+ if (vector.has(id)) {
+ vector.remove(id);
+ result.removed++;
+ }
+ continue;
+ }
+ let embedding: Float32Array;
+ try {
+ embedding = base64ToFloat32(row.e!);
+ } catch {
+ result.skipped++;
+ continue;
+ }
+ if (embedding.length === 0 || (expectedDimensions > 0 && embedding.length !== expectedDimensions)) {
+ result.skipped++;
+ continue;
+ }
+ vector.add(id, typeof row.s === "string" ? row.s : "", embedding);
+ result.added++;
+ }
+ } finally {
+ this.logSuppressed = false;
+ }
+ if (result.added + result.removed > 0 || result.cleared) this.scheduleSave();
+ return result;
+ }
+
+ async readBackfillMarker(): Promise<string | null> {
+ if (!this.vector) return null;
+ try {
+ const marker = await this.kv.get<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY);
+ return marker && typeof marker.since === "string" && !Number.isNaN(Date.parse(marker.since)) ? marker.since : null;
+ } catch (err) {
+ logger.warn("index persistence: vector backfill marker read failed", { message: errorMessage(err) });
+ return null;
+ }
+ }
+
+ async markBackfillSince(since: string): Promise<void> {
+ if (!this.vector || Number.isNaN(Date.parse(since))) return;
+ const current = await this.readBackfillMarker();
+ if (current !== null && Date.parse(current) <= Date.parse(since)) return;
+ await this.kv.set<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY, { v: 1, since });
+ }
+
+ async clearBackfillMarker(): Promise<void> {
+ if (!this.vector) return;
+ await this.kv.delete(KV.bm25Index, BACKFILL_MARKER_KEY);
+ }
+
+ async flushPendingLog(): Promise<void> {
+ await Promise.all([...this.logChains.values()]);
+ }
+
+ private nextSeq(): number {
+ this.seq = Math.max(this.seq + 1, this.now() * 1000);
+ return this.seq;
+ }
+
+ private logChange(id: string | null, entry: VectorEntry | null): void {
+ if (this.logSuppressed) return;
+ const key = id ?? PENDING_CLEAR_KEY;
+ const q = this.nextSeq();
+ let row: PendingVectorRow;
+ if (id === null) {
+ row = { q, c: 1 };
+ } else if (entry) {
+ row = {
+ q,
+ id,
+ s: entry.sessionId,
+ k: id.startsWith(MEMORY_ID_PREFIX) ? "memory" : "observation",
+ e: float32ToBase64(entry.embedding),
+ };
+ } else {
+ row = { q, id, t: 1 };
+ }
+ this.pendingLog.set(key, q);
+ this.enqueueLog(key, async () => {
+ await this.kv.set<PendingVectorRow>(KV.vectorPendingLog, key, row);
+ }).then(
+ () => {
+ this.pendingLogError = null;
+ },
+ (err) => {
+ this.pendingLogError = errorMessage(err);
+ this.logFailure(err);
+ },
+ );
+ this.maybeEarlySave();
+ }
+
+ private enqueueLog(key: string, op: () => Promise<void>): Promise<void> {
+ const run = (this.logChains.get(key) ?? Promise.resolve()).then(op);
+ const tail = run.then(
+ () => undefined,
+ () => undefined,
+ );
+ this.logChains.set(key, tail);
+ void tail.then(() => {
+ if (this.logChains.get(key) === tail) this.logChains.delete(key);
+ });
+ return run;
+ }
+
+ private maybeEarlySave(): void {
+ if (this.stopped || this.running || this.earlySaveQueued) return;
+ if (this.pendingLog.size < PENDING_LOG_SAVE_THRESHOLD) return;
+ if (this.now() - this.lastSaveAt < EARLY_SAVE_MIN_GAP_MS) return;
+ this.earlySaveQueued = true;
+ setTimeout(() => {
+ this.earlySaveQueued = false;
+ this.save().catch((err) => this.logFailure(err));
+ }, 0);
+ }
+
+ private deletePendingKey(key: string, q: number | undefined): Promise<void> {
+ return this.enqueueLog(key, async () => {
+ if (this.pendingLog.get(key) !== q) return;
+ await this.kv.delete(KV.vectorPendingLog, key);
+ if (this.pendingLog.get(key) === q) this.pendingLog.delete(key);
+ });
+ }
+
+ private async clearCoveredPendingLog(coveredSeq: number): Promise<void> {
+ const clearSeq = this.pendingLog.get(PENDING_CLEAR_KEY);
+ if (clearSeq !== undefined ? clearSeq <= coveredSeq : this.pendingLogUnknown) {
+ await this.deletePendingKey(PENDING_CLEAR_KEY, clearSeq);
+ }
+ const covered = [...this.pendingLog].filter(([key, q]) => key !== PENDING_CLEAR_KEY && q <= coveredSeq);
+ const failures = await inBatches(covered, WRITE_CONCURRENCY, ([key, q]) => this.deletePendingKey(key, q));
+ if (failures.length > 0) {
+ throw new Error(`${failures.length} of ${covered.length} pending vector log deletes failed: ${errorMessage(failures[0])}`);
+ }
+ this.pendingLogUnknown = false;
+ }
+
stop(): void {
this.stopped = true;
this.clearTimer();
@@ -321,8 +535,13 @@ export class IndexPersistence {
this.markedDuringRunAt = null;
this.lastSaveAt = this.now();
this.firstSaveAttempted = true;
+ const coveredSeq = this.seq;
try {
await this.writeChanges(vector);
+ await this.clearCoveredPendingLog(coveredSeq).catch((err) => {
+ this.pendingLogError = errorMessage(err);
+ this.logFailure(err);
+ });
this.leg.lastSavedAt = new Date(this.now()).toISOString();
this.leg.lastError = null;
this.leg.lastErrorAt = null;
diff --git a/src/state/schema.ts b/src/state/schema.ts
index 207be7df..c2b3925d 100644
--- a/src/state/schema.ts
+++ b/src/state/schema.ts
@@ -11,6 +11,7 @@ export const KV = {
health: "mem:health",
embeddings: (obsId: string) => `mem:emb:${obsId}`,
bm25Index: "mem:index:bm25",
+ vectorPendingLog: "mem:index:vec-pending",
relations: "mem:relations",
profiles: "mem:profiles",
claudeBridge: "mem:claude-bridge",
diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts
index 2425bef8..ddd79a4a 100644
--- a/src/state/vector-index.ts
+++ b/src/state/vector-index.ts
@@ -36,17 +36,29 @@ function cosineSimilarity(a: Float32Array, b: Float32Array): number {
export type VectorEntry = { embedding: Float32Array; sessionId: string };
+export type VectorChangeListener = (obsId: string | null, entry: VectorEntry | null) => void;
+
export class VectorIndex {
private vectors: Map<string, VectorEntry> = new Map();
private changes: Map<string, boolean> = new Map();
+ private listener: VectorChangeListener | null = null;
+
+ setChangeListener(listener: VectorChangeListener | null): void {
+ this.listener = listener;
+ }
add(obsId: string, sessionId: string, embedding: Float32Array): void {
- this.vectors.set(obsId, { embedding, sessionId });
+ const entry = { embedding, sessionId };
+ this.vectors.set(obsId, entry);
this.changes.set(obsId, true);
+ this.listener?.(obsId, entry);
}
remove(obsId: string): void {
- if (this.vectors.delete(obsId)) this.changes.set(obsId, false);
+ if (this.vectors.delete(obsId)) {
+ this.changes.set(obsId, false);
+ this.listener?.(obsId, null);
+ }
}
has(obsId: string): boolean {
@@ -147,8 +159,10 @@ export class VectorIndex {
}
clear(): void {
+ const hadVectors = this.vectors.size > 0;
for (const obsId of this.vectors.keys()) this.changes.set(obsId, false);
this.vectors.clear();
+ if (hadVectors) this.listener?.(null, null);
}
restoreFrom(other: VectorIndex): void {
diff --git a/src/viewer/index.html b/src/viewer/index.html
index 20a85c93..1e43ea3d 100644
--- a/src/viewer/index.html
+++ b/src/viewer/index.html
@@ -7164,11 +7164,20 @@
(b ? esc(fmtNum(b.observations) + ' observations + ' + fmtNum(b.memories) + ' memories + ' + fmtNum(b.lessons) + ' lessons. ') : '') + esc(BM25_EXPLAINER));
html += hrow('Vector index', hasVectors ? esc(fmtNum(idx.vectorDocuments) + ' vectors') : 'off', 'vector',
hasVectors ? 'Meaning-based search. Each vector is one embedded record.' : 'No embedding provider, so search is keyword only. Set an embedding provider key (for example OPENAI_API_KEY) and restart.');
- html += hrow('Waiting for embeddings', esc(fmtNum(idx.pendingVectorBackfill || 0)), 'pendingBackfill', idx.pendingVectorBackfill ? 'Embedded in the background, capped per start by AGENTMEMORY_VECTOR_BACKFILL_MAX.' : '');
+ var bfState = ri.vectorBackfillState;
+ var bfNote = bfState === 'paused' ? 'Paused because the embedding provider did not return vectors. Retried on the next start.' :
+ bfState === 'waiting-for-opt-in' ? 'Waiting for AGENTMEMORY_VECTOR_BACKFILL=all and a restart.' :
+ 'Embedded in the background in batches of AGENTMEMORY_VECTOR_BACKFILL_MAX. If agentmemory stops first, it continues on the next start.';
+ html += hrow('Waiting for embeddings', esc(fmtNum(idx.pendingVectorBackfill || 0) + (bfState && bfState !== 'idle' && idx.pendingVectorBackfill ? ' · ' + (bfState === 'waiting-for-opt-in' ? 'waiting for opt-in' : bfState) : '')), 'pendingBackfill', idx.pendingVectorBackfill ? esc(bfNote) : '');
html += hrow('Missing from index', ri.missingObservations === null || ri.missingObservations === undefined ? 'not checked yet' : esc(fmtNum(ri.missingObservations)), 'missingFromIndex');
html += hrow('Keyword rebuild', ri.bm25Incomplete ? '<span style="color:var(--accent);">incomplete</span>' : 'complete', null, 'Rebuilt from stored data at every start, so it is never saved to disk.');
if (p) {
html += hrow('Vector save', esc(legText(p.vector)), null, 'Saved at most every ' + esc(durationText(p.saveIntervalMs || 600000)) + ' (AGENTMEMORY_INDEX_SAVE_INTERVAL_MS). ' + esc(fmtNum(p.buckets) + ' bucket' + (p.buckets === 1 ? '' : 's') + ', ' + fmtNum(p.pendingChanges) + ' unsaved change' + (p.pendingChanges === 1 ? '' : 's') + (p.saving ? ', saving now' : '') + '.'));
+ if (typeof p.pendingLog === 'number') {
+ html += hrow('Pending vector log', esc(fmtNum(p.pendingLog) + ' vector change' + (p.pendingLog === 1 ? '' : 's')), null,
+ p.pendingLogError ? '<span style="color:var(--accent);overflow-wrap:anywhere;">' + esc('Log writes failing: ' + p.pendingLogError) + '</span>' :
+ 'New and removed vectors are written here right away and replayed after a crash, so nothing is embedded twice. Each save empties it.');
+ }
}
html += hrow('Graph', esc(fmtNum(c.graphNodes) + ' nodes · ' + fmtNum(c.graphEdges) + ' edges'), 'graph', store.graph && store.graph.updatedAt ? 'Snapshot rebuilt ' + esc(timeAgo(store.graph.updatedAt)) + '.' : '');
if (r && r.graphCompaction) html += graphCompactionRowHtml(r.graphCompaction);
diff --git a/test/index-first-checkpoint.test.ts b/test/index-first-checkpoint.test.ts
index b0e02eaa..bbcef2ee 100644
--- a/test/index-first-checkpoint.test.ts
+++ b/test/index-first-checkpoint.test.ts
@@ -14,7 +14,7 @@ function mockKV() {
writes,
get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,
set: async <T>(scope: string, key: string, data: T): Promise<T> => {
- writes.push({ scope, key });
+ if (scope !== "mem:index:vec-pending") writes.push({ scope, key });
if (!store.has(scope)) store.set(scope, new Map());
store.get(scope)!.set(key, data);
return data;
diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts
index 9a28a668..62747017 100644
--- a/test/index-persistence-throttle.test.ts
+++ b/test/index-persistence-throttle.test.ts
@@ -91,6 +91,7 @@ describe("IndexPersistence save throttling", () => {
const slowKv = {
...kv,
set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === "mem:index:vec-pending") return kv.set(scope, key, data);
inFlight++;
maxInFlight = Math.max(maxInFlight, inFlight);
if (gate) await gate;
diff --git a/test/status.test.ts b/test/status.test.ts
index c3ed308e..4950f73e 100644
--- a/test/status.test.ts
+++ b/test/status.test.ts
@@ -508,3 +508,32 @@ describe("markLlmFunctions", () => {
expect(markLlmFunctions(metrics, "llm").some((m) => m.offWithoutLlm)).toBe(false);
});
});
+
+describe("vector durability rows", () => {
+ it("shows the pending vector log, a paused backlog and skipped repeats in plain language", () => {
+ const report = evaluateStatus(
+ inputs({
+ index: { ...inputs().index, pendingVectorBackfill: 7, vectorBackfillState: "paused" },
+ indexPersistence: {
+ saveIntervalMs: 600000,
+ saving: false,
+ buckets: 2,
+ pendingChanges: 3,
+ vector: { lastSavedAt: "2026-09-24T11:59:00.000Z", dirtySince: null, lastError: null, lastErrorAt: null },
+ vectorCountShortfall: null,
+ pendingLog: 3,
+ pendingLogError: "state write timed out",
+ },
+ observeDedup: { skippedSinceStart: 4, windowSeconds: 300 },
+ }),
+ );
+ expect(codes(report)).toContain("index-pending-log-failing");
+ const backlog = report.problems.find((p) => p.code === "index-vector-backfill-pending");
+ expect(backlog?.fix).toContain("retried on the next start");
+ expect(report.observeDedup).toEqual({ skippedSinceStart: 4, windowSeconds: 300 });
+ const html = renderStatusHtml(report, "n");
+ expect(html).toContain("3 vector changes");
+ expect(html).toContain("paused, retried on the next start");
+ expect(html).toContain("4 since start");
+ });
+});
diff --git a/test/vector-pending-log.test.ts b/test/vector-pending-log.test.ts
new file mode 100644
index 00000000..8bbf7ca5
--- /dev/null
+++ b/test/vector-pending-log.test.ts
@@ -0,0 +1,343 @@
+import { describe, it, expect, afterEach, vi } from "vitest";
+import { IndexPersistence } from "../src/state/index-persistence.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import {
+ backfillVectorBacklog,
+ getPendingVectorBackfillCount,
+ getSearchIndex,
+ rebuildKeywordIndex,
+ setEmbeddingProvider,
+ setIndexPersistence,
+ setVectorIndex,
+} from "../src/functions/search.js";
+import type { CompressedObservation, EmbeddingProvider, Session } from "../src/types.js";
+
+const PENDING_SCOPE = "mem:index:vec-pending";
+const INDEX_SCOPE = "mem:index:bm25";
+
+function mockKV(options: { shuffle?: boolean } = {}) {
+ const store = new Map<string, Map<string, unknown>>();
+ const ops: Array<{ op: "set" | "delete" | "list"; scope: string; key?: string }> = [];
+ return {
+ store,
+ ops,
+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ ops.push({ op: "set", scope, key });
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, JSON.parse(JSON.stringify(data)));
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ ops.push({ op: "delete", scope, key });
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ ops.push({ op: "list", scope });
+ const rows = Array.from(store.get(scope)?.values() ?? []);
+ if (options.shuffle) rows.reverse();
+ return rows as T[];
+ },
+ };
+}
+
+type MockKV = ReturnType<typeof mockKV>;
+
+function v(seed: number): Float32Array {
+ return new Float32Array([seed, seed + 0.5, seed + 1]);
+}
+
+function pendingRows(kv: MockKV): Map<string, Record<string, unknown>> {
+ return (kv.store.get(PENDING_SCOPE) ?? new Map()) as Map<string, Record<string, unknown>>;
+}
+
+async function boot(kv: MockKV, dims = 3): Promise<{ vector: VectorIndex; persistence: IndexPersistence }> {
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000, bucketSize: 4 });
+ const loaded = await persistence.load();
+ if (loaded.vector && loaded.vector.size > 0) vector.restoreFrom(loaded.vector);
+ await persistence.replayPendingLog(dims);
+ return { vector, persistence };
+}
+
+describe("pending vector log", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ getSearchIndex().clear();
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ setIndexPersistence(null);
+ });
+
+ it("writes a compact entry for every added vector and a tombstone for every removal", async () => {
+ const kv = mockKV();
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000 });
+
+ vector.add("obs_a", "ses_1", v(1));
+ vector.add("mem_b", "ses_2", v(2));
+ vector.remove("obs_a");
+ await persistence.flushPendingLog();
+
+ const rows = pendingRows(kv);
+ expect(rows.size).toBe(2);
+ expect(rows.get("obs_a")).toMatchObject({ id: "obs_a", t: 1 });
+ expect(rows.get("mem_b")).toMatchObject({ id: "mem_b", s: "ses_2", k: "memory" });
+ expect(typeof rows.get("mem_b")!.e).toBe("string");
+ expect(rows.get("obs_a")!.q as number).toBeGreaterThan(rows.get("mem_b")!.q as number);
+ expect(persistence.status().pendingLog).toBe(2);
+ });
+
+ it("does nothing when there is no vector index", async () => {
+ const kv = mockKV();
+ const persistence = new IndexPersistence(kv as never, null);
+ const standalone = new VectorIndex();
+ standalone.add("obs_a", "ses_1", v(1));
+ await persistence.save();
+ const replay = await persistence.replayPendingLog(3);
+
+ expect(kv.store.get(PENDING_SCOPE)).toBeUndefined();
+ expect(kv.ops.filter((o) => o.scope === PENDING_SCOPE)).toEqual([]);
+ expect(replay.entries).toBe(0);
+ expect(persistence.status().pendingLog).toBe(0);
+ });
+
+ it("replays adds and removals after a force-kill without any embedding calls", async () => {
+ const kv = mockKV();
+ const first = await boot(kv);
+ first.vector.add("obs_1", "ses_1", v(1));
+ first.vector.add("obs_2", "ses_1", v(2));
+ await first.persistence.save();
+ expect(pendingRows(kv).size).toBe(0);
+
+ first.vector.add("obs_3", "ses_1", v(3));
+ first.vector.add("obs_4", "ses_1", v(4));
+ first.vector.remove("obs_1");
+ await first.persistence.flushPendingLog();
+ first.persistence.stop();
+
+ const embed = vi.fn();
+ const second = await boot(kv);
+ expect(embed).not.toHaveBeenCalled();
+ expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);
+ expect(Array.from(second.vector.get("obs_3")!.embedding)).toEqual(Array.from(v(3)));
+ expect(second.persistence.status().pendingLog).toBe(3);
+
+ await second.persistence.save();
+ expect(pendingRows(kv).size).toBe(0);
+ const third = await boot(kv);
+ expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);
+ });
+
+ it("skips logged vectors whose dimension does not match the active provider", async () => {
+ const kv = mockKV();
+ const first = await boot(kv);
+ first.vector.add("obs_ok", "ses_1", v(1));
+ first.vector.add("obs_wide", "ses_1", new Float32Array([1, 2, 3, 4]));
+ await first.persistence.flushPendingLog();
+
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector);
+ await persistence.load();
+ const replay = await persistence.replayPendingLog(3);
+ expect(replay).toMatchObject({ added: 1, skipped: 1 });
+ expect(vector.has("obs_wide")).toBe(false);
+ });
+
+ it("clears only the entries a snapshot covered", async () => {
+ const kv = mockKV();
+ let release: () => void = () => undefined;
+ let gate: Promise<void> | null = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ const slow = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope !== PENDING_SCOPE && gate) await gate;
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(slow as never, vector, { saveIntervalMs: 600_000 });
+ vector.add("obs_before", "ses_1", v(1));
+ await persistence.flushPendingLog();
+
+ const saving = persistence.save();
+ vector.add("obs_during", "ses_1", v(2));
+ await persistence.flushPendingLog();
+ gate = null;
+ release();
+ await saving;
+
+ expect([...pendingRows(kv).keys()]).toEqual(["obs_during"]);
+ expect(persistence.status().pendingLog).toBe(1);
+ });
+
+ it("keeps every entry when the snapshot write fails", async () => {
+ const kv = mockKV();
+ const failing = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === INDEX_SCOPE) throw new Error("state write refused");
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(failing as never, vector, { saveIntervalMs: 600_000 });
+ vector.add("obs_1", "ses_1", v(1));
+ vector.add("obs_2", "ses_1", v(2));
+ await persistence.save();
+
+ expect(persistence.status().vector?.lastError).toContain("state write refused");
+ expect(pendingRows(kv).size).toBe(2);
+ expect(kv.ops.filter((o) => o.op === "delete" && o.scope === PENDING_SCOPE)).toEqual([]);
+ });
+
+ it("orders replay by sequence, not by list order, including a full clear", async () => {
+ const kv = mockKV({ shuffle: true });
+ const first = await boot(kv);
+ first.vector.add("obs_old", "ses_1", v(1));
+ await first.persistence.save();
+ first.vector.add("obs_gone", "ses_1", v(2));
+ first.vector.clear();
+ first.vector.add("obs_new", "ses_1", v(3));
+ first.vector.add("obs_flip", "ses_1", v(4));
+ first.vector.remove("obs_flip");
+ first.vector.add("obs_flip", "ses_1", v(5));
+ await first.persistence.flushPendingLog();
+
+ const second = await boot(kv);
+ expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);
+ expect(Array.from(second.vector.get("obs_flip")!.embedding)).toEqual(Array.from(v(5)));
+
+ await second.persistence.save();
+ expect(pendingRows(kv).size).toBe(0);
+ const third = await boot(kv);
+ expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);
+ });
+
+ it("saves early once the log grows past its bound", async () => {
+ vi.useFakeTimers();
+ const kv = mockKV();
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000 });
+ await vi.advanceTimersByTimeAsync(6_000);
+ const metaSet = () => kv.ops.some((o) => o.op === "set" && o.key === "vectors:meta");
+ for (let i = 0; i < 520; i++) vector.add(`obs_${i}`, "ses_1", v(i));
+ expect(metaSet()).toBe(false);
+ await vi.advanceTimersByTimeAsync(10);
+ expect(persistence.status().saving || metaSet()).toBe(true);
+ await persistence.save();
+ await persistence.flushPendingLog();
+
+ expect(kv.ops.some((o) => o.op === "set" && o.key === "vectors:meta")).toBe(true);
+ expect(pendingRows(kv).size).toBe(0);
+ });
+});
+
+describe("durable vector backlog", () => {
+ afterEach(() => {
+ getSearchIndex().clear();
+ setVectorIndex(null);
+ setEmbeddingProvider(null);
+ setIndexPersistence(null);
+ });
+
+ function seed(kv: MockKV, count: number, timestamp: string): void {
+ const sessionId = "ses_0";
+ kv.store.set("mem:sessions", new Map([[sessionId, { id: sessionId, project: "p", cwd: "/tmp", startedAt: timestamp, status: "completed", observationCount: count } as Session]]));
+ const scope = new Map<string, unknown>();
+ for (let i = 0; i < count; i++) {
+ const id = `obs_${String(i).padStart(3, "0")}`;
+ scope.set(id, {
+ id,
+ sessionId,
+ timestamp,
+ type: "file_edit",
+ title: `change ${i}`,
+ facts: [],
+ narrative: `narrative for change ${i}`,
+ concepts: [],
+ files: [],
+ importance: 5,
+ } satisfies CompressedObservation);
+ }
+ kv.store.set(`mem:obs:${sessionId}`, scope);
+ }
+
+ function provider(failAfterCalls: number | null, calls: { n: number }): EmbeddingProvider {
+ return {
+ name: "stub",
+ dimensions: 3,
+ embed: async () => new Float32Array([0.1, 0.2, 0.3]),
+ embedBatch: async (texts: string[]) => {
+ calls.n++;
+ if (failAfterCalls !== null && calls.n > failAfterCalls) throw new Error("provider down");
+ return texts.map(() => new Float32Array([0.1, 0.2, 0.3]));
+ },
+ } as EmbeddingProvider;
+ }
+
+ it("keeps the backfill marker until the backlog is done and resumes after a restart", async () => {
+ process.env.REBUILD_EMBED_BATCH_SIZE = "5";
+ try {
+ const kv = mockKV();
+ seed(kv, 12, "2026-09-02T00:00:00.000Z");
+
+ const first = await boot(kv);
+ setVectorIndex(first.vector);
+ setIndexPersistence(first.persistence);
+ const calls = { n: 0 };
+ setEmbeddingProvider(provider(1, calls));
+ await first.persistence.markBackfillSince("2026-09-01T00:00:00.000Z");
+ const rebuilt = await rebuildKeywordIndex(kv as never, "2026-09-01T00:00:00.000Z");
+ expect(rebuilt.vectorJobs).toHaveLength(12);
+
+ const run = await backfillVectorBacklog(rebuilt.vectorJobs, { batchSize: 5, pauseMs: 0 });
+ expect(run).toMatchObject({ added: 5, complete: false, remaining: 7 });
+ expect(getPendingVectorBackfillCount()).toBe(7);
+ expect(await first.persistence.readBackfillMarker()).toBe("2026-09-01T00:00:00.000Z");
+ await first.persistence.flushPendingLog();
+ first.persistence.stop();
+
+ await first.persistence.markBackfillSince("2026-09-03T00:00:00.000Z");
+ expect(await first.persistence.readBackfillMarker()).toBe("2026-09-01T00:00:00.000Z");
+
+ getSearchIndex().clear();
+ const second = await boot(kv);
+ expect(second.vector.size).toBe(5);
+ setVectorIndex(second.vector);
+ setIndexPersistence(second.persistence);
+ const resumedCalls = { n: 0 };
+ setEmbeddingProvider(provider(null, resumedCalls));
+ const since = await second.persistence.readBackfillMarker();
+ const resumed = await rebuildKeywordIndex(kv as never, since);
+ expect(resumed.vectorJobs).toHaveLength(7);
+
+ const done = await backfillVectorBacklog(resumed.vectorJobs, { batchSize: 5, pauseMs: 0 });
+ expect(done).toMatchObject({ added: 7, complete: true, remaining: 0 });
+ expect(second.vector.size).toBe(12);
+ expect(resumedCalls.n).toBe(2);
+ await second.persistence.clearBackfillMarker();
+ expect(await second.persistence.readBackfillMarker()).toBeNull();
+ } finally {
+ delete process.env.REBUILD_EMBED_BATCH_SIZE;
+ }
+ });
+
+ it("does not cap incremental recovery at AGENTMEMORY_VECTOR_BACKFILL_MAX", async () => {
+ process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "4";
+ try {
+ const kv = mockKV();
+ seed(kv, 10, "2026-09-02T00:00:00.000Z");
+ setVectorIndex(new VectorIndex());
+ setEmbeddingProvider(provider(null, { n: 0 }));
+ const rebuilt = await rebuildKeywordIndex(kv as never, "2026-09-01T00:00:00.000Z");
+ expect(rebuilt.vectorJobs).toHaveLength(10);
+ const run = await backfillVectorBacklog(rebuilt.vectorJobs, { pauseMs: 0 });
+ expect(run).toMatchObject({ added: 10, complete: true });
+ } finally {
+ delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX;
+ }
+ });
+});