Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

perf(index): throttle index saves and report each leg's save state (#1415)

Rohit Ghumarecfc193408c09
diff --git a/.env.example b/.env.example
index 9219a338..4b9deb0b 100644
--- a/.env.example
+++ b/.env.example
@@ -129,6 +129,7 @@
# AGENTMEMORY_REFLECT=true # Periodically auto-synthesize lessons from memories
# AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging.
# AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental).
+# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately.
# -----------------------------------------------------------------------------
# 6. CLI / runtime knobs
diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md
index d152a7c6..5fe07156 100644
--- a/plugin/skills/agentmemory-config/REFERENCE.md
+++ b/plugin/skills/agentmemory-config/REFERENCE.md
@@ -3,7 +3,7 @@
Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.
<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->
-Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 43 recognized variables:
+Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables:
- `AGENTMEMORY_AGENT_SCOPE`
- `AGENTMEMORY_ALLOW_AGENT_SDK`
@@ -27,6 +27,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e
- `AGENTMEMORY_III_VERSION`
- `AGENTMEMORY_IMAGE_EMBEDDINGS`
- `AGENTMEMORY_IMAGE_STORE_MAX_BYTES`
+- `AGENTMEMORY_INDEX_SAVE_INTERVAL_MS`
- `AGENTMEMORY_INJECT_CONTEXT`
- `AGENTMEMORY_LLM_NOTHINK`
- `AGENTMEMORY_LLM_TIMEOUT_MS`
diff --git a/src/config.ts b/src/config.ts
index a6c62ce5..ac82bd44 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -467,6 +467,16 @@ export function getConsolidationCooldownMs(): number {
return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS;
}
+export const INDEX_SAVE_INTERVAL_DEFAULT_MS = 600_000;
+
+export function getIndexSaveIntervalMs(): number {
+ const raw = safeParseInt(
+ getMergedEnv()["AGENTMEMORY_INDEX_SAVE_INTERVAL_MS"],
+ INDEX_SAVE_INTERVAL_DEFAULT_MS,
+ );
+ return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS;
+}
+
export function isStandaloneMcp(): boolean {
return getMergedEnv()["STANDALONE_MCP"] === "true";
}
diff --git a/src/functions/search.ts b/src/functions/search.ts
index 2b8858bd..8abce07f 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -1,4 +1,5 @@
import type { IIIClient } from 'iii-sdk'
+import type { IndexPersistenceStatus } from "../state/index-persistence.js";
import type { CompactSearchResult, CompressedObservation, Memory, SearchResult, Session } from '../types.js'
import { KV } from '../state/schema.js'
import { StateKV } from '../state/kv.js'
@@ -75,17 +76,22 @@ export function vectorIndexRemove(id: string): void {
// Wired by src/index.ts after IndexPersistence is constructed; no-op
// until then so unit tests that exercise the delete paths in
// isolation don't need to wire persistence.
-let indexPersistence: {
+type IndexPersistenceHook = {
scheduleSave: () => void;
save: () => Promise<void>;
-} | null = null;
+ status?: () => IndexPersistenceStatus;
+};
-export function setIndexPersistence(
- p: { scheduleSave: () => void; save: () => Promise<void> } | null,
-): void {
+let indexPersistence: IndexPersistenceHook | null = null;
+
+export function setIndexPersistence(p: IndexPersistenceHook | null): void {
indexPersistence = p;
}
+export function getIndexPersistenceStatus(): IndexPersistenceStatus | null {
+ return indexPersistence?.status?.() ?? null;
+}
+
export function scheduleIndexSave(): void {
indexPersistence?.scheduleSave();
}
diff --git a/src/functions/status.ts b/src/functions/status.ts
index 552fe6c0..0b3ab531 100644
--- a/src/functions/status.ts
+++ b/src/functions/status.ts
@@ -1,3 +1,5 @@
+import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js";
+
export type StatusLevel = "ok" | "info" | "warn" | "error";
export interface StatusProblem {
@@ -58,6 +60,7 @@ export interface StatusInputs {
};
graph: GraphStatsInput | null;
graphExtractionEnabled: boolean;
+ indexPersistence?: IndexPersistenceStatus | null;
}
export interface StatusReport {
@@ -72,6 +75,7 @@ export interface StatusReport {
health: StatusInputs["health"];
provider: { llm: string; embeddings: string; circuitBreaker: StatusInputs["circuitBreaker"] };
index: StatusInputs["index"];
+ indexPersistence: IndexPersistenceStatus | null;
graph: (GraphStatsInput & { ageSeconds: number | null; extractionEnabled: boolean }) | null;
functions: Array<FunctionMetricInput & { failureRate: number }>;
flags: StatusFlag[];
@@ -81,6 +85,7 @@ export interface StatusReport {
const FAILURE_RATE_THRESHOLD = 0.2;
const FAILURE_MIN_CALLS = 5;
const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60;
+const VECTOR_STRING_WARN_RATIO = 0.8;
const LEVEL_RANK: Record<StatusLevel, number> = { ok: 0, info: 1, warn: 2, error: 3 };
const FUNCTION_FIXES: Record<string, string> = {
@@ -168,6 +173,43 @@ export function evaluateStatus(input: StatusInputs): StatusReport {
});
}
+ const persistence = input.indexPersistence ?? null;
+ if (persistence) {
+ const legs: Array<[string, IndexLegStatus | null]> = [
+ ["BM25", persistence.bm25],
+ ["vector", persistence.vector],
+ ];
+ for (const [label, leg] of legs) {
+ if (!leg) continue;
+ if (leg.lastError) {
+ problems.push({
+ level: "error",
+ code: "index-save-failing",
+ message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`,
+ fix: "Check the server log for the failing state write. The next save retries automatically.",
+ });
+ }
+ const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined);
+ if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {
+ problems.push({
+ level: "warn",
+ code: "index-save-stale",
+ message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,
+ fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",
+ });
+ }
+ }
+ const vectorChars = persistence.vector?.serializedChars ?? null;
+ if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) {
+ problems.push({
+ level: "warn",
+ code: "vector-index-near-string-limit",
+ message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`,
+ fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.",
+ });
+ }
+ }
+
let graph: StatusReport["graph"] = null;
if (input.graph) {
const ageSeconds = secondsBetween(input.now, input.graph.updatedAt);
@@ -218,6 +260,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport {
circuitBreaker: input.circuitBreaker,
},
index: input.index,
+ indexPersistence: persistence,
graph,
functions,
flags: input.flags,
@@ -248,6 +291,29 @@ function row(label: string, value: string): string {
return `<tr><th>${escapeHtml(label)}</th><td>${value}</td></tr>`;
}
+function legSummary(report: StatusReport, leg: IndexLegStatus): string {
+ if (leg.lastError) return `failing: ${leg.lastError}`;
+ const saved = leg.lastSavedAt ? `saved ${formatDuration(secondsBetween(new Date(report.checkedAt), leg.lastSavedAt))} ago` : "not saved since start";
+ return leg.dirtySince ? `${saved}, unsaved changes pending` : saved;
+}
+
+function indexPersistenceRows(report: StatusReport): string {
+ const persistence = report.indexPersistence;
+ if (!persistence) return "";
+ let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25)));
+ if (persistence.vector) {
+ rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector)));
+ const chars = persistence.vector.serializedChars;
+ if (chars !== null) {
+ rows += row(
+ "Vector index size",
+ escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`),
+ );
+ }
+ }
+ return rows;
+}
+
export function renderStatusHtml(report: StatusReport, styleNonce: string): string {
const problems = report.problems.length
? report.problems
@@ -330,6 +396,7 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off")
${row("Observations indexed", escapeHtml(idx.observationsIndexed))}
${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))}
${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}
+${indexPersistenceRows(report)}
</table>
<h2>Knowledge graph</h2><table>
${graph
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 19b05cb5..28a24d5a 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -4,8 +4,9 @@ import type { StateKV } from "./kv.js";
import { KV, generateId } from "./schema.js";
import { logger } from "../logger.js";
import { safeAudit } from "../functions/audit.js";
+import { getIndexSaveIntervalMs } from "../config.js";
-const DEBOUNCE_MS = 5000;
+export const V8_MAX_STRING_CHARS = 536_870_888;
const FAILURE_LOG_THROTTLE_MS = 60_000;
const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence";
const BM25_KEY = "data";
@@ -41,8 +42,31 @@ type IndexShardManifest = {
type IndexPersistenceOptions = {
shardChars?: number;
createGeneration?: () => string;
+ saveIntervalMs?: number;
+ now?: () => number;
};
+export type IndexLeg = "bm25" | "vector";
+
+export interface IndexLegStatus {
+ lastSavedAt: string | null;
+ lastError: string | null;
+ lastErrorAt: string | null;
+ dirtySince: string | null;
+ serializedChars: number | null;
+}
+
+export interface IndexPersistenceStatus {
+ saveIntervalMs: number;
+ saving: boolean;
+ bm25: IndexLegStatus;
+ vector: IndexLegStatus | null;
+}
+
+function emptyLegStatus(): IndexLegStatus {
+ return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null };
+}
+
function shardChars(options: IndexPersistenceOptions): number {
const configured = options.shardChars;
if (typeof configured !== "number" || !Number.isFinite(configured)) {
@@ -81,39 +105,72 @@ function isValidShardDescriptor(
export class IndexPersistence {
private timer: ReturnType<typeof setTimeout> | null = null;
- private lastFailureLogAt = 0;
+ private lastFailureLogAt = new Map<IndexLeg, number>();
+ private running: Promise<void> | null = null;
+ private queued: Promise<void> | null = null;
+ private stopped = false;
+ private lastSaveAt: number;
+ private dirtyEpoch = 0;
+ private markedDuringRunAt: number | null = null;
+ private legs: Record<IndexLeg, IndexLegStatus> = {
+ bm25: emptyLegStatus(),
+ vector: emptyLegStatus(),
+ };
+ private readonly saveIntervalMs: number;
+ private readonly now: () => number;
constructor(
private kv: StateKV,
private bm25: SearchIndex,
private vector: VectorIndex | null,
private options: IndexPersistenceOptions = {},
- ) {}
+ ) {
+ this.now = options.now ?? Date.now;
+ const interval = options.saveIntervalMs;
+ this.saveIntervalMs =
+ typeof interval === "number" && Number.isFinite(interval) && interval > 0
+ ? interval
+ : getIndexSaveIntervalMs();
+ this.lastSaveAt = this.now();
+ }
scheduleSave(): void {
- if (this.timer) clearTimeout(this.timer);
- // setTimeout discards the returned promise, so any rejection inside
- // save() would surface as unhandledRejection and crash the process
- // under sustained iii-engine write timeouts (issue #204). Funnel
- // rejections through logFailure() instead.
+ if (this.stopped) return;
+ const now = this.now();
+ this.dirtyEpoch++;
+ if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now;
+ for (const leg of this.activeLegs()) {
+ if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString();
+ }
+ if (this.timer) return;
+ const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now);
this.timer = setTimeout(() => {
- this.save().catch((err) => this.logFailure(err));
- }, DEBOUNCE_MS);
+ this.timer = null;
+ this.save().catch((err) => this.logFailure("bm25", err));
+ }, delay);
}
- async save(): Promise<void> {
- if (this.timer) {
- clearTimeout(this.timer);
- this.timer = null;
- }
- try {
- await this.saveBm25Index(this.bm25.serialize());
- if (this.vector) {
- await this.saveVectorIndex(this.vector.serialize());
- }
- } catch (err) {
- this.logFailure(err);
+ save(): Promise<void> {
+ this.clearTimer();
+ if (this.queued) return this.queued;
+ if (this.running) {
+ const queued = this.running.then(() => {
+ this.queued = null;
+ return this.startRun();
+ });
+ this.queued = queued;
+ return queued;
}
+ return this.startRun();
+ }
+
+ status(): IndexPersistenceStatus {
+ return {
+ saveIntervalMs: this.saveIntervalMs,
+ saving: this.running !== null,
+ bm25: { ...this.legs.bm25 },
+ vector: this.vector ? { ...this.legs.vector } : null,
+ };
}
async load(): Promise<{
@@ -137,27 +194,80 @@ export class IndexPersistence {
}
stop(): void {
+ this.stopped = true;
+ this.clearTimer();
+ }
+
+ private clearTimer(): void {
if (this.timer) {
clearTimeout(this.timer);
this.timer = null;
}
}
- private logFailure(err: unknown): void {
- const now = Date.now();
- // Throttle: persistence failures under load arrive in bursts
- // (iii-engine queue pressure). Logging every debounce flush adds
- // noise without information.
- if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return;
- this.lastFailureLogAt = now;
+ private activeLegs(): IndexLeg[] {
+ return this.vector ? ["bm25", "vector"] : ["bm25"];
+ }
+
+ private startRun(): Promise<void> {
+ const run = this.runSave().finally(() => {
+ if (this.running === run) this.running = null;
+ });
+ this.running = run;
+ return run;
+ }
+
+ private async runSave(): Promise<void> {
+ this.markedDuringRunAt = null;
+ this.lastSaveAt = this.now();
+ await this.saveLeg("bm25", async () => {
+ const serialized = this.bm25.serialize();
+ this.legs.bm25.serializedChars = serialized.length;
+ await this.saveBm25Index(serialized);
+ });
+ const vector = this.vector;
+ if (vector) {
+ await this.saveLeg("vector", async () => {
+ const serialized = vector.serialize();
+ this.legs.vector.serializedChars = serialized.length;
+ await this.saveVectorIndex(serialized);
+ });
+ }
+ }
+
+ private async saveLeg(leg: IndexLeg, run: () => Promise<void>): Promise<void> {
+ const status = this.legs[leg];
+ const epoch = this.dirtyEpoch;
+ try {
+ await run();
+ status.lastSavedAt = new Date(this.now()).toISOString();
+ status.lastError = null;
+ status.lastErrorAt = null;
+ if (this.dirtyEpoch === epoch) {
+ status.dirtySince = null;
+ } else if (this.markedDuringRunAt !== null) {
+ status.dirtySince = new Date(this.markedDuringRunAt).toISOString();
+ }
+ } catch (err) {
+ status.lastError = errorMessage(err);
+ status.lastErrorAt = new Date(this.now()).toISOString();
+ if (status.dirtySince === null) status.dirtySince = status.lastErrorAt;
+ this.logFailure(leg, err);
+ }
+ }
+
+ private logFailure(leg: IndexLeg, err: unknown): void {
+ const now = this.now();
+ if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return;
+ this.lastFailureLogAt.set(leg, now);
const code = (err as { code?: string })?.code;
const message = err instanceof Error ? err.message : String(err);
- logger.warn("index persistence: failed to save BM25/vector index", {
+ logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, {
code,
message,
hint:
code === "TIMEOUT"
- ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next debounce flush"
+ ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save"
: undefined,
});
}
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index d8a447f2..2fc2f328 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -12,7 +12,7 @@ import type { ResilientProvider } from "../providers/resilient.js";
import { III_PINNED_VERSION, VERSION } from "../version.js";
import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js";
import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js";
-import { findUnindexedObservations, getSearchIndex, getVectorIndex } from "../functions/search.js";
+import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js";
import { timingSafeCompare } from "../auth.js";
import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js";
import { renderViewerDocument } from "../viewer/document.js";
@@ -373,6 +373,7 @@ export function registerApiTriggers(
},
graph,
graphExtractionEnabled: isGraphExtractionEnabled(),
+ indexPersistence: getIndexPersistenceStatus(),
});
const accept = req.headers?.["accept"] ?? req.headers?.["Accept"];
const format = req.query_params?.["format"];
diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts
new file mode 100644
index 00000000..54eff133
--- /dev/null
+++ b/test/index-persistence-throttle.test.ts
@@ -0,0 +1,335 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js";
+import { SearchIndex } from "../src/state/search-index.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js";
+import type { CompressedObservation } from "../src/types.js";
+
+const BM25_SCOPE = "mem:index:bm25";
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ const sets: Array<{ scope: string; key: string }> = [];
+ return {
+ sets,
+ store,
+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ sets.push({ scope, key });
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => Array.from(store.get(scope)?.values() ?? []) as T[],
+ };
+}
+
+function obs(id: string, title: string): CompressedObservation {
+ return {
+ id,
+ sessionId: "ses_1",
+ timestamp: new Date().toISOString(),
+ type: "file_edit",
+ title,
+ facts: [],
+ narrative: `${title} narrative`,
+ concepts: [],
+ files: [],
+ importance: 5,
+ };
+}
+
+function manifestSaves(kv: ReturnType<typeof mockKV>, key: string): number {
+ return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length;
+}
+
+describe("IndexPersistence save throttling", () => {
+ let kv: ReturnType<typeof mockKV>;
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ kv = mockKV();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("saves at most once per interval however often changes are scheduled", async () => {
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });
+
+ for (let i = 0; i < 50; i++) persistence.scheduleSave();
+ await vi.advanceTimersByTimeAsync(59_000);
+ expect(manifestSaves(kv, "data:manifest")).toBe(0);
+
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(manifestSaves(kv, "data:manifest")).toBe(1);
+
+ for (let i = 0; i < 50; i++) persistence.scheduleSave();
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(manifestSaves(kv, "data:manifest")).toBe(1);
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(manifestSaves(kv, "data:manifest")).toBe(2);
+ });
+
+ it("an explicit save runs immediately and cancels the pending timer", async () => {
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });
+
+ persistence.scheduleSave();
+ await persistence.save();
+ expect(manifestSaves(kv, "data:manifest")).toBe(1);
+
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(manifestSaves(kv, "data:manifest")).toBe(1);
+ });
+
+ it("never runs two saves at once and coalesces requests made during a save into one", async () => {
+ let release: () => void = () => undefined;
+ let inFlight = 0;
+ let maxInFlight = 0;
+ let gate: Promise<void> | null = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ const slowKv = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ inFlight++;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ if (gate) await gate;
+ const result = await kv.set(scope, key, data);
+ inFlight--;
+ return result;
+ },
+ };
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });
+
+ const first = persistence.save();
+ const second = persistence.save();
+ const third = persistence.save();
+ expect(second).toBe(third);
+ expect(persistence.status().saving).toBe(true);
+
+ gate = null;
+ release();
+ await Promise.all([first, second, third]);
+
+ expect(manifestSaves(kv, "data:manifest")).toBe(2);
+ expect(maxInFlight).toBe(1);
+ expect(persistence.status().saving).toBe(false);
+ });
+
+ it("a failing BM25 leg does not stop the vector leg from saving", async () => {
+ const failingKv = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed");
+ return kv.set(scope, key, data);
+ },
+ };
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const vector = new VectorIndex();
+ vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
+ const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 });
+
+ persistence.scheduleSave();
+ await persistence.save();
+
+ const status = persistence.status();
+ expect(status.bm25.lastError).toBe("bm25 manifest write failed");
+ expect(status.bm25.dirtySince).not.toBeNull();
+ expect(status.vector?.lastError).toBeNull();
+ expect(status.vector?.lastSavedAt).not.toBeNull();
+ expect(status.vector?.dirtySince).toBeNull();
+ expect(status.vector?.serializedChars).toBeGreaterThan(0);
+
+ const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load();
+ expect(loaded.bm25).toBeNull();
+ expect(loaded.vector?.size).toBe(1);
+ });
+
+ it("keeps a leg dirty when a change arrives while it is being saved", async () => {
+ let release: () => void = () => undefined;
+ const gate = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ let held = false;
+ const slowKv = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!held) {
+ held = true;
+ await gate;
+ }
+ return kv.set(scope, key, data);
+ },
+ };
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });
+
+ const saving = persistence.save();
+ persistence.scheduleSave();
+ release();
+ await saving;
+
+ expect(persistence.status().bm25.lastSavedAt).not.toBeNull();
+ expect(persistence.status().bm25.dirtySince).not.toBeNull();
+
+ await persistence.save();
+ expect(persistence.status().bm25.dirtySince).toBeNull();
+ });
+
+ it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => {
+ let release: () => void = () => undefined;
+ const gate = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ let heldBm25Manifest = false;
+ const slowKv = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) {
+ heldBm25Manifest = true;
+ await gate;
+ }
+ return kv.set(scope, key, data);
+ },
+ };
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const vector = new VectorIndex();
+ vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
+ const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 });
+
+ const saving = persistence.save();
+ persistence.scheduleSave();
+ release();
+ await saving;
+
+ const status = persistence.status();
+ expect(status.bm25.dirtySince).not.toBeNull();
+ expect(status.vector?.dirtySince).toBeNull();
+ expect(status.vector?.lastSavedAt).not.toBeNull();
+ });
+
+ it("stop prevents later scheduled saves", async () => {
+ const bm25 = new SearchIndex();
+ bm25.add(obs("obs_1", "alpha"));
+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 });
+ persistence.stop();
+ persistence.scheduleSave();
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(manifestSaves(kv, "data:manifest")).toBe(0);
+ });
+});
+
+function statusInputs(overrides: Partial<StatusInputs> = {}): StatusInputs {
+ return {
+ now: new Date("2026-09-25T12:00:00.000Z"),
+ version: "0.9.29",
+ engineVersion: "0.22.1",
+ uptimeSeconds: 100,
+ ports: { rest: 3111, streams: 3112, viewer: 3113 },
+ health: { status: "healthy", alerts: [], notes: [], connectionState: "connected" },
+ circuitBreaker: null,
+ functionMetrics: [],
+ provider: "llm",
+ embeddingProvider: "embeddings",
+ flags: [],
+ index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 },
+ graph: null,
+ graphExtractionEnabled: false,
+ ...overrides,
+ };
+}
+
+const cleanLeg = {
+ lastSavedAt: "2026-09-25T11:59:00.000Z",
+ lastError: null,
+ lastErrorAt: null,
+ dirtySince: null,
+ serializedChars: 1000,
+};
+
+describe("status reports index persistence", () => {
+ it("is quiet when both legs saved cleanly", () => {
+ const report = evaluateStatus(
+ statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }),
+ );
+ expect(report.problems).toEqual([]);
+ expect(report.indexPersistence?.vector?.serializedChars).toBe(1000);
+ });
+
+ it("reports a failing leg as an error", () => {
+ const report = evaluateStatus(
+ statusInputs({
+ indexPersistence: {
+ saveIntervalMs: 600_000,
+ saving: false,
+ bm25: cleanLeg,
+ vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" },
+ },
+ }),
+ );
+ expect(report.status).toBe("error");
+ expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]);
+ expect(report.problems[0].message).toContain("vector");
+ });
+
+ it("warns when the vector index passes 80% of the Node.js string limit", () => {
+ const report = evaluateStatus(
+ statusInputs({
+ indexPersistence: {
+ saveIntervalMs: 600_000,
+ saving: false,
+ bm25: cleanLeg,
+ vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) },
+ },
+ }),
+ );
+ expect(report.status).toBe("warn");
+ expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]);
+ expect(report.problems[0].message).toContain("85%");
+ });
+
+ it("warns when unsaved changes are older than twice the save interval", () => {
+ const report = evaluateStatus(
+ statusInputs({
+ indexPersistence: {
+ saveIntervalMs: 600_000,
+ saving: false,
+ bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" },
+ vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" },
+ },
+ }),
+ );
+ expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]);
+ expect(report.problems[0].message).toContain("BM25");
+ });
+
+ it("shows save state and vector size on the status page", () => {
+ const report = evaluateStatus(
+ statusInputs({
+ indexPersistence: {
+ saveIntervalMs: 600_000,
+ saving: false,
+ bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" },
+ vector: { ...cleanLeg, serializedChars: 53_687_089 },
+ },
+ }),
+ );
+ const html = renderStatusHtml(report, "n");
+ expect(html).toContain("<th>BM25 save</th><td>saved 1m ago, unsaved changes pending</td>");
+ expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)");
+ });
+});
diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts
index ba47c3ef..802084bd 100644
--- a/test/index-persistence.test.ts
+++ b/test/index-persistence.test.ts
@@ -670,7 +670,7 @@ describe("IndexPersistence", () => {
it("scheduleSave debounces multiple calls", async () => {
const bm25 = new SearchIndex();
- const persistence = new IndexPersistence(kv as never, bm25, null);
+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 });
persistence.scheduleSave();
persistence.scheduleSave();
@@ -721,7 +721,7 @@ describe("IndexPersistence", () => {
};
const bm25 = new SearchIndex();
bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));
- const persistence = new IndexPersistence(failingKv as never, bm25, null);
+ const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });
let unhandled = false;
const onUnhandled = () => {
@@ -751,7 +751,7 @@ describe("IndexPersistence", () => {
};
const bm25 = new SearchIndex();
bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));
- const persistence = new IndexPersistence(failingKv as never, bm25, null);
+ const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });
await expect(persistence.save()).resolves.toBeUndefined();
});