#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
perf(index): throttle index saves and report each leg's save state (#1415)
Rohit Ghumarecfc193408c09
diff --git a/.env.example b/.env.exampleindex 9219a338..4b9deb0b 100644--- a/.env.example+++ b/.env.example@@ -129,6 +129,7 @@# AGENTMEMORY_REFLECT=true # Periodically auto-synthesize lessons from memories# AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging.# AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental).+# AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately.# -----------------------------------------------------------------------------# 6. CLI / runtime knobsdiff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.mdindex d152a7c6..5fe07156 100644--- a/plugin/skills/agentmemory-config/REFERENCE.md+++ b/plugin/skills/agentmemory-config/REFERENCE.md@@ -3,7 +3,7 @@Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded.<!-- AUTOGEN:env START - generated by scripts/skills/generate.ts, do not edit by hand -->-Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 43 recognized variables:+Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables:- `AGENTMEMORY_AGENT_SCOPE`- `AGENTMEMORY_ALLOW_AGENT_SDK`@@ -27,6 +27,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e- `AGENTMEMORY_III_VERSION`- `AGENTMEMORY_IMAGE_EMBEDDINGS`- `AGENTMEMORY_IMAGE_STORE_MAX_BYTES`+- `AGENTMEMORY_INDEX_SAVE_INTERVAL_MS`- `AGENTMEMORY_INJECT_CONTEXT`- `AGENTMEMORY_LLM_NOTHINK`- `AGENTMEMORY_LLM_TIMEOUT_MS`diff --git a/src/config.ts b/src/config.tsindex a6c62ce5..ac82bd44 100644--- a/src/config.ts+++ b/src/config.ts@@ -467,6 +467,16 @@ export function getConsolidationCooldownMs(): number {return raw >= 0 ? raw : CONSOLIDATION_COOLDOWN_DEFAULT_MS;}+export const INDEX_SAVE_INTERVAL_DEFAULT_MS = 600_000;++export function getIndexSaveIntervalMs(): number {+ const raw = safeParseInt(+ getMergedEnv()["AGENTMEMORY_INDEX_SAVE_INTERVAL_MS"],+ INDEX_SAVE_INTERVAL_DEFAULT_MS,+ );+ return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS;+}+export function isStandaloneMcp(): boolean {return getMergedEnv()["STANDALONE_MCP"] === "true";}diff --git a/src/functions/search.ts b/src/functions/search.tsindex 2b8858bd..8abce07f 100644--- a/src/functions/search.ts+++ b/src/functions/search.ts@@ -1,4 +1,5 @@import type { IIIClient } from 'iii-sdk'+import type { IndexPersistenceStatus } from "../state/index-persistence.js";import type { CompactSearchResult, CompressedObservation, Memory, SearchResult, Session } from '../types.js'import { KV } from '../state/schema.js'import { StateKV } from '../state/kv.js'@@ -75,17 +76,22 @@ export function vectorIndexRemove(id: string): void {// Wired by src/index.ts after IndexPersistence is constructed; no-op// until then so unit tests that exercise the delete paths in// isolation don't need to wire persistence.-let indexPersistence: {+type IndexPersistenceHook = {scheduleSave: () => void;save: () => Promise<void>;-} | null = null;+ status?: () => IndexPersistenceStatus;+};-export function setIndexPersistence(- p: { scheduleSave: () => void; save: () => Promise<void> } | null,-): void {+let indexPersistence: IndexPersistenceHook | null = null;++export function setIndexPersistence(p: IndexPersistenceHook | null): void {indexPersistence = p;}+export function getIndexPersistenceStatus(): IndexPersistenceStatus | null {+ return indexPersistence?.status?.() ?? null;+}+export function scheduleIndexSave(): void {indexPersistence?.scheduleSave();}diff --git a/src/functions/status.ts b/src/functions/status.tsindex 552fe6c0..0b3ab531 100644--- a/src/functions/status.ts+++ b/src/functions/status.ts@@ -1,3 +1,5 @@+import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js";+export type StatusLevel = "ok" | "info" | "warn" | "error";export interface StatusProblem {@@ -58,6 +60,7 @@ export interface StatusInputs {};graph: GraphStatsInput | null;graphExtractionEnabled: boolean;+ indexPersistence?: IndexPersistenceStatus | null;}export interface StatusReport {@@ -72,6 +75,7 @@ export interface StatusReport {health: StatusInputs["health"];provider: { llm: string; embeddings: string; circuitBreaker: StatusInputs["circuitBreaker"] };index: StatusInputs["index"];+ indexPersistence: IndexPersistenceStatus | null;graph: (GraphStatsInput & { ageSeconds: number | null; extractionEnabled: boolean }) | null;functions: Array<FunctionMetricInput & { failureRate: number }>;flags: StatusFlag[];@@ -81,6 +85,7 @@ export interface StatusReport {const FAILURE_RATE_THRESHOLD = 0.2;const FAILURE_MIN_CALLS = 5;const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60;+const VECTOR_STRING_WARN_RATIO = 0.8;const LEVEL_RANK: Record<StatusLevel, number> = { ok: 0, info: 1, warn: 2, error: 3 };const FUNCTION_FIXES: Record<string, string> = {@@ -168,6 +173,43 @@ export function evaluateStatus(input: StatusInputs): StatusReport {});}+ const persistence = input.indexPersistence ?? null;+ if (persistence) {+ const legs: Array<[string, IndexLegStatus | null]> = [+ ["BM25", persistence.bm25],+ ["vector", persistence.vector],+ ];+ for (const [label, leg] of legs) {+ if (!leg) continue;+ if (leg.lastError) {+ problems.push({+ level: "error",+ code: "index-save-failing",+ message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`,+ fix: "Check the server log for the failing state write. The next save retries automatically.",+ });+ }+ const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined);+ if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) {+ problems.push({+ level: "warn",+ code: "index-save-stale",+ message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`,+ fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.",+ });+ }+ }+ const vectorChars = persistence.vector?.serializedChars ?? null;+ if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) {+ problems.push({+ level: "warn",+ code: "vector-index-near-string-limit",+ message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`,+ fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.",+ });+ }+ }+let graph: StatusReport["graph"] = null;if (input.graph) {const ageSeconds = secondsBetween(input.now, input.graph.updatedAt);@@ -218,6 +260,7 @@ export function evaluateStatus(input: StatusInputs): StatusReport {circuitBreaker: input.circuitBreaker,},index: input.index,+ indexPersistence: persistence,graph,functions,flags: input.flags,@@ -248,6 +291,29 @@ function row(label: string, value: string): string {return `<tr><th>${escapeHtml(label)}</th><td>${value}</td></tr>`;}+function legSummary(report: StatusReport, leg: IndexLegStatus): string {+ if (leg.lastError) return `failing: ${leg.lastError}`;+ const saved = leg.lastSavedAt ? `saved ${formatDuration(secondsBetween(new Date(report.checkedAt), leg.lastSavedAt))} ago` : "not saved since start";+ return leg.dirtySince ? `${saved}, unsaved changes pending` : saved;+}++function indexPersistenceRows(report: StatusReport): string {+ const persistence = report.indexPersistence;+ if (!persistence) return "";+ let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25)));+ if (persistence.vector) {+ rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector)));+ const chars = persistence.vector.serializedChars;+ if (chars !== null) {+ rows += row(+ "Vector index size",+ escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`),+ );+ }+ }+ return rows;+}+export function renderStatusHtml(report: StatusReport, styleNonce: string): string {const problems = report.problems.length? report.problems@@ -330,6 +396,7 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off")${row("Observations indexed", escapeHtml(idx.observationsIndexed))}${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))}${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))}+${indexPersistenceRows(report)}</table><h2>Knowledge graph</h2><table>${graphdiff --git a/src/state/index-persistence.ts b/src/state/index-persistence.tsindex 19b05cb5..28a24d5a 100644--- a/src/state/index-persistence.ts+++ b/src/state/index-persistence.ts@@ -4,8 +4,9 @@ import type { StateKV } from "./kv.js";import { KV, generateId } from "./schema.js";import { logger } from "../logger.js";import { safeAudit } from "../functions/audit.js";+import { getIndexSaveIntervalMs } from "../config.js";-const DEBOUNCE_MS = 5000;+export const V8_MAX_STRING_CHARS = 536_870_888;const FAILURE_LOG_THROTTLE_MS = 60_000;const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence";const BM25_KEY = "data";@@ -41,8 +42,31 @@ type IndexShardManifest = {type IndexPersistenceOptions = {shardChars?: number;createGeneration?: () => string;+ saveIntervalMs?: number;+ now?: () => number;};+export type IndexLeg = "bm25" | "vector";++export interface IndexLegStatus {+ lastSavedAt: string | null;+ lastError: string | null;+ lastErrorAt: string | null;+ dirtySince: string | null;+ serializedChars: number | null;+}++export interface IndexPersistenceStatus {+ saveIntervalMs: number;+ saving: boolean;+ bm25: IndexLegStatus;+ vector: IndexLegStatus | null;+}++function emptyLegStatus(): IndexLegStatus {+ return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null };+}+function shardChars(options: IndexPersistenceOptions): number {const configured = options.shardChars;if (typeof configured !== "number" || !Number.isFinite(configured)) {@@ -81,39 +105,72 @@ function isValidShardDescriptor(export class IndexPersistence {private timer: ReturnType<typeof setTimeout> | null = null;- private lastFailureLogAt = 0;+ private lastFailureLogAt = new Map<IndexLeg, number>();+ private running: Promise<void> | null = null;+ private queued: Promise<void> | null = null;+ private stopped = false;+ private lastSaveAt: number;+ private dirtyEpoch = 0;+ private markedDuringRunAt: number | null = null;+ private legs: Record<IndexLeg, IndexLegStatus> = {+ bm25: emptyLegStatus(),+ vector: emptyLegStatus(),+ };+ private readonly saveIntervalMs: number;+ private readonly now: () => number;constructor(private kv: StateKV,private bm25: SearchIndex,private vector: VectorIndex | null,private options: IndexPersistenceOptions = {},- ) {}+ ) {+ this.now = options.now ?? Date.now;+ const interval = options.saveIntervalMs;+ this.saveIntervalMs =+ typeof interval === "number" && Number.isFinite(interval) && interval > 0+ ? interval+ : getIndexSaveIntervalMs();+ this.lastSaveAt = this.now();+ }scheduleSave(): void {- if (this.timer) clearTimeout(this.timer);- // setTimeout discards the returned promise, so any rejection inside- // save() would surface as unhandledRejection and crash the process- // under sustained iii-engine write timeouts (issue #204). Funnel- // rejections through logFailure() instead.+ if (this.stopped) return;+ const now = this.now();+ this.dirtyEpoch++;+ if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now;+ for (const leg of this.activeLegs()) {+ if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString();+ }+ if (this.timer) return;+ const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now);this.timer = setTimeout(() => {- this.save().catch((err) => this.logFailure(err));- }, DEBOUNCE_MS);+ this.timer = null;+ this.save().catch((err) => this.logFailure("bm25", err));+ }, delay);}- async save(): Promise<void> {- if (this.timer) {- clearTimeout(this.timer);- this.timer = null;- }- try {- await this.saveBm25Index(this.bm25.serialize());- if (this.vector) {- await this.saveVectorIndex(this.vector.serialize());- }- } catch (err) {- this.logFailure(err);+ save(): Promise<void> {+ this.clearTimer();+ if (this.queued) return this.queued;+ if (this.running) {+ const queued = this.running.then(() => {+ this.queued = null;+ return this.startRun();+ });+ this.queued = queued;+ return queued;}+ return this.startRun();+ }++ status(): IndexPersistenceStatus {+ return {+ saveIntervalMs: this.saveIntervalMs,+ saving: this.running !== null,+ bm25: { ...this.legs.bm25 },+ vector: this.vector ? { ...this.legs.vector } : null,+ };}async load(): Promise<{@@ -137,27 +194,80 @@ export class IndexPersistence {}stop(): void {+ this.stopped = true;+ this.clearTimer();+ }++ private clearTimer(): void {if (this.timer) {clearTimeout(this.timer);this.timer = null;}}- private logFailure(err: unknown): void {- const now = Date.now();- // Throttle: persistence failures under load arrive in bursts- // (iii-engine queue pressure). Logging every debounce flush adds- // noise without information.- if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return;- this.lastFailureLogAt = now;+ private activeLegs(): IndexLeg[] {+ return this.vector ? ["bm25", "vector"] : ["bm25"];+ }++ private startRun(): Promise<void> {+ const run = this.runSave().finally(() => {+ if (this.running === run) this.running = null;+ });+ this.running = run;+ return run;+ }++ private async runSave(): Promise<void> {+ this.markedDuringRunAt = null;+ this.lastSaveAt = this.now();+ await this.saveLeg("bm25", async () => {+ const serialized = this.bm25.serialize();+ this.legs.bm25.serializedChars = serialized.length;+ await this.saveBm25Index(serialized);+ });+ const vector = this.vector;+ if (vector) {+ await this.saveLeg("vector", async () => {+ const serialized = vector.serialize();+ this.legs.vector.serializedChars = serialized.length;+ await this.saveVectorIndex(serialized);+ });+ }+ }++ private async saveLeg(leg: IndexLeg, run: () => Promise<void>): Promise<void> {+ const status = this.legs[leg];+ const epoch = this.dirtyEpoch;+ try {+ await run();+ status.lastSavedAt = new Date(this.now()).toISOString();+ status.lastError = null;+ status.lastErrorAt = null;+ if (this.dirtyEpoch === epoch) {+ status.dirtySince = null;+ } else if (this.markedDuringRunAt !== null) {+ status.dirtySince = new Date(this.markedDuringRunAt).toISOString();+ }+ } catch (err) {+ status.lastError = errorMessage(err);+ status.lastErrorAt = new Date(this.now()).toISOString();+ if (status.dirtySince === null) status.dirtySince = status.lastErrorAt;+ this.logFailure(leg, err);+ }+ }++ private logFailure(leg: IndexLeg, err: unknown): void {+ const now = this.now();+ if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return;+ this.lastFailureLogAt.set(leg, now);const code = (err as { code?: string })?.code;const message = err instanceof Error ? err.message : String(err);- logger.warn("index persistence: failed to save BM25/vector index", {+ logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, {code,message,hint:code === "TIMEOUT"- ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next debounce flush"+ ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save": undefined,});}diff --git a/src/triggers/api.ts b/src/triggers/api.tsindex d8a447f2..2fc2f328 100644--- a/src/triggers/api.ts+++ b/src/triggers/api.ts@@ -12,7 +12,7 @@ import type { ResilientProvider } from "../providers/resilient.js";import { III_PINNED_VERSION, VERSION } from "../version.js";import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js";import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js";-import { findUnindexedObservations, getSearchIndex, getVectorIndex } from "../functions/search.js";+import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js";import { timingSafeCompare } from "../auth.js";import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js";import { renderViewerDocument } from "../viewer/document.js";@@ -373,6 +373,7 @@ export function registerApiTriggers(},graph,graphExtractionEnabled: isGraphExtractionEnabled(),+ indexPersistence: getIndexPersistenceStatus(),});const accept = req.headers?.["accept"] ?? req.headers?.["Accept"];const format = req.query_params?.["format"];diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.tsnew file mode 100644index 00000000..54eff133--- /dev/null+++ b/test/index-persistence-throttle.test.ts@@ -0,0 +1,335 @@+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";+import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js";+import { SearchIndex } from "../src/state/search-index.js";+import { VectorIndex } from "../src/state/vector-index.js";+import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js";+import type { CompressedObservation } from "../src/types.js";++const BM25_SCOPE = "mem:index:bm25";++function mockKV() {+ const store = new Map<string, Map<string, unknown>>();+ const sets: Array<{ scope: string; key: string }> = [];+ return {+ sets,+ store,+ get: async <T>(scope: string, key: string): Promise<T | null> => (store.get(scope)?.get(key) as T) ?? null,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ sets.push({ scope, key });+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, data);+ return data;+ },+ delete: async (scope: string, key: string): Promise<void> => {+ store.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> => Array.from(store.get(scope)?.values() ?? []) as T[],+ };+}++function obs(id: string, title: string): CompressedObservation {+ return {+ id,+ sessionId: "ses_1",+ timestamp: new Date().toISOString(),+ type: "file_edit",+ title,+ facts: [],+ narrative: `${title} narrative`,+ concepts: [],+ files: [],+ importance: 5,+ };+}++function manifestSaves(kv: ReturnType<typeof mockKV>, key: string): number {+ return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length;+}++describe("IndexPersistence save throttling", () => {+ let kv: ReturnType<typeof mockKV>;++ beforeEach(() => {+ vi.useFakeTimers();+ kv = mockKV();+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("saves at most once per interval however often changes are scheduled", async () => {+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });++ for (let i = 0; i < 50; i++) persistence.scheduleSave();+ await vi.advanceTimersByTimeAsync(59_000);+ expect(manifestSaves(kv, "data:manifest")).toBe(0);++ await vi.advanceTimersByTimeAsync(1_000);+ expect(manifestSaves(kv, "data:manifest")).toBe(1);++ for (let i = 0; i < 50; i++) persistence.scheduleSave();+ await vi.advanceTimersByTimeAsync(30_000);+ expect(manifestSaves(kv, "data:manifest")).toBe(1);+ await vi.advanceTimersByTimeAsync(30_000);+ expect(manifestSaves(kv, "data:manifest")).toBe(2);+ });++ it("an explicit save runs immediately and cancels the pending timer", async () => {+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 });++ persistence.scheduleSave();+ await persistence.save();+ expect(manifestSaves(kv, "data:manifest")).toBe(1);++ await vi.advanceTimersByTimeAsync(120_000);+ expect(manifestSaves(kv, "data:manifest")).toBe(1);+ });++ it("never runs two saves at once and coalesces requests made during a save into one", async () => {+ let release: () => void = () => undefined;+ let inFlight = 0;+ let maxInFlight = 0;+ let gate: Promise<void> | null = new Promise<void>((resolve) => {+ release = resolve;+ });+ const slowKv = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ if (gate) await gate;+ const result = await kv.set(scope, key, data);+ inFlight--;+ return result;+ },+ };+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });++ const first = persistence.save();+ const second = persistence.save();+ const third = persistence.save();+ expect(second).toBe(third);+ expect(persistence.status().saving).toBe(true);++ gate = null;+ release();+ await Promise.all([first, second, third]);++ expect(manifestSaves(kv, "data:manifest")).toBe(2);+ expect(maxInFlight).toBe(1);+ expect(persistence.status().saving).toBe(false);+ });++ it("a failing BM25 leg does not stop the vector leg from saving", async () => {+ const failingKv = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed");+ return kv.set(scope, key, data);+ },+ };+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const vector = new VectorIndex();+ vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));+ const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 });++ persistence.scheduleSave();+ await persistence.save();++ const status = persistence.status();+ expect(status.bm25.lastError).toBe("bm25 manifest write failed");+ expect(status.bm25.dirtySince).not.toBeNull();+ expect(status.vector?.lastError).toBeNull();+ expect(status.vector?.lastSavedAt).not.toBeNull();+ expect(status.vector?.dirtySince).toBeNull();+ expect(status.vector?.serializedChars).toBeGreaterThan(0);++ const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load();+ expect(loaded.bm25).toBeNull();+ expect(loaded.vector?.size).toBe(1);+ });++ it("keeps a leg dirty when a change arrives while it is being saved", async () => {+ let release: () => void = () => undefined;+ const gate = new Promise<void>((resolve) => {+ release = resolve;+ });+ let held = false;+ const slowKv = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (!held) {+ held = true;+ await gate;+ }+ return kv.set(scope, key, data);+ },+ };+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 });++ const saving = persistence.save();+ persistence.scheduleSave();+ release();+ await saving;++ expect(persistence.status().bm25.lastSavedAt).not.toBeNull();+ expect(persistence.status().bm25.dirtySince).not.toBeNull();++ await persistence.save();+ expect(persistence.status().bm25.dirtySince).toBeNull();+ });++ it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => {+ let release: () => void = () => undefined;+ const gate = new Promise<void>((resolve) => {+ release = resolve;+ });+ let heldBm25Manifest = false;+ const slowKv = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) {+ heldBm25Manifest = true;+ await gate;+ }+ return kv.set(scope, key, data);+ },+ };+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const vector = new VectorIndex();+ vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));+ const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 });++ const saving = persistence.save();+ persistence.scheduleSave();+ release();+ await saving;++ const status = persistence.status();+ expect(status.bm25.dirtySince).not.toBeNull();+ expect(status.vector?.dirtySince).toBeNull();+ expect(status.vector?.lastSavedAt).not.toBeNull();+ });++ it("stop prevents later scheduled saves", async () => {+ const bm25 = new SearchIndex();+ bm25.add(obs("obs_1", "alpha"));+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 });+ persistence.stop();+ persistence.scheduleSave();+ await vi.advanceTimersByTimeAsync(10_000);+ expect(manifestSaves(kv, "data:manifest")).toBe(0);+ });+});++function statusInputs(overrides: Partial<StatusInputs> = {}): StatusInputs {+ return {+ now: new Date("2026-09-25T12:00:00.000Z"),+ version: "0.9.29",+ engineVersion: "0.22.1",+ uptimeSeconds: 100,+ ports: { rest: 3111, streams: 3112, viewer: 3113 },+ health: { status: "healthy", alerts: [], notes: [], connectionState: "connected" },+ circuitBreaker: null,+ functionMetrics: [],+ provider: "llm",+ embeddingProvider: "embeddings",+ flags: [],+ index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 },+ graph: null,+ graphExtractionEnabled: false,+ ...overrides,+ };+}++const cleanLeg = {+ lastSavedAt: "2026-09-25T11:59:00.000Z",+ lastError: null,+ lastErrorAt: null,+ dirtySince: null,+ serializedChars: 1000,+};++describe("status reports index persistence", () => {+ it("is quiet when both legs saved cleanly", () => {+ const report = evaluateStatus(+ statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }),+ );+ expect(report.problems).toEqual([]);+ expect(report.indexPersistence?.vector?.serializedChars).toBe(1000);+ });++ it("reports a failing leg as an error", () => {+ const report = evaluateStatus(+ statusInputs({+ indexPersistence: {+ saveIntervalMs: 600_000,+ saving: false,+ bm25: cleanLeg,+ vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" },+ },+ }),+ );+ expect(report.status).toBe("error");+ expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]);+ expect(report.problems[0].message).toContain("vector");+ });++ it("warns when the vector index passes 80% of the Node.js string limit", () => {+ const report = evaluateStatus(+ statusInputs({+ indexPersistence: {+ saveIntervalMs: 600_000,+ saving: false,+ bm25: cleanLeg,+ vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) },+ },+ }),+ );+ expect(report.status).toBe("warn");+ expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]);+ expect(report.problems[0].message).toContain("85%");+ });++ it("warns when unsaved changes are older than twice the save interval", () => {+ const report = evaluateStatus(+ statusInputs({+ indexPersistence: {+ saveIntervalMs: 600_000,+ saving: false,+ bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" },+ vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" },+ },+ }),+ );+ expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]);+ expect(report.problems[0].message).toContain("BM25");+ });++ it("shows save state and vector size on the status page", () => {+ const report = evaluateStatus(+ statusInputs({+ indexPersistence: {+ saveIntervalMs: 600_000,+ saving: false,+ bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" },+ vector: { ...cleanLeg, serializedChars: 53_687_089 },+ },+ }),+ );+ const html = renderStatusHtml(report, "n");+ expect(html).toContain("<th>BM25 save</th><td>saved 1m ago, unsaved changes pending</td>");+ expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)");+ });+});diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.tsindex ba47c3ef..802084bd 100644--- a/test/index-persistence.test.ts+++ b/test/index-persistence.test.ts@@ -670,7 +670,7 @@ describe("IndexPersistence", () => {it("scheduleSave debounces multiple calls", async () => {const bm25 = new SearchIndex();- const persistence = new IndexPersistence(kv as never, bm25, null);+ const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 });persistence.scheduleSave();persistence.scheduleSave();@@ -721,7 +721,7 @@ describe("IndexPersistence", () => {};const bm25 = new SearchIndex();bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));- const persistence = new IndexPersistence(failingKv as never, bm25, null);+ const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });let unhandled = false;const onUnhandled = () => {@@ -751,7 +751,7 @@ describe("IndexPersistence", () => {};const bm25 = new SearchIndex();bm25.add(makeObs({ id: "obs_1", title: "auth handler" }));- const persistence = new IndexPersistence(failingKv as never, bm25, null);+ const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 });await expect(persistence.save()).resolves.toBeUndefined();});