#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix(release): wait for vector persistence and recovery (#1520)
Rohit Ghumare7ee162844956
diff --git a/scripts/release-gate/recovery.mjs b/scripts/release-gate/recovery.mjsnew file mode 100644index 00000000..b2ca81db--- /dev/null+++ b/scripts/release-gate/recovery.mjs@@ -0,0 +1,69 @@+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { join } from "node:path";++export function decodeRkyvJsonScope(bytes) {+ const invalid = () => { throw new SyntaxError("Invalid rkyv 0.8 32-bit little-endian JSON scope"); };+ if (bytes.length < 8 || bytes.length % 4 !== 0) invalid();+ const root = bytes.length - 8;+ let length;+ if ((bytes[root] & 0xc0) === 0x80) {+ const encodedLength = bytes.readUInt32LE(root);+ length = (encodedLength & 0x3f) | ((encodedLength & 0xffffff00) >>> 2);+ const start = root + bytes.readInt32LE(root + 4);+ if (length <= 8 || start !== 0 || length > root || root - length > 3) invalid();+ } else {+ if (root !== 0) invalid();+ const end = bytes.indexOf(0xff);+ length = end === -1 ? 8 : end;+ if (!bytes.subarray(length).every((byte) => byte === 0xff)) invalid();+ }+ let json;+ try {+ json = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length));+ } catch {+ invalid();+ }+ const value = JSON.parse(json);+ if (value === null || typeof value !== "object" || Array.isArray(value)) invalid();+ return value;+}++export function isVectorLogAcknowledged(status, count, checkpointCount) {+ const persistence = status.indexPersistence;+ const pending = count - checkpointCount;+ return status.index?.vectorDocuments === count && persistence?.saving === false &&+ persistence.pendingChanges === pending && persistence.pendingLog === pending &&+ persistence.pendingLogAcknowledged === pending && persistence.pendingLogError === null;+}++export function hasPersistedVectorFixture(dataDir, sessionId, checkpointId, pendingIds, dimensions) {+ const readScope = (scope) => decodeRkyvJsonScope(readFileSync(join(dataDir, "state_store.db", `${encodeURIComponent(scope)}.bin`)));+ const matchesVector = (row, id) => row?.id === id && row.s === sessionId &&+ typeof row.e === "string" && Buffer.from(row.e, "base64").length === dimensions * 4;+ try {+ if (readScope("mem:sessions")[sessionId]?.id !== sessionId) return false;+ const meta = readScope("mem:index:bm25")["vectors:meta"];+ if (meta?.v !== 3 || meta.count !== 1 || meta.bucketCount !== 1) return false;+ if (!matchesVector(readScope("mem:index:bm25:vec:0000")[checkpointId], checkpointId)) return false;+ const pending = readScope("mem:index:vec-pending");+ const observations = readScope(`mem:obs:${sessionId}`);+ return [checkpointId, ...pendingIds].every((id) => observations[id]?.id === id) &&+ pendingIds.every((id) => matchesVector(pending[id], id) && Number.isFinite(pending[id].q) && !pending[id].t && !pending[id].c);+ } catch (err) {+ if (err.code === "ENOENT" || err instanceof SyntaxError) return false;+ throw err;+ }+}++export function isVectorRecoverySettled(status) {+ return typeof status.index?.vectorDocuments === "number" && status.index.keywordRebuildRunning === false;+}++export function assertRecoveredVectors(status, expectedCount, embeddingInputs) {+ const index = status.index;+ assert.equal(index.vectorDocuments, expectedCount, `${index.vectorDocuments} vectors after the crash, expected ${expectedCount}`);+ assert.equal(index.pendingVectorBackfill, 0, `${index.pendingVectorBackfill} documents wait for vector backfill after the crash`);+ assert.equal(index.vectorBackfillState, "idle", `vector backfill is ${index.vectorBackfillState} after the crash`);+ assert.equal(embeddingInputs, 0, `the restart re-embedded ${embeddingInputs} inputs instead of loading saved vectors`);+}diff --git a/scripts/release-gate/run.mjs b/scripts/release-gate/run.mjsindex 65a76e7f..6b76b2b6 100644--- a/scripts/release-gate/run.mjs+++ b/scripts/release-gate/run.mjs@@ -21,6 +21,7 @@ import net from "node:net";import { homedir, tmpdir, platform, arch } from "node:os";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";+import { assertRecoveredVectors, hasPersistedVectorFixture, isVectorLogAcknowledged, isVectorRecoverySettled } from "./recovery.mjs";const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");const FORBIDDEN_PORTS = new Set([3111, 3112, 3113, 4098, 4131, 4132, 4133, 49134]);@@ -32,7 +33,7 @@ const SCENARIOS = [["offline", "hooks run while the service is down and the spool is recovered on restart"],["dedup", "a replayed host event after a force kill stays one observation"],["deadletter", "an accepted capture that fails processing survives a restart and is visible as a dead letter"],- ["vectors", "vectors made before the first checkpoint survive a crash"],+ ["vectors", "checkpointed and pending vectors survive a crash without re-embedding"],["stopflush", "agentmemory stop then start loses nothing"],["status", "viewer serves and /agentmemory/status explains every problem"],["roundtrip", "export then import into a fresh home round-trips"],@@ -367,6 +368,7 @@ class Instance {this.name = name;this.dir = join(ROOT, name);this.home = join(this.dir, "home");+ this.dataDir = join(this.dir, "data");this.cwd = join(this.dir, "cwd");this.project = join(this.dir, "projects", "gate-app");for (const d of [this.home, this.cwd, this.project]) mkdirSync(d, { recursive: true });@@ -391,6 +393,7 @@ class Instance {OPENAI_BASE_URL: `http://127.0.0.1:${gate.fake.port}`,OPENAI_EMBEDDING_DIMENSIONS: String(gate.dims),AGENTMEMORY_URL: `http://127.0.0.1:${this.port}`,+ AGENTMEMORY_DATA_DIR: this.dataDir,...extra,};}@@ -519,6 +522,7 @@ class Instance {CLAUDE_PLUGIN_ROOT: join(gate.pkgDir, "plugin"),CLAUDE_PROJECT_DIR: this.project,AGENTMEMORY_URL: `http://127.0.0.1:${this.port}`,+ AGENTMEMORY_DATA_DIR: this.dataDir,},cwd: this.project,input: JSON.stringify({ hook_event_name: event, cwd: this.project, transcript_path: join(this.dir, "transcript.jsonl"), ...payload }),@@ -907,33 +911,46 @@ async function main() {await ensureUp(instB);const sid = `gate-vectors-${randomBytes(4).toString("hex")}`;const count = 20;- for (let i = 0; i < count; i++) await instB.hook("PostToolUse", instB.toolEvent(sid, marker(`vec${i}x`)));+ await instB.hook("PostToolUse", instB.toolEvent(sid, marker("vec0x")));const t0 = Date.now();const st = await poll("the first vector checkpoint", async () => {const s = await instB.status();const p = s.indexPersistence;- const durable = p && !p.saving && (p.pendingChanges === 0 || (p.pendingLog ?? 0) >= p.pendingChanges);- return (s.index?.vectorDocuments ?? 0) >= count && p?.vector?.lastSavedAt && !p.firstCheckpointPending && durable ? s : null;+ return s.index?.vectorDocuments === 1 && p?.vector?.lastSavedAt && !p.firstCheckpointPending && !p.saving && p.pendingChanges === 0 ? s : null;}, { timeoutMs: 60_000, intervalMs: 500 });d.firstCheckpointWaitMs = Date.now() - t0;d.saveIntervalMs = st.indexPersistence.saveIntervalMs;- const vectorsBefore = st.index.vectorDocuments;- d.vectorsBefore = vectorsBefore;+ d.checkpointVectors = st.index.vectorDocuments;+ const checkpointId = (await instB.observations(sid))[0]?.id;+ assert(checkpointId, "the checkpoint observation is missing");assert(d.saveIntervalMs > d.firstCheckpointWaitMs, "the checkpoint only happened at the regular save interval, so the first-checkpoint boundary was not exercised");- await sleep(STATE_FLUSH_WAIT_MS);+ for (let i = 1; i < count; i++) await instB.hook("PostToolUse", instB.toolEvent(sid, marker(`vec${i}x`)));+ const before = await poll("every pending vector write to be acknowledged", async () => {+ const s = await instB.status();+ return isVectorLogAcknowledged(s, count, d.checkpointVectors) ? s : null;+ }, { timeoutMs: 60_000, intervalMs: 500 });+ const vectorsBefore = before.index.vectorDocuments;+ d.vectorsBefore = vectorsBefore;+ d.pendingLogBeforeCrash = before.indexPersistence.pendingLog;+ d.pendingLogAcknowledged = before.indexPersistence.pendingLogAcknowledged;+ const pendingIds = (await instB.observations(sid)).map((observation) => observation.id).filter((id) => id !== checkpointId);+ assert(pendingIds.length === count - 1, `${pendingIds.length} pending observations, expected ${count - 1}`);+ await poll("checkpoint and pending vectors to be present on disk", async () =>+ hasPersistedVectorFixture(instB.dataDir, sid, checkpointId, pendingIds, gate.dims),+ { timeoutMs: 60_000, intervalMs: 250 });+ d.persistedVectorsBeforeCrash = pendingIds.length + 1;const embedBefore = gate.fake.stats.inputs;await instB.forceKill();d.restartMs = await instB.start();- const after = await poll("vector index to load", async () => {+ const after = await poll("startup index recovery to finish", async () => {const s = await instB.status();- return typeof s.index?.vectorDocuments === "number" && s.index.vectorDocuments > 0 ? s : null;+ return isVectorRecoverySettled(s) ? s : null;}, { timeoutMs: 30_000 });d.vectorsAfter = after.index.vectorDocuments;d.pendingVectorBackfill = after.index.pendingVectorBackfill;+ d.vectorBackfillState = after.index.vectorBackfillState;d.embeddingsAfterRestart = gate.fake.stats.inputs - embedBefore;- assert(d.vectorsAfter === vectorsBefore, `${d.vectorsAfter} vectors after the crash, expected ${vectorsBefore}`);- assert(d.pendingVectorBackfill === 0, `${d.pendingVectorBackfill} documents wait for vector backfill after the crash`);- assert(d.embeddingsAfterRestart === 0, `the restart re-embedded ${d.embeddingsAfterRestart} inputs instead of loading saved vectors`);+ assertRecoveredVectors(after, vectorsBefore, d.embeddingsAfterRestart);await instB.stop();});diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.tsindex 9f1cd810..46c565d3 100644--- a/src/state/index-persistence.ts+++ b/src/state/index-persistence.ts@@ -100,6 +100,7 @@ export interface IndexPersistenceStatus {vector: IndexLegStatus | null;vectorCountShortfall: VectorCountShortfall | null;pendingLog?: number;+ pendingLogAcknowledged?: number;pendingLogError?: string | null;}@@ -199,6 +200,7 @@ export class IndexPersistence {private vectorCountShortfall: VectorCountShortfall | null = null;private seq = 0;private pendingLog: Map<string, number> = new Map();+ private acknowledgedLog: Map<string, number> = new Map();private logChains: Map<string, Promise<void>> = new Map();private logSuppressed = false;private pendingLogUnknown = false;@@ -259,6 +261,7 @@ export class IndexPersistence {vector: this.vector ? { ...this.leg } : null,vectorCountShortfall: this.vectorCountShortfall,pendingLog: this.vector ? this.pendingLog.size : 0,+ pendingLogAcknowledged: this.vector ? this.acknowledgedLog.size : 0,pendingLogError: this.pendingLogError,};}@@ -283,6 +286,7 @@ export class IndexPersistence {result.entries++;const key = row.c === 1 ? PENDING_CLEAR_KEY : row.id!;if ((this.pendingLog.get(key) ?? -Infinity) < row.q) this.pendingLog.set(key, row.q);+ if (this.pendingLog.get(key) === row.q) this.acknowledgedLog.set(key, row.q);if (row.q > this.seq) this.seq = row.q;if (row.c === 1) {vector.clear();@@ -369,8 +373,10 @@ export class IndexPersistence {row = { q, id, t: 1 };}this.pendingLog.set(key, q);+ this.acknowledgedLog.delete(key);this.enqueueLog(key, async () => {await this.kv.set<PendingVectorRow>(KV.vectorPendingLog, key, row);+ if (this.pendingLog.get(key) === q) this.acknowledgedLog.set(key, q);}).then(() => {this.pendingLogError = null;@@ -411,7 +417,10 @@ export class IndexPersistence {return this.enqueueLog(key, async () => {if (this.pendingLog.get(key) !== q) return;await this.kv.delete(KV.vectorPendingLog, key);- if (this.pendingLog.get(key) === q) this.pendingLog.delete(key);+ if (this.pendingLog.get(key) === q) {+ this.pendingLog.delete(key);+ this.acknowledgedLog.delete(key);+ }});}diff --git a/test/release-gate-durability.test.ts b/test/release-gate-durability.test.tsnew file mode 100644index 00000000..97a2530c--- /dev/null+++ b/test/release-gate-durability.test.ts@@ -0,0 +1,124 @@+import { afterEach, describe, expect, it } from "vitest";+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { decodeRkyvJsonScope, hasPersistedVectorFixture, isVectorLogAcknowledged } from "../scripts/release-gate/recovery.mjs";++const dirs: string[] = [];++afterEach(() => {+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });+});++function archiveScope(value: unknown) {+ const json = Buffer.from(JSON.stringify(value));+ if (json.length <= 8) {+ const bytes = Buffer.alloc(8, 0xff);+ json.copy(bytes);+ return bytes;+ }+ const root = Math.ceil(json.length / 4) * 4;+ const bytes = Buffer.alloc(root + 8);+ json.copy(bytes);+ bytes.writeUInt32LE((json.length & 0x3f) | 0x80 | ((json.length >>> 6) << 8), root);+ bytes.writeInt32LE(-root, root + 4);+ return bytes;+}++function fixture() {+ const dir = mkdtempSync(join(tmpdir(), "vector-durability-"));+ dirs.push(dir);+ mkdirSync(join(dir, "state_store.db"));+ const rows = Object.fromEntries(["checkpoint", "pending"].map((id) => [id, {+ id, s: "session", e: Buffer.from(new Float32Array([1, 2, 3]).buffer).toString("base64"), q: 1,+ }]));+ const scopes: Record<string, unknown> = {+ "mem:sessions": { session: { id: "session" } },+ "mem:index:bm25": { "vectors:meta": { v: 3, count: 1, bucketCount: 1 } },+ "mem:index:bm25:vec:0000": { checkpoint: rows.checkpoint },+ "mem:index:vec-pending": { pending: rows.pending },+ "mem:obs:session": { checkpoint: { id: "checkpoint" }, pending: { id: "pending" } },+ };+ const write = (scope: string, value: unknown) => writeFileSync(join(dir, "state_store.db", `${encodeURIComponent(scope)}.bin`), archiveScope(value));+ const check = () => hasPersistedVectorFixture(dir, "session", "checkpoint", ["pending"], 3);+ return { dir, scopes, rows, write, check };+}++describe("release gate vector durability", () => {+ it("decodes a scope captured from the pinned engine", () => {+ const bytes = Buffer.from("7b22766563746f72733a6d657461223a7b226275636b6574436f756e74223a312c22636f756e74223a32302c2273617665644174223a22323032362d31302d31305430393a32363a31372e3836345a222c2276223a337d7d98010000a8ffffff", "hex");+ expect(decodeRkyvJsonScope(bytes)).toEqual({+ "vectors:meta": { bucketCount: 1, count: 20, savedAt: "2026-10-10T09:26:17.864Z", v: 3 },+ });+ expect(decodeRkyvJsonScope(Buffer.from("7b7dffffffffffff", "hex"))).toEqual({});+ });++ it.each([+ (bytes: Buffer) => bytes.subarray(0, bytes.length - 1),+ (bytes: Buffer) => { bytes.writeInt32LE(-1, bytes.length - 4); return bytes; },+ (bytes: Buffer) => { bytes.writeUInt32LE(0xffffffbf, bytes.length - 8); return bytes; },+ (bytes: Buffer) => { bytes[0] = 0xff; return bytes; },+ () => archiveScope([]),+ () => Buffer.from("7b7dff00ffffffff", "hex"),+ ])("rejects invalid archive boundaries, UTF-8 and scope values", (corrupt) => {+ expect(() => decodeRkyvJsonScope(corrupt(archiveScope({ pending: { q: 1 } })))).toThrow(SyntaxError);+ });++ it("requires successful acknowledgments rather than the pending count", () => {+ const status = {+ index: { vectorDocuments: 20 },+ indexPersistence: { saving: false, pendingChanges: 19, pendingLog: 19, pendingLogAcknowledged: 0, pendingLogError: null as string | null },+ };+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(false);+ status.indexPersistence.pendingLogAcknowledged = 18;+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(false);+ status.indexPersistence.pendingLogAcknowledged = 19;+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(true);+ status.indexPersistence.pendingLogError = "write refused";+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(false);+ });++ it("waits for the actual checkpoint and every pending row to reach disk", () => {+ const { scopes, write, check } = fixture();+ expect(check()).toBe(false);+ for (const scope of ["mem:index:bm25", "mem:index:bm25:vec:0000", "mem:obs:session"]) write(scope, scopes[scope]);+ expect(check()).toBe(false);+ write("mem:index:vec-pending", {});+ expect(check()).toBe(false);+ write("mem:index:vec-pending", scopes["mem:index:vec-pending"]);+ expect(check()).toBe(false);+ write("mem:sessions", scopes["mem:sessions"]);+ expect(check()).toBe(true);+ });++ it("retries a file observed during a partial rewrite", () => {+ const { dir, scopes, write, check } = fixture();+ for (const [scope, value] of Object.entries(scopes)) write(scope, value);+ writeFileSync(join(dir, "state_store.db", "mem%3Aindex%3Avec-pending.bin"), '{"pending":');+ expect(check()).toBe(false);+ write("mem:index:vec-pending", scopes["mem:index:vec-pending"]);+ expect(check()).toBe(true);+ });++ it.each([+ ["mem:sessions", {}],+ ["mem:index:bm25", { "vectors:meta": { v: 3, count: 0, bucketCount: 1 } }],+ ["mem:index:bm25:vec:0000", {}],+ ["mem:obs:session", { checkpoint: { id: "checkpoint" } }],+ ])("rejects incomplete persisted scope %s", (scope, value) => {+ const { scopes, write, check } = fixture();+ for (const [name, row] of Object.entries(scopes)) write(name, row);+ write(scope as string, value);+ expect(check()).toBe(false);+ });++ it.each([{ s: "another-session" }, { id: "another-id" }, { e: "AA==" }, { q: null }, { t: 1 }, { c: 1 }])(+ "rejects mismatched or incomplete pending rows: %o",+ (override) => {+ const { scopes, rows, write, check } = fixture();+ for (const [scope, value] of Object.entries(scopes)) write(scope, value);+ write("mem:index:vec-pending", { pending: { ...rows.pending, ...override } });+ expect(check()).toBe(false);+ },+ );+});diff --git a/test/release-gate-recovery.test.ts b/test/release-gate-recovery.test.tsnew file mode 100644index 00000000..9e21d08c--- /dev/null+++ b/test/release-gate-recovery.test.ts@@ -0,0 +1,154 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { assertRecoveredVectors, isVectorRecoverySettled } from "../scripts/release-gate/recovery.mjs";+import { IndexPersistence } from "../src/state/index-persistence.js";+import { VectorIndex } from "../src/state/vector-index.js";+import { KV } from "../src/state/schema.js";+import { createStatusReporter } from "../src/triggers/api.js";+import {+ getSearchIndex,+ markKeywordRebuildPending,+ rebuildKeywordIndex,+ setEmbeddingProvider,+ setIndexPersistence,+ setPendingVectorBackfillCount,+ setVectorBackfillState,+ setVectorIndex,+} from "../src/functions/search.js";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++function storeKV() {+ const scopes = new Map<string, Map<string, unknown>>();+ return {+ backend: "file" as const,+ get: async <T>(scope: string, key: string): Promise<T | null> =>+ (scopes.get(scope)?.get(key) as T) ?? null,+ set: async <T>(scope: string, key: string, value: T): Promise<T> => {+ if (!scopes.has(scope)) scopes.set(scope, new Map());+ scopes.get(scope)!.set(key, structuredClone(value));+ return value;+ },+ delete: async (scope: string, key: string): Promise<void> => {+ scopes.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> =>+ [...(scopes.get(scope)?.values() ?? [])] as T[],+ };+}++function settledStatus(vectorDocuments = 20) {+ return {+ index: {+ vectorDocuments,+ keywordRebuildRunning: false,+ pendingVectorBackfill: 0,+ vectorBackfillState: "idle",+ },+ };+}++const persistenceInstances: IndexPersistence[] = [];++afterEach(async () => {+ for (const persistence of persistenceInstances.splice(0)) persistence.stop();+ setVectorIndex(null);+ setIndexPersistence(null);+ setEmbeddingProvider(null);+ setPendingVectorBackfillCount(0);+ setVectorBackfillState("idle");+ await rebuildKeywordIndex(storeKV() as never);+ getSearchIndex().clear();+});++describe("release gate vector recovery", () => {+ it("waits while eight checkpoint vectors are visible before twelve pending vectors replay", async () => {+ const kv = storeKV();+ const firstVector = new VectorIndex();+ const first = new IndexPersistence(kv as never, firstVector, { saveIntervalMs: 600_000 });+ persistenceInstances.push(first);+ await kv.set(KV.sessions, "ses_recovery", {+ id: "ses_recovery", project: "recovery", startedAt: "2026-10-10T00:00:00Z", observationCount: 20,+ });+ for (let i = 0; i < 20; i++) {+ const id = `obs_recovery_${i}`;+ await kv.set(KV.observations("ses_recovery"), id, {+ id, sessionId: "ses_recovery", title: `recovery ${i}`, narrative: `saved content ${i}`,+ type: "decision", facts: [], concepts: [], files: [], importance: 5, timestamp: "2026-10-10T00:00:00Z",+ });+ firstVector.add(id, "ses_recovery", new Float32Array([i, i + 1, i + 2]));+ if (i === 7) await first.save();+ }+ await first.flushPendingLog();+ first.stop();+ expect(firstVector.size).toBe(20);+ expect(await kv.list(KV.vectorPendingLog)).toHaveLength(12);++ let releaseReplay!: () => void;+ const pendingRead = new Promise<void>((resolve) => { releaseReplay = resolve; });+ const delayedKV = {+ ...kv,+ list: async <T>(scope: string): Promise<T[]> => {+ if (scope === KV.vectorPendingLog) await pendingRead;+ return kv.list<T>(scope);+ },+ };+ const vector = new VectorIndex();+ const restored = new IndexPersistence(delayedKV as never, vector, { saveIntervalMs: 600_000 });+ persistenceInstances.push(restored);+ setVectorIndex(vector);+ setIndexPersistence(restored);+ const embed = vi.fn(async () => new Float32Array([1, 2, 3]));+ const embedBatch = vi.fn(async () => []);+ setEmbeddingProvider({ name: "recovery-test", dimensions: 3, embed, embedBatch });+ markKeywordRebuildPending();+ const loaded = await restored.load();+ expect(loaded.vector?.size).toBe(8);+ vector.restoreFrom(loaded.vector!);+ const replay = restored.replayPendingLog(3);+ const report = createStatusReporter({ trigger: async () => null } as never, delayedKV as never, {});++ try {+ const during = await report({ health: null });+ expect(during.index).toMatchObject({ vectorDocuments: 8, keywordRebuildRunning: true });+ expect(isVectorRecoverySettled(during)).toBe(false);++ releaseReplay();+ expect(await replay).toMatchObject({ added: 12, skipped: 0 });+ const keyword = await rebuildKeywordIndex(delayedKV as never, loaded.savedAt);+ setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending);+ const after = await report({ health: null });+ expect(after.index).toMatchObject({ vectorDocuments: 20, keywordRebuildRunning: false });+ expect(isVectorRecoverySettled(after)).toBe(true);+ expect(() => assertRecoveredVectors(after, 20, embed.mock.calls.length + embedBatch.mock.calls.length)).not.toThrow();+ } finally {+ releaseReplay();+ await replay;+ }+ });++ it.each([0, 8, 21])("fails a settled count of %i immediately instead of waiting for the desired count", (count) => {+ const status = settledStatus(count);+ expect(isVectorRecoverySettled(status)).toBe(true);+ expect(() => assertRecoveredVectors(status, 20, 0)).toThrow(`${count} vectors after the crash, expected 20`);+ });++ it.each([{}, { index: { vectorDocuments: 20 } }, { index: { vectorDocuments: 20, keywordRebuildRunning: true } }])(+ "does not infer readiness from an absent or unfinished rebuild state",+ (status) => { expect(isVectorRecoverySettled(status)).toBe(false); },+ );++ it("fails when a recovered count required new embeddings", () => {+ expect(() => assertRecoveredVectors(settledStatus(), 20, 12)).toThrow("the restart re-embedded 12 inputs");+ });++ it("fails when vector backfill is pending or running", () => {+ const pending = settledStatus();+ pending.index.pendingVectorBackfill = 12;+ expect(() => assertRecoveredVectors(pending, 20, 0)).toThrow("12 documents wait for vector backfill");+ const running = settledStatus();+ running.index.vectorBackfillState = "running";+ expect(() => assertRecoveredVectors(running, 20, 0)).toThrow("vector backfill is running");+ });+});diff --git a/test/vector-pending-log.test.ts b/test/vector-pending-log.test.tsindex 8bbf7ca5..64ef481e 100644--- a/test/vector-pending-log.test.ts+++ b/test/vector-pending-log.test.ts@@ -86,6 +86,107 @@ describe("pending vector log", () => {expect(typeof rows.get("mem_b")!.e).toBe("string");expect(rows.get("obs_a")!.q as number).toBeGreaterThan(rows.get("mem_b")!.q as number);expect(persistence.status().pendingLog).toBe(2);+ expect(persistence.status().pendingLogAcknowledged).toBe(2);+ });++ it("acknowledges a pending row only after its write succeeds even beyond five seconds", async () => {+ vi.useFakeTimers();+ const kv = mockKV();+ const slow = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === PENDING_SCOPE) await new Promise((resolve) => setTimeout(resolve, 6_000));+ return kv.set(scope, key, data);+ },+ };+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(slow as never, vector, { saveIntervalMs: 600_000 });+ vector.add("obs_slow", "ses_1", v(1));++ await vi.advanceTimersByTimeAsync(5_001);+ expect(persistence.status()).toMatchObject({ pendingLog: 1, pendingLogAcknowledged: 0 });+ expect(pendingRows(kv).size).toBe(0);++ await vi.advanceTimersByTimeAsync(999);+ await persistence.flushPendingLog();+ expect(persistence.status().pendingLogAcknowledged).toBe(1);+ });++ it("counts successful keys independently when writes finish out of order", async () => {+ vi.useFakeTimers();+ const kv = mockKV();+ const delayed = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === PENDING_SCOPE) await new Promise((resolve) => setTimeout(resolve, key === "obs_first" ? 6_000 : 1_000));+ return kv.set(scope, key, data);+ },+ };+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(delayed as never, vector, { saveIntervalMs: 600_000 });+ vector.add("obs_first", "ses_1", v(1));+ vector.add("obs_second", "ses_1", v(2));++ await vi.advanceTimersByTimeAsync(1_000);+ expect([...pendingRows(kv).keys()]).toEqual(["obs_second"]);+ expect(persistence.status()).toMatchObject({ pendingLog: 2, pendingLogAcknowledged: 1 });++ await vi.advanceTimersByTimeAsync(5_000);+ await persistence.flushPendingLog();+ expect(persistence.status().pendingLogAcknowledged).toBe(2);+ });++ it("does not acknowledge a failed row when an unrelated success clears the error", async () => {+ const kv = mockKV();+ const failing = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === PENDING_SCOPE && key === "obs_failed") throw new Error("pending write refused");+ return kv.set(scope, key, data);+ },+ };+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(failing as never, vector, { saveIntervalMs: 600_000 });+ vector.add("obs_failed", "ses_1", v(1));+ await persistence.flushPendingLog();+ expect(persistence.status().pendingLogError).toContain("pending write refused");++ vector.add("obs_ok", "ses_1", v(2));+ await persistence.flushPendingLog();+ expect(persistence.status()).toMatchObject({ pendingLog: 2, pendingLogAcknowledged: 1, pendingLogError: null });+ expect([...pendingRows(kv).keys()]).toEqual(["obs_ok"]);+ });++ it("requires the latest same-key write acknowledgment and invalidates an earlier success", async () => {+ vi.useFakeTimers();+ const kv = mockKV();+ const slow = {+ ...kv,+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (scope === PENDING_SCOPE) await new Promise((resolve) => setTimeout(resolve, 6_000));+ return kv.set(scope, key, data);+ },+ };+ const vector = new VectorIndex();+ const persistence = new IndexPersistence(slow as never, vector, { saveIntervalMs: 600_000 });+ vector.add("obs_latest", "ses_1", v(1));+ await vi.advanceTimersByTimeAsync(0);+ vector.add("obs_latest", "ses_1", v(2));++ await vi.advanceTimersByTimeAsync(6_000);+ expect(pendingRows(kv).size).toBe(1);+ expect(persistence.status()).toMatchObject({ pendingLog: 1, pendingLogAcknowledged: 0 });++ await vi.advanceTimersByTimeAsync(6_000);+ await persistence.flushPendingLog();+ expect(persistence.status().pendingLogAcknowledged).toBe(1);++ vector.remove("obs_latest");+ expect(persistence.status().pendingLogAcknowledged).toBe(0);+ await vi.advanceTimersByTimeAsync(6_000);+ await persistence.flushPendingLog();+ expect(persistence.status().pendingLogAcknowledged).toBe(1);+ expect(pendingRows(kv).get("obs_latest")).toMatchObject({ t: 1 });});it("does nothing when there is no vector index", async () => {@@ -100,6 +201,7 @@ describe("pending vector log", () => {expect(kv.ops.filter((o) => o.scope === PENDING_SCOPE)).toEqual([]);expect(replay.entries).toBe(0);expect(persistence.status().pendingLog).toBe(0);+ expect(persistence.status().pendingLogAcknowledged).toBe(0);});it("replays adds and removals after a force-kill without any embedding calls", async () => {@@ -122,9 +224,11 @@ describe("pending vector log", () => {expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);expect(Array.from(second.vector.get("obs_3")!.embedding)).toEqual(Array.from(v(3)));expect(second.persistence.status().pendingLog).toBe(3);+ expect(second.persistence.status().pendingLogAcknowledged).toBe(3);await second.persistence.save();expect(pendingRows(kv).size).toBe(0);+ expect(second.persistence.status().pendingLogAcknowledged).toBe(0);const third = await boot(kv);expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);});@@ -171,6 +275,7 @@ describe("pending vector log", () => {expect([...pendingRows(kv).keys()]).toEqual(["obs_during"]);expect(persistence.status().pendingLog).toBe(1);+ expect(persistence.status().pendingLogAcknowledged).toBe(1);});it("keeps every entry when the snapshot write fails", async () => {@@ -190,6 +295,7 @@ describe("pending vector log", () => {expect(persistence.status().vector?.lastError).toContain("state write refused");expect(pendingRows(kv).size).toBe(2);+ expect(persistence.status().pendingLogAcknowledged).toBe(2);expect(kv.ops.filter((o) => o.op === "delete" && o.scope === PENDING_SCOPE)).toEqual([]);});@@ -209,9 +315,11 @@ describe("pending vector log", () => {const second = await boot(kv);expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);expect(Array.from(second.vector.get("obs_flip")!.embedding)).toEqual(Array.from(v(5)));+ expect(second.persistence.status().pendingLogAcknowledged).toBe(pendingRows(kv).size);await second.persistence.save();expect(pendingRows(kv).size).toBe(0);+ expect(second.persistence.status().pendingLogAcknowledged).toBe(0);const third = await boot(kv);expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);});