Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(release): wait for vector persistence and recovery (#1520)

Rohit Ghumare7ee162844956
diff --git a/scripts/release-gate/recovery.mjs b/scripts/release-gate/recovery.mjs
new file mode 100644
index 00000000..b2ca81db
--- /dev/null
+++ b/scripts/release-gate/recovery.mjs
@@ -0,0 +1,69 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+export function decodeRkyvJsonScope(bytes) {
+ const invalid = () => { throw new SyntaxError("Invalid rkyv 0.8 32-bit little-endian JSON scope"); };
+ if (bytes.length < 8 || bytes.length % 4 !== 0) invalid();
+ const root = bytes.length - 8;
+ let length;
+ if ((bytes[root] & 0xc0) === 0x80) {
+ const encodedLength = bytes.readUInt32LE(root);
+ length = (encodedLength & 0x3f) | ((encodedLength & 0xffffff00) >>> 2);
+ const start = root + bytes.readInt32LE(root + 4);
+ if (length <= 8 || start !== 0 || length > root || root - length > 3) invalid();
+ } else {
+ if (root !== 0) invalid();
+ const end = bytes.indexOf(0xff);
+ length = end === -1 ? 8 : end;
+ if (!bytes.subarray(length).every((byte) => byte === 0xff)) invalid();
+ }
+ let json;
+ try {
+ json = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length));
+ } catch {
+ invalid();
+ }
+ const value = JSON.parse(json);
+ if (value === null || typeof value !== "object" || Array.isArray(value)) invalid();
+ return value;
+}
+
+export function isVectorLogAcknowledged(status, count, checkpointCount) {
+ const persistence = status.indexPersistence;
+ const pending = count - checkpointCount;
+ return status.index?.vectorDocuments === count && persistence?.saving === false &&
+ persistence.pendingChanges === pending && persistence.pendingLog === pending &&
+ persistence.pendingLogAcknowledged === pending && persistence.pendingLogError === null;
+}
+
+export function hasPersistedVectorFixture(dataDir, sessionId, checkpointId, pendingIds, dimensions) {
+ const readScope = (scope) => decodeRkyvJsonScope(readFileSync(join(dataDir, "state_store.db", `${encodeURIComponent(scope)}.bin`)));
+ const matchesVector = (row, id) => row?.id === id && row.s === sessionId &&
+ typeof row.e === "string" && Buffer.from(row.e, "base64").length === dimensions * 4;
+ try {
+ if (readScope("mem:sessions")[sessionId]?.id !== sessionId) return false;
+ const meta = readScope("mem:index:bm25")["vectors:meta"];
+ if (meta?.v !== 3 || meta.count !== 1 || meta.bucketCount !== 1) return false;
+ if (!matchesVector(readScope("mem:index:bm25:vec:0000")[checkpointId], checkpointId)) return false;
+ const pending = readScope("mem:index:vec-pending");
+ const observations = readScope(`mem:obs:${sessionId}`);
+ return [checkpointId, ...pendingIds].every((id) => observations[id]?.id === id) &&
+ pendingIds.every((id) => matchesVector(pending[id], id) && Number.isFinite(pending[id].q) && !pending[id].t && !pending[id].c);
+ } catch (err) {
+ if (err.code === "ENOENT" || err instanceof SyntaxError) return false;
+ throw err;
+ }
+}
+
+export function isVectorRecoverySettled(status) {
+ return typeof status.index?.vectorDocuments === "number" && status.index.keywordRebuildRunning === false;
+}
+
+export function assertRecoveredVectors(status, expectedCount, embeddingInputs) {
+ const index = status.index;
+ assert.equal(index.vectorDocuments, expectedCount, `${index.vectorDocuments} vectors after the crash, expected ${expectedCount}`);
+ assert.equal(index.pendingVectorBackfill, 0, `${index.pendingVectorBackfill} documents wait for vector backfill after the crash`);
+ assert.equal(index.vectorBackfillState, "idle", `vector backfill is ${index.vectorBackfillState} after the crash`);
+ assert.equal(embeddingInputs, 0, `the restart re-embedded ${embeddingInputs} inputs instead of loading saved vectors`);
+}
diff --git a/scripts/release-gate/run.mjs b/scripts/release-gate/run.mjs
index 65a76e7f..6b76b2b6 100644
--- a/scripts/release-gate/run.mjs
+++ b/scripts/release-gate/run.mjs
@@ -21,6 +21,7 @@ import net from "node:net";
import { homedir, tmpdir, platform, arch } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
+import { assertRecoveredVectors, hasPersistedVectorFixture, isVectorLogAcknowledged, isVectorRecoverySettled } from "./recovery.mjs";
const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
const FORBIDDEN_PORTS = new Set([3111, 3112, 3113, 4098, 4131, 4132, 4133, 49134]);
@@ -32,7 +33,7 @@ const SCENARIOS = [
["offline", "hooks run while the service is down and the spool is recovered on restart"],
["dedup", "a replayed host event after a force kill stays one observation"],
["deadletter", "an accepted capture that fails processing survives a restart and is visible as a dead letter"],
- ["vectors", "vectors made before the first checkpoint survive a crash"],
+ ["vectors", "checkpointed and pending vectors survive a crash without re-embedding"],
["stopflush", "agentmemory stop then start loses nothing"],
["status", "viewer serves and /agentmemory/status explains every problem"],
["roundtrip", "export then import into a fresh home round-trips"],
@@ -367,6 +368,7 @@ class Instance {
this.name = name;
this.dir = join(ROOT, name);
this.home = join(this.dir, "home");
+ this.dataDir = join(this.dir, "data");
this.cwd = join(this.dir, "cwd");
this.project = join(this.dir, "projects", "gate-app");
for (const d of [this.home, this.cwd, this.project]) mkdirSync(d, { recursive: true });
@@ -391,6 +393,7 @@ class Instance {
OPENAI_BASE_URL: `http://127.0.0.1:${gate.fake.port}`,
OPENAI_EMBEDDING_DIMENSIONS: String(gate.dims),
AGENTMEMORY_URL: `http://127.0.0.1:${this.port}`,
+ AGENTMEMORY_DATA_DIR: this.dataDir,
...extra,
};
}
@@ -519,6 +522,7 @@ class Instance {
CLAUDE_PLUGIN_ROOT: join(gate.pkgDir, "plugin"),
CLAUDE_PROJECT_DIR: this.project,
AGENTMEMORY_URL: `http://127.0.0.1:${this.port}`,
+ AGENTMEMORY_DATA_DIR: this.dataDir,
},
cwd: this.project,
input: JSON.stringify({ hook_event_name: event, cwd: this.project, transcript_path: join(this.dir, "transcript.jsonl"), ...payload }),
@@ -907,33 +911,46 @@ async function main() {
await ensureUp(instB);
const sid = `gate-vectors-${randomBytes(4).toString("hex")}`;
const count = 20;
- for (let i = 0; i < count; i++) await instB.hook("PostToolUse", instB.toolEvent(sid, marker(`vec${i}x`)));
+ await instB.hook("PostToolUse", instB.toolEvent(sid, marker("vec0x")));
const t0 = Date.now();
const st = await poll("the first vector checkpoint", async () => {
const s = await instB.status();
const p = s.indexPersistence;
- const durable = p && !p.saving && (p.pendingChanges === 0 || (p.pendingLog ?? 0) >= p.pendingChanges);
- return (s.index?.vectorDocuments ?? 0) >= count && p?.vector?.lastSavedAt && !p.firstCheckpointPending && durable ? s : null;
+ return s.index?.vectorDocuments === 1 && p?.vector?.lastSavedAt && !p.firstCheckpointPending && !p.saving && p.pendingChanges === 0 ? s : null;
}, { timeoutMs: 60_000, intervalMs: 500 });
d.firstCheckpointWaitMs = Date.now() - t0;
d.saveIntervalMs = st.indexPersistence.saveIntervalMs;
- const vectorsBefore = st.index.vectorDocuments;
- d.vectorsBefore = vectorsBefore;
+ d.checkpointVectors = st.index.vectorDocuments;
+ const checkpointId = (await instB.observations(sid))[0]?.id;
+ assert(checkpointId, "the checkpoint observation is missing");
assert(d.saveIntervalMs > d.firstCheckpointWaitMs, "the checkpoint only happened at the regular save interval, so the first-checkpoint boundary was not exercised");
- await sleep(STATE_FLUSH_WAIT_MS);
+ for (let i = 1; i < count; i++) await instB.hook("PostToolUse", instB.toolEvent(sid, marker(`vec${i}x`)));
+ const before = await poll("every pending vector write to be acknowledged", async () => {
+ const s = await instB.status();
+ return isVectorLogAcknowledged(s, count, d.checkpointVectors) ? s : null;
+ }, { timeoutMs: 60_000, intervalMs: 500 });
+ const vectorsBefore = before.index.vectorDocuments;
+ d.vectorsBefore = vectorsBefore;
+ d.pendingLogBeforeCrash = before.indexPersistence.pendingLog;
+ d.pendingLogAcknowledged = before.indexPersistence.pendingLogAcknowledged;
+ const pendingIds = (await instB.observations(sid)).map((observation) => observation.id).filter((id) => id !== checkpointId);
+ assert(pendingIds.length === count - 1, `${pendingIds.length} pending observations, expected ${count - 1}`);
+ await poll("checkpoint and pending vectors to be present on disk", async () =>
+ hasPersistedVectorFixture(instB.dataDir, sid, checkpointId, pendingIds, gate.dims),
+ { timeoutMs: 60_000, intervalMs: 250 });
+ d.persistedVectorsBeforeCrash = pendingIds.length + 1;
const embedBefore = gate.fake.stats.inputs;
await instB.forceKill();
d.restartMs = await instB.start();
- const after = await poll("vector index to load", async () => {
+ const after = await poll("startup index recovery to finish", async () => {
const s = await instB.status();
- return typeof s.index?.vectorDocuments === "number" && s.index.vectorDocuments > 0 ? s : null;
+ return isVectorRecoverySettled(s) ? s : null;
}, { timeoutMs: 30_000 });
d.vectorsAfter = after.index.vectorDocuments;
d.pendingVectorBackfill = after.index.pendingVectorBackfill;
+ d.vectorBackfillState = after.index.vectorBackfillState;
d.embeddingsAfterRestart = gate.fake.stats.inputs - embedBefore;
- assert(d.vectorsAfter === vectorsBefore, `${d.vectorsAfter} vectors after the crash, expected ${vectorsBefore}`);
- assert(d.pendingVectorBackfill === 0, `${d.pendingVectorBackfill} documents wait for vector backfill after the crash`);
- assert(d.embeddingsAfterRestart === 0, `the restart re-embedded ${d.embeddingsAfterRestart} inputs instead of loading saved vectors`);
+ assertRecoveredVectors(after, vectorsBefore, d.embeddingsAfterRestart);
await instB.stop();
});
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 9f1cd810..46c565d3 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -100,6 +100,7 @@ export interface IndexPersistenceStatus {
vector: IndexLegStatus | null;
vectorCountShortfall: VectorCountShortfall | null;
pendingLog?: number;
+ pendingLogAcknowledged?: number;
pendingLogError?: string | null;
}
@@ -199,6 +200,7 @@ export class IndexPersistence {
private vectorCountShortfall: VectorCountShortfall | null = null;
private seq = 0;
private pendingLog: Map<string, number> = new Map();
+ private acknowledgedLog: Map<string, number> = new Map();
private logChains: Map<string, Promise<void>> = new Map();
private logSuppressed = false;
private pendingLogUnknown = false;
@@ -259,6 +261,7 @@ export class IndexPersistence {
vector: this.vector ? { ...this.leg } : null,
vectorCountShortfall: this.vectorCountShortfall,
pendingLog: this.vector ? this.pendingLog.size : 0,
+ pendingLogAcknowledged: this.vector ? this.acknowledgedLog.size : 0,
pendingLogError: this.pendingLogError,
};
}
@@ -283,6 +286,7 @@ export class IndexPersistence {
result.entries++;
const key = row.c === 1 ? PENDING_CLEAR_KEY : row.id!;
if ((this.pendingLog.get(key) ?? -Infinity) < row.q) this.pendingLog.set(key, row.q);
+ if (this.pendingLog.get(key) === row.q) this.acknowledgedLog.set(key, row.q);
if (row.q > this.seq) this.seq = row.q;
if (row.c === 1) {
vector.clear();
@@ -369,8 +373,10 @@ export class IndexPersistence {
row = { q, id, t: 1 };
}
this.pendingLog.set(key, q);
+ this.acknowledgedLog.delete(key);
this.enqueueLog(key, async () => {
await this.kv.set<PendingVectorRow>(KV.vectorPendingLog, key, row);
+ if (this.pendingLog.get(key) === q) this.acknowledgedLog.set(key, q);
}).then(
() => {
this.pendingLogError = null;
@@ -411,7 +417,10 @@ export class IndexPersistence {
return this.enqueueLog(key, async () => {
if (this.pendingLog.get(key) !== q) return;
await this.kv.delete(KV.vectorPendingLog, key);
- if (this.pendingLog.get(key) === q) this.pendingLog.delete(key);
+ if (this.pendingLog.get(key) === q) {
+ this.pendingLog.delete(key);
+ this.acknowledgedLog.delete(key);
+ }
});
}
diff --git a/test/release-gate-durability.test.ts b/test/release-gate-durability.test.ts
new file mode 100644
index 00000000..97a2530c
--- /dev/null
+++ b/test/release-gate-durability.test.ts
@@ -0,0 +1,124 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { decodeRkyvJsonScope, hasPersistedVectorFixture, isVectorLogAcknowledged } from "../scripts/release-gate/recovery.mjs";
+
+const dirs: string[] = [];
+
+afterEach(() => {
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function archiveScope(value: unknown) {
+ const json = Buffer.from(JSON.stringify(value));
+ if (json.length <= 8) {
+ const bytes = Buffer.alloc(8, 0xff);
+ json.copy(bytes);
+ return bytes;
+ }
+ const root = Math.ceil(json.length / 4) * 4;
+ const bytes = Buffer.alloc(root + 8);
+ json.copy(bytes);
+ bytes.writeUInt32LE((json.length & 0x3f) | 0x80 | ((json.length >>> 6) << 8), root);
+ bytes.writeInt32LE(-root, root + 4);
+ return bytes;
+}
+
+function fixture() {
+ const dir = mkdtempSync(join(tmpdir(), "vector-durability-"));
+ dirs.push(dir);
+ mkdirSync(join(dir, "state_store.db"));
+ const rows = Object.fromEntries(["checkpoint", "pending"].map((id) => [id, {
+ id, s: "session", e: Buffer.from(new Float32Array([1, 2, 3]).buffer).toString("base64"), q: 1,
+ }]));
+ const scopes: Record<string, unknown> = {
+ "mem:sessions": { session: { id: "session" } },
+ "mem:index:bm25": { "vectors:meta": { v: 3, count: 1, bucketCount: 1 } },
+ "mem:index:bm25:vec:0000": { checkpoint: rows.checkpoint },
+ "mem:index:vec-pending": { pending: rows.pending },
+ "mem:obs:session": { checkpoint: { id: "checkpoint" }, pending: { id: "pending" } },
+ };
+ const write = (scope: string, value: unknown) => writeFileSync(join(dir, "state_store.db", `${encodeURIComponent(scope)}.bin`), archiveScope(value));
+ const check = () => hasPersistedVectorFixture(dir, "session", "checkpoint", ["pending"], 3);
+ return { dir, scopes, rows, write, check };
+}
+
+describe("release gate vector durability", () => {
+ it("decodes a scope captured from the pinned engine", () => {
+ const bytes = Buffer.from("7b22766563746f72733a6d657461223a7b226275636b6574436f756e74223a312c22636f756e74223a32302c2273617665644174223a22323032362d31302d31305430393a32363a31372e3836345a222c2276223a337d7d98010000a8ffffff", "hex");
+ expect(decodeRkyvJsonScope(bytes)).toEqual({
+ "vectors:meta": { bucketCount: 1, count: 20, savedAt: "2026-10-10T09:26:17.864Z", v: 3 },
+ });
+ expect(decodeRkyvJsonScope(Buffer.from("7b7dffffffffffff", "hex"))).toEqual({});
+ });
+
+ it.each([
+ (bytes: Buffer) => bytes.subarray(0, bytes.length - 1),
+ (bytes: Buffer) => { bytes.writeInt32LE(-1, bytes.length - 4); return bytes; },
+ (bytes: Buffer) => { bytes.writeUInt32LE(0xffffffbf, bytes.length - 8); return bytes; },
+ (bytes: Buffer) => { bytes[0] = 0xff; return bytes; },
+ () => archiveScope([]),
+ () => Buffer.from("7b7dff00ffffffff", "hex"),
+ ])("rejects invalid archive boundaries, UTF-8 and scope values", (corrupt) => {
+ expect(() => decodeRkyvJsonScope(corrupt(archiveScope({ pending: { q: 1 } })))).toThrow(SyntaxError);
+ });
+
+ it("requires successful acknowledgments rather than the pending count", () => {
+ const status = {
+ index: { vectorDocuments: 20 },
+ indexPersistence: { saving: false, pendingChanges: 19, pendingLog: 19, pendingLogAcknowledged: 0, pendingLogError: null as string | null },
+ };
+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(false);
+ status.indexPersistence.pendingLogAcknowledged = 18;
+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(false);
+ status.indexPersistence.pendingLogAcknowledged = 19;
+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(true);
+ status.indexPersistence.pendingLogError = "write refused";
+ expect(isVectorLogAcknowledged(status, 20, 1)).toBe(false);
+ });
+
+ it("waits for the actual checkpoint and every pending row to reach disk", () => {
+ const { scopes, write, check } = fixture();
+ expect(check()).toBe(false);
+ for (const scope of ["mem:index:bm25", "mem:index:bm25:vec:0000", "mem:obs:session"]) write(scope, scopes[scope]);
+ expect(check()).toBe(false);
+ write("mem:index:vec-pending", {});
+ expect(check()).toBe(false);
+ write("mem:index:vec-pending", scopes["mem:index:vec-pending"]);
+ expect(check()).toBe(false);
+ write("mem:sessions", scopes["mem:sessions"]);
+ expect(check()).toBe(true);
+ });
+
+ it("retries a file observed during a partial rewrite", () => {
+ const { dir, scopes, write, check } = fixture();
+ for (const [scope, value] of Object.entries(scopes)) write(scope, value);
+ writeFileSync(join(dir, "state_store.db", "mem%3Aindex%3Avec-pending.bin"), '{"pending":');
+ expect(check()).toBe(false);
+ write("mem:index:vec-pending", scopes["mem:index:vec-pending"]);
+ expect(check()).toBe(true);
+ });
+
+ it.each([
+ ["mem:sessions", {}],
+ ["mem:index:bm25", { "vectors:meta": { v: 3, count: 0, bucketCount: 1 } }],
+ ["mem:index:bm25:vec:0000", {}],
+ ["mem:obs:session", { checkpoint: { id: "checkpoint" } }],
+ ])("rejects incomplete persisted scope %s", (scope, value) => {
+ const { scopes, write, check } = fixture();
+ for (const [name, row] of Object.entries(scopes)) write(name, row);
+ write(scope as string, value);
+ expect(check()).toBe(false);
+ });
+
+ it.each([{ s: "another-session" }, { id: "another-id" }, { e: "AA==" }, { q: null }, { t: 1 }, { c: 1 }])(
+ "rejects mismatched or incomplete pending rows: %o",
+ (override) => {
+ const { scopes, rows, write, check } = fixture();
+ for (const [scope, value] of Object.entries(scopes)) write(scope, value);
+ write("mem:index:vec-pending", { pending: { ...rows.pending, ...override } });
+ expect(check()).toBe(false);
+ },
+ );
+});
diff --git a/test/release-gate-recovery.test.ts b/test/release-gate-recovery.test.ts
new file mode 100644
index 00000000..9e21d08c
--- /dev/null
+++ b/test/release-gate-recovery.test.ts
@@ -0,0 +1,154 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { assertRecoveredVectors, isVectorRecoverySettled } from "../scripts/release-gate/recovery.mjs";
+import { IndexPersistence } from "../src/state/index-persistence.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import { KV } from "../src/state/schema.js";
+import { createStatusReporter } from "../src/triggers/api.js";
+import {
+ getSearchIndex,
+ markKeywordRebuildPending,
+ rebuildKeywordIndex,
+ setEmbeddingProvider,
+ setIndexPersistence,
+ setPendingVectorBackfillCount,
+ setVectorBackfillState,
+ setVectorIndex,
+} from "../src/functions/search.js";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+function storeKV() {
+ const scopes = new Map<string, Map<string, unknown>>();
+ return {
+ backend: "file" as const,
+ get: async <T>(scope: string, key: string): Promise<T | null> =>
+ (scopes.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, value: T): Promise<T> => {
+ if (!scopes.has(scope)) scopes.set(scope, new Map());
+ scopes.get(scope)!.set(key, structuredClone(value));
+ return value;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ scopes.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> =>
+ [...(scopes.get(scope)?.values() ?? [])] as T[],
+ };
+}
+
+function settledStatus(vectorDocuments = 20) {
+ return {
+ index: {
+ vectorDocuments,
+ keywordRebuildRunning: false,
+ pendingVectorBackfill: 0,
+ vectorBackfillState: "idle",
+ },
+ };
+}
+
+const persistenceInstances: IndexPersistence[] = [];
+
+afterEach(async () => {
+ for (const persistence of persistenceInstances.splice(0)) persistence.stop();
+ setVectorIndex(null);
+ setIndexPersistence(null);
+ setEmbeddingProvider(null);
+ setPendingVectorBackfillCount(0);
+ setVectorBackfillState("idle");
+ await rebuildKeywordIndex(storeKV() as never);
+ getSearchIndex().clear();
+});
+
+describe("release gate vector recovery", () => {
+ it("waits while eight checkpoint vectors are visible before twelve pending vectors replay", async () => {
+ const kv = storeKV();
+ const firstVector = new VectorIndex();
+ const first = new IndexPersistence(kv as never, firstVector, { saveIntervalMs: 600_000 });
+ persistenceInstances.push(first);
+ await kv.set(KV.sessions, "ses_recovery", {
+ id: "ses_recovery", project: "recovery", startedAt: "2026-10-10T00:00:00Z", observationCount: 20,
+ });
+ for (let i = 0; i < 20; i++) {
+ const id = `obs_recovery_${i}`;
+ await kv.set(KV.observations("ses_recovery"), id, {
+ id, sessionId: "ses_recovery", title: `recovery ${i}`, narrative: `saved content ${i}`,
+ type: "decision", facts: [], concepts: [], files: [], importance: 5, timestamp: "2026-10-10T00:00:00Z",
+ });
+ firstVector.add(id, "ses_recovery", new Float32Array([i, i + 1, i + 2]));
+ if (i === 7) await first.save();
+ }
+ await first.flushPendingLog();
+ first.stop();
+ expect(firstVector.size).toBe(20);
+ expect(await kv.list(KV.vectorPendingLog)).toHaveLength(12);
+
+ let releaseReplay!: () => void;
+ const pendingRead = new Promise<void>((resolve) => { releaseReplay = resolve; });
+ const delayedKV = {
+ ...kv,
+ list: async <T>(scope: string): Promise<T[]> => {
+ if (scope === KV.vectorPendingLog) await pendingRead;
+ return kv.list<T>(scope);
+ },
+ };
+ const vector = new VectorIndex();
+ const restored = new IndexPersistence(delayedKV as never, vector, { saveIntervalMs: 600_000 });
+ persistenceInstances.push(restored);
+ setVectorIndex(vector);
+ setIndexPersistence(restored);
+ const embed = vi.fn(async () => new Float32Array([1, 2, 3]));
+ const embedBatch = vi.fn(async () => []);
+ setEmbeddingProvider({ name: "recovery-test", dimensions: 3, embed, embedBatch });
+ markKeywordRebuildPending();
+ const loaded = await restored.load();
+ expect(loaded.vector?.size).toBe(8);
+ vector.restoreFrom(loaded.vector!);
+ const replay = restored.replayPendingLog(3);
+ const report = createStatusReporter({ trigger: async () => null } as never, delayedKV as never, {});
+
+ try {
+ const during = await report({ health: null });
+ expect(during.index).toMatchObject({ vectorDocuments: 8, keywordRebuildRunning: true });
+ expect(isVectorRecoverySettled(during)).toBe(false);
+
+ releaseReplay();
+ expect(await replay).toMatchObject({ added: 12, skipped: 0 });
+ const keyword = await rebuildKeywordIndex(delayedKV as never, loaded.savedAt);
+ setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending);
+ const after = await report({ health: null });
+ expect(after.index).toMatchObject({ vectorDocuments: 20, keywordRebuildRunning: false });
+ expect(isVectorRecoverySettled(after)).toBe(true);
+ expect(() => assertRecoveredVectors(after, 20, embed.mock.calls.length + embedBatch.mock.calls.length)).not.toThrow();
+ } finally {
+ releaseReplay();
+ await replay;
+ }
+ });
+
+ it.each([0, 8, 21])("fails a settled count of %i immediately instead of waiting for the desired count", (count) => {
+ const status = settledStatus(count);
+ expect(isVectorRecoverySettled(status)).toBe(true);
+ expect(() => assertRecoveredVectors(status, 20, 0)).toThrow(`${count} vectors after the crash, expected 20`);
+ });
+
+ it.each([{}, { index: { vectorDocuments: 20 } }, { index: { vectorDocuments: 20, keywordRebuildRunning: true } }])(
+ "does not infer readiness from an absent or unfinished rebuild state",
+ (status) => { expect(isVectorRecoverySettled(status)).toBe(false); },
+ );
+
+ it("fails when a recovered count required new embeddings", () => {
+ expect(() => assertRecoveredVectors(settledStatus(), 20, 12)).toThrow("the restart re-embedded 12 inputs");
+ });
+
+ it("fails when vector backfill is pending or running", () => {
+ const pending = settledStatus();
+ pending.index.pendingVectorBackfill = 12;
+ expect(() => assertRecoveredVectors(pending, 20, 0)).toThrow("12 documents wait for vector backfill");
+ const running = settledStatus();
+ running.index.vectorBackfillState = "running";
+ expect(() => assertRecoveredVectors(running, 20, 0)).toThrow("vector backfill is running");
+ });
+});
diff --git a/test/vector-pending-log.test.ts b/test/vector-pending-log.test.ts
index 8bbf7ca5..64ef481e 100644
--- a/test/vector-pending-log.test.ts
+++ b/test/vector-pending-log.test.ts
@@ -86,6 +86,107 @@ describe("pending vector log", () => {
expect(typeof rows.get("mem_b")!.e).toBe("string");
expect(rows.get("obs_a")!.q as number).toBeGreaterThan(rows.get("mem_b")!.q as number);
expect(persistence.status().pendingLog).toBe(2);
+ expect(persistence.status().pendingLogAcknowledged).toBe(2);
+ });
+
+ it("acknowledges a pending row only after its write succeeds even beyond five seconds", async () => {
+ vi.useFakeTimers();
+ const kv = mockKV();
+ const slow = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === PENDING_SCOPE) await new Promise((resolve) => setTimeout(resolve, 6_000));
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(slow as never, vector, { saveIntervalMs: 600_000 });
+ vector.add("obs_slow", "ses_1", v(1));
+
+ await vi.advanceTimersByTimeAsync(5_001);
+ expect(persistence.status()).toMatchObject({ pendingLog: 1, pendingLogAcknowledged: 0 });
+ expect(pendingRows(kv).size).toBe(0);
+
+ await vi.advanceTimersByTimeAsync(999);
+ await persistence.flushPendingLog();
+ expect(persistence.status().pendingLogAcknowledged).toBe(1);
+ });
+
+ it("counts successful keys independently when writes finish out of order", async () => {
+ vi.useFakeTimers();
+ const kv = mockKV();
+ const delayed = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === PENDING_SCOPE) await new Promise((resolve) => setTimeout(resolve, key === "obs_first" ? 6_000 : 1_000));
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(delayed as never, vector, { saveIntervalMs: 600_000 });
+ vector.add("obs_first", "ses_1", v(1));
+ vector.add("obs_second", "ses_1", v(2));
+
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect([...pendingRows(kv).keys()]).toEqual(["obs_second"]);
+ expect(persistence.status()).toMatchObject({ pendingLog: 2, pendingLogAcknowledged: 1 });
+
+ await vi.advanceTimersByTimeAsync(5_000);
+ await persistence.flushPendingLog();
+ expect(persistence.status().pendingLogAcknowledged).toBe(2);
+ });
+
+ it("does not acknowledge a failed row when an unrelated success clears the error", async () => {
+ const kv = mockKV();
+ const failing = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === PENDING_SCOPE && key === "obs_failed") throw new Error("pending write refused");
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(failing as never, vector, { saveIntervalMs: 600_000 });
+ vector.add("obs_failed", "ses_1", v(1));
+ await persistence.flushPendingLog();
+ expect(persistence.status().pendingLogError).toContain("pending write refused");
+
+ vector.add("obs_ok", "ses_1", v(2));
+ await persistence.flushPendingLog();
+ expect(persistence.status()).toMatchObject({ pendingLog: 2, pendingLogAcknowledged: 1, pendingLogError: null });
+ expect([...pendingRows(kv).keys()]).toEqual(["obs_ok"]);
+ });
+
+ it("requires the latest same-key write acknowledgment and invalidates an earlier success", async () => {
+ vi.useFakeTimers();
+ const kv = mockKV();
+ const slow = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === PENDING_SCOPE) await new Promise((resolve) => setTimeout(resolve, 6_000));
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(slow as never, vector, { saveIntervalMs: 600_000 });
+ vector.add("obs_latest", "ses_1", v(1));
+ await vi.advanceTimersByTimeAsync(0);
+ vector.add("obs_latest", "ses_1", v(2));
+
+ await vi.advanceTimersByTimeAsync(6_000);
+ expect(pendingRows(kv).size).toBe(1);
+ expect(persistence.status()).toMatchObject({ pendingLog: 1, pendingLogAcknowledged: 0 });
+
+ await vi.advanceTimersByTimeAsync(6_000);
+ await persistence.flushPendingLog();
+ expect(persistence.status().pendingLogAcknowledged).toBe(1);
+
+ vector.remove("obs_latest");
+ expect(persistence.status().pendingLogAcknowledged).toBe(0);
+ await vi.advanceTimersByTimeAsync(6_000);
+ await persistence.flushPendingLog();
+ expect(persistence.status().pendingLogAcknowledged).toBe(1);
+ expect(pendingRows(kv).get("obs_latest")).toMatchObject({ t: 1 });
});
it("does nothing when there is no vector index", async () => {
@@ -100,6 +201,7 @@ describe("pending vector log", () => {
expect(kv.ops.filter((o) => o.scope === PENDING_SCOPE)).toEqual([]);
expect(replay.entries).toBe(0);
expect(persistence.status().pendingLog).toBe(0);
+ expect(persistence.status().pendingLogAcknowledged).toBe(0);
});
it("replays adds and removals after a force-kill without any embedding calls", async () => {
@@ -122,9 +224,11 @@ describe("pending vector log", () => {
expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);
expect(Array.from(second.vector.get("obs_3")!.embedding)).toEqual(Array.from(v(3)));
expect(second.persistence.status().pendingLog).toBe(3);
+ expect(second.persistence.status().pendingLogAcknowledged).toBe(3);
await second.persistence.save();
expect(pendingRows(kv).size).toBe(0);
+ expect(second.persistence.status().pendingLogAcknowledged).toBe(0);
const third = await boot(kv);
expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_2", "obs_3", "obs_4"]);
});
@@ -171,6 +275,7 @@ describe("pending vector log", () => {
expect([...pendingRows(kv).keys()]).toEqual(["obs_during"]);
expect(persistence.status().pendingLog).toBe(1);
+ expect(persistence.status().pendingLogAcknowledged).toBe(1);
});
it("keeps every entry when the snapshot write fails", async () => {
@@ -190,6 +295,7 @@ describe("pending vector log", () => {
expect(persistence.status().vector?.lastError).toContain("state write refused");
expect(pendingRows(kv).size).toBe(2);
+ expect(persistence.status().pendingLogAcknowledged).toBe(2);
expect(kv.ops.filter((o) => o.op === "delete" && o.scope === PENDING_SCOPE)).toEqual([]);
});
@@ -209,9 +315,11 @@ describe("pending vector log", () => {
const second = await boot(kv);
expect([...second.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);
expect(Array.from(second.vector.get("obs_flip")!.embedding)).toEqual(Array.from(v(5)));
+ expect(second.persistence.status().pendingLogAcknowledged).toBe(pendingRows(kv).size);
await second.persistence.save();
expect(pendingRows(kv).size).toBe(0);
+ expect(second.persistence.status().pendingLogAcknowledged).toBe(0);
const third = await boot(kv);
expect([...third.vector.entries()].map(([id]) => id).sort()).toEqual(["obs_flip", "obs_new"]);
});