Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: preserve vector recovery across checkpoints and restarts (#1562)

Rohit Ghumare7bf9c53e4c85
diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml
index e0e305d8..f2b8c1c9 100644
--- a/.github/workflows/release-gate.yml
+++ b/.github/workflows/release-gate.yml
@@ -45,6 +45,18 @@ jobs:
- run: npm install --legacy-peer-deps --no-audit --no-fund
- run: npm run release:gate -- --out "${{ runner.temp }}/release-gate"
- run: npm run test:startup:packaged -- --skip-build --out "${{ runner.temp }}/startup-gate"
+ - name: Test packaged vector recovery
+ shell: bash
+ run: |
+ pack_dir="$(mktemp -d "${{ runner.temp }}/vector-recovery-pack.XXXXXX")"
+ npm pack --pack-destination "$pack_dir"
+ tarballs=("$pack_dir"/agentmemory-agentmemory-*.tgz)
+ test "${#tarballs[@]}" -eq 1
+ test -f "${tarballs[0]}"
+ npm run test:vector-recovery:packaged -- \
+ --tarball "${tarballs[0]}" \
+ --dependencies "$GITHUB_WORKSPACE/node_modules" \
+ --out "${{ runner.temp }}/vector-recovery-gate"
- run: npm run test:summary-recovery:packaged -- --skip-build --out "${{ runner.temp }}/summary-recovery-gate"
- name: Verify bounded graph migration with the packaged CLI
run: |
@@ -59,6 +71,7 @@ jobs:
path: |
${{ runner.temp }}/release-gate
${{ runner.temp }}/startup-gate
+ ${{ runner.temp }}/vector-recovery-gate
${{ runner.temp }}/summary-recovery-gate
${{ runner.temp }}/graph-index-gate
if-no-files-found: ignore
diff --git a/docs/reliability/source-and-vector-recovery.md b/docs/reliability/source-and-vector-recovery.md
index 94501179..03c1a62a 100644
--- a/docs/reliability/source-and-vector-recovery.md
+++ b/docs/reliability/source-and-vector-recovery.md
@@ -22,6 +22,16 @@ Five seconds is a scheduling bound, not a disk-durability guarantee. Completion
Status distinguishes active vector backfill from backfill waiting for explicit opt-in. Missing vectors do not silently trigger unlimited embedding calls. Keyword recovery remains independent of vector coverage.
+## Resuming missing vector coverage
+
+A full-store recovery marker stays in StateKV until the planned recovery is complete and its vector checkpoint succeeds. The checkpoint retains the active cutoff and widens its fallback to full recovery before completion, so a crash during marker deletion cannot lose a queued record whose timestamp moved backwards. Coverage is checked again after deletion, including snapshot replacements that keep old timestamps. Dimension resets and unreadable or wrong-dimension pending entries require full recovery. They no longer reuse the discarded checkpoint's timestamp as a cutoff.
+
+Full recovery requires `AGENTMEMORY_VECTOR_BACKFILL=all` on each start. Each start processes at most `AGENTMEMORY_VECTOR_BACKFILL_MAX` missing records (default 500); remaining records stay pending and resume on the next start. Existing vectors are reused. After completion, the checkpoint conservatively retains full recovery intent until a later ordinary checkpoint clears it, including after incremental recovery. If the process restarts before then, newly missing historical vectors still require the full-recovery opt-in; already-covered records incur no embedding calls. Removing the opt-in pauses old-record embeddings, and keyword search remains available. Explicit all-mode also checks old missing records in checkpoints created before this recovery marker existed.
+
+Incremental recovery plans only records after its saved cutoff and processes successive bounded batches without requiring full-recovery opt-in for that run. Before embeddings start, it saves full restart intent. If a queued record is replaced with an older timestamp, that ID stays in the active coverage check. An interrupted incremental run therefore requires full-recovery opt-in on the next start; this prevents historical replacements from disappearing behind the old cutoff. If the restart intent cannot be saved, background embeddings and vector checkpoints stay paused. Provider failures, incomplete source scans and failed vector checkpoints retain recovery intent. A failed recovery-marker read or write pauses vector checkpoints and backfill until restart, preserving the previous cutoff while keyword search and source capture stay available. Repair access to the state store and restart to retry. The marker uses the existing index state scope and accepts the previous incremental marker format.
+
+Run `npm run test:vector-recovery:packaged -- --tarball /path/to/package.tgz --engine /path/to/iii` after building and packing. The gate installs the tarball in an isolated home, runs the pinned native engine and a deterministic local embedding endpoint, and checks full recovery across forced restarts, opt-in withdrawal, provider outage, dimension changes, an empty saved checkpoint, an absent marker with checkpoint recovery intent, writes during backfill, and keyword search plus capture during recovery quarantine. `--dependencies /path/to/node_modules` reuses cached dependencies for offline runs; it does not establish clean-install compatibility.
+
## Verification
Validation used main `ab3e4efd282659b87d31b36c8514c6bc6b871f0b` as the baseline and an installed package built from the fix branch. Synthetic records and a deterministic local embedding fixture exercised persistence, not semantic retrieval quality.
diff --git a/package.json b/package.json
index 0242d4fc..77063fdb 100644
--- a/package.json
+++ b/package.json
@@ -25,6 +25,7 @@
"test:watch": "vitest --exclude test/integration.test.ts",
"test:integration": "vitest run test/integration.test.ts",
"test:all": "vitest run",
+ "test:vector-recovery:packaged": "node scripts/release-gate/vector-recovery.mjs",
"test:startup:packaged": "node scripts/release-gate/startup.mjs",
"test:summary-recovery:packaged": "node scripts/release-gate/summary-recovery.mjs",
"test:graph-index:packaged": "node scripts/release-gate/graph-index.mjs",
diff --git a/scripts/release-gate/vector-recovery.mjs b/scripts/release-gate/vector-recovery.mjs
new file mode 100644
index 00000000..a54fbb00
--- /dev/null
+++ b/scripts/release-gate/vector-recovery.mjs
@@ -0,0 +1,258 @@
+#!/usr/bin/env node
+import assert from "node:assert/strict";
+import { spawn, execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, symlinkSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import net from "node:net";
+import { arch, homedir, platform, tmpdir } from "node:os";
+import { delimiter, dirname, join, resolve } from "node:path";
+import { registerWorker } from "iii-sdk";
+import { decodeRkyvJsonScope } from "./recovery.mjs";
+
+const options = {};
+for (let i = 2; i < process.argv.length; i++) {
+ const key = process.argv[i];
+ assert(["--tarball", "--engine", "--out", "--dependencies"].includes(key), `Unknown argument ${key}`);
+ assert(process.argv[i + 1], `Missing value for ${key}`);
+ options[key.slice(2)] = resolve(process.argv[++i]);
+}
+assert(options.tarball, "Pass --tarball with the built npm package to test");
+const out = options.out ?? mkdtempSync(join(tmpdir(), "agentmemory-vector-recovery-"));
+mkdirSync(out, { recursive: true });
+const root = mkdtempSync(join(out, "run-"));
+const home = join(root, "home"), runtime = join(home, ".agentmemory"), data = join(root, "data"), prefix = join(root, "prefix");
+const modules = join(prefix, "node_modules"), pkg = join(modules, "@agentmemory/agentmemory");
+for (const dir of [runtime, data, prefix]) mkdirSync(dir, { recursive: true });
+if (options.dependencies) {
+ mkdirSync(pkg, { recursive: true });
+ execFileSync("tar", ["-xzf", options.tarball, "--strip-components=1", "-C", pkg]);
+ for (const entry of readdirSync(options.dependencies, { withFileTypes: true })) {
+ if (entry.name.startsWith(".")) continue;
+ if (entry.name.startsWith("@")) {
+ mkdirSync(join(modules, entry.name), { recursive: true });
+ for (const name of readdirSync(join(options.dependencies, entry.name))) {
+ const target = join(modules, entry.name, name);
+ if (!existsSync(target)) symlinkSync(join(options.dependencies, entry.name, name), target);
+ }
+ } else symlinkSync(join(options.dependencies, entry.name), join(modules, entry.name));
+ }
+} else {
+ writeFileSync(join(prefix, "package.json"), '{"name":"vector-recovery-gate","private":true}\n');
+ execFileSync("npm", ["install", "--ignore-scripts", "--omit=optional", "--no-audit", "--no-fund", options.tarball], { cwd: prefix, stdio: "inherit" });
+}
+const manifest = JSON.parse(readFileSync(join(pkg, "package.json"), "utf8"));
+const engine = options.engine ?? process.env.AGENTMEMORY_GATE_ENGINE ?? join(homedir(), ".cache/agentmemory-release-gate", `iii-${manifest.dependencies["iii-sdk"]}-${platform()}-${arch()}`);
+assert(execFileSync(engine, ["--version"], { encoding: "utf8" }).includes(manifest.dependencies["iii-sdk"]), "Pinned native engine version required");
+const hash = (path) => createHash("sha256").update(readFileSync(path)).digest("hex");
+const cli = join(pkg, "dist/cli.mjs"), worker = join(pkg, "dist/index.mjs");
+const originalHashes = [options.tarball, cli, worker].map(hash);
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function poll(label, check, timeout = 30_000) {
+ const deadline = Date.now() + timeout;
+ let error;
+ do { try { const value = await check(); if (value) return value; } catch (err) { error = err; } await sleep(100); } while (Date.now() < deadline);
+ throw new Error(`${label} timed out: ${error?.message ?? "not ready"}`);
+}
+const free = (port) => new Promise((resolve) => {
+ const server = net.createServer(); server.once("error", () => resolve(false));
+ server.listen(port, "127.0.0.1", () => server.close(() => resolve(true)));
+});
+let port;
+for (let i = 0; i < 100; i++) {
+ const candidate = 12_000 + Math.floor(Math.random() * 7_000);
+ if ((await Promise.all([0, 1, 2, 3, 6353, 46023].map((n) => free(candidate + n)))).every(Boolean)) { port = candidate; break; }
+}
+assert(port, "No free port block");
+const ports = [0, 1, 2, 3, 6353, 46023].map((n) => port + n);
+let dimensions = 3, failEmbeddings = false, holdNext = false, releaseHeld, inputs = 0, secret = "", workerProcess, engineProcess;
+const children = new Set();
+const fixture = createServer(async (req, res) => {
+ try {
+ assert.equal(req.url, "/v1/embeddings");
+ const chunks = []; for await (const chunk of req) chunks.push(chunk);
+ const body = JSON.parse(Buffer.concat(chunks).toString());
+ assert(Array.isArray(body.input)); inputs += body.input.length;
+ if (holdNext) { holdNext = false; await new Promise((resolve) => { releaseHeld = resolve; }); }
+ else if (failEmbeddings) { res.writeHead(503); res.end("fixture unavailable"); return; }
+ res.setHeader("content-type", "application/json");
+ res.end(JSON.stringify({ data: body.input.map((_, index) => ({ index, embedding: Array(dimensions).fill(1) })) }));
+ } catch (error) { res.writeHead(400); res.end(error.message); }
+});
+await new Promise((resolve, reject) => { fixture.once("error", reject); fixture.listen(0, "127.0.0.1", resolve); });
+const fixturePort = fixture.address().port;
+let config = readFileSync(join(pkg, "iii-config.yaml"), "utf8").split(" - name: iii-exec")[0];
+config = config.replaceAll("3111", String(port)).replaceAll("3112", String(port + 1)).replaceAll("3113", String(port + 2)).replaceAll("./data/", `${data}/`);
+config += ` - name: iii-worker-manager\n config:\n port: ${port + 46023}\n host: 127.0.0.1\n`;
+const configPath = join(root, "iii-config.yaml"); writeFileSync(configPath, config);
+const env = (extra = {}) => ({
+ HOME: home, USERPROFILE: home, PATH: [dirname(process.execPath), "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(delimiter), TMPDIR: root,
+ CI: "1", NO_COLOR: "1", LANG: "C.UTF-8", OTEL_ENABLED: "false", III_TELEMETRY_ENABLED: "false",
+ AGENTMEMORY_DATA_DIR: data, AGENTMEMORY_RUNTIME_DIR: runtime, III_REST_PORT: String(port), III_STREAM_PORT: String(port + 1), III_VIEWER_PORT: String(port + 2),
+ III_ENGINE_PORT: String(port + 46023), III_ENGINE_URL: `ws://127.0.0.1:${port + 46023}`, AGENTMEMORY_METRICS_PORT: String(port + 6353),
+ LLM_PROVIDER: "none", EMBEDDING_PROVIDER: "openai", OPENAI_EMBEDDING_API_KEY: "local-fixture-only", OPENAI_EMBEDDING_DIMENSIONS: String(dimensions),
+ OPENAI_EMBEDDING_BASE_URL: `http://127.0.0.1:${fixture.address().port}/v1`, AGENTMEMORY_VECTOR_BACKFILL_MAX: "2", AGENTMEMORY_GRAPH_WEIGHT: "0",
+ AGENTMEMORY_GRAPH_COMPACT_ON_BOOT: "false", AGENTMEMORY_AUTO_COMPRESS: "false", AGENTMEMORY_SESSION_SWEEP_ENABLED: "false", AGENTMEMORY_FINALIZE_IDLE_MS: "0",
+ GRAPH_EXTRACTION_ENABLED: "false", CONSOLIDATION_ENABLED: "false", AGENTMEMORY_REFLECT: "false",
+ SNAPSHOT_ENABLED: "true", SNAPSHOT_DIR: join(root, "snapshots"), SNAPSHOT_INTERVAL: "3600", ...extra,
+});
+function launch(command, args, extra, name) {
+ const child = spawn(command, args, { env: env(extra), cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"] });
+ children.add(child); child.text = "";
+ const log = (chunk) => { child.text += chunk; appendFileSync(join(root, `${name}.log`), chunk); };
+ child.stdout.on("data", log); child.stderr.on("data", log); child.on("error", (error) => log(String(error)));
+ child.done = new Promise((resolve) => child.on("close", (code, signal) => { children.delete(child); resolve({ code, signal }); }));
+ return child;
+}
+function kill(child, signal = "SIGKILL") {
+ if (!child || child.exitCode !== null || child.signalCode !== null) return;
+ try { process.kill(-child.pid, signal); } catch { try { child.kill(signal); } catch {} }
+}
+async function rest(path, body, timeoutMs = 3_000) {
+ const response = await fetch(`http://127.0.0.1:${port}/agentmemory${path}`, { method: body ? "POST" : "GET", headers: { "content-type": "application/json", ...(secret ? { authorization: `Bearer ${secret}` } : {}) }, body: body ? JSON.stringify(body) : undefined, signal: AbortSignal.timeout(timeoutMs) });
+ assert(response.ok, `${path}: HTTP ${response.status}`); return response.json();
+}
+function persisted() {
+ try { return decodeRkyvJsonScope(readFileSync(join(data, "state_store.db", `${encodeURIComponent("mem:index:bm25")}.bin`))); } catch { return {}; }
+}
+async function state(functionId, payload) {
+ const client = registerWorker(`ws://127.0.0.1:${port + 46023}`, { workerName: "vector-recovery-fixture", invocationTimeoutMs: 5_000, enableMetricsReporting: false, otel: { enabled: false } });
+ try { return await client.trigger({ function_id: functionId, payload }); }
+ finally { await client.shutdown(); }
+}
+async function stop(graceful = false) {
+ if (graceful && workerProcess) { kill(workerProcess, "SIGTERM"); await Promise.race([workerProcess.done, sleep(5_000)]); }
+ kill(workerProcess); if (workerProcess) await Promise.race([workerProcess.done, sleep(2_000)]);
+ workerProcess = undefined;
+ if (graceful) await sleep(3_000);
+ kill(engineProcess); if (engineProcess) await Promise.race([engineProcess.done, sleep(2_000)]);
+ engineProcess = undefined;
+ await poll("ports closed", async () => (await Promise.all(ports.map(free))).every(Boolean), 5_000);
+}
+async function start(name, extra = {}, failure = false) {
+ inputs = 0;
+ engineProcess = launch(engine, ["--config", configPath, "--no-update-check"], extra, `${name}-engine`);
+ await poll("native engine", async () => (await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(500) })).status > 0);
+ workerProcess = launch(process.execPath, [cli, "--no-engine", "--port", String(port), "--data-dir", data], extra, name);
+ if (failure) {
+ const result = await Promise.race([workerProcess.done, sleep(10_000).then(() => ({ timeout: true }))]);
+ assert.equal(result.code, 1, JSON.stringify(result)); assert(workerProcess.text.includes("wrong dimension"));
+ assert.equal(inputs, 0); await stop(); return;
+ }
+ await poll("packaged CLI", async () => { if (!workerProcess.text.includes("REST API")) return false; const live = await rest("/livez"); return live.viewerPort || live.viewerSkipped; });
+ secret = readFileSync(join(runtime, "secret"), "utf8").trim();
+}
+const results = [];
+async function check(name, vectors, pending, state, expectedInputs) {
+ const status = await poll(name, async () => {
+ const value = await rest("/status");
+ return value.index?.keywordRebuildRunning === false && value.index.vectorDocuments === vectors && value.index.pendingVectorBackfill === pending && value.index.vectorBackfillState === state ? value : null;
+ });
+ assert.equal(inputs, expectedInputs, `${name} embedding inputs`);
+ if (state !== "waiting-for-opt-in") await poll("durable checkpoint and recovery marker", () => {
+ const saved = persisted(); return saved["vectors:meta"]?.count === vectors && (pending ? saved["vectors:backfill"]?.since === null : !saved["vectors:backfill"]);
+ });
+ const record = { name, vectors, pending, state, embeddingInputs: inputs, persistence: status.indexPersistence };
+ results.push(record); console.log(JSON.stringify(record));
+}
+let fatal, cleanupError;
+try {
+ await start("seed", { EMBEDDING_PROVIDER: "none" });
+ const timestamp = "2020-01-01T00:00:00.000Z";
+ const rows = Array.from({ length: 3 }, (_, i) => ({ id: `obs_${i}`, sessionId: "fixture", timestamp, type: "discovery", title: `Cobalt fixture ${i}`, narrative: `Cobalt historical record ${i}`, facts: [], concepts: [], files: [], importance: 5 }));
+ const imported = await rest("/import", { exportData: { version: manifest.version, exportedAt: timestamp, sessions: [{ id: "fixture", project: "fixture", cwd: root, startedAt: timestamp, status: "completed", observationCount: 3 }], observations: { fixture: rows }, memories: [], summaries: [] } });
+ assert.equal(imported.success, true); await sleep(3_000); await stop();
+ failEmbeddings = true;
+ await start("provider-down", { AGENTMEMORY_VECTOR_BACKFILL: "all" });
+ const failed = await poll("provider failure remains pending", async () => { const s = await rest("/status"); return s.index?.vectorBackfillState === "paused" ? s : null; });
+ assert.equal(failed.index.pendingVectorBackfill, 3); assert.equal(failed.index.vectorDocuments, 0); await sleep(3_000); await stop(); failEmbeddings = false;
+ await start("first-cap", { AGENTMEMORY_VECTOR_BACKFILL: "all" }); await check("first-cap", 2, 1, "paused", 2);
+ assert.equal(persisted()["vectors:meta"].backfillSince, null);
+ await state("state::delete", { scope: "mem:index:bm25", key: "vectors:backfill" });
+ await poll("marker absent with checkpoint fallback", () => !persisted()["vectors:backfill"]);
+ await stop();
+ await start("opt-in-withdrawn"); await check("opt-in-withdrawn", 2, 1, "waiting-for-opt-in", 0); await stop();
+ await start("resume-cap", { AGENTMEMORY_VECTOR_BACKFILL: "all" }); await check("resume-cap", 3, 0, "idle", 1); await stop();
+ dimensions = 4;
+ await start("dimension-refusal", {}, true);
+ await start("dimension-reset", { AGENTMEMORY_DROP_STALE_INDEX: "true" }); await check("dimension-reset", 0, 3, "waiting-for-opt-in", 0); await stop(true);
+ assert.equal(persisted()["vectors:meta"].count, 0); assert.equal(persisted()["vectors:backfill"].since, null);
+ await start("empty-checkpoint"); await check("empty-checkpoint", 0, 3, "waiting-for-opt-in", 0); await stop();
+ await start("dimension-cap", { AGENTMEMORY_VECTOR_BACKFILL: "all" }); await check("dimension-cap", 2, 1, "paused", 2); await stop();
+ await start("dimension-complete", { AGENTMEMORY_VECTOR_BACKFILL: "all" }); await check("dimension-complete", 3, 0, "idle", 1); await stop();
+ await start("completed-restart"); await check("completed-restart", 3, 0, "idle", 0);
+ assert((await rest("/search", { query: "Cobalt", limit: 3 })).results.length > 0);
+ failEmbeddings = true;
+ assert.equal((await rest("/remember", { content: "Amber baseline migration uses durable retries", project: "fixture" })).success, true);
+ await sleep(3_000); await stop();
+ failEmbeddings = false; holdNext = true;
+ await start("concurrent-write", { AGENTMEMORY_VECTOR_BACKFILL: "all" });
+ await poll("held recovery request", () => releaseHeld);
+ failEmbeddings = true;
+ assert.equal((await rest("/remember", { content: "Zircon late canonical record needs vector coverage", project: "fixture" })).success, true);
+ failEmbeddings = false; releaseHeld(); releaseHeld = undefined;
+ await check("concurrent-write", 4, 1, "paused", 4); await stop();
+ await start("concurrent-write-restart", { AGENTMEMORY_VECTOR_BACKFILL: "all" });
+ await check("concurrent-write-restart", 5, 0, "idle", 1);
+ await state("state::set", { scope: "mem:index:bm25", key: "vectors:backfill", value: { v: 2, since: "invalid fixture timestamp" } });
+ await poll("invalid marker persisted", () => persisted()["vectors:backfill"]?.since === "invalid fixture timestamp");
+ await stop();
+ await start("marker-quarantine");
+ await poll("vector quarantine with ready keywords", async () => {
+ const status = await rest("/status");
+ return status.index?.keywordRebuildRunning === false && status.index.vectorBackfillState === "paused" && status.indexPersistence.vector.lastError?.includes("Invalid persisted vector backfill marker");
+ });
+ assert((await rest("/search", { query: "Cobalt", limit: 3 })).results.length > 0);
+ assert.equal((await rest("/remember", { content: "Indigo captured while optional recovery is quarantined", project: "fixture" })).success, true);
+ await sleep(3_000);
+ assert.equal(persisted()["vectors:meta"].count, 5);
+ await state("state::set", { scope: "mem:index:bm25", key: "vectors:backfill", value: { v: 2, since: null } });
+ await poll("repaired marker persisted", () => persisted()["vectors:backfill"]?.since === null);
+ await stop(true);
+ assert.equal(persisted()["vectors:meta"].count, 5, "shutdown must not advance a quarantined checkpoint");
+ results.push({ name: "marker-quarantine", keywordsAvailable: true, captureAvailable: true, checkpointUnchanged: true });
+ await start("marker-repaired", { AGENTMEMORY_VECTOR_BACKFILL: "all" });
+ await check("marker-repaired", 6, 0, "idle", 1);
+ assert.equal((await rest("/remember", { content: "Violet stable checkpoint clears completed recovery", project: "fixture" })).success, true);
+ const target = { ...rows[0], id: "obs_incremental", sessionId: "incremental", timestamp: "2099-01-01T00:00:00.000Z", title: "Onyx current state" };
+ failEmbeddings = true;
+ assert.equal((await rest("/import", { exportData: { version: manifest.version, exportedAt: timestamp, sessions: [{ id: "incremental", project: "fixture", cwd: root, startedAt: timestamp, status: "completed", observationCount: 1 }], observations: { incremental: [target] }, memories: [], summaries: [] } })).success, true);
+ await stop(true); failEmbeddings = false;
+ assert.equal(persisted()["vectors:meta"].count, 7);
+ assert(!Object.hasOwn(persisted()["vectors:meta"], "backfillSince"), "ordinary checkpoint clears completion fallback");
+ const snapshots = join(root, "snapshots"); mkdirSync(snapshots, { recursive: true });
+ writeFileSync(join(snapshots, "state.json"), JSON.stringify({ observations: { incremental: [{ ...target, timestamp, title: "Onyx restored historical state" }] } }));
+ execFileSync("git", ["init", "--quiet", snapshots]);
+ execFileSync("git", ["-C", snapshots, "add", "state.json"]);
+ execFileSync("git", ["-C", snapshots, "-c", "user.name=Local fixture", "-c", "user.email=fixture@localhost", "commit", "--quiet", "-m", "Historical snapshot fixture"]);
+ const commitHash = execFileSync("git", ["-C", snapshots, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+ holdNext = true;
+ await start("incremental-snapshot");
+ await poll("held incremental recovery", () => releaseHeld);
+ failEmbeddings = true;
+ assert.equal((await rest("/snapshot/restore", { commitHash }, 30_000)).success, true);
+ failEmbeddings = false; releaseHeld(); releaseHeld = undefined;
+ await check("incremental-snapshot", 7, 1, "paused", 4); await stop();
+ await start("incremental-snapshot-gated");
+ await check("incremental-snapshot-gated", 7, 1, "waiting-for-opt-in", 0); await stop();
+ await start("incremental-snapshot-recovered", { AGENTMEMORY_VECTOR_BACKFILL: "all" });
+ await check("incremental-snapshot-recovered", 8, 0, "idle", 1);
+ const recall = await rest("/search", { query: "Onyx restored historical", limit: 100, format: "full" });
+ writeFileSync(join(root, "snapshot-recall.json"), JSON.stringify(recall, null, 2) + "\n");
+ const restored = recall.results.find((row) => row.observation?.id === target.id)?.observation;
+ assert.equal(restored?.title, "Onyx restored historical state");
+ assert.equal(restored?.timestamp, timestamp);
+} catch (error) { fatal = error.stack; console.error(fatal); }
+finally {
+ try { await stop(); } catch (error) { cleanupError = error.stack ?? String(error); }
+ finally {
+ for (const child of children) kill(child);
+ fixture.closeAllConnections();
+ await new Promise((resolve) => fixture.close(resolve));
+ await Promise.all([...children].map((child) => Promise.race([child.done, sleep(2_000)])));
+ }
+}
+const unchanged = [options.tarball, cli, worker].every((path, i) => hash(path) === originalHashes[i]);
+const summary = { ok: !fatal && !cleanupError && unchanged, root, tarball: options.tarball, tarballSha256: originalHashes[0], installedArtifactsUnmodified: unchanged, cachedDependencies: Boolean(options.dependencies), node: process.version, platform: `${platform()}-${arch()}`, engine: manifest.dependencies["iii-sdk"], results, fatal, cleanupError, portsClosed: (await Promise.all([...ports, fixturePort].map(free))).every(Boolean) };
+writeFileSync(join(root, "summary.json"), JSON.stringify(summary, null, 2) + "\n");
+console.log(`Summary: ${join(root, "summary.json")}`); process.exit(summary.ok && summary.portsClosed ? 0 : 1);
diff --git a/src/functions/search.ts b/src/functions/search.ts
index 88a8078c..7e8bf9ba 100644
--- a/src/functions/search.ts
+++ b/src/functions/search.ts
@@ -584,11 +584,11 @@ async function runKeywordRebuild(
): void => {
if (!backfillEligible || vi?.has(id)) return
if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return
- if (backfillGated) {
+ if (backfillGated || (wholeStoreBackfill && vectorJobs.length >= backfillCap)) {
fullBackfillPending++
return
}
- if (!backfillActive || (wholeStoreBackfill && vectorJobs.length >= backfillCap)) return
+ if (!backfillActive) return
vectorJobs.push({ id, sessionId, text, revision: idx.watchRevision(id), context: { kind, logId: id } })
}
@@ -652,17 +652,20 @@ async function embedBackfillJobs(
remainingAfter: number,
): Promise<{ ok: number; fail: number; skipped: number }> {
const batchSize = getRebuildEmbedBatchSize()
+ const initialVector = vectorIndex
+ const initialProvider = currentEmbeddingProvider
let ok = 0
let fail = 0
let skipped = 0
let batchesSinceSave = 0
setPendingVectorBackfillCount(remainingAfter + jobs.length)
for (let offset = 0; offset < jobs.length; offset += batchSize) {
+ if (vectorIndex !== initialVector || currentEmbeddingProvider !== initialProvider) break
const result = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize))
ok += result.ok
fail += result.fail
skipped += result.skipped
- setPendingVectorBackfillCount(remainingAfter + jobs.length - Math.min(jobs.length, offset + batchSize))
+ setPendingVectorBackfillCount(remainingAfter + fail + jobs.length - Math.min(jobs.length, offset + batchSize))
batchesSinceSave++
if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) {
await flushIndexSave()
@@ -681,15 +684,27 @@ export type VectorBacklogResult = { added: number; failed: number; remaining: nu
export async function backfillVectorBacklog(
jobs: VectorBackfillJob[],
- options: { batchSize?: number; pauseMs?: number } = {},
+ options: { batchSize?: number; pauseMs?: number; remainingAfter?: number; remainingCount?: () => number } = {},
): Promise<VectorBacklogResult> {
const batchSize = options.batchSize && options.batchSize > 0 ? options.batchSize : getVectorBackfillMax()
const pauseMs = options.pauseMs ?? VECTOR_BACKLOG_PAUSE_MS
+ const deferred = options.remainingAfter ?? 0
+ const initialVector = vectorIndex
+ const initialProvider = currentEmbeddingProvider
+ const replaced = () => vectorIndex !== initialVector || currentEmbeddingProvider !== initialProvider
+ const countRemaining = () => options.remainingCount?.() ?? deferred + jobs.filter((job) =>
+ !vectorIndex?.has(job.id) && (!job.revision || getSearchIndex().has(job.id))).length
+ const interrupted = (): VectorBacklogResult => {
+ const remaining = countRemaining()
+ setPendingVectorBackfillCount(remaining)
+ return { added, failed, remaining, complete: false }
+ }
let added = 0
let failed = 0
for (let offset = 0; offset < jobs.length; offset += batchSize) {
if (offset > 0 && pauseMs > 0) await delay(pauseMs)
- const after = Math.max(0, jobs.length - offset - batchSize)
+ if (replaced()) return interrupted()
+ const after = deferred + failed + Math.max(0, jobs.length - offset - batchSize)
const batch = jobs.slice(offset, offset + batchSize).filter((job) => !vectorIndex?.has(job.id) && (!job.revision || job.revision.isCurrent()))
if (batch.length === 0) {
setPendingVectorBackfillCount(after)
@@ -698,14 +713,16 @@ export async function backfillVectorBacklog(
const result = await embedBackfillJobs(batch, after)
added += result.ok
failed += result.fail
+ if (replaced()) return interrupted()
if (result.ok === 0 && (result.fail > 0 || result.skipped === 0)) {
- const remaining = after + failed + (result.fail === 0 ? batch.length : 0)
+ const remaining = countRemaining()
setPendingVectorBackfillCount(remaining)
return { added, failed, remaining, complete: false }
}
}
- setPendingVectorBackfillCount(failed)
- return { added, failed, remaining: failed, complete: failed === 0 }
+ const remaining = countRemaining()
+ setPendingVectorBackfillCount(remaining)
+ return { added, failed, remaining, complete: remaining === 0 }
}
export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void {
diff --git a/src/index.ts b/src/index.ts
index e54bf591..678ac4f0 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -15,6 +15,7 @@ import {
isContextInjectionEnabled,
isDropStaleIndexEnabled,
getVectorBackfillMax,
+ isVectorBackfillAllEnabled,
getAuditRetentionMonths,
getStateBackend,
isSessionSweepEnabled,
@@ -45,9 +46,9 @@ import { registerDiskSizeManager } from "./functions/disk-size-manager.js";
import { registerCompressFunction } from "./functions/compress.js";
import {
registerSearchFunction,
- backfillVectors,
backfillVectorBacklog,
isBm25RebuildIncomplete,
+ isMemoryIndexReady,
rebuildKeywordIndex,
markKeywordRebuildPending,
getSearchIndex,
@@ -454,13 +455,14 @@ async function main() {
: undefined,
});
- const indexPersistence = new IndexPersistence(kv, vectorIndex);
+ const indexPersistence = new IndexPersistence(kv, vectorIndex, { requireRecoveryPlan: true });
setIndexPersistence(indexPersistence);
const loaded = await indexPersistence.load().catch((err) => {
console.warn(`[agentmemory] Failed to load persisted vector index:`, err);
return null;
});
+ let vectorBackfillSince = await indexPersistence.prepareBackfill(loaded, isVectorBackfillAllEnabled());
if (loaded?.vector && vectorIndex && loaded.vector.size > 0) {
// Persisted vectors carry whatever dimension the provider had when
// they were written. If the active provider declares a different
@@ -485,6 +487,7 @@ async function main() {
const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", ");
const dropStale = isDropStaleIndexEnabled();
if (dropStale) {
+ vectorBackfillSince = await indexPersistence.prepareBackfill(loaded, true);
for (const [obsId] of loaded.vector.entries()) vectorIndex.markRemoved(obsId);
console.warn(
`[agentmemory] Persisted vector index has ${mismatches.length} of ` +
@@ -518,6 +521,8 @@ async function main() {
if (vectorIndex && loaded && loaded.state !== "unavailable") {
const replay = await indexPersistence.replayPendingLog(embeddingProvider?.dimensions ?? 0);
+ if (replay.skipped > 0) vectorBackfillSince = null;
+ if (indexPersistence.isRecoveryPaused()) vectorBackfillSince = undefined;
if (replay.entries > 0) {
bootLog(
`Recovered ${replay.added} vectors and ${replay.removed} removals written after the last index save, without calling the embedding provider` +
@@ -528,78 +533,65 @@ async function main() {
const auditMigration = startAuditMigration(kv).catch(() => {});
- const vectorCountShortfall =
- Boolean(loaded?.vector) &&
- loaded?.expectedCount !== undefined &&
- loaded.vector!.size < loaded.expectedCount;
- let vectorBackfillSince =
- !loaded || loaded.state === "unavailable"
- ? undefined
- : loaded.state === "none" || vectorCountShortfall
- ? null
- : loaded.savedAt;
- const incrementalBackfill = typeof vectorBackfillSince === "string" && Boolean(vectorIndex && embeddingProvider);
- if (incrementalBackfill) {
- const since = vectorBackfillSince as string;
- const marker = await indexPersistence.readBackfillMarker();
- if (marker !== null && Date.parse(marker) < Date.parse(since)) vectorBackfillSince = marker;
- await indexPersistence.markBackfillSince(vectorBackfillSince as string).catch((err) => {
- bootWarn(`Could not save the vector backfill marker: ${err instanceof Error ? err.message : String(err)}`);
- });
- }
const keywordStart = Date.now();
try {
const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince);
bootLog(
`Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`,
);
- setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending);
- setVectorBackfillState(keyword.fullBackfillPending > 0 ? "waiting-for-opt-in" : "idle");
+ const pending = keyword.vectorJobs.length + keyword.fullBackfillPending;
+ const backlogScanComplete = !isBm25RebuildIncomplete() && isMemoryIndexReady();
+ const optedIn = isVectorBackfillAllEnabled();
+ const activeIds = new Set(keyword.vectorJobs.map((job) => job.id));
+ const hasVector = (id: string) => Boolean(vectorIndex?.has(id));
+ const countRemaining = () => bm25Index.countMissingDocuments(hasVector, vectorBackfillSince, activeIds);
+ setPendingVectorBackfillCount(indexPersistence.isRecoveryPaused() ? countRemaining() : pending);
+ if (indexPersistence.isRecoveryPaused()) {
+ setVectorBackfillState("paused");
+ } else if (vectorBackfillSince !== undefined) {
+ setVectorBackfillState(keyword.fullBackfillPending > 0 && !optedIn ? "waiting-for-opt-in" : backlogScanComplete ? "idle" : "paused");
+ }
if (keyword.fullBackfillPending > 0) {
- bootLog(
- `Vector backfill needs ${keyword.fullBackfillPending} embeddings but a full backfill was not started ` +
- `(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`,
- );
+ bootLog(optedIn
+ ? `Vector backfill has ${keyword.fullBackfillPending} additional documents beyond this start's limit; restart to resume.`
+ : `Vector backfill needs ${keyword.fullBackfillPending} embeddings (set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`);
}
- const backlogScanComplete = !isBm25RebuildIncomplete();
- if (incrementalBackfill && backlogScanComplete && keyword.vectorJobs.length === 0) {
- await indexPersistence.clearBackfillMarker().catch(() => {});
+ if (vectorBackfillSince !== undefined && backlogScanComplete && pending === 0) {
+ if (!(await indexPersistence.completeBackfill(() => countRemaining() === 0))) {
+ setPendingVectorBackfillCount(countRemaining());
+ setVectorBackfillState("paused");
+ }
}
- if (incrementalBackfill && keyword.vectorJobs.length > 0) {
+ if (keyword.vectorJobs.length > 0 && !(await indexPersistence.beginBackfill())) {
+ setVectorBackfillState("paused");
+ activeIds.clear();
+ for (const job of keyword.vectorJobs) job.revision?.release();
+ } else if (keyword.vectorJobs.length > 0) {
setVectorBackfillState("running");
- bootLog(
- `Re-embedding ${keyword.vectorJobs.length} vectors missing since the last index save, ${getVectorBackfillMax()} per batch in the background`,
- );
- void backfillVectorBacklog(keyword.vectorJobs)
+ bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors, ${getVectorBackfillMax()} per batch in the background`);
+ void backfillVectorBacklog(keyword.vectorJobs, {
+ remainingAfter: keyword.fullBackfillPending,
+ remainingCount: countRemaining,
+ })
.then(async (result) => {
- if (result.complete && backlogScanComplete) await indexPersistence.clearBackfillMarker().catch(() => {});
- setVectorBackfillState(result.complete ? "idle" : "paused");
+ const complete = result.complete && backlogScanComplete && await indexPersistence.completeBackfill(() => countRemaining() === 0);
+ const remaining = countRemaining();
+ setPendingVectorBackfillCount(remaining);
+ setVectorBackfillState(complete ? "idle" : "paused");
if (result.added > 0) bootLog(`Vector index backfilled: ${result.added} entries`);
- if (!result.complete) {
- bootWarn(
- `Vector backfill stopped with ${result.remaining} documents still missing a vector. They are retried on the next start.`,
- );
- }
+ if (!complete) bootWarn(`Vector recovery remains incomplete (${remaining} documents pending); it resumes on the next start.`);
})
.catch((err) => {
setVectorBackfillState("paused");
console.warn(`[agentmemory] Failed to backfill vectors:`, err);
- });
- } else if (keyword.vectorJobs.length > 0) {
- setVectorBackfillState("running");
- bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`);
- void backfillVectors(keyword.vectorJobs)
- .then((count) => {
- setPendingVectorBackfillCount(keyword.fullBackfillPending);
- setVectorBackfillState(keyword.fullBackfillPending > 0 ? "waiting-for-opt-in" : "idle");
- if (count > 0) bootLog(`Vector index backfilled: ${count} entries`);
})
- .catch((err) => {
- setVectorBackfillState("idle");
- console.warn(`[agentmemory] Failed to backfill vectors:`, err);
+ .finally(() => {
+ activeIds.clear();
+ for (const job of keyword.vectorJobs) job.revision?.release();
});
}
} catch (err) {
+ setVectorBackfillState("paused");
console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err);
}
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 46c565d3..a09af90f 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -41,6 +41,7 @@ type VectorMeta = {
bucketCount: number;
savedAt: string;
count: number;
+ backfillSince?: string | null;
};
type PersistedVector = {
@@ -60,8 +61,8 @@ type PendingVectorRow = {
};
type BackfillMarker = {
- v: 1;
- since: string;
+ v: 1 | 2;
+ since: string | null;
};
export interface PendingReplayResult {
@@ -76,6 +77,7 @@ type IndexPersistenceOptions = {
saveIntervalMs?: number;
bucketSize?: number;
now?: () => number;
+ requireRecoveryPlan?: boolean;
};
export interface IndexLegStatus {
@@ -111,6 +113,7 @@ export interface VectorLoadResult {
state: VectorLoadState;
savedAt: string | null;
expectedCount?: number;
+ backfillSince?: string | null;
}
export function vectorBucketScope(bucket: number): string {
@@ -125,6 +128,15 @@ function errorMessage(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
+function earliestBackfillSince(...cutoffs: Array<string | null | undefined>): string | null | undefined {
+ let earliest: string | undefined;
+ for (const cutoff of cutoffs) {
+ if (cutoff === null) return null;
+ if (cutoff !== undefined && (earliest === undefined || Date.parse(cutoff) < Date.parse(earliest))) earliest = cutoff;
+ }
+ return earliest;
+}
+
function generationTime(generation: string | undefined): string | null {
const stamp = generation?.split("_")[1];
if (!stamp) return null;
@@ -186,8 +198,8 @@ export class IndexPersistence {
private lastSaveAt: number;
private dirtyEpoch = 0;
private markedDuringRunAt: number | null = null;
- private metaWritten = false;
- private firstSaveAttempted = false;
+ private hasVectorCheckpoint = false;
+ private firstVectorSaveAttempted = false;
private nextSaveAt: number | null = null;
private leg: IndexLegStatus = emptyLegStatus();
private readonly saveIntervalMs: number;
@@ -206,12 +218,20 @@ export class IndexPersistence {
private pendingLogUnknown = false;
private pendingLogError: string | null = null;
private earlySaveQueued = false;
+ private backfillQueue: Promise<void> = Promise.resolve();
+ private recoveryError: string | null = null;
+ private backfillRevision = 0;
+ private recoveryReady: boolean;
+ private backfillSince: string | null | undefined;
+ private checkpointBackfillSince: string | null | undefined;
+ private completingBackfill = false;
constructor(
private kv: StateKV,
private vector: VectorIndex | null,
options: IndexPersistenceOptions = {},
) {
+ this.recoveryReady = !options.requireRecoveryPlan || !vector;
this.vector?.setChangeListener((id, entry) => this.logChange(id, entry));
this.now = options.now ?? Date.now;
const interval = options.saveIntervalMs;
@@ -228,7 +248,7 @@ export class IndexPersistence {
}
scheduleSave(): void {
- if (this.stopped || !this.vector?.pendingChanges) return;
+ if (this.stopped || !this.recoveryReady || this.recoveryError || !this.vector?.pendingChanges) return;
const now = this.now();
this.dirtyEpoch++;
if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now;
@@ -253,7 +273,7 @@ export class IndexPersistence {
status(): IndexPersistenceStatus {
return {
saveIntervalMs: this.saveIntervalMs,
- firstCheckpointPending: Boolean(this.vector && !this.metaWritten && (this.vector.pendingChanges > 0 || this.running)),
+ firstCheckpointPending: Boolean(this.vector && !this.hasVectorCheckpoint && (this.vector.pendingChanges > 0 || this.running)),
nextSaveAt: this.nextSaveAt === null ? null : new Date(this.nextSaveAt).toISOString(),
saving: this.running !== null,
buckets: this.bucketCountInUse(),
@@ -269,7 +289,7 @@ export class IndexPersistence {
async replayPendingLog(expectedDimensions = 0): Promise<PendingReplayResult> {
const result: PendingReplayResult = { entries: 0, added: 0, removed: 0, cleared: false, skipped: 0 };
const vector = this.vector;
- if (!vector) return result;
+ if (!vector || this.recoveryError) return result;
let rows: unknown[];
try {
rows = await this.kv.list<unknown>(KV.vectorPendingLog);
@@ -318,31 +338,130 @@ export class IndexPersistence {
} finally {
this.logSuppressed = false;
}
+ if (result.skipped > 0) {
+ try { await this.markBackfillSince(null); } catch (err) { this.pauseRecovery(err); }
+ }
if (result.added + result.removed > 0 || result.cleared) this.scheduleSave();
return result;
}
- async readBackfillMarker(): Promise<string | null> {
- if (!this.vector) return null;
+ async readBackfillMarker(): Promise<string | null | undefined> {
+ if (!this.vector) return undefined;
+ const marker = await this.kv.get<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY);
+ if (!marker) return undefined;
+ if (marker.v === 2 && marker.since === null) return null;
+ if ((marker.v === 1 || marker.v === 2) && typeof marker.since === "string" && Number.isFinite(Date.parse(marker.since))) return marker.since;
+ throw new Error("Invalid persisted vector backfill marker");
+ }
+
+ markBackfillSince(since: string | null): Promise<void> {
+ this.backfillRevision++;
+ return this.queueBackfillChange(async () => {
+ if (!this.vector || (since !== null && !Number.isFinite(Date.parse(since)))) return;
+ const current = await this.readBackfillMarker();
+ const next = earliestBackfillSince(current, this.backfillSince, since)!;
+ if (current !== next) await this.kv.set<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY, { v: 2, since: next });
+ this.backfillSince = next;
+ });
+ }
+
+ private queueBackfillChange(change: () => Promise<void>): Promise<void> {
+ const run = this.backfillQueue.then(change);
+ this.backfillQueue = run.catch(() => {});
+ return run;
+ }
+
+ isRecoveryPaused(): boolean {
+ return this.recoveryError !== null || !this.recoveryReady;
+ }
+
+ private pauseRecovery(error: unknown): void {
+ this.recoveryReady = false;
+ this.recoveryError = `Vector recovery paused until restart: ${errorMessage(error)}`;
+ this.leg.lastError = this.recoveryError;
+ this.leg.lastErrorAt = new Date(this.now()).toISOString();
+ this.clearTimer();
+ logger.warn(this.recoveryError);
+ }
+
+ async prepareBackfill(loaded: VectorLoadResult | null, forceFull = false): Promise<string | null | undefined> {
+ if (!this.vector || this.recoveryError) return undefined;
+ if (!loaded || loaded.state === "unavailable") {
+ this.pauseRecovery(new Error("Persisted vector checkpoint is unavailable"));
+ return undefined;
+ }
+ const shortfall = loaded.expectedCount !== undefined && (loaded.vector?.size ?? 0) < loaded.expectedCount;
+ const since = forceFull || loaded.state === "none" || shortfall || !Number.isFinite(Date.parse(loaded.savedAt ?? "")) ? null : loaded.savedAt;
+ try {
+ const fallback = loaded.backfillSince;
+ if (fallback !== undefined && fallback !== null && (typeof fallback !== "string" || !Number.isFinite(Date.parse(fallback)))) {
+ throw new Error("Invalid persisted vector recovery cutoff");
+ }
+ await this.markBackfillSince(earliestBackfillSince(since, fallback)!);
+ const marker = await this.readBackfillMarker();
+ this.recoveryReady = true;
+ this.scheduleSave();
+ return marker;
+ } catch (err) {
+ this.pauseRecovery(err);
+ return undefined;
+ }
+ }
+
+ async beginBackfill(): Promise<boolean> {
+ if (this.isRecoveryPaused()) return false;
try {
- const marker = await this.kv.get<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY);
- return marker && typeof marker.since === "string" && !Number.isNaN(Date.parse(marker.since)) ? marker.since : null;
+ await this.markBackfillSince(null);
+ return true;
} catch (err) {
- logger.warn("index persistence: vector backfill marker read failed", { message: errorMessage(err) });
- return null;
+ this.pauseRecovery(err);
+ return false;
}
}
- async markBackfillSince(since: string): Promise<void> {
- if (!this.vector || Number.isNaN(Date.parse(since))) return;
- const current = await this.readBackfillMarker();
- if (current !== null && Date.parse(current) <= Date.parse(since)) return;
- await this.kv.set<BackfillMarker>(KV.bm25Index, BACKFILL_MARKER_KEY, { v: 1, since });
+ private checkpointRecoveryCutoff(): string | null | undefined {
+ return this.completingBackfill ? null : this.backfillSince;
+ }
+
+ private needsRecoveryCheckpoint(): boolean {
+ const cutoff = this.checkpointRecoveryCutoff();
+ return cutoff !== undefined && cutoff !== this.checkpointBackfillSince;
+ }
+
+ async completeBackfill(isCovered: () => boolean = () => true): Promise<boolean> {
+ if (this.isRecoveryPaused()) return false;
+ const revision = this.backfillRevision;
+ const since = this.backfillSince;
+ this.completingBackfill = since !== undefined;
+ await this.save();
+ if (this.leg.lastError || this.vector?.pendingChanges || !isCovered()) return false;
+ let cleared = false;
+ try {
+ await this.queueBackfillChange(async () => {
+ if (this.recoveryError || this.backfillRevision !== revision || !isCovered()) return;
+ await this.kv.delete(KV.bm25Index, BACKFILL_MARKER_KEY);
+ cleared = true;
+ });
+ if (!cleared) return false;
+ if (this.backfillRevision !== revision || !isCovered()) {
+ if (since !== undefined) await this.markBackfillSince(since);
+ return false;
+ }
+ // The checkpoint retains the cutoff through the delete/crash window. A later
+ // ordinary checkpoint can remove it once this recovery has completed.
+ this.backfillSince = undefined;
+ this.completingBackfill = false;
+ return true;
+ } catch (err) {
+ this.pauseRecovery(err);
+ return false;
+ }
}
- async clearBackfillMarker(): Promise<void> {
- if (!this.vector) return;
- await this.kv.delete(KV.bm25Index, BACKFILL_MARKER_KEY);
+ clearBackfillMarker(): Promise<void> {
+ return this.queueBackfillChange(async () => {
+ if (this.vector) await this.kv.delete(KV.bm25Index, BACKFILL_MARKER_KEY);
+ });
}
async flushPendingLog(): Promise<void> {
@@ -355,7 +474,7 @@ export class IndexPersistence {
}
private logChange(id: string | null, entry: VectorEntry | null): void {
- if (this.logSuppressed) return;
+ if (this.logSuppressed || !this.recoveryReady || this.recoveryError) return;
const key = id ?? PENDING_CLEAR_KEY;
const q = this.nextSeq();
let row: PendingVectorRow;
@@ -403,7 +522,7 @@ export class IndexPersistence {
}
private maybeEarlySave(): void {
- if (this.stopped || this.running || this.earlySaveQueued) return;
+ if (this.stopped || !this.recoveryReady || this.recoveryError || this.running || this.earlySaveQueued) return;
if (this.pendingLog.size < PENDING_LOG_SAVE_THRESHOLD) return;
if (this.now() - this.lastSaveAt < EARLY_SAVE_MIN_GAP_MS) return;
this.earlySaveQueued = true;
@@ -459,7 +578,8 @@ export class IndexPersistence {
if (meta && meta.v === 3 && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0) {
const loaded = await this.loadBuckets(meta.bucketCount);
if (!loaded) return { vector: null, state: "unavailable", savedAt: null };
- this.metaWritten = true;
+ this.hasVectorCheckpoint = loaded.size > 0;
+ this.checkpointBackfillSince = meta.backfillSince;
await this.removeLegacyVectorSnapshotIfPresent();
await this.removeLegacyBm25Snapshot();
const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined;
@@ -472,7 +592,7 @@ export class IndexPersistence {
...this.vectorCountShortfall,
});
}
- return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null, expectedCount };
+ return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null, expectedCount, backfillSince: meta.backfillSince };
}
return this.migrateLegacy();
@@ -514,10 +634,10 @@ export class IndexPersistence {
}
private queueSave(): void {
- if (this.stopped || this.timer || !this.vector?.pendingChanges) return;
+ if (this.stopped || !this.recoveryReady || this.recoveryError || this.timer || !this.vector?.pendingChanges) return;
const now = this.now();
const remaining = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now);
- const delay = !this.metaWritten && !this.firstSaveAttempted
+ const delay = !this.hasVectorCheckpoint && !this.firstVectorSaveAttempted
? Math.min(FIRST_CHECKPOINT_DELAY_MS, remaining)
: remaining;
this.nextSaveAt = now + delay;
@@ -539,11 +659,15 @@ export class IndexPersistence {
private async runSave(): Promise<void> {
const vector = this.vector;
- if (!vector?.pendingChanges) return;
+ if (!vector || this.isRecoveryPaused()) return;
+ if (!vector.pendingChanges && !this.needsRecoveryCheckpoint()) return;
+ const hasVectorChanges = vector.pendingChanges > 0;
const epoch = this.dirtyEpoch;
this.markedDuringRunAt = null;
- this.lastSaveAt = this.now();
- this.firstSaveAttempted = true;
+ if (hasVectorChanges) {
+ this.lastSaveAt = this.now();
+ this.firstVectorSaveAttempted = true;
+ }
const coveredSeq = this.seq;
try {
await this.writeChanges(vector);
@@ -563,7 +687,7 @@ export class IndexPersistence {
this.leg.lastError = errorMessage(err);
this.leg.lastErrorAt = new Date(this.now()).toISOString();
if (this.leg.dirtySince === null) this.leg.dirtySince = this.leg.lastErrorAt;
- this.lastSaveAt = this.now();
+ if (hasVectorChanges) this.lastSaveAt = this.now();
this.clearTimer();
this.logFailure(err);
}
@@ -571,7 +695,7 @@ export class IndexPersistence {
private async writeChanges(vector: VectorIndex): Promise<void> {
const changes = vector.takeChanges();
- if (changes.size === 0) return;
+ if (changes.size === 0 && !this.needsRecoveryCheckpoint()) return;
const failed = new Map<string, boolean>();
const failures = await inBatches([...changes], WRITE_CONCURRENCY, async ([id, present]) => {
const entry = present ? vector.get(id) : undefined;
@@ -601,18 +725,22 @@ export class IndexPersistence {
`${failures.length} of ${changes.size} vector writes failed: ${errorMessage(failures[0])}`,
);
}
+ const backfillSince = this.checkpointRecoveryCutoff();
try {
await this.kv.set<VectorMeta>(KV.bm25Index, VECTOR_META_KEY, {
v: 3,
bucketCount: this.bucketCountInUse(),
savedAt: new Date(this.now()).toISOString(),
count: vector.size,
+ ...(backfillSince !== undefined ? { backfillSince } : {}),
});
+ this.checkpointBackfillSince = backfillSince;
} catch (err) {
vector.returnChanges(changes);
throw err;
}
- this.metaWritten = true;
+ this.hasVectorCheckpoint = this.bucketOfId.size > 0;
+ if (!this.hasVectorCheckpoint) this.firstVectorSaveAttempted = false;
}
private async loadBuckets(bucketCount: number): Promise<VectorIndex | null> {
@@ -677,6 +805,7 @@ export class IndexPersistence {
const staging = new VectorIndex();
for (const [id, entry] of legacy.entries()) staging.add(id, entry.sessionId, entry.embedding);
try {
+ await this.markBackfillSince(savedAt);
await this.writeChanges(staging);
} catch (err) {
logger.warn("index persistence: migrating the legacy vector snapshot failed; it stays in place and the next save retries", {
diff --git a/src/state/search-index.ts b/src/state/search-index.ts
index 902ef33b..bf4abbef 100644
--- a/src/state/search-index.ts
+++ b/src/state/search-index.ts
@@ -9,6 +9,7 @@ interface IndexEntry {
sessionId: string;
termCount: number;
layer: SearchResultLayer;
+ timestamp?: string;
}
export interface SearchIndexRevision {
@@ -45,6 +46,7 @@ export class SearchIndex {
sessionId: obs.sessionId,
termCount,
layer,
+ timestamp: obs.timestamp,
});
this.docTermCounts.set(obs.id, termFreq);
this.totalDocLength += termCount;
@@ -113,6 +115,17 @@ export class SearchIndex {
return this.entries.get(id)?.layer;
}
+ countMissingDocuments(hasDocument: (id: string) => boolean, since?: string | null, activeIds?: ReadonlySet<string>): number {
+ const cutoff = typeof since === "string" ? Date.parse(since) : Number.NaN;
+ let missing = 0;
+ for (const entry of this.entries.values()) {
+ if (entry.layer !== "observation" && entry.layer !== "memory") continue;
+ if (Number.isFinite(cutoff) && !(Date.parse(entry.timestamp ?? "") > cutoff) && !activeIds?.has(entry.obsId)) continue;
+ if (!hasDocument(entry.obsId)) missing++;
+ }
+ return missing;
+ }
+
observationCountsBySession(): Map<string, number> {
const counts = new Map<string, number>();
for (const entry of this.entries.values()) {
diff --git a/test/index-first-checkpoint.test.ts b/test/index-first-checkpoint.test.ts
index bbcef2ee..9ad1e91b 100644
--- a/test/index-first-checkpoint.test.ts
+++ b/test/index-first-checkpoint.test.ts
@@ -137,9 +137,11 @@ describe("first vector checkpoint", () => {
expect((await new IndexPersistence(kv as never, new VectorIndex()).load()).vector?.size).toBe(1);
});
- it.each(["entry", "metadata"])("retries a failed first %s write at the normal interval", async (failure) => {
+ it.each([
+ ["entry", false], ["metadata", false], ["entry", true], ["metadata", true],
+ ])("retries a failed first %s write at the normal interval (after recovery: %s)", async (failure, afterRecovery) => {
const kv = mockKV();
- let fail = true;
+ let fail = false;
const failingKV = {
...kv,
set: async <T>(scope: string, key: string, data: T): Promise<T> => {
@@ -150,13 +152,20 @@ describe("first vector checkpoint", () => {
},
};
const vector = new VectorIndex();
- add(vector, "obs_1");
const persistence = new IndexPersistence(failingKV as never, vector, { saveIntervalMs: 60_000 });
+ if (afterRecovery) {
+ await persistence.prepareBackfill(await persistence.load());
+ expect(await persistence.completeBackfill()).toBe(true);
+ }
+ const previousMeta = await kv.get(INDEX_SCOPE, META_KEY);
+ const previousSavedAt = persistence.status().vector?.lastSavedAt;
+ fail = true;
+ add(vector, "obs_1");
persistence.scheduleSave();
await vi.advanceTimersByTimeAsync(5_000);
expect(persistence.status()).toMatchObject({ firstCheckpointPending: true, pendingChanges: 1 });
- expect(persistence.status().vector?.lastSavedAt).toBeNull();
- expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull();
+ expect(persistence.status().vector?.lastSavedAt).toBe(previousSavedAt);
+ expect(await kv.get(INDEX_SCOPE, META_KEY)).toEqual(previousMeta);
const writesAfterFailure = kv.writes.length;
fail = false;
await vi.advanceTimersByTimeAsync(59_999);
diff --git a/test/index-keyword-rebuild.test.ts b/test/index-keyword-rebuild.test.ts
index f0080675..58afa778 100644
--- a/test/index-keyword-rebuild.test.ts
+++ b/test/index-keyword-rebuild.test.ts
@@ -209,6 +209,7 @@ describe("rebuildKeywordIndex", () => {
process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "2";
const capped = await rebuildKeywordIndex(kv as never, null);
expect(capped.vectorJobs.length).toBe(2);
+ expect(capped.fullBackfillPending).toBe(2);
} finally {
delete process.env.AGENTMEMORY_VECTOR_BACKFILL;
delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX;
diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts
index 62747017..3764a402 100644
--- a/test/index-persistence-throttle.test.ts
+++ b/test/index-persistence-throttle.test.ts
@@ -68,6 +68,88 @@ describe("IndexPersistence save throttling", () => {
expect(metaSaves(kv)).toBe(2);
});
+ it.each([false, true])("checkpoints the first captured vector early after metadata-only recovery (restart: %s)", async (restart) => {
+ let vector = new VectorIndex();
+ let persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000, requireRecoveryPlan: true });
+ await persistence.prepareBackfill(await persistence.load());
+ expect(await persistence.completeBackfill()).toBe(true);
+ expect(metaSaves(kv)).toBe(1);
+
+ if (restart) {
+ persistence.stop();
+ vector = new VectorIndex();
+ persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000, requireRecoveryPlan: true });
+ await persistence.prepareBackfill(await persistence.load());
+ expect(await persistence.completeBackfill()).toBe(true);
+ }
+
+ await vi.advanceTimersByTimeAsync(20_000);
+ touch(vector, "obs_first");
+ persistence.scheduleSave();
+ expect(persistence.status().firstCheckpointPending).toBe(true);
+ await vi.advanceTimersByTimeAsync(4_999);
+ expect(metaSaves(kv)).toBe(1);
+ await vi.advanceTimersByTimeAsync(1);
+ expect(metaSaves(kv)).toBe(2);
+ expect(persistence.status().pendingChanges).toBe(0);
+ expect(persistence.status().firstCheckpointPending).toBe(false);
+
+ touch(vector, "obs_second");
+ persistence.scheduleSave();
+ await vi.advanceTimersByTimeAsync(5_000);
+ expect(metaSaves(kv)).toBe(2);
+ persistence.stop();
+ });
+
+ it("rearms the early checkpoint after clearing all persisted vectors", async () => {
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 600_000 });
+ touch(vector, "obs_old");
+ await persistence.save();
+ vector.clear();
+ await persistence.save();
+ expect(metaSaves(kv)).toBe(2);
+
+ touch(vector, "obs_replacement");
+ persistence.scheduleSave();
+ await vi.advanceTimersByTimeAsync(5_000);
+ expect(metaSaves(kv)).toBe(3);
+ expect(persistence.status().pendingChanges).toBe(0);
+ persistence.stop();
+ });
+
+ it("keeps a vector added during metadata-only recovery eligible for the early checkpoint", async () => {
+ let release!: () => void;
+ let entered!: () => void;
+ const gate = new Promise<void>((resolve) => { release = resolve; });
+ const writing = new Promise<void>((resolve) => { entered = resolve; });
+ let held = false;
+ const slowKV = {
+ ...kv,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (key === META_KEY && !held) {
+ held = true;
+ entered();
+ await gate;
+ }
+ return kv.set(scope, key, data);
+ },
+ };
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(slowKV as never, vector, { saveIntervalMs: 600_000, requireRecoveryPlan: true });
+ await persistence.prepareBackfill(await persistence.load());
+ const completion = persistence.completeBackfill();
+ await writing;
+ touch(vector, "obs_concurrent");
+ persistence.scheduleSave();
+ release();
+ expect(await completion).toBe(false);
+ await vi.advanceTimersByTimeAsync(5_000);
+ expect(metaSaves(kv)).toBe(2);
+ expect(persistence.status().pendingChanges).toBe(0);
+ persistence.stop();
+ });
+
it("an explicit save runs immediately and cancels the pending timer", async () => {
const vector = new VectorIndex();
const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 });
diff --git a/test/search-index-write-race.test.ts b/test/search-index-write-race.test.ts
index d234da44..e36a2747 100644
--- a/test/search-index-write-race.test.ts
+++ b/test/search-index-write-race.test.ts
@@ -146,12 +146,12 @@ describe("search indexing async write ownership", () => {
getSearchIndex().add(observation("changed", "newer"));
getSearchIndex().remove("deleted");
const result = await backfillVectorBacklog(rebuilt.vectorJobs, { pauseMs: 0 });
- expect(result).toEqual({ added: 0, failed: 0, remaining: 0, complete: true });
- expect(getPendingVectorBackfillCount()).toBe(0);
+ expect(result).toEqual({ added: 0, failed: 0, remaining: 1, complete: false });
+ expect(getPendingVectorBackfillCount()).toBe(1);
expect(embedding.embedBatch).not.toHaveBeenCalled();
});
- it("completes an entirely superseded in-flight backlog even when its provider rejects", async () => {
+ it("retains edited rows without replacement vectors when a superseded in-flight provider rejects", async () => {
getSearchIndex().add(observation("a")); getSearchIndex().add(observation("b"));
const pending = deferred<Float32Array[]>();
const entered = deferred<void>();
@@ -161,8 +161,8 @@ describe("search indexing async write ownership", () => {
await entered.promise;
getSearchIndex().remove("a"); getSearchIndex().add(observation("b", "newer"));
pending.reject(new Error("provider offline"));
- expect(await backlog).toEqual({ added: 0, failed: 0, remaining: 0, complete: true });
- expect(getPendingVectorBackfillCount()).toBe(0);
+ expect(await backlog).toEqual({ added: 0, failed: 0, remaining: 1, complete: false });
+ expect(getPendingVectorBackfillCount()).toBe(1);
expect(vectors.size).toBe(0);
jobs.forEach((j) => j.revision.release());
});
diff --git a/test/vector-backfill-recovery.test.ts b/test/vector-backfill-recovery.test.ts
new file mode 100644
index 00000000..10b00928
--- /dev/null
+++ b/test/vector-backfill-recovery.test.ts
@@ -0,0 +1,483 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { IndexPersistence } from "../src/state/index-persistence.js";
+import { VectorIndex } from "../src/state/vector-index.js";
+import { backfillVectorBacklog, getPendingVectorBackfillCount, getSearchIndex, indexRecords, rebuildKeywordIndex, setEmbeddingProvider, setIndexPersistence, setVectorIndex, vectorIndexRemove } from "../src/functions/search.js";
+import type { CompressedObservation } from "../src/types.js";
+import { KV } from "../src/state/schema.js";
+
+function storage() {
+ const rows = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string) => (rows.get(scope)?.get(key) ?? null) as T | null,
+ set: async <T>(scope: string, key: string, value: T) => {
+ if (!rows.has(scope)) rows.set(scope, new Map());
+ rows.get(scope)!.set(key, structuredClone(value));
+ return value;
+ },
+ delete: async (scope: string, key: string) => { rows.get(scope)?.delete(key); },
+ list: async <T>(scope: string) => [...(rows.get(scope)?.values() ?? [])] as T[],
+ };
+}
+const provider = (dimensions = 3) => ({
+ name: "fixture", dimensions,
+ embed: vi.fn(async () => new Float32Array(dimensions).fill(1)),
+ embedBatch: vi.fn(async (texts: string[]) => texts.map(() => new Float32Array(dimensions).fill(1))),
+});
+async function seed(kv: ReturnType<typeof storage>, count = 3) {
+ await kv.set(KV.sessions, "session", { id: "session", project: "fixture", observationCount: count });
+ for (let i = 0; i < count; i++) await kv.set(KV.observations("session"), `obs_${i}`, {
+ id: `obs_${i}`, sessionId: "session", timestamp: "2020-01-01T00:00:00.000Z", type: "discovery",
+ title: `fixture ${i}`, narrative: `record ${i}`, facts: [], concepts: [], files: [], importance: 5,
+ });
+}
+const active: IndexPersistence[] = [];
+async function boot(kv: ReturnType<typeof storage>, embedding = provider()) {
+ getSearchIndex().clear();
+ const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector);
+ active.push(persistence);
+ setVectorIndex(vector); setEmbeddingProvider(embedding); setIndexPersistence(persistence);
+ const loaded = await persistence.load();
+ if (loaded.vector) vector.restoreFrom(loaded.vector);
+ await persistence.replayPendingLog(embedding.dimensions);
+ return { vector, persistence, loaded, embedding };
+}
+afterEach(() => {
+ active.splice(0).forEach((p) => p.stop());
+ vi.unstubAllEnvs(); getSearchIndex().clear(); setVectorIndex(null); setEmbeddingProvider(null); setIndexPersistence(null);
+});
+
+describe("vector recovery across checkpoints", () => {
+ it("retains a whole-store backlog beyond the cap and resumes old rows after a checkpoint and restart", async () => {
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL_MAX", "2");
+ const kv = storage(); await seed(kv);
+ const first = await boot(kv);
+ await first.persistence.markBackfillSince(null);
+ const plan = await first.persistence.readBackfillMarker();
+ expect(plan).toBeNull();
+ const scan = await rebuildKeywordIndex(kv as never, plan);
+ const result = await backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0, remainingAfter: scan.fullBackfillPending });
+ expect(result).toMatchObject({ added: 2, remaining: 1, complete: false });
+ expect(getPendingVectorBackfillCount()).toBe(1);
+ first.persistence.stop();
+ const second = await boot(kv);
+ expect(second.vector.size).toBe(2);
+ const resumed = await rebuildKeywordIndex(kv as never, await second.persistence.readBackfillMarker());
+ expect(resumed.vectorJobs.map((job) => job.id)).toEqual(["obs_2"]);
+ expect(await backfillVectorBacklog(resumed.vectorJobs, { pauseMs: 0 })).toMatchObject({ added: 1, remaining: 0, complete: true });
+ expect(second.vector.size).toBe(3);
+ });
+ it("keeps a dimension reset as whole-store recovery even after saving an empty checkpoint", async () => {
+ const kv = storage(); await seed(kv);
+ const first = await boot(kv, provider(2));
+ first.vector.add("obs_0", "session", new Float32Array([1, 1]));
+ await first.persistence.save(); first.persistence.stop();
+ const second = await boot(kv);
+ const since = await second.persistence.prepareBackfill(second.loaded, true);
+ for (const [id] of second.vector.entries()) second.vector.remove(id);
+ await second.persistence.save(); second.persistence.stop();
+ expect(since).toBeNull();
+ const third = await boot(kv);
+ const resumed = await third.persistence.prepareBackfill(third.loaded);
+ const scan = await rebuildKeywordIndex(kv as never, resumed);
+ expect(scan.vectorJobs).toHaveLength(0);
+ expect(scan.fullBackfillPending).toBe(3);
+ expect(third.embedding.embedBatch).not.toHaveBeenCalled();
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ expect((await rebuildKeywordIndex(kv as never, resumed)).vectorJobs).toHaveLength(3);
+ });
+
+ it("retains recovery intent when a provider is replaced during embedding", async () => {
+ const kv = storage(); await seed(kv);
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ const first = await boot(kv);
+ let release!: (embeddings: Float32Array[]) => void;
+ let entered!: () => void;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ first.embedding.embedBatch.mockImplementationOnce(async () => {
+ entered(); return new Promise<Float32Array[]>((resolve) => { release = resolve; });
+ });
+ const scan = await rebuildKeywordIndex(kv as never, null);
+ const pending = backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0 });
+ await called;
+ setEmbeddingProvider(provider(4));
+ release(scan.vectorJobs.map(() => new Float32Array(3).fill(1)));
+ expect(await pending).toMatchObject({ added: 0, complete: false, remaining: 3 });
+ expect(first.vector.size).toBe(0);
+ expect(getPendingVectorBackfillCount()).toBe(3);
+ });
+
+ it("does not let a concurrent incremental marker overwrite whole-store recovery", async () => {
+ const kv = storage();
+ let release!: () => void;
+ let entered!: () => void;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ const delayed = { ...kv, set: async <T>(scope: string, key: string, value: T) => {
+ if (key === "vectors:backfill" && (value as { since?: unknown }).since !== null) {
+ entered(); await new Promise<void>((resolve) => { release = resolve; });
+ }
+ return kv.set(scope, key, value);
+ } };
+ const p = new IndexPersistence(delayed as never, new VectorIndex()); active.push(p);
+ const incremental = p.markBackfillSince("2026-01-01T00:00:00Z");
+ await called;
+ const full = p.markBackfillSince(null);
+ await new Promise((resolve) => setImmediate(resolve));
+ release(); await Promise.all([incremental, full]);
+ expect(await p.readBackfillMarker()).toBeNull();
+ });
+
+ it("preserves legacy incremental markers and refuses to advance recovery when marker storage fails", async () => {
+ const kv = storage();
+ await kv.set(KV.bm25Index, "vectors:backfill", { v: 1, since: "2020-01-01T00:00:00Z" });
+ const p = new IndexPersistence(kv as never, new VectorIndex()); active.push(p);
+ const loaded = { state: "buckets" as const, savedAt: "2026-01-01T00:00:00Z", vector: new VectorIndex() };
+ expect(await p.prepareBackfill(loaded)).toBe("2020-01-01T00:00:00Z");
+ const broken = new IndexPersistence({ ...kv, get: async () => { throw new Error("state read failed"); } } as never, new VectorIndex()); active.push(broken);
+ expect(await broken.prepareBackfill(loaded)).toBeUndefined();
+ expect(broken.isRecoveryPaused()).toBe(true);
+ expect(await p.readBackfillMarker()).toBe("2020-01-01T00:00:00Z");
+ const unavailable = await p.prepareBackfill({ vector: null, state: "unavailable", savedAt: null }, true);
+ expect(unavailable).toBeUndefined();
+ expect(await p.readBackfillMarker()).toBe("2020-01-01T00:00:00Z");
+ });
+
+ it("keeps deferred rows and partial provider failures pending across restart without repeated successful embeddings", async () => {
+ const kv = storage(); await seed(kv, 5);
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all"); vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL_MAX", "3");
+ const first = await boot(kv);
+ first.embedding.embedBatch.mockImplementationOnce(async (texts) => texts.map((_, i) => new Float32Array(i === 1 ? 2 : 3).fill(1)));
+ const plan = await first.persistence.prepareBackfill(first.loaded);
+ const scan = await rebuildKeywordIndex(kv as never, plan);
+ const result = await backfillVectorBacklog(scan.vectorJobs, { remainingAfter: scan.fullBackfillPending, pauseMs: 0 });
+ expect(result).toEqual({ added: 2, failed: 1, remaining: 3, complete: false });
+ expect(getPendingVectorBackfillCount()).toBe(3);
+ first.persistence.stop();
+ const second = await boot(kv);
+ const resumed = await rebuildKeywordIndex(kv as never, await second.persistence.prepareBackfill(second.loaded));
+ expect(resumed.vectorJobs.map((job) => job.id)).toEqual(["obs_1", "obs_3", "obs_4"]);
+ expect(await backfillVectorBacklog(resumed.vectorJobs, { pauseMs: 0 })).toMatchObject({ added: 3, complete: true });
+ expect(second.embedding.embedBatch).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not acknowledge recovery until its vector checkpoint succeeds", async () => {
+ const kv = storage(); let fail = true;
+ const broken = { ...kv, set: async <T>(scope: string, key: string, value: T) => {
+ if (fail && scope.startsWith(`${KV.bm25Index}:vec:`)) throw new Error("checkpoint unavailable");
+ return kv.set(scope, key, value);
+ } };
+ const vector = new VectorIndex(); const persistence = new IndexPersistence(broken as never, vector); active.push(persistence);
+ await persistence.markBackfillSince(null);
+ vector.add("obs_1", "session", new Float32Array([1, 1, 1]));
+ expect(await persistence.completeBackfill()).toBe(false);
+ expect(await persistence.readBackfillMarker()).toBeNull();
+ fail = false;
+ expect(await persistence.completeBackfill()).toBe(true);
+ expect(await persistence.readBackfillMarker()).toBeUndefined();
+ const restarted = await boot(kv);
+ expect(restarted.vector.has("obs_1")).toBe(true);
+ });
+
+ it.each(["edited", "new"] as const)("keeps %s canonical rows missing after a concurrent embedding failure recoverable on restart", async (kind) => {
+ const kv = storage(); await seed(kv, 2);
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ const first = await boot(kv);
+ let release!: (embeddings: Float32Array[]) => void;
+ let entered!: () => void;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ first.embedding.embedBatch.mockImplementationOnce(async () => {
+ entered(); return new Promise<Float32Array[]>((resolve) => { release = resolve; });
+ }).mockRejectedValueOnce(new Error("live embedding unavailable"));
+ const since = await first.persistence.prepareBackfill(first.loaded);
+ const scan = await rebuildKeywordIndex(kv as never, since);
+ const running = backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0,
+ remainingCount: () => getSearchIndex().countMissingDocuments((id) => first.vector.has(id), since),
+ });
+ await called;
+ const id = kind === "edited" ? "obs_0" : "obs_new";
+ const updated: CompressedObservation = { id, sessionId: "session", timestamp: new Date().toISOString(), type: "discovery", title: "updated Cobalt", narrative: "new canonical content", facts: [], concepts: [], files: [], importance: 5 };
+ await kv.set(KV.observations("session"), id, updated);
+ await indexRecords([updated], []);
+ release(scan.vectorJobs.map(() => new Float32Array(3).fill(1)));
+ expect(await running).toMatchObject({ remaining: 1, complete: false });
+ expect(first.vector.has(id)).toBe(false);
+ expect(await first.persistence.readBackfillMarker()).toBeNull();
+ first.persistence.stop();
+ const second = await boot(kv);
+ const resumed = await rebuildKeywordIndex(kv as never, await second.persistence.prepareBackfill(second.loaded));
+ expect(resumed.vectorJobs.map((job) => job.id)).toEqual([id]);
+ expect(await backfillVectorBacklog(resumed.vectorJobs, { pauseMs: 0 })).toMatchObject({ added: 1, complete: true });
+ });
+
+ it.each(["read", "write"] as const)("keeps keyword search available and vector checkpoint unchanged after recovery marker %s fails", async (operation) => {
+ const kv = storage(); await seed(kv);
+ const first = await boot(kv);
+ first.vector.add("obs_0", "session", new Float32Array([1, 1, 1]));
+ await first.persistence.save(); first.persistence.stop();
+ const original = await kv.get(KV.bm25Index, "vectors:meta");
+ const faulty = { ...kv,
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ if (operation === "read" && key === "vectors:backfill") throw new Error("marker read unavailable");
+ return kv.get<T>(scope, key);
+ },
+ set: async <T>(scope: string, key: string, value: T): Promise<T> => {
+ if (operation === "write" && key === "vectors:backfill") throw new Error("marker write unavailable");
+ return kv.set(scope, key, value);
+ },
+ };
+ const second = await boot(faulty);
+ const since = await second.persistence.prepareBackfill(second.loaded, true);
+ expect(since).toBeUndefined(); expect(second.persistence.isRecoveryPaused()).toBe(true);
+ const scan = await rebuildKeywordIndex(faulty as never, since);
+ expect(scan.documents).toBe(3); expect(scan.vectorJobs).toHaveLength(0);
+ expect(getSearchIndex().search("fixture")).toHaveLength(3);
+ second.vector.add("obs_1", "session", new Float32Array([1, 1, 1]));
+ second.persistence.scheduleSave(); await second.persistence.save(); await second.persistence.flushPendingLog();
+ expect(await second.persistence.completeBackfill()).toBe(false);
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toEqual(original);
+ expect(await kv.list(KV.vectorPendingLog)).toEqual([]);
+ expect(second.persistence.status().vector?.lastError).toContain("paused until restart");
+ second.persistence.stop();
+ const third = await boot(kv);
+ expect(third.vector.size).toBe(1);
+ expect(await third.persistence.prepareBackfill(third.loaded, true)).toBeNull();
+ });
+
+ it.each(["coverage", "intent"] as const)("retains recovery if %s changes during the awaited checkpoint", async (change) => {
+ const kv = storage(); let release!: () => void; let entered!: () => void;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ const blocked = { ...kv, set: async <T>(scope: string, key: string, value: T) => {
+ if (scope.startsWith(`${KV.bm25Index}:vec:`)) { entered(); await new Promise<void>((resolve) => { release = resolve; }); }
+ return kv.set(scope, key, value);
+ } };
+ const vector = new VectorIndex(); const persistence = new IndexPersistence(blocked as never, vector); active.push(persistence);
+ await persistence.markBackfillSince(null);
+ vector.add("obs_1", "session", new Float32Array([1, 1, 1]));
+ let covered = true;
+ const completing = persistence.completeBackfill(() => covered);
+ await called;
+ if (change === "coverage") covered = false;
+ else await persistence.markBackfillSince(null);
+ release();
+ expect(await completing).toBe(false);
+ expect(await persistence.readBackfillMarker()).toBeNull();
+ });
+
+ it("defers early capture checkpoints until startup has established durable recovery intent", async () => {
+ const kv = storage(); const vector = new VectorIndex();
+ const persistence = new IndexPersistence(kv as never, vector, { requireRecoveryPlan: true }); active.push(persistence);
+ const loaded = await persistence.load();
+ vector.add("early", "session", new Float32Array([1, 1, 1]));
+ persistence.scheduleSave(); await persistence.save(); await persistence.flushPendingLog();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toBeNull();
+ expect(await kv.list(KV.vectorPendingLog)).toEqual([]);
+ expect(await persistence.prepareBackfill(loaded)).toBeNull();
+ await persistence.save();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toMatchObject({ count: 1 });
+ expect(await persistence.readBackfillMarker()).toBeNull();
+ });
+
+ it("resumes checkpoint recovery intent when the separate marker is absent, then clears it on a later stable checkpoint", async () => {
+ const kv = storage(); await seed(kv);
+ const first = await boot(kv);
+ await first.persistence.prepareBackfill(first.loaded);
+ first.vector.add("obs_0", "session", new Float32Array([1, 1, 1]));
+ await first.persistence.save();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toMatchObject({ backfillSince: null });
+ await kv.delete(KV.bm25Index, "vectors:backfill"); first.persistence.stop();
+ const second = await boot(kv);
+ const since = await second.persistence.prepareBackfill(second.loaded);
+ expect(since).toBeNull();
+ const gated = await rebuildKeywordIndex(kv as never, since);
+ expect(gated.fullBackfillPending).toBe(2); expect(second.embedding.embedBatch).not.toHaveBeenCalled();
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ const scan = await rebuildKeywordIndex(kv as never, since);
+ await backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0 });
+ expect(await second.persistence.completeBackfill()).toBe(true);
+ const completed = await kv.get(KV.bm25Index, "vectors:meta");
+ expect(completed).toMatchObject({ backfillSince: null });
+ await second.persistence.save();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toEqual(completed);
+ second.vector.add("obs_later", "session", new Float32Array([1, 1, 1]));
+ await second.persistence.save();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).not.toHaveProperty("backfillSince");
+ });
+
+ it.each([false, true])("retains old-timestamp snapshot edits during marker deletion when restoring the marker fails=%s", async (failRestore) => {
+ const kv = storage(); await seed(kv, 1);
+ let release!: () => void; let entered!: () => void; let deleted = false;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ const blocked = { ...kv,
+ delete: async (scope: string, key: string) => {
+ if (key === "vectors:backfill") { entered(); await new Promise<void>((resolve) => { release = resolve; }); }
+ await kv.delete(scope, key);
+ if (key === "vectors:backfill") deleted = true;
+ },
+ set: async <T>(scope: string, key: string, value: T) => {
+ if (key === "vectors:backfill" && deleted && failRestore) throw new Error("marker restore unavailable");
+ return kv.set(scope, key, value);
+ },
+ };
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ const first = await boot(blocked);
+ const since = await first.persistence.prepareBackfill(first.loaded);
+ const scan = await rebuildKeywordIndex(blocked as never, since);
+ await backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0 });
+ const completing = first.persistence.completeBackfill(() => getSearchIndex().countMissingDocuments((id) => first.vector.has(id), since) === 0);
+ await called;
+ const updated = { ...(await kv.get<CompressedObservation>(KV.observations("session"), "obs_0"))!, title: "Cobalt changed during deletion" };
+ await kv.set(KV.observations("session"), updated.id, updated);
+ getSearchIndex().remove(updated.id);
+ vectorIndexRemove(updated.id);
+ first.embedding.embedBatch.mockRejectedValueOnce(new Error("live embedding unavailable"));
+ await indexRecords([updated], []);
+ await first.persistence.flushPendingLog();
+ release();
+ expect(await completing).toBe(false);
+ expect(first.persistence.isRecoveryPaused()).toBe(failRestore);
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toMatchObject({ backfillSince: null });
+ first.persistence.stop();
+ const second = await boot(kv);
+ const resumed = await rebuildKeywordIndex(kv as never, await second.persistence.prepareBackfill(second.loaded));
+ expect(resumed.vectorJobs.map((job) => job.id)).toEqual(["obs_0"]);
+ expect(await backfillVectorBacklog(resumed.vectorJobs, { pauseMs: 0 })).toMatchObject({ added: 1, complete: true });
+ });
+
+ it("pauses vector persistence for invalid checkpoint recovery intent without advancing its cutoff", async () => {
+ const kv = storage();
+ const meta = { v: 3, bucketCount: 0, savedAt: "2026-01-01T00:00:00Z", count: 0, backfillSince: "invalid cutoff" };
+ await kv.set(KV.bm25Index, "vectors:meta", meta);
+ const first = await boot(kv);
+ expect(await first.persistence.prepareBackfill(first.loaded)).toBeUndefined();
+ expect(first.persistence.isRecoveryPaused()).toBe(true);
+ expect(first.persistence.status().vector?.lastError).toContain("Invalid persisted vector recovery cutoff");
+ first.vector.add("obs_new", "session", new Float32Array([1, 1, 1]));
+ await first.persistence.save();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toEqual(meta);
+ });
+
+ it.each([["embedding", false], ["deletion", false], ["deletion", true]] as const)("retains incremental jobs moved behind the cutoff during %s with marker restoration failure=%s", async (phase, failRestore) => {
+ const kv = storage(); await seed(kv, 2);
+ const original = await boot(kv);
+ original.vector.add("obs_0", "session", new Float32Array([1, 1, 1]));
+ await original.persistence.save(); original.persistence.stop();
+ const target = { ...(await kv.get<CompressedObservation>(KV.observations("session"), "obs_0"))!, id: "obs_target", timestamp: "2099-01-01T00:00:00Z" };
+ await kv.set(KV.observations("session"), target.id, target);
+ let release!: () => void; let entered!: () => void;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ let markerDeleted = false;
+ const blocked = { ...kv, delete: async (scope: string, key: string) => {
+ if (phase === "deletion" && key === "vectors:backfill") { entered(); await new Promise<void>((resolve) => { release = resolve; }); }
+ await kv.delete(scope, key);
+ if (key === "vectors:backfill") markerDeleted = true;
+ }, set: async <T>(scope: string, key: string, value: T) => {
+ if (key === "vectors:backfill" && markerDeleted && failRestore) throw new Error("promotion unavailable");
+ return kv.set(scope, key, value);
+ } };
+ const first = await boot(blocked);
+ let since = await first.persistence.prepareBackfill(first.loaded);
+ expect(typeof since).toBe("string");
+ const scan = await rebuildKeywordIndex(blocked as never, since);
+ expect(scan.vectorJobs.map((job) => job.id)).toEqual([target.id]);
+ const ids = new Set(scan.vectorJobs.map((job) => job.id));
+ const missing = () => getSearchIndex().countMissingDocuments((id) => first.vector.has(id), since, ids);
+ expect(await first.persistence.beginBackfill()).toBe(true);
+ expect(await first.persistence.readBackfillMarker()).toBeNull();
+ if (phase === "embedding") first.embedding.embedBatch.mockImplementationOnce(async (texts) => {
+ entered(); await new Promise<void>((resolve) => { release = resolve; });
+ return texts.map(() => new Float32Array(3).fill(1));
+ });
+ const backfill = backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0, remainingCount: missing });
+ let completing: Promise<boolean> | undefined;
+ if (phase === "deletion") {
+ expect(await backfill).toMatchObject({ added: 1, complete: true });
+ completing = first.persistence.completeBackfill(() => missing() === 0);
+ }
+ await called;
+ const updated = { ...target, timestamp: "2020-01-01T00:00:00Z", title: "older snapshot content" };
+ await kv.set(KV.observations("session"), target.id, updated);
+ getSearchIndex().remove(target.id); vectorIndexRemove(target.id);
+ first.embedding.embedBatch.mockRejectedValueOnce(new Error("replacement embedding unavailable"));
+ await indexRecords([updated], []);
+ if (phase === "embedding") {
+ await first.persistence.save();
+ expect(await first.persistence.readBackfillMarker()).toBeNull();
+ } else expect(await kv.get(KV.bm25Index, "vectors:meta")).toMatchObject({ backfillSince: null });
+ release();
+ if (completing) expect(await completing).toBe(false);
+ else expect(await backfill).toMatchObject({ complete: false });
+ expect(await first.persistence.readBackfillMarker()).toBe(failRestore ? undefined : null);
+ expect(first.persistence.isRecoveryPaused()).toBe(failRestore);
+ first.persistence.stop();
+ const restarted = await boot(kv);
+ const resumed = await rebuildKeywordIndex(kv as never, await restarted.persistence.prepareBackfill(restarted.loaded));
+ expect(resumed.vectorJobs).toHaveLength(0);
+ expect(resumed.fullBackfillPending).toBe(2);
+ expect(restarted.embedding.embedBatch).not.toHaveBeenCalled();
+ vi.stubEnv("AGENTMEMORY_VECTOR_BACKFILL", "all");
+ const optedIn = await rebuildKeywordIndex(kv as never, await restarted.persistence.readBackfillMarker());
+ expect(optedIn.vectorJobs.map((job) => job.id).sort()).toEqual(["obs_1", "obs_target"]);
+ });
+
+ it("pauses before embedding or checkpointing when the incremental recovery barrier cannot be persisted", async () => {
+ const kv = storage(); let fail = false;
+ const faulty = { ...kv, set: async <T>(scope: string, key: string, value: T) => {
+ if (fail && key === "vectors:backfill") throw new Error("promotion unavailable");
+ return kv.set(scope, key, value);
+ } };
+ const vector = new VectorIndex(); const persistence = new IndexPersistence(faulty as never, vector); active.push(persistence);
+ await persistence.markBackfillSince("2020-01-01T00:00:00Z");
+ vector.add("obs_0", "session", new Float32Array([1, 1, 1])); await persistence.save();
+ const previous = await kv.get(KV.bm25Index, "vectors:meta");
+ fail = true;
+ expect(await persistence.beginBackfill()).toBe(false);
+ vector.add("obs_1", "session", new Float32Array([1, 1, 1])); await persistence.save();
+ expect(persistence.isRecoveryPaused()).toBe(true);
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toEqual(previous);
+ });
+
+ it("processes only the planned incremental rows without opt-in after saving conservative restart intent", async () => {
+ const kv = storage(); await seed(kv, 2);
+ const original = await boot(kv);
+ original.vector.add("obs_0", "session", new Float32Array([1, 1, 1])); await original.persistence.save(); original.persistence.stop();
+ const next = { ...(await kv.get<CompressedObservation>(KV.observations("session"), "obs_0"))!, id: "obs_next", timestamp: "2099-01-01T00:00:00Z" };
+ await kv.set(KV.observations("session"), next.id, next);
+ const first = await boot(kv); const since = await first.persistence.prepareBackfill(first.loaded);
+ const scan = await rebuildKeywordIndex(kv as never, since);
+ expect(scan.vectorJobs.map((job) => job.id)).toEqual(["obs_next"]);
+ expect(await first.persistence.beginBackfill()).toBe(true);
+ expect(await first.persistence.readBackfillMarker()).toBeNull();
+ expect(await backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0 })).toMatchObject({ added: 1, complete: true });
+ expect(first.embedding.embedBatch).toHaveBeenCalledTimes(1);
+ expect(first.vector.has("obs_1")).toBe(false);
+ expect(await first.persistence.completeBackfill()).toBe(true);
+ expect(await first.persistence.readBackfillMarker()).toBeUndefined();
+ });
+
+ it("recovers an incremental replacement interrupted before any later vector checkpoint", async () => {
+ const kv = storage(); await seed(kv, 2);
+ const original = await boot(kv);
+ original.vector.add("obs_0", "session", new Float32Array([1, 1, 1])); await original.persistence.save(); original.persistence.stop();
+ const checkpoint = await kv.get(KV.bm25Index, "vectors:meta");
+ const target = { ...(await kv.get<CompressedObservation>(KV.observations("session"), "obs_0"))!, id: "obs_target", timestamp: "2099-01-01T00:00:00Z" };
+ await kv.set(KV.observations("session"), target.id, target);
+ const first = await boot(kv);
+ const scan = await rebuildKeywordIndex(kv as never, await first.persistence.prepareBackfill(first.loaded));
+ expect(await first.persistence.beginBackfill()).toBe(true);
+ let release!: (embeddings: Float32Array[]) => void; let entered!: () => void;
+ const called = new Promise<void>((resolve) => { entered = resolve; });
+ first.embedding.embedBatch.mockImplementationOnce(async () => { entered(); return new Promise<Float32Array[]>((resolve) => { release = resolve; }); }).mockRejectedValueOnce(new Error("live unavailable"));
+ const pending = backfillVectorBacklog(scan.vectorJobs, { pauseMs: 0 }); await called;
+ const restored = { ...target, timestamp: "2020-01-01T00:00:00Z", title: "restored old version" };
+ await kv.set(KV.observations("session"), restored.id, restored); await indexRecords([restored], []);
+ first.persistence.stop();
+ expect(await kv.get(KV.bm25Index, "vectors:meta")).toEqual(checkpoint);
+ const restarted = await boot(kv);
+ const resumed = await rebuildKeywordIndex(kv as never, await restarted.persistence.prepareBackfill(restarted.loaded));
+ expect(resumed.vectorJobs).toHaveLength(0); expect(resumed.fullBackfillPending).toBe(2);
+ expect(restarted.embedding.embedBatch).not.toHaveBeenCalled();
+ release([new Float32Array([1, 1, 1])]); await pending;
+ });
+
+});
diff --git a/test/vector-pending-log.test.ts b/test/vector-pending-log.test.ts
index 64ef481e..8f39300d 100644
--- a/test/vector-pending-log.test.ts
+++ b/test/vector-pending-log.test.ts
@@ -427,7 +427,7 @@ describe("durable vector backlog", () => {
expect(second.vector.size).toBe(12);
expect(resumedCalls.n).toBe(2);
await second.persistence.clearBackfillMarker();
- expect(await second.persistence.readBackfillMarker()).toBeNull();
+ expect(await second.persistence.readBackfillMarker()).toBeUndefined();
} finally {
delete process.env.REBUILD_EMBED_BATCH_SIZE;
}