Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: shard persisted search indexes

Rokurolize33a23d31946d
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 7e484f57..9dde418b 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
import { SearchIndex } from "./search-index.js";
import { VectorIndex } from "./vector-index.js";
import type { StateKV } from "./kv.js";
@@ -6,6 +7,39 @@ import { logger } from "../logger.js";
const DEBOUNCE_MS = 5000;
const FAILURE_LOG_THROTTLE_MS = 60_000;
+const BM25_KEY = "data";
+const BM25_MANIFEST_KEY = "data:manifest";
+const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`;
+const VECTOR_KEY = "vectors";
+const VECTOR_MANIFEST_KEY = "vectors:manifest";
+const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`;
+const INDEX_SHARD_KEY = "data";
+const DEFAULT_INDEX_SHARD_CHARS = 2_000_000;
+
+type IndexShardManifest = {
+ v: 1;
+ generation?: string;
+ shards: Array<{ scope: string; key: string; chars: number }>;
+ chars: number;
+};
+
+type IndexPersistenceOptions = {
+ shardChars?: number;
+ createGeneration?: () => string;
+};
+
+function shardChars(options: IndexPersistenceOptions): number {
+ const configured = options.shardChars;
+ return typeof configured === "number" &&
+ Number.isFinite(configured) &&
+ configured > 0
+ ? Math.floor(configured)
+ : DEFAULT_INDEX_SHARD_CHARS;
+}
+
+function createIndexGeneration(): string {
+ return `${Date.now().toString(36)}-${randomUUID().replace(/-/g, "")}`;
+}
export class IndexPersistence {
private timer: ReturnType<typeof setTimeout> | null = null;
@@ -15,6 +49,7 @@ export class IndexPersistence {
private kv: StateKV,
private bm25: SearchIndex,
private vector: VectorIndex | null,
+ private options: IndexPersistenceOptions = {},
) {}
scheduleSave(): void {
@@ -34,9 +69,9 @@ export class IndexPersistence {
this.timer = null;
}
try {
- await this.kv.set(KV.bm25Index, "data", this.bm25.serialize());
+ await this.saveBm25Index(this.bm25.serialize());
if (this.vector && this.vector.size > 0) {
- await this.kv.set(KV.bm25Index, "vectors", this.vector.serialize());
+ await this.saveVectorIndex(this.vector.serialize());
}
} catch (err) {
this.logFailure(err);
@@ -50,16 +85,12 @@ export class IndexPersistence {
let bm25: SearchIndex | null = null;
let vector: VectorIndex | null = null;
- const bm25Data = await this.kv
- .get<string>(KV.bm25Index, "data")
- .catch(() => null);
+ const bm25Data = await this.loadBm25Data();
if (bm25Data && typeof bm25Data === "string") {
bm25 = SearchIndex.deserialize(bm25Data);
}
- const vecData = await this.kv
- .get<string>(KV.bm25Index, "vectors")
- .catch(() => null);
+ const vecData = await this.loadVectorData();
if (vecData && typeof vecData === "string") {
vector = VectorIndex.deserialize(vecData);
}
@@ -92,4 +123,153 @@ export class IndexPersistence {
: undefined,
});
}
+
+ private async saveBm25Index(serialized: string): Promise<void> {
+ await this.saveShardedIndex(
+ serialized,
+ BM25_MANIFEST_KEY,
+ BM25_KEY,
+ BM25_SHARD_SCOPE_PREFIX,
+ );
+ }
+
+ private async saveVectorIndex(serialized: string): Promise<void> {
+ await this.saveShardedIndex(
+ serialized,
+ VECTOR_MANIFEST_KEY,
+ VECTOR_KEY,
+ VECTOR_SHARD_SCOPE_PREFIX,
+ );
+ }
+
+ private async saveShardedIndex(
+ serialized: string,
+ manifestKey: string,
+ legacyKey: string,
+ scopePrefix: string,
+ ): Promise<void> {
+ const previous = await this.kv
+ .get<IndexShardManifest>(KV.bm25Index, manifestKey)
+ .catch(() => null);
+ const generation =
+ this.options.createGeneration?.() ?? createIndexGeneration();
+ const chunkChars = shardChars(this.options);
+ const shards: IndexShardManifest["shards"] = [];
+
+ for (let offset = 0; offset < serialized.length; offset += chunkChars) {
+ const shardIndex = shards.length;
+ const scope = `${scopePrefix}${generation}:${String(shardIndex).padStart(
+ 5,
+ "0",
+ )}`;
+ const chunk = serialized.slice(offset, offset + chunkChars);
+ shards.push({ scope, key: INDEX_SHARD_KEY, chars: chunk.length });
+ try {
+ await this.kv.set(scope, INDEX_SHARD_KEY, chunk);
+ } catch (err) {
+ await this.deleteShards(shards);
+ throw err;
+ }
+ }
+
+ await this.kv.set<IndexShardManifest>(KV.bm25Index, manifestKey, {
+ v: 1,
+ generation,
+ shards,
+ chars: serialized.length,
+ });
+
+ await this.kv.delete(KV.bm25Index, legacyKey).catch(() => {});
+ if (previous?.v === 1 && Array.isArray(previous.shards)) {
+ const currentShardIds = new Set(
+ shards.map((shard) => `${shard.scope}\0${shard.key}`),
+ );
+ for (const shard of previous.shards) {
+ if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue;
+ await this.deleteShards([shard]);
+ }
+ }
+ }
+
+ private async deleteShards(
+ shards: IndexShardManifest["shards"],
+ ): Promise<void> {
+ for (const shard of shards) {
+ await this.kv.delete(shard.scope, shard.key).catch(() => {});
+ }
+ }
+
+ private async loadBm25Data(): Promise<string | null> {
+ return this.loadShardedData(BM25_KEY, BM25_MANIFEST_KEY, "BM25");
+ }
+
+ private async loadVectorData(): Promise<string | null> {
+ return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector");
+ }
+
+ private async loadShardedData(
+ legacyKey: string,
+ manifestKey: string,
+ label: string,
+ ): Promise<string | null> {
+ const manifest = await this.kv
+ .get<IndexShardManifest>(KV.bm25Index, manifestKey)
+ .catch(() => null);
+ if (manifest !== null) {
+ return this.loadManifestData(manifest, label);
+ }
+
+ const legacy = await this.kv
+ .get<string>(KV.bm25Index, legacyKey)
+ .catch(() => null);
+ if (legacy && typeof legacy === "string") return legacy;
+ return null;
+ }
+
+ private async loadManifestData(
+ manifest: IndexShardManifest,
+ label: string,
+ ): Promise<string | null> {
+ if (
+ manifest.v !== 1 ||
+ !Array.isArray(manifest.shards) ||
+ manifest.shards.length === 0
+ ) {
+ logger.warn(`index persistence: ${label} shard manifest invalid`);
+ return null;
+ }
+ const chunks: string[] = [];
+ let chars = 0;
+ for (const shard of manifest.shards) {
+ const chunk = await this.kv
+ .get<string>(shard.scope, shard.key)
+ .catch(() => null);
+ if (typeof chunk !== "string") {
+ logger.warn(`index persistence: ${label} shard missing`, {
+ scope: shard.scope,
+ key: shard.key,
+ });
+ return null;
+ }
+ if (chunk.length !== shard.chars) {
+ logger.warn(`index persistence: ${label} shard length mismatch`, {
+ scope: shard.scope,
+ key: shard.key,
+ expected: shard.chars,
+ actual: chunk.length,
+ });
+ return null;
+ }
+ chunks.push(chunk);
+ chars += chunk.length;
+ }
+ if (chars !== manifest.chars) {
+ logger.warn(`index persistence: ${label} total length mismatch`, {
+ expected: manifest.chars,
+ actual: chars,
+ });
+ return null;
+ }
+ return chunks.join("");
+ }
}
diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts
index fd60ab24..537f0255 100644
--- a/test/index-persistence.test.ts
+++ b/test/index-persistence.test.ts
@@ -4,6 +4,19 @@ import { SearchIndex } from "../src/state/search-index.js";
import { VectorIndex } from "../src/state/vector-index.js";
import type { CompressedObservation } from "../src/types.js";
+const BM25_SCOPE = "mem:index:bm25";
+const BM25_LEGACY_KEY = "data";
+const BM25_MANIFEST_KEY = "data:manifest";
+const VECTOR_LEGACY_KEY = "vectors";
+const VECTOR_MANIFEST_KEY = "vectors:manifest";
+
+type TestIndexShardManifest = {
+ v: 1;
+ generation?: string;
+ shards: Array<{ scope: string; key: string; chars: number }>;
+ chars: number;
+};
+
function mockKV() {
const store = new Map<string, Map<string, unknown>>();
return {
@@ -25,6 +38,8 @@ function mockKV() {
};
}
+type MockKV = ReturnType<typeof mockKV>;
+
function makeObs(
overrides: Partial<CompressedObservation> = {},
): CompressedObservation {
@@ -44,6 +59,27 @@ function makeObs(
};
}
+function makeBm25(id: string, title: string): SearchIndex {
+ const bm25 = new SearchIndex();
+ bm25.add(makeObs({ id, title, narrative: `${title} narrative` }));
+ return bm25;
+}
+
+function makeVector(id = "obs_1"): VectorIndex {
+ const vector = new VectorIndex();
+ vector.add(id, "ses_1", new Float32Array([0.1, 0.2, 0.3]));
+ return vector;
+}
+
+async function getBm25Manifest(kv: MockKV): Promise<TestIndexShardManifest> {
+ const manifest = await kv.get<TestIndexShardManifest>(
+ BM25_SCOPE,
+ BM25_MANIFEST_KEY,
+ );
+ expect(manifest).not.toBeNull();
+ return manifest!;
+}
+
describe("IndexPersistence", () => {
let kv: ReturnType<typeof mockKV>;
@@ -70,10 +106,90 @@ describe("IndexPersistence", () => {
expect(results.length).toBe(1);
});
+ it("saves BM25 index shards outside the BM25 metadata scope", async () => {
+ const bm25 = new SearchIndex();
+ bm25.add(
+ makeObs({
+ id: "obs_1",
+ title: "auth handler ".repeat(40),
+ narrative: "JWT middleware validation ".repeat(40),
+ }),
+ );
+
+ const persistence = new IndexPersistence(kv as never, bm25, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_bm25",
+ });
+ await persistence.save();
+
+ const manifest = await getBm25Manifest(kv);
+ expect(manifest.generation).toBe("gen_bm25");
+ expect(manifest.shards.length).toBeGreaterThan(1);
+ expect(manifest.shards[0].scope).toContain(":gen_bm25:");
+ await expect(kv.get(BM25_SCOPE, BM25_LEGACY_KEY)).resolves.toBeNull();
+ await expect(
+ kv.get(manifest.shards[0].scope, manifest.shards[0].key),
+ ).resolves.toEqual(expect.any(String));
+
+ const loaded = await persistence.load();
+ expect(loaded.bm25).not.toBeNull();
+ expect(loaded.bm25!.search("auth").length).toBe(1);
+ });
+
+ it("loads legacy monolithic BM25 and vector snapshots", async () => {
+ const bm25 = makeBm25("obs_1", "legacy auth handler");
+ const vector = makeVector("obs_1");
+ await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, bm25.serialize());
+ await kv.set(BM25_SCOPE, VECTOR_LEGACY_KEY, vector.serialize());
+
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+
+ expect(loaded.bm25).not.toBeNull();
+ expect(loaded.bm25!.search("legacy").length).toBe(1);
+ expect(loaded.vector).not.toBeNull();
+ expect(loaded.vector!.size).toBe(1);
+ });
+
+ it("loads sharded manifests that omit optional generation metadata", async () => {
+ const bm25 = makeBm25("obs_1", "deterministic shard auth");
+ const serialized = bm25.serialize();
+ const chunks = [serialized.slice(0, 50), serialized.slice(50)];
+ await kv.set("mem:index:bm25:bm25:00000", "data", chunks[0]);
+ await kv.set("mem:index:bm25:bm25:00001", "data", chunks[1]);
+ await kv.set<TestIndexShardManifest>(BM25_SCOPE, BM25_MANIFEST_KEY, {
+ v: 1,
+ chars: serialized.length,
+ shards: [
+ {
+ scope: "mem:index:bm25:bm25:00000",
+ key: "data",
+ chars: chunks[0].length,
+ },
+ {
+ scope: "mem:index:bm25:bm25:00001",
+ key: "data",
+ chars: chunks[1].length,
+ },
+ ],
+ });
+
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+
+ expect(loaded.bm25).not.toBeNull();
+ expect(loaded.bm25!.search("deterministic").length).toBe(1);
+ });
+
it("saves and loads vector index round-trip", async () => {
const bm25 = new SearchIndex();
- const vector = new VectorIndex();
- vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3]));
+ const vector = makeVector();
const persistence = new IndexPersistence(kv as never, bm25, vector);
await persistence.save();
@@ -83,6 +199,346 @@ describe("IndexPersistence", () => {
expect(loaded.vector!.size).toBe(1);
});
+ it("saves vector index shards outside the BM25 scope", async () => {
+ const bm25 = new SearchIndex();
+ const vector = new VectorIndex();
+ vector.add(
+ "obs_1",
+ "ses_1",
+ new Float32Array(Array.from({ length: 32 }, (_, i) => i)),
+ );
+
+ const persistence = new IndexPersistence(kv as never, bm25, vector, {
+ shardChars: 40,
+ createGeneration: () => "gen_vector",
+ });
+ await persistence.save();
+
+ const manifest = await kv.get<TestIndexShardManifest>(
+ BM25_SCOPE,
+ VECTOR_MANIFEST_KEY,
+ );
+ expect(manifest).not.toBeNull();
+ expect(manifest!.generation).toBe("gen_vector");
+ expect(manifest!.shards.length).toBeGreaterThan(1);
+ expect(manifest!.shards[0].scope).toContain(":gen_vector:");
+ await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull();
+ await expect(
+ kv.get(manifest!.shards[0].scope, manifest!.shards[0].key),
+ ).resolves.toEqual(expect.any(String));
+
+ const loaded = await persistence.load();
+ expect(loaded.vector).not.toBeNull();
+ expect(loaded.vector!.size).toBe(1);
+ });
+
+ it("avoids one oversized state::set string payload for persisted indexes", async () => {
+ const maxStringPayloadChars = 80;
+ const bm25 = new SearchIndex();
+ bm25.add(
+ makeObs({
+ id: "obs_1",
+ title: "large persisted snapshot ".repeat(40),
+ narrative: "oversized state set reproduction ".repeat(40),
+ }),
+ );
+ const vector = new VectorIndex();
+ vector.add(
+ "obs_1",
+ "ses_1",
+ new Float32Array(Array.from({ length: 64 }, (_, i) => i / 10)),
+ );
+ const guardedKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (
+ typeof data === "string" &&
+ data.length > maxStringPayloadChars
+ ) {
+ throw new Error(`oversized state::set payload: ${scope}/${key}`);
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+
+ await new IndexPersistence(guardedKv as never, bm25, vector, {
+ shardChars: maxStringPayloadChars,
+ createGeneration: () => "gen_payload_limit",
+ }).save();
+
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("oversized").length).toBe(1);
+ expect(loaded.vector!.size).toBe(1);
+ });
+
+ it("keeps the previous generation when a shard write fails before manifest commit", async () => {
+ const previous = makeBm25("obs_old", "alpha previous snapshot");
+ await new IndexPersistence(kv as never, previous, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_old",
+ }).save();
+ const previousManifest = await getBm25Manifest(kv);
+
+ let newShardWrites = 0;
+ const failingKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope.includes(":gen_new:")) {
+ newShardWrites += 1;
+ if (newShardWrites === 2) throw new Error("shard write failed");
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+
+ const next = makeBm25("obs_new", "bravo new snapshot");
+ await new IndexPersistence(failingKv as never, next, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_new",
+ }).save();
+
+ await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
+ previousManifest,
+ );
+ await expect(
+ kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
+ ).resolves.toBeNull();
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("alpha").length).toBe(1);
+ expect(loaded.bm25!.search("bravo").length).toBe(0);
+ });
+
+ it("keeps the previous generation when manifest set rejects before commit", async () => {
+ const previous = makeBm25("obs_old", "alpha previous snapshot");
+ await new IndexPersistence(kv as never, previous, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_old",
+ }).save();
+ const previousManifest = await getBm25Manifest(kv);
+
+ const failingKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {
+ throw new Error("manifest write failed");
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+
+ const next = makeBm25("obs_new", "bravo new snapshot");
+ await new IndexPersistence(failingKv as never, next, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_new",
+ }).save();
+
+ await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
+ previousManifest,
+ );
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("alpha").length).toBe(1);
+ expect(loaded.bm25!.search("bravo").length).toBe(0);
+ });
+
+ it("keeps a generation loadable when manifest set commits before rejecting", async () => {
+ const previous = makeBm25("obs_old", "alpha previous snapshot");
+ await new IndexPersistence(kv as never, previous, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_old",
+ }).save();
+
+ const failingKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {
+ await kv.set(scope, key, data);
+ throw new Error("manifest write timed out after commit");
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+
+ const next = makeBm25("obs_new", "bravo new snapshot");
+ await new IndexPersistence(failingKv as never, next, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_new",
+ }).save();
+
+ const manifest = await getBm25Manifest(kv);
+ expect(manifest.generation).toBe("gen_new");
+ await expect(
+ kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
+ ).resolves.toEqual(expect.any(String));
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("bravo").length).toBe(1);
+ });
+
+ it("deletes a shard that committed before set rejected", async () => {
+ const previous = makeBm25("obs_old", "alpha previous snapshot");
+ await new IndexPersistence(kv as never, previous, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_old",
+ }).save();
+ const previousManifest = await getBm25Manifest(kv);
+
+ const failingKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === "mem:index:bm25:bm25:gen_new:00000") {
+ await kv.set(scope, key, data);
+ throw new Error("state::set timed out after commit");
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+
+ const next = makeBm25("obs_new", "bravo new snapshot");
+ await new IndexPersistence(failingKv as never, next, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_new",
+ }).save();
+
+ await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
+ previousManifest,
+ );
+ await expect(
+ kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
+ ).resolves.toBeNull();
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("alpha").length).toBe(1);
+ expect(loaded.bm25!.search("bravo").length).toBe(0);
+ });
+
+ it("loads the new generation even when old generation cleanup fails", async () => {
+ const previous = makeBm25("obs_old", "alpha previous snapshot");
+ await new IndexPersistence(kv as never, previous, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_old",
+ }).save();
+
+ const cleanupKv = {
+ ...kv,
+ delete: vi.fn(async () => {
+ throw new Error("cleanup failed");
+ }),
+ };
+ const next = makeBm25("obs_new", "bravo new snapshot");
+ await new IndexPersistence(cleanupKv as never, next, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_new",
+ }).save();
+
+ const manifest = await getBm25Manifest(kv);
+ expect(manifest.generation).toBe("gen_new");
+ expect(cleanupKv.delete).toHaveBeenCalled();
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("bravo").length).toBe(1);
+ expect(loaded.bm25!.search("alpha").length).toBe(0);
+ });
+
+ it("keeps the previous vector generation when vector save fails after BM25 publish", async () => {
+ const previousBm25 = makeBm25("obs_old", "alpha previous snapshot");
+ const previousVector = makeVector("obs_old");
+ await new IndexPersistence(kv as never, previousBm25, previousVector, {
+ shardChars: 80,
+ createGeneration: () => "gen_old",
+ }).save();
+
+ const failingKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) {
+ throw new Error("vector manifest write failed");
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+ const nextBm25 = makeBm25("obs_new", "bravo new snapshot");
+ const nextVector = new VectorIndex();
+ nextVector.add("obs_new", "ses_1", new Float32Array([0.4, 0.5, 0.6]));
+
+ await new IndexPersistence(failingKv as never, nextBm25, nextVector, {
+ shardChars: 80,
+ createGeneration: () => "gen_new",
+ }).save();
+
+ await expect(
+ kv.get("mem:index:bm25:vectors:gen_new:00000", "data"),
+ ).resolves.toEqual(expect.any(String));
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+ expect(loaded.bm25!.search("bravo").length).toBe(1);
+ expect(loaded.vector!.size).toBe(1);
+ expect(
+ loaded.vector!.search(new Float32Array([0.1, 0.2, 0.3]))[0]?.obsId,
+ ).toBe("obs_old");
+ });
+
+ it("fails closed when a manifest shard is missing", async () => {
+ const bm25 = makeBm25("obs_1", "alpha sharded snapshot");
+ await new IndexPersistence(kv as never, bm25, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_missing",
+ }).save();
+ const manifest = await getBm25Manifest(kv);
+ await kv.delete(manifest.shards[0].scope, manifest.shards[0].key);
+
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+
+ expect(loaded.bm25).toBeNull();
+ });
+
+ it("fails closed when a manifest shard length mismatches", async () => {
+ const bm25 = makeBm25("obs_1", "alpha sharded snapshot");
+ await new IndexPersistence(kv as never, bm25, null, {
+ shardChars: 80,
+ createGeneration: () => "gen_mismatch",
+ }).save();
+ const manifest = await getBm25Manifest(kv);
+ const firstShard = manifest.shards[0];
+ const chunk = await kv.get<string>(firstShard.scope, firstShard.key);
+ await kv.set(firstShard.scope, firstShard.key, `${chunk}x`);
+
+ const loaded = await new IndexPersistence(
+ kv as never,
+ new SearchIndex(),
+ null,
+ ).load();
+
+ expect(loaded.bm25).toBeNull();
+ });
+
it("scheduleSave debounces multiple calls", async () => {
const bm25 = new SearchIndex();
const persistence = new IndexPersistence(kv as never, bm25, null);
@@ -91,12 +547,12 @@ describe("IndexPersistence", () => {
persistence.scheduleSave();
persistence.scheduleSave();
- await expect(kv.get("mem:index:bm25", "data")).resolves.toBeNull();
+ await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toBeNull();
vi.advanceTimersByTime(5000);
await vi.runAllTimersAsync();
- const saved = await kv.get<string>("mem:index:bm25", "data");
+ const saved = await kv.get<string>(BM25_SCOPE, BM25_MANIFEST_KEY);
expect(saved).not.toBeNull();
});
@@ -109,7 +565,7 @@ describe("IndexPersistence", () => {
persistence.stop();
vi.advanceTimersByTime(10000);
- const saved = await kv.get<string>("mem:index:bm25", "data");
+ const saved = await kv.get<string>(BM25_SCOPE, BM25_MANIFEST_KEY);
expect(saved).toBeNull();
});