Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: address shard publication review

Rokurolize80088effa285
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts
index 9dde418b..a97d8d4d 100644
--- a/src/state/index-persistence.ts
+++ b/src/state/index-persistence.ts
@@ -30,11 +30,11 @@ type IndexPersistenceOptions = {
function shardChars(options: IndexPersistenceOptions): number {
const configured = options.shardChars;
- return typeof configured === "number" &&
- Number.isFinite(configured) &&
- configured > 0
- ? Math.floor(configured)
- : DEFAULT_INDEX_SHARD_CHARS;
+ if (typeof configured !== "number" || !Number.isFinite(configured)) {
+ return DEFAULT_INDEX_SHARD_CHARS;
+ }
+ const wholeChars = Math.floor(configured);
+ return wholeChars >= 1 ? wholeChars : DEFAULT_INDEX_SHARD_CHARS;
}
function createIndexGeneration(): string {
@@ -172,12 +172,24 @@ export class IndexPersistence {
}
}
- await this.kv.set<IndexShardManifest>(KV.bm25Index, manifestKey, {
+ const nextManifest: IndexShardManifest = {
v: 1,
generation,
shards,
chars: serialized.length,
- });
+ };
+ try {
+ await this.kv.set<IndexShardManifest>(
+ KV.bm25Index,
+ manifestKey,
+ nextManifest,
+ );
+ } catch (err) {
+ if (!(await this.isManifestPublished(manifestKey, nextManifest))) {
+ await this.deleteShards(shards);
+ }
+ throw err;
+ }
await this.kv.delete(KV.bm25Index, legacyKey).catch(() => {});
if (previous?.v === 1 && Array.isArray(previous.shards)) {
@@ -199,6 +211,33 @@ export class IndexPersistence {
}
}
+ private async isManifestPublished(
+ manifestKey: string,
+ expected: IndexShardManifest,
+ ): Promise<boolean> {
+ const published = await this.kv
+ .get<IndexShardManifest>(KV.bm25Index, manifestKey)
+ .catch(() => null);
+ if (
+ published?.v !== 1 ||
+ published.generation !== expected.generation ||
+ published.chars !== expected.chars ||
+ !Array.isArray(published.shards) ||
+ published.shards.length !== expected.shards.length
+ ) {
+ return false;
+ }
+ return published.shards.every((shard, index) => {
+ const expectedShard = expected.shards[index];
+ if (!expectedShard) return false;
+ return (
+ shard.scope === expectedShard.scope &&
+ shard.key === expectedShard.key &&
+ shard.chars === expectedShard.chars
+ );
+ });
+ }
+
private async loadBm25Data(): Promise<string | null> {
return this.loadShardedData(BM25_KEY, BM25_MANIFEST_KEY, "BM25");
}
diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts
index 537f0255..2ccd061b 100644
--- a/test/index-persistence.test.ts
+++ b/test/index-persistence.test.ts
@@ -275,6 +275,33 @@ describe("IndexPersistence", () => {
expect(loaded.vector!.size).toBe(1);
});
+ it("falls back to the default shard size for fractional values below one", async () => {
+ const bm25 = makeBm25("obs_fraction", "fractional shard config");
+ let newShardWrites = 0;
+ const guardedKv = {
+ ...kv,
+ set: vi.fn(async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (scope.includes(":gen_fraction:")) {
+ newShardWrites += 1;
+ if (newShardWrites > 3) {
+ throw new Error("fractional shard size caused zero-width shards");
+ }
+ }
+ return kv.set(scope, key, data);
+ }),
+ };
+
+ await new IndexPersistence(guardedKv as never, bm25, null, {
+ shardChars: 0.5,
+ createGeneration: () => "gen_fraction",
+ }).save();
+
+ const manifest = await getBm25Manifest(kv);
+ expect(manifest.generation).toBe("gen_fraction");
+ expect(manifest.shards.length).toBe(1);
+ expect(newShardWrites).toBe(1);
+ });
+
it("keeps the previous generation when a shard write fails before manifest commit", async () => {
const previous = makeBm25("obs_old", "alpha previous snapshot");
await new IndexPersistence(kv as never, previous, null, {
@@ -343,6 +370,9 @@ describe("IndexPersistence", () => {
await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual(
previousManifest,
);
+ await expect(
+ kv.get("mem:index:bm25:bm25:gen_new:00000", "data"),
+ ).resolves.toBeNull();
const loaded = await new IndexPersistence(
kv as never,
new SearchIndex(),
@@ -488,7 +518,7 @@ describe("IndexPersistence", () => {
await expect(
kv.get("mem:index:bm25:vectors:gen_new:00000", "data"),
- ).resolves.toEqual(expect.any(String));
+ ).resolves.toBeNull();
const loaded = await new IndexPersistence(
kv as never,
new SearchIndex(),