#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix: harden sharded index persistence
Rokurolize0914ad6ac0ea
diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.tsindex a97d8d4d..cadef6bf 100644--- a/src/state/index-persistence.ts+++ b/src/state/index-persistence.ts@@ -1,12 +1,13 @@-import { randomUUID } from "node:crypto";import { SearchIndex } from "./search-index.js";import { VectorIndex } from "./vector-index.js";import type { StateKV } from "./kv.js";-import { KV } from "./schema.js";+import { KV, generateId } from "./schema.js";import { logger } from "../logger.js";+import { safeAudit } from "../functions/audit.js";const DEBOUNCE_MS = 5000;const FAILURE_LOG_THROTTLE_MS = 60_000;+const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence";const BM25_KEY = "data";const BM25_MANIFEST_KEY = "data:manifest";const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`;@@ -38,7 +39,30 @@ function shardChars(options: IndexPersistenceOptions): number {}function createIndexGeneration(): string {- return `${Date.now().toString(36)}-${randomUUID().replace(/-/g, "")}`;+ return generateId("idx");+}++function statePath(scope: string, key: string): string {+ return `${scope}/${key}`;+}++function errorMessage(err: unknown): string {+ return err instanceof Error ? err.message : String(err);+}++function isValidShardDescriptor(+ shard: unknown,+): shard is IndexShardManifest["shards"][number] {+ if (!shard || typeof shard !== "object") return false;+ const candidate = shard as { scope?: unknown; key?: unknown; chars?: unknown };+ return (+ typeof candidate.scope === "string" &&+ candidate.scope.length > 0 &&+ typeof candidate.key === "string" &&+ candidate.key.length > 0 &&+ Number.isInteger(candidate.chars) &&+ candidate.chars >= 0+ );}export class IndexPersistence {@@ -70,7 +94,7 @@ export class IndexPersistence {}try {await this.saveBm25Index(this.bm25.serialize());- if (this.vector && this.vector.size > 0) {+ if (this.vector) {await this.saveVectorIndex(this.vector.serialize());}} catch (err) {@@ -166,8 +190,17 @@ export class IndexPersistence {shards.push({ scope, key: INDEX_SHARD_KEY, chars: chunk.length });try {await this.kv.set(scope, INDEX_SHARD_KEY, chunk);+ await this.auditIndexPersistence("shard_write", [+ statePath(scope, INDEX_SHARD_KEY),+ ], {+ scope,+ key: INDEX_SHARD_KEY,+ manifestKey,+ generation,+ chars: chunk.length,+ });} catch (err) {- await this.deleteShards(shards);+ await this.deleteShards(shards, "shard_write_rollback");throw err;}}@@ -184,30 +217,87 @@ export class IndexPersistence {manifestKey,nextManifest,);+ await this.auditIndexPersistence("manifest_publish", [+ statePath(KV.bm25Index, manifestKey),+ ], {+ manifestKey,+ generation,+ chars: serialized.length,+ shards: shards.length,+ result: "committed",+ });} catch (err) {- if (!(await this.isManifestPublished(manifestKey, nextManifest))) {- await this.deleteShards(shards);+ if (await this.isManifestPublished(manifestKey, nextManifest)) {+ await this.auditIndexPersistence("manifest_publish", [+ statePath(KV.bm25Index, manifestKey),+ ], {+ manifestKey,+ generation,+ chars: serialized.length,+ shards: shards.length,+ result: "committed_after_error",+ error: errorMessage(err),+ });+ } else {+ await this.deleteShards(shards, "manifest_publish_rollback");}throw err;}- await this.kv.delete(KV.bm25Index, legacyKey).catch(() => {});+ await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup");if (previous?.v === 1 && Array.isArray(previous.shards)) {const currentShardIds = new Set(shards.map((shard) => `${shard.scope}\0${shard.key}`),);for (const shard of previous.shards) {if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue;- await this.deleteShards([shard]);+ await this.deleteShards([shard], "previous_generation_cleanup");}}}+ private async auditIndexPersistence(+ action: string,+ targetIds: string[],+ details: Record<string, unknown>,+ ): Promise<void> {+ await safeAudit(+ this.kv,+ "index_persist",+ INDEX_PERSISTENCE_FUNCTION_ID,+ targetIds,+ { action, ...details },+ );+ }++ private async deleteKey(+ scope: string,+ key: string,+ reason: string,+ ): Promise<void> {+ let result = "deleted";+ let error: string | undefined;+ try {+ await this.kv.delete(scope, key);+ } catch (err) {+ result = "failed";+ error = errorMessage(err);+ }+ await this.auditIndexPersistence("delete", [statePath(scope, key)], {+ scope,+ key,+ reason,+ result,+ error,+ });+ }+private async deleteShards(shards: IndexShardManifest["shards"],+ reason: string,): Promise<void> {for (const shard of shards) {- await this.kv.delete(shard.scope, shard.key).catch(() => {});+ await this.deleteKey(shard.scope, shard.key, reason);}}@@ -251,20 +341,46 @@ export class IndexPersistence {manifestKey: string,label: string,): Promise<string | null> {- const manifest = await this.kv- .get<IndexShardManifest>(KV.bm25Index, manifestKey)- .catch(() => null);- if (manifest !== null) {- return this.loadManifestData(manifest, label);+ const manifest = await this.readIndexValue<IndexShardManifest>(+ KV.bm25Index,+ manifestKey,+ label,+ "manifest",+ );+ if (!manifest.ok) return null;+ if (manifest.value !== null) {+ return this.loadManifestData(manifest.value, label);}- const legacy = await this.kv- .get<string>(KV.bm25Index, legacyKey)- .catch(() => null);- if (legacy && typeof legacy === "string") return legacy;+ const legacy = await this.readIndexValue<string>(+ KV.bm25Index,+ legacyKey,+ label,+ "legacy",+ );+ if (!legacy.ok) return null;+ if (legacy.value && typeof legacy.value === "string") return legacy.value;return null;}+ private async readIndexValue<T>(+ scope: string,+ key: string,+ label: string,+ source: "manifest" | "legacy",+ ): Promise<{ ok: true; value: T | null } | { ok: false }> {+ try {+ return { ok: true, value: await this.kv.get<T>(scope, key) };+ } catch (err) {+ logger.warn(`index persistence: ${label} ${source} read failed`, {+ scope,+ key,+ message: errorMessage(err),+ });+ return { ok: false };+ }+ }+private async loadManifestData(manifest: IndexShardManifest,label: string,@@ -272,17 +388,28 @@ export class IndexPersistence {if (manifest.v !== 1 ||!Array.isArray(manifest.shards) ||- manifest.shards.length === 0+ manifest.shards.length === 0 ||+ !Number.isInteger(manifest.chars) ||+ manifest.chars < 0) {logger.warn(`index persistence: ${label} shard manifest invalid`);return null;}+ for (const shard of manifest.shards) {+ if (!isValidShardDescriptor(shard)) {+ logger.warn(`index persistence: ${label} shard manifest invalid`);+ return null;+ }+ }+ const loadedShards = await Promise.all(+ manifest.shards.map(async (shard) => ({+ shard,+ chunk: await this.kv.get<string>(shard.scope, shard.key).catch(() => null),+ })),+ );const chunks: string[] = [];let chars = 0;- for (const shard of manifest.shards) {- const chunk = await this.kv- .get<string>(shard.scope, shard.key)- .catch(() => null);+ for (const { shard, chunk } of loadedShards) {if (typeof chunk !== "string") {logger.warn(`index persistence: ${label} shard missing`, {scope: shard.scope,diff --git a/src/types.ts b/src/types.tsindex b734a4d2..76b77485 100644--- a/src/types.ts+++ b/src/types.ts@@ -540,6 +540,7 @@ export interface AuditEntry {| "crystallize"| "diagnose"| "heal"+ | "index_persist"| "facet_tag"| "lesson_save"| "lesson_recall"diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.tsindex 2ccd061b..41877deb 100644--- a/test/index-persistence.test.ts+++ b/test/index-persistence.test.ts@@ -154,6 +154,48 @@ describe("IndexPersistence", () => {expect(loaded.vector!.size).toBe(1);});+ it("fails closed instead of falling back when manifest reads fail", async () => {+ const legacy = makeBm25("obs_legacy", "legacy stale snapshot");+ await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, legacy.serialize());+ const failingKv = {+ ...kv,+ get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {+ if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) {+ throw new Error("manifest backend unavailable");+ }+ return kv.get(scope, key);+ }),+ };++ const loaded = await new IndexPersistence(+ failingKv as never,+ new SearchIndex(),+ null,+ ).load();++ expect(loaded.bm25).toBeNull();+ });++ it("fails closed when legacy snapshot reads fail", async () => {+ const failingKv = {+ ...kv,+ get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {+ if (scope === BM25_SCOPE && key === BM25_LEGACY_KEY) {+ throw new Error("legacy backend unavailable");+ }+ return kv.get(scope, key);+ }),+ };++ const loaded = await new IndexPersistence(+ failingKv as never,+ new SearchIndex(),+ null,+ ).load();++ expect(loaded.bm25).toBeNull();+ });+it("loads sharded manifests that omit optional generation metadata", async () => {const bm25 = makeBm25("obs_1", "deterministic shard auth");const serialized = bm25.serialize();@@ -232,6 +274,37 @@ describe("IndexPersistence", () => {expect(loaded.vector!.size).toBe(1);});+ it("persists empty vector snapshots so cleared vectors do not reload", async () => {+ const previousBm25 = makeBm25("obs_old", "alpha previous snapshot");+ const previousVector = makeVector("obs_old");+ await new IndexPersistence(kv as never, previousBm25, previousVector, {+ shardChars: 80,+ createGeneration: () => "gen_old",+ }).save();++ const nextBm25 = makeBm25("obs_new", "bravo new snapshot");+ const emptyVector = new VectorIndex();+ await new IndexPersistence(kv as never, nextBm25, emptyVector, {+ shardChars: 80,+ createGeneration: () => "gen_empty",+ }).save();++ const vectorManifest = await kv.get<TestIndexShardManifest>(+ BM25_SCOPE,+ VECTOR_MANIFEST_KEY,+ );+ expect(vectorManifest).not.toBeNull();+ expect(vectorManifest!.generation).toBe("gen_empty");+ const loaded = await new IndexPersistence(+ kv as never,+ new SearchIndex(),+ null,+ ).load();+ expect(loaded.bm25!.search("bravo").length).toBe(1);+ expect(loaded.vector).not.toBeNull();+ expect(loaded.vector!.size).toBe(0);+ });+it("avoids one oversized state::set string payload for persisted indexes", async () => {const maxStringPayloadChars = 80;const bm25 = new SearchIndex();@@ -569,6 +642,32 @@ describe("IndexPersistence", () => {expect(loaded.bm25).toBeNull();});+ it("fails closed before reading invalid shard descriptors", async () => {+ await kv.set<TestIndexShardManifest>(BM25_SCOPE, BM25_MANIFEST_KEY, {+ v: 1,+ chars: 10,+ shards: [{ scope: "", key: "data", chars: 10 }],+ });+ const guardedKv = {+ ...kv,+ get: vi.fn(async <T>(scope: string, key: string): Promise<T | null> => {+ if (scope === "") {+ throw new Error("invalid shard descriptor was read");+ }+ return kv.get(scope, key);+ }),+ };++ const loaded = await new IndexPersistence(+ guardedKv as never,+ new SearchIndex(),+ null,+ ).load();++ expect(loaded.bm25).toBeNull();+ expect(guardedKv.get).not.toHaveBeenCalledWith("", "data");+ });+it("scheduleSave debounces multiple calls", async () => {const bm25 = new SearchIndex();const persistence = new IndexPersistence(kv as never, bm25, null);