#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix(memory): resolve CodeRabbit linter & concurrency bugs
Tanmay-0082b7adc061c52
diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.tsindex 72e7d8db..e9efce49 100644--- a/src/functions/auto-forget.ts+++ b/src/functions/auto-forget.ts@@ -154,7 +154,7 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {result.lowValueObs.push(obs.id);if (!dryRun) {if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData);- if (obs.imageRef) await decrementImageRef(kv, sdk, obs.imageRef);+ if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef);await kv.delete(KV.observations(sessions[i].id), obs.id).catch(() => { });diff --git a/src/functions/disk-size-manager.ts b/src/functions/disk-size-manager.tsindex b4e8c300..957866dd 100644--- a/src/functions/disk-size-manager.ts+++ b/src/functions/disk-size-manager.ts@@ -3,6 +3,7 @@ import { getContext } from "iii-sdk";import { KV } from "../state/schema.js";import { StateKV } from "../state/kv.js";import { getMaxBytes } from "../utils/image-store.js";+import { withKeyedLock } from "../state/keyed-mutex.js";const DISK_SIZE_KEY = "system:currentDiskSize";@@ -19,22 +20,24 @@ export function registerDiskSizeManager(sdk: ISdk, kv: StateKV): void {return { success: false, error: "deltaBytes must be a finite number" };}- const currentTotal = (await kv.get<number>(KV.state, DISK_SIZE_KEY)) || 0;- let newTotal = currentTotal + data.deltaBytes;+ return withKeyedLock(DISK_SIZE_KEY, async () => {+ const currentTotal = (await kv.get<number>(KV.state, DISK_SIZE_KEY)) || 0;+ let newTotal = currentTotal + data.deltaBytes;- if (newTotal < 0) newTotal = 0;+ if (newTotal < 0) newTotal = 0;- await kv.set(KV.state, DISK_SIZE_KEY, newTotal);+ await kv.set(KV.state, DISK_SIZE_KEY, newTotal);- if (data.deltaBytes > 0 && newTotal > getMaxBytes()) {- sdk.triggerVoid("mem::image-quota-cleanup", {});- ctx.logger.info("[agentmemory] Disk quota exceeded, cleanup triggered", {- currentBytes: newTotal,- maxBytes: getMaxBytes(),- });- }+ if (data.deltaBytes > 0 && newTotal > getMaxBytes()) {+ sdk.triggerVoid("mem::image-quota-cleanup", {});+ ctx.logger.info("[agentmemory] Disk quota exceeded, cleanup triggered", {+ currentBytes: newTotal,+ maxBytes: getMaxBytes(),+ });+ }- return { success: true, currentTotal: newTotal };+ return { success: true, currentTotal: newTotal };+ });},);}diff --git a/src/functions/evict.ts b/src/functions/evict.tsindex f5a1458e..dc9415a3 100644--- a/src/functions/evict.ts+++ b/src/functions/evict.ts@@ -96,7 +96,7 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {stats.lowImportanceObs++;if (!dryRun) {if (o.imageData) await decrementImageRef(kv, sdk, o.imageData);- if (o.imageRef) await decrementImageRef(kv, sdk, o.imageRef);+ if (o.imageRef && o.imageRef !== o.imageData) await decrementImageRef(kv, sdk, o.imageRef);await kv.delete(KV.observations(session.id), o.id).catch(() => {});@@ -123,7 +123,7 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {if (!dryRun) {for (const o of toEvict) {if (o.imageData) await decrementImageRef(kv, sdk, o.imageData);- if (o.imageRef) await decrementImageRef(kv, sdk, o.imageRef);+ if (o.imageRef && o.imageRef !== o.imageData) await decrementImageRef(kv, sdk, o.imageRef);await kv.delete(KV.observations(o.sessionId), o.id).catch(() => {});diff --git a/src/functions/image-quota-cleanup.ts b/src/functions/image-quota-cleanup.tsindex 2c1ead5d..545208ea 100644--- a/src/functions/image-quota-cleanup.ts+++ b/src/functions/image-quota-cleanup.ts@@ -6,10 +6,8 @@ import { readdir, stat } from "node:fs/promises";import { join } from "node:path";import { IMAGES_DIR, getMaxBytes, deleteImage } from "../utils/image-store.js";import { getImageRefCount } from "./image-refs.js";+import { withKeyedLock } from "../state/keyed-mutex.js";-const LOCK_KEY = "system:cleanupLockTimestamp";-const LOCK_TTL_MS = 60_000;-const HEARTBEAT_INTERVAL_MS = 30_000;const GRACE_PERIOD_MS = 30_000;export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {@@ -22,13 +20,7 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {const ctx = getContext();const now = Date.now();- const lockTime = await kv.get<number>(KV.state, LOCK_KEY);- if (lockTime && (now - lockTime) < LOCK_TTL_MS) {- return { success: true, skipped: true, reason: "locked" };- }- await kv.set(KV.state, LOCK_KEY, now);-- try {+ return withKeyedLock("system:cleanupLock", async () => {let totalSize = 0;const fileStats: Array<{ filePath: string; size: number; mtimeMs: number }> = [];@@ -57,7 +49,6 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {let totalToFree = totalSize - limit;let evicted = 0;let freedBytes = 0;- let lastHeartbeat = Date.now();for (const f of fileStats) {if (totalToFree <= 0) break;@@ -66,29 +57,26 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {continue;}- let refCount = 0;- try {- refCount = await getImageRefCount(kv, f.filePath);- } catch (err) {- ctx.logger.error(`[agentmemory] Failed to read refCount for ${f.filePath}:`, err);- }-- if (refCount > 0) {- continue;- }+ await withKeyedLock(`imgRef:${f.filePath}`, async () => {+ let refCount = 0;+ try {+ refCount = await getImageRefCount(kv, f.filePath);+ } catch (err) {+ ctx.logger.error(`[agentmemory] Failed to read refCount for ${f.filePath}:`, err);+ }- const { deletedBytes } = await deleteImage(f.filePath);- if (deletedBytes > 0) {- sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });- totalToFree -= deletedBytes;- freedBytes += deletedBytes;- evicted++;- }+ if (refCount > 0) {+ return;+ }- if (Date.now() - lastHeartbeat > HEARTBEAT_INTERVAL_MS) {- await kv.set(KV.state, LOCK_KEY, Date.now());- lastHeartbeat = Date.now();- }+ const { deletedBytes } = await deleteImage(f.filePath);+ if (deletedBytes > 0) {+ sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });+ totalToFree -= deletedBytes;+ freedBytes += deletedBytes;+ evicted++;+ }+ });}if (evicted > 0) {@@ -97,9 +85,7 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {}return { success: true, evicted, freedBytes };- } finally {- await kv.delete(KV.state, LOCK_KEY);- }+ });},);}diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.tsindex fb15e279..857bd1c5 100644--- a/src/functions/image-refs.ts+++ b/src/functions/image-refs.ts@@ -2,6 +2,7 @@ import type { ISdk } from "iii-sdk";import { KV } from "../state/schema.js";import { StateKV } from "../state/kv.js";import { deleteImage, touchImage } from "../utils/image-store.js";+import { withKeyedLock } from "../state/keyed-mutex.js";export async function getImageRefCount(kv: StateKV, filePath: string): Promise<number> {const count = await kv.get<number>(KV.imageRefs, filePath);@@ -9,20 +10,24 @@ export async function getImageRefCount(kv: StateKV, filePath: string): Promise<n}export async function incrementImageRef(kv: StateKV, filePath: string): Promise<void> {- const current = await getImageRefCount(kv, filePath);- await kv.set(KV.imageRefs, filePath, current + 1);- await touchImage(filePath);+ return withKeyedLock(`imgRef:${filePath}`, async () => {+ const current = await getImageRefCount(kv, filePath);+ await kv.set(KV.imageRefs, filePath, current + 1);+ await touchImage(filePath);+ });}export async function decrementImageRef(kv: StateKV, sdk: ISdk, filePath: string): Promise<void> {- const current = await getImageRefCount(kv, filePath);- if (current <= 1) {- await kv.delete(KV.imageRefs, filePath);- const { deletedBytes } = await deleteImage(filePath);- if (deletedBytes > 0) {- sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });+ return withKeyedLock(`imgRef:${filePath}`, async () => {+ const current = await getImageRefCount(kv, filePath);+ if (current <= 1) {+ await kv.delete(KV.imageRefs, filePath);+ const { deletedBytes } = await deleteImage(filePath);+ if (deletedBytes > 0) {+ sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });+ }+ } else {+ await kv.set(KV.imageRefs, filePath, current - 1);}- } else {- await kv.set(KV.imageRefs, filePath, current - 1);- }+ });}diff --git a/src/utils/image-store.ts b/src/utils/image-store.tsindex 05469484..3197e2c7 100644--- a/src/utils/image-store.ts+++ b/src/utils/image-store.ts@@ -1,5 +1,5 @@import { homedir } from "node:os";-import { join, resolve } from "node:path";+import { join, resolve, sep } from "node:path";import { existsSync } from "node:fs";import { mkdir, writeFile, unlink, utimes, stat } from "node:fs/promises";import { createHash } from "node:crypto";@@ -14,7 +14,8 @@ export function getMaxBytes(): number {export function isManagedImagePath(filePath: string): boolean {const resolved = resolve(filePath);- return resolved.startsWith(IMAGES_DIR + "/") || resolved === IMAGES_DIR;+ const normalizedImagesDir = resolve(IMAGES_DIR);+ return resolved.startsWith(normalizedImagesDir + sep) || resolved === normalizedImagesDir;}function contentHash(data: string): string {@@ -47,6 +48,10 @@ export async function saveImageToDisk(base64Data: string): Promise<{ filePath: sconst hash = contentHash(cleanBase64);const filePath = join(IMAGES_DIR, `${hash}.${ext}`);+ if (existsSync(filePath)) {+ return { filePath, bytesWritten: 0 };+ }+const buffer = Buffer.from(cleanBase64, "base64");await writeFile(filePath, buffer);diff --git a/test/multimodal.test.ts b/test/multimodal.test.tsindex 38f0b452..afcb3ed4 100644--- a/test/multimodal.test.ts+++ b/test/multimodal.test.ts@@ -191,30 +191,36 @@ describe("Disk Size Manager", () => {it("should trigger cleanup when total exceeds max bytes", async () => {const originalEnv = process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES;- process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = "5000";-- const localKv = mockKV() as any;- const localTriggerVoid = vi.fn();- const localSdk = { triggerVoid: localTriggerVoid } as any;-- let managerCallback: any = null;- const sdkMocker = {- ...localSdk,- registerFunction: vi.fn((config: any, cb: any) => {- if (config.id === "mem::disk-size-delta") managerCallback = cb;- }),- };-- const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");- registerDiskSizeManager(sdkMocker, localKv);-- await managerCallback({ deltaBytes: 3000 });- expect(localTriggerVoid).not.toHaveBeenCalled();-- await managerCallback({ deltaBytes: 3000 });- expect(localTriggerVoid).toHaveBeenCalledWith("mem::image-quota-cleanup", {});-- process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = originalEnv || "";+ try {+ process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = "5000";++ const localKv = mockKV() as any;+ const localTriggerVoid = vi.fn();+ const localSdk = { triggerVoid: localTriggerVoid } as any;++ let managerCallback: any = null;+ const sdkMocker = {+ ...localSdk,+ registerFunction: vi.fn((config: any, cb: any) => {+ if (config.id === "mem::disk-size-delta") managerCallback = cb;+ }),+ };++ const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");+ registerDiskSizeManager(sdkMocker, localKv);++ await managerCallback({ deltaBytes: 3000 });+ expect(localTriggerVoid).not.toHaveBeenCalled();++ await managerCallback({ deltaBytes: 3000 });+ expect(localTriggerVoid).toHaveBeenCalledWith("mem::image-quota-cleanup", {});+ } finally {+ if (originalEnv === undefined) {+ delete process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES;+ } else {+ process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = originalEnv;+ }+ }});it("should clamp negative totals to zero", async () => {@@ -257,21 +263,59 @@ describe("Disk Size Manager", () => {});describe("Image Refs", () => {- it("should increment and decrement ref counts correctly", async () => {+ it("should increment and decrement ref counts correctly with deletion parity", async () => {const localKv = mockKV() as any;const localTriggerVoid = vi.fn();const localSdk = { triggerVoid: localTriggerVoid } as any;+ const { saveImageToDisk } = await import("../src/utils/image-store.js");const { incrementImageRef, decrementImageRef, getImageRefCount } = await import("../src/functions/image-refs.js");- await incrementImageRef(localKv, "/fake/path/test.png");- expect(await getImageRefCount(localKv, "/fake/path/test.png")).toBe(1);+ const result = await saveImageToDisk(+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="+ );+ const testFile = result.filePath;++ expect(existsSync(testFile)).toBe(true);++ // Initial state+ expect(await getImageRefCount(localKv, testFile)).toBe(0);++ // Increment to 1+ await incrementImageRef(localKv, testFile);+ expect(await getImageRefCount(localKv, testFile)).toBe(1);++ // Increment to 2 (shared image)+ await incrementImageRef(localKv, testFile);+ expect(await getImageRefCount(localKv, testFile)).toBe(2);++ // Decrement from 2 to 1+ await decrementImageRef(localKv, localSdk, testFile);+ expect(await getImageRefCount(localKv, testFile)).toBe(1);++ // (c) shared image with refcount >= 2 is NOT deleted when one decrements+ expect(existsSync(testFile)).toBe(true);+ expect(localTriggerVoid).not.toHaveBeenCalled();- await incrementImageRef(localKv, "/fake/path/test.png");- expect(await getImageRefCount(localKv, "/fake/path/test.png")).toBe(2);+ // (a) decrementing to zero triggers image file deletion and negative delta+ await decrementImageRef(localKv, localSdk, testFile);+ expect(await getImageRefCount(localKv, testFile)).toBe(0);+ expect(existsSync(testFile)).toBe(false);++ const deltaCalls = localTriggerVoid.mock.calls.filter(+ (c: any[]) => c[0] === "mem::disk-size-delta"+ );+ expect(deltaCalls.length).toBe(1);+ expect(deltaCalls[0][1].deltaBytes).toBeLessThan(0);- await decrementImageRef(localKv, localSdk, "/fake/path/test.png");- expect(await getImageRefCount(localKv, "/fake/path/test.png")).toBe(1);+ // (b) decrementing an already-zero/unknown ref is a no-op+ localTriggerVoid.mockClear();+ await decrementImageRef(localKv, localSdk, "/fake/unknown/path.png");+ expect(await getImageRefCount(localKv, "/fake/unknown/path.png")).toBe(0);+ const noOpDeltaCalls = localTriggerVoid.mock.calls.filter(+ (c: any[]) => c[0] === "mem::disk-size-delta"+ );+ expect(noOpDeltaCalls.length).toBe(0);});});@@ -305,7 +349,7 @@ describe("Image Store", () => {expect(testFilePath).toBeDefined();expect(existsSync(testFilePath!)).toBe(true);- const result = await deleteImage(testFilePath);+ const result = await deleteImage(testFilePath!);expect(result.deletedBytes).toBeGreaterThan(0);expect(existsSync(testFilePath!)).toBe(false);