Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(memory): resolve CodeRabbit linter & concurrency bugs

Tanmay-0082b7adc061c52
diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts
index 72e7d8db..e9efce49 100644
--- a/src/functions/auto-forget.ts
+++ b/src/functions/auto-forget.ts
@@ -154,7 +154,7 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
result.lowValueObs.push(obs.id);
if (!dryRun) {
if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData);
- if (obs.imageRef) await decrementImageRef(kv, sdk, obs.imageRef);
+ if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef);
await kv
.delete(KV.observations(sessions[i].id), obs.id)
.catch(() => { });
diff --git a/src/functions/disk-size-manager.ts b/src/functions/disk-size-manager.ts
index b4e8c300..957866dd 100644
--- a/src/functions/disk-size-manager.ts
+++ b/src/functions/disk-size-manager.ts
@@ -3,6 +3,7 @@ import { getContext } from "iii-sdk";
import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { getMaxBytes } from "../utils/image-store.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
const DISK_SIZE_KEY = "system:currentDiskSize";
@@ -19,22 +20,24 @@ export function registerDiskSizeManager(sdk: ISdk, kv: StateKV): void {
return { success: false, error: "deltaBytes must be a finite number" };
}
- const currentTotal = (await kv.get<number>(KV.state, DISK_SIZE_KEY)) || 0;
- let newTotal = currentTotal + data.deltaBytes;
+ return withKeyedLock(DISK_SIZE_KEY, async () => {
+ const currentTotal = (await kv.get<number>(KV.state, DISK_SIZE_KEY)) || 0;
+ let newTotal = currentTotal + data.deltaBytes;
- if (newTotal < 0) newTotal = 0;
+ if (newTotal < 0) newTotal = 0;
- await kv.set(KV.state, DISK_SIZE_KEY, newTotal);
+ await kv.set(KV.state, DISK_SIZE_KEY, newTotal);
- if (data.deltaBytes > 0 && newTotal > getMaxBytes()) {
- sdk.triggerVoid("mem::image-quota-cleanup", {});
- ctx.logger.info("[agentmemory] Disk quota exceeded, cleanup triggered", {
- currentBytes: newTotal,
- maxBytes: getMaxBytes(),
- });
- }
+ if (data.deltaBytes > 0 && newTotal > getMaxBytes()) {
+ sdk.triggerVoid("mem::image-quota-cleanup", {});
+ ctx.logger.info("[agentmemory] Disk quota exceeded, cleanup triggered", {
+ currentBytes: newTotal,
+ maxBytes: getMaxBytes(),
+ });
+ }
- return { success: true, currentTotal: newTotal };
+ return { success: true, currentTotal: newTotal };
+ });
},
);
}
diff --git a/src/functions/evict.ts b/src/functions/evict.ts
index f5a1458e..dc9415a3 100644
--- a/src/functions/evict.ts
+++ b/src/functions/evict.ts
@@ -96,7 +96,7 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
stats.lowImportanceObs++;
if (!dryRun) {
if (o.imageData) await decrementImageRef(kv, sdk, o.imageData);
- if (o.imageRef) await decrementImageRef(kv, sdk, o.imageRef);
+ if (o.imageRef && o.imageRef !== o.imageData) await decrementImageRef(kv, sdk, o.imageRef);
await kv
.delete(KV.observations(session.id), o.id)
.catch(() => {});
@@ -123,7 +123,7 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
if (!dryRun) {
for (const o of toEvict) {
if (o.imageData) await decrementImageRef(kv, sdk, o.imageData);
- if (o.imageRef) await decrementImageRef(kv, sdk, o.imageRef);
+ if (o.imageRef && o.imageRef !== o.imageData) await decrementImageRef(kv, sdk, o.imageRef);
await kv
.delete(KV.observations(o.sessionId), o.id)
.catch(() => {});
diff --git a/src/functions/image-quota-cleanup.ts b/src/functions/image-quota-cleanup.ts
index 2c1ead5d..545208ea 100644
--- a/src/functions/image-quota-cleanup.ts
+++ b/src/functions/image-quota-cleanup.ts
@@ -6,10 +6,8 @@ import { readdir, stat } from "node:fs/promises";
import { join } from "node:path";
import { IMAGES_DIR, getMaxBytes, deleteImage } from "../utils/image-store.js";
import { getImageRefCount } from "./image-refs.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
-const LOCK_KEY = "system:cleanupLockTimestamp";
-const LOCK_TTL_MS = 60_000;
-const HEARTBEAT_INTERVAL_MS = 30_000;
const GRACE_PERIOD_MS = 30_000;
export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {
@@ -22,13 +20,7 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {
const ctx = getContext();
const now = Date.now();
- const lockTime = await kv.get<number>(KV.state, LOCK_KEY);
- if (lockTime && (now - lockTime) < LOCK_TTL_MS) {
- return { success: true, skipped: true, reason: "locked" };
- }
- await kv.set(KV.state, LOCK_KEY, now);
-
- try {
+ return withKeyedLock("system:cleanupLock", async () => {
let totalSize = 0;
const fileStats: Array<{ filePath: string; size: number; mtimeMs: number }> = [];
@@ -57,7 +49,6 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {
let totalToFree = totalSize - limit;
let evicted = 0;
let freedBytes = 0;
- let lastHeartbeat = Date.now();
for (const f of fileStats) {
if (totalToFree <= 0) break;
@@ -66,29 +57,26 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {
continue;
}
- let refCount = 0;
- try {
- refCount = await getImageRefCount(kv, f.filePath);
- } catch (err) {
- ctx.logger.error(`[agentmemory] Failed to read refCount for ${f.filePath}:`, err);
- }
-
- if (refCount > 0) {
- continue;
- }
+ await withKeyedLock(`imgRef:${f.filePath}`, async () => {
+ let refCount = 0;
+ try {
+ refCount = await getImageRefCount(kv, f.filePath);
+ } catch (err) {
+ ctx.logger.error(`[agentmemory] Failed to read refCount for ${f.filePath}:`, err);
+ }
- const { deletedBytes } = await deleteImage(f.filePath);
- if (deletedBytes > 0) {
- sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
- totalToFree -= deletedBytes;
- freedBytes += deletedBytes;
- evicted++;
- }
+ if (refCount > 0) {
+ return;
+ }
- if (Date.now() - lastHeartbeat > HEARTBEAT_INTERVAL_MS) {
- await kv.set(KV.state, LOCK_KEY, Date.now());
- lastHeartbeat = Date.now();
- }
+ const { deletedBytes } = await deleteImage(f.filePath);
+ if (deletedBytes > 0) {
+ sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
+ totalToFree -= deletedBytes;
+ freedBytes += deletedBytes;
+ evicted++;
+ }
+ });
}
if (evicted > 0) {
@@ -97,9 +85,7 @@ export function registerImageQuotaCleanup(sdk: ISdk, kv: StateKV): void {
}
return { success: true, evicted, freedBytes };
- } finally {
- await kv.delete(KV.state, LOCK_KEY);
- }
+ });
},
);
}
diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.ts
index fb15e279..857bd1c5 100644
--- a/src/functions/image-refs.ts
+++ b/src/functions/image-refs.ts
@@ -2,6 +2,7 @@ import type { ISdk } from "iii-sdk";
import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { deleteImage, touchImage } from "../utils/image-store.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
export async function getImageRefCount(kv: StateKV, filePath: string): Promise<number> {
const count = await kv.get<number>(KV.imageRefs, filePath);
@@ -9,20 +10,24 @@ export async function getImageRefCount(kv: StateKV, filePath: string): Promise<n
}
export async function incrementImageRef(kv: StateKV, filePath: string): Promise<void> {
- const current = await getImageRefCount(kv, filePath);
- await kv.set(KV.imageRefs, filePath, current + 1);
- await touchImage(filePath);
+ return withKeyedLock(`imgRef:${filePath}`, async () => {
+ const current = await getImageRefCount(kv, filePath);
+ await kv.set(KV.imageRefs, filePath, current + 1);
+ await touchImage(filePath);
+ });
}
export async function decrementImageRef(kv: StateKV, sdk: ISdk, filePath: string): Promise<void> {
- const current = await getImageRefCount(kv, filePath);
- if (current <= 1) {
- await kv.delete(KV.imageRefs, filePath);
- const { deletedBytes } = await deleteImage(filePath);
- if (deletedBytes > 0) {
- sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
+ return withKeyedLock(`imgRef:${filePath}`, async () => {
+ const current = await getImageRefCount(kv, filePath);
+ if (current <= 1) {
+ await kv.delete(KV.imageRefs, filePath);
+ const { deletedBytes } = await deleteImage(filePath);
+ if (deletedBytes > 0) {
+ sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
+ }
+ } else {
+ await kv.set(KV.imageRefs, filePath, current - 1);
}
- } else {
- await kv.set(KV.imageRefs, filePath, current - 1);
- }
+ });
}
diff --git a/src/utils/image-store.ts b/src/utils/image-store.ts
index 05469484..3197e2c7 100644
--- a/src/utils/image-store.ts
+++ b/src/utils/image-store.ts
@@ -1,5 +1,5 @@
import { homedir } from "node:os";
-import { join, resolve } from "node:path";
+import { join, resolve, sep } from "node:path";
import { existsSync } from "node:fs";
import { mkdir, writeFile, unlink, utimes, stat } from "node:fs/promises";
import { createHash } from "node:crypto";
@@ -14,7 +14,8 @@ export function getMaxBytes(): number {
export function isManagedImagePath(filePath: string): boolean {
const resolved = resolve(filePath);
- return resolved.startsWith(IMAGES_DIR + "/") || resolved === IMAGES_DIR;
+ const normalizedImagesDir = resolve(IMAGES_DIR);
+ return resolved.startsWith(normalizedImagesDir + sep) || resolved === normalizedImagesDir;
}
function contentHash(data: string): string {
@@ -47,6 +48,10 @@ export async function saveImageToDisk(base64Data: string): Promise<{ filePath: s
const hash = contentHash(cleanBase64);
const filePath = join(IMAGES_DIR, `${hash}.${ext}`);
+ if (existsSync(filePath)) {
+ return { filePath, bytesWritten: 0 };
+ }
+
const buffer = Buffer.from(cleanBase64, "base64");
await writeFile(filePath, buffer);
diff --git a/test/multimodal.test.ts b/test/multimodal.test.ts
index 38f0b452..afcb3ed4 100644
--- a/test/multimodal.test.ts
+++ b/test/multimodal.test.ts
@@ -191,30 +191,36 @@ describe("Disk Size Manager", () => {
it("should trigger cleanup when total exceeds max bytes", async () => {
const originalEnv = process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES;
- process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = "5000";
-
- const localKv = mockKV() as any;
- const localTriggerVoid = vi.fn();
- const localSdk = { triggerVoid: localTriggerVoid } as any;
-
- let managerCallback: any = null;
- const sdkMocker = {
- ...localSdk,
- registerFunction: vi.fn((config: any, cb: any) => {
- if (config.id === "mem::disk-size-delta") managerCallback = cb;
- }),
- };
-
- const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");
- registerDiskSizeManager(sdkMocker, localKv);
-
- await managerCallback({ deltaBytes: 3000 });
- expect(localTriggerVoid).not.toHaveBeenCalled();
-
- await managerCallback({ deltaBytes: 3000 });
- expect(localTriggerVoid).toHaveBeenCalledWith("mem::image-quota-cleanup", {});
-
- process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = originalEnv || "";
+ try {
+ process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = "5000";
+
+ const localKv = mockKV() as any;
+ const localTriggerVoid = vi.fn();
+ const localSdk = { triggerVoid: localTriggerVoid } as any;
+
+ let managerCallback: any = null;
+ const sdkMocker = {
+ ...localSdk,
+ registerFunction: vi.fn((config: any, cb: any) => {
+ if (config.id === "mem::disk-size-delta") managerCallback = cb;
+ }),
+ };
+
+ const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");
+ registerDiskSizeManager(sdkMocker, localKv);
+
+ await managerCallback({ deltaBytes: 3000 });
+ expect(localTriggerVoid).not.toHaveBeenCalled();
+
+ await managerCallback({ deltaBytes: 3000 });
+ expect(localTriggerVoid).toHaveBeenCalledWith("mem::image-quota-cleanup", {});
+ } finally {
+ if (originalEnv === undefined) {
+ delete process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES;
+ } else {
+ process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = originalEnv;
+ }
+ }
});
it("should clamp negative totals to zero", async () => {
@@ -257,21 +263,59 @@ describe("Disk Size Manager", () => {
});
describe("Image Refs", () => {
- it("should increment and decrement ref counts correctly", async () => {
+ it("should increment and decrement ref counts correctly with deletion parity", async () => {
const localKv = mockKV() as any;
const localTriggerVoid = vi.fn();
const localSdk = { triggerVoid: localTriggerVoid } as any;
+ const { saveImageToDisk } = await import("../src/utils/image-store.js");
const { incrementImageRef, decrementImageRef, getImageRefCount } = await import("../src/functions/image-refs.js");
- await incrementImageRef(localKv, "/fake/path/test.png");
- expect(await getImageRefCount(localKv, "/fake/path/test.png")).toBe(1);
+ const result = await saveImageToDisk(
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="
+ );
+ const testFile = result.filePath;
+
+ expect(existsSync(testFile)).toBe(true);
+
+ // Initial state
+ expect(await getImageRefCount(localKv, testFile)).toBe(0);
+
+ // Increment to 1
+ await incrementImageRef(localKv, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(1);
+
+ // Increment to 2 (shared image)
+ await incrementImageRef(localKv, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(2);
+
+ // Decrement from 2 to 1
+ await decrementImageRef(localKv, localSdk, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(1);
+
+ // (c) shared image with refcount >= 2 is NOT deleted when one decrements
+ expect(existsSync(testFile)).toBe(true);
+ expect(localTriggerVoid).not.toHaveBeenCalled();
- await incrementImageRef(localKv, "/fake/path/test.png");
- expect(await getImageRefCount(localKv, "/fake/path/test.png")).toBe(2);
+ // (a) decrementing to zero triggers image file deletion and negative delta
+ await decrementImageRef(localKv, localSdk, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(0);
+ expect(existsSync(testFile)).toBe(false);
+
+ const deltaCalls = localTriggerVoid.mock.calls.filter(
+ (c: any[]) => c[0] === "mem::disk-size-delta"
+ );
+ expect(deltaCalls.length).toBe(1);
+ expect(deltaCalls[0][1].deltaBytes).toBeLessThan(0);
- await decrementImageRef(localKv, localSdk, "/fake/path/test.png");
- expect(await getImageRefCount(localKv, "/fake/path/test.png")).toBe(1);
+ // (b) decrementing an already-zero/unknown ref is a no-op
+ localTriggerVoid.mockClear();
+ await decrementImageRef(localKv, localSdk, "/fake/unknown/path.png");
+ expect(await getImageRefCount(localKv, "/fake/unknown/path.png")).toBe(0);
+ const noOpDeltaCalls = localTriggerVoid.mock.calls.filter(
+ (c: any[]) => c[0] === "mem::disk-size-delta"
+ );
+ expect(noOpDeltaCalls.length).toBe(0);
});
});
@@ -305,7 +349,7 @@ describe("Image Store", () => {
expect(testFilePath).toBeDefined();
expect(existsSync(testFilePath!)).toBe(true);
- const result = await deleteImage(testFilePath);
+ const result = await deleteImage(testFilePath!);
expect(result.deletedBytes).toBeGreaterThan(0);
expect(existsSync(testFilePath!)).toBe(false);