Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(memory): address multimodal review feedback (bugs, perf, cleanup)

Tanmay-00859f7611a2307
diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts
index d1da615a..cbc92c52 100644
--- a/src/functions/auto-forget.ts
+++ b/src/functions/auto-forget.ts
@@ -29,6 +29,7 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
const ctx = getContext();
const dryRun = data?.dryRun ?? false;
const now = Date.now();
+ const { deleteImage } = await import("../utils/image-store.js");
const result: AutoForgetResult = {
ttlExpired: [],
@@ -46,9 +47,8 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
result.ttlExpired.push(mem.id);
deletedIds.add(mem.id);
if (!dryRun) {
- if ((mem as any).imageRef) {
- const { deleteImage } = await import("../utils/image-store.js");
- deleteImage((mem as any).imageRef);
+ if (mem.imageRef) {
+ deleteImage(mem.imageRef);
}
await kv.delete(KV.memories, mem.id);
}
@@ -153,9 +153,8 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
if (age > 180 * MS_PER_DAY && (obs.importance ?? 5) <= 2) {
result.lowValueObs.push(obs.id);
if (!dryRun) {
- const { deleteImage } = await import("../utils/image-store.js");
- if ((obs as any).imageData) deleteImage((obs as any).imageData);
- if ((obs as any).imageRef) deleteImage((obs as any).imageRef);
+ if (obs.imageData) deleteImage(obs.imageData);
+ if (obs.imageRef) deleteImage(obs.imageRef);
await kv
.delete(KV.observations(sessions[i].id), obs.id)
.catch(() => { });
diff --git a/src/functions/evict.ts b/src/functions/evict.ts
index 935580ec..9c7ea5f0 100644
--- a/src/functions/evict.ts
+++ b/src/functions/evict.ts
@@ -43,6 +43,7 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
async (data: { dryRun?: boolean }): Promise<EvictionStats> => {
const ctx = getContext();
const dryRun = data?.dryRun ?? false;
+ const { deleteImage } = await import("../utils/image-store.js");
const configOverride = await kv
.get<Partial<EvictionConfig>>(KV.config, "eviction")
@@ -94,9 +95,8 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
) {
stats.lowImportanceObs++;
if (!dryRun) {
- const { deleteImage } = await import("../utils/image-store.js");
- if ((o as any).imageData) deleteImage((o as any).imageData);
- if ((o as any).imageRef) deleteImage((o as any).imageRef);
+ if (o.imageData) deleteImage(o.imageData);
+ if (o.imageRef) deleteImage(o.imageRef);
await kv
.delete(KV.observations(session.id), o.id)
.catch(() => {});
@@ -122,9 +122,8 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
stats.capEvictions += toEvict.length;
if (!dryRun) {
for (const o of toEvict) {
- const { deleteImage } = await import("../utils/image-store.js");
- if ((o as any).imageData) deleteImage((o as any).imageData);
- if ((o as any).imageRef) deleteImage((o as any).imageRef);
+ if (o.imageData) deleteImage(o.imageData);
+ if (o.imageRef) deleteImage(o.imageRef);
await kv
.delete(KV.observations(o.sessionId), o.id)
.catch(() => {});
@@ -142,9 +141,8 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
stats.expiredMemories++;
evictedMemIds.add(mem.id);
if (!dryRun) {
- if ((mem as any).imageRef) {
- const { deleteImage } = await import("../utils/image-store.js");
- deleteImage((mem as any).imageRef);
+ if (mem.imageRef) {
+ deleteImage(mem.imageRef);
}
await kv.delete(KV.memories, mem.id).catch(() => {});
}
@@ -160,9 +158,8 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
if (age > cfg.lowImportanceMaxDays * MS_PER_DAY) {
stats.nonLatestMemories++;
if (!dryRun) {
- if ((mem as any).imageRef) {
- const { deleteImage } = await import("../utils/image-store.js");
- deleteImage((mem as any).imageRef);
+ if (mem.imageRef) {
+ deleteImage(mem.imageRef);
}
await kv.delete(KV.memories, mem.id).catch(() => {});
}
diff --git a/src/functions/observe.ts b/src/functions/observe.ts
index 3a552f03..3c66c696 100644
--- a/src/functions/observe.ts
+++ b/src/functions/observe.ts
@@ -21,7 +21,7 @@ export function extractImage(d: unknown): string | undefined {
if (typeof obj["image_path"] === "string") return obj["image_path"];
if (typeof obj["imageBase64"] === "string") return obj["imageBase64"];
if (typeof obj["imagePath"] === "string") return obj["imagePath"];
-
+
for (const key of Object.keys(obj)) {
const match = extractImage(obj[key]);
if (match) return match;
@@ -95,6 +95,8 @@ export function registerObserveFunction(
raw: sanitizedRaw,
};
+ let extractedImage: string | undefined;
+
if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) {
const d = sanitizedRaw as Record<string, unknown>;
if (
@@ -109,15 +111,18 @@ export function registerObserveFunction(
raw.userPrompt = d["prompt"] as string | undefined;
}
- const hiddenImage = extractImage(sanitizedRaw);
- if (hiddenImage) {
+ extractedImage = extractImage(sanitizedRaw);
+ if (extractedImage) {
raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image";
}
+ } else if (typeof sanitizedRaw === "string") {
+ extractedImage = extractImage(sanitizedRaw);
+ if (extractedImage) {
+ raw.modality = "image";
+ }
}
- const pendingImageData = (raw.modality === "image" || raw.modality === "mixed")
- ? extractImage(typeof sanitizedRaw === "object" ? sanitizedRaw : undefined)
- : undefined;
+ const pendingImageData = extractedImage;
return withKeyedLock(`obs:${payload.sessionId}`, async () => {
if (maxObservationsPerSession && maxObservationsPerSession > 0) {
@@ -135,7 +140,17 @@ export function registerObserveFunction(
raw.imageData = saveImageToDisk(pendingImageData);
}
- await kv.set(KV.observations(payload.sessionId), obsId, raw);
+ try {
+
+ await kv.set(KV.observations(payload.sessionId), obsId, raw);
+
+ } catch (error) {
+ if (raw.imageData) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ deleteImage(raw.imageData);
+ }
+ throw error;
+ }
if (dedupMap && dedupHash) {
dedupMap.record(dedupHash);
diff --git a/src/functions/retention.ts b/src/functions/retention.ts
index 8322bb8f..154b393d 100644
--- a/src/functions/retention.ts
+++ b/src/functions/retention.ts
@@ -194,6 +194,7 @@ export function registerRetentionFunctions(
const ctx = getContext();
const threshold = data.threshold ?? DEFAULT_DECAY.tierThresholds.cold;
const maxEvict = data.maxEvict ?? 50;
+ const { deleteImage } = await import("../utils/image-store.js");
const allScores = await kv.list<RetentionScore>(KV.retentionScores);
const candidates = allScores
@@ -217,9 +218,8 @@ export function registerRetentionFunctions(
for (const candidate of candidates) {
try {
const mem = await kv.get<Memory>(KV.memories, candidate.memoryId);
- if (mem && (mem as any).imageRef) {
- const { deleteImage } = await import("../utils/image-store.js");
- deleteImage((mem as any).imageRef);
+ if (mem && mem.imageRef) {
+ deleteImage(mem.imageRef);
}
await kv.delete(KV.memories, candidate.memoryId);
await kv.delete(KV.retentionScores, candidate.memoryId);
diff --git a/src/providers/agent-sdk.ts b/src/providers/agent-sdk.ts
index 47ab67f4..9dfaf0b0 100644
--- a/src/providers/agent-sdk.ts
+++ b/src/providers/agent-sdk.ts
@@ -26,8 +26,7 @@ export class AgentSDKProvider implements MemoryProvider {
let result = ''
for await (const msg of messages) {
if (msg.type === 'result') {
- const payload = (msg as any).result;
- result = typeof payload === 'string' ? payload : JSON.stringify(payload)
+ result = (msg as any).result
}
}
return result
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 341b77c1..636065fc 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -1208,26 +1208,31 @@ export function registerApiTriggers(
sdk.registerFunction(
{ id: "api::routine-create" },
- async (req: ApiRequest): Promise<Response> => {
+ async (
+ req: ApiRequest<{
+ name: string;
+ description?: string;
+ steps: unknown[];
+ tags?: string[];
+ frozen?: boolean;
+ sourceProceduralIds?: string[];
+ }>,
+ ): Promise<Response> => {
const authErr = checkAuth(req, secret);
if (authErr) return authErr;
- const body = (req.body ?? {}) as Record<string, unknown>;
- const name = typeof body.name === "string" ? body.name.trim() : "";
- const steps = body.steps;
- if (!name || !Array.isArray(steps) || steps.length === 0) {
- return { status_code: 400, body: { error: "name and steps are required" } };
+ if (!req.body?.name || !req.body?.steps) {
+ return {
+ status_code: 400,
+ body: { error: "name and steps are required" },
+ };
}
const payload = {
- name,
- description: typeof body.description === "string" ? body.description : undefined,
- steps,
- tags: Array.isArray(body.tags)
- ? body.tags.filter((t): t is string => typeof t === "string")
- : undefined,
- frozen: typeof body.frozen === "boolean" ? body.frozen : undefined,
- sourceProceduralIds: Array.isArray(body.sourceProceduralIds)
- ? body.sourceProceduralIds.filter((id): id is string => typeof id === "string")
- : undefined,
+ name: req.body.name,
+ description: req.body.description,
+ steps: req.body.steps,
+ tags: req.body.tags,
+ frozen: req.body.frozen,
+ sourceProceduralIds: req.body.sourceProceduralIds,
};
const result = await sdk.trigger("mem::routine-create", payload);
return { status_code: 201, body: result };
diff --git a/src/types.ts b/src/types.ts
index 5070717a..a797c7c1 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -39,8 +39,8 @@ export interface CompressedObservation {
importance: number;
confidence?: number;
imageRef?: string;
+ imageData?: string;
imageDescription?: string;
- imageThumbBase64?: string;
modality?: "text" | "image" | "mixed";
}
@@ -59,6 +59,7 @@ export type ObservationType =
| "subagent"
| "notification"
| "task"
+ | "image"
| "other";
export interface Memory {
@@ -79,6 +80,8 @@ export interface Memory {
sourceObservationIds?: string[];
isLatest: boolean;
forgetAfter?: string;
+ imageRef?: string;
+ imageData?: string;
}
export interface SessionSummary {