Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix: resolve conflicts with main, address qodo-ai security findings

Rohit Ghumare2edc02feb571
diff --cc src/functions/auto-forget.ts
index e9efce49,1287c98b..49493572
--- a/src/functions/auto-forget.ts
+++ b/src/functions/auto-forget.ts
@@@ -1,8 -1,10 +1,10 @@@
import type { ISdk } from "iii-sdk";
- import { getContext } from "iii-sdk";
import type { Memory, CompressedObservation, Session } from "../types.js";
-import { KV, jaccardSimilarity } from "../state/schema.js";
+import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
+ import { recordAudit } from "./audit.js";
+ import { deleteAccessLog } from "./access-tracker.js";
+ import { logger } from "../logger.js";
const MS_PER_DAY = 24 * 60 * 60 * 1000;
const CONTRADICTION_THRESHOLD = 0.9;
@@@ -19,17 -21,10 +21,11 @@@ interface AutoForgetResult
}
export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
- sdk.registerFunction(
- {
- id: "mem::auto-forget",
- description:
- "Auto-forget expired (TTL), contradictory, and low-value data",
- },
+ sdk.registerFunction("mem::auto-forget",
async (data: { dryRun?: boolean }): Promise<AutoForgetResult> => {
- const ctx = getContext();
const dryRun = data?.dryRun ?? false;
const now = Date.now();
+ const { decrementImageRef } = await import("./image-refs.js");
const result: AutoForgetResult = {
ttlExpired: [],
@@@ -47,10 -42,13 +43,16 @@@
result.ttlExpired.push(mem.id);
deletedIds.add(mem.id);
if (!dryRun) {
+ if (mem.imageRef) {
+ await decrementImageRef(kv, sdk, mem.imageRef);
+ }
await kv.delete(KV.memories, mem.id);
+ await recordAudit(kv, "delete", "mem::auto-forget", [mem.id], {
+ resource: "memory",
+ reason: "auto-forget TTL",
+ timestamp: mem.forgetAfter,
+ });
+ await deleteAccessLog(kv, mem.id);
}
}
}
@@@ -153,11 -157,15 +161,25 @@@
if (age > 180 * MS_PER_DAY && (obs.importance ?? 5) <= 2) {
result.lowValueObs.push(obs.id);
if (!dryRun) {
- if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData);
- if (obs.imageRef && obs.imageRef !== obs.imageData) await decrementImageRef(kv, sdk, obs.imageRef);
-- await kv
-- .delete(KV.observations(sessions[i].id), obs.id)
- .catch(() => { });
- .catch(() => {});
- await recordAudit(kv, "delete", "mem::auto-forget", [obs.id], {
- resource: "observation",
- reason: "auto-forget low-value observation",
- sessionId: sessions[i].id,
- timestamp: obs.timestamp,
- });
++ let deletedOk = false;
++ try {
++ await kv.delete(KV.observations(sessions[i].id), obs.id);
++ deletedOk = true;
++ } catch {
++ deletedOk = false;
++ }
++ if (deletedOk) {
++ if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData);
++ if (obs.imageRef && obs.imageRef !== obs.imageData) {
++ await decrementImageRef(kv, sdk, obs.imageRef);
++ }
++ await recordAudit(kv, "delete", "mem::auto-forget", [obs.id], {
++ resource: "observation",
++ reason: "auto-forget low-value observation",
++ sessionId: sessions[i].id,
++ timestamp: obs.timestamp,
++ });
++ }
}
}
}
diff --cc src/functions/compress-synthetic.ts
index 00000000,f931ba00..93f5f902
mode 000000,100644..100644
--- a/src/functions/compress-synthetic.ts
+++ b/src/functions/compress-synthetic.ts
@@@ -1,0 -1,102 +1,105 @@@
+ import type {
+ RawObservation,
+ CompressedObservation,
+ ObservationType,
+ } from "../types.js";
+
+ // Zero-LLM compression path. Converts a RawObservation into a
+ // CompressedObservation using only heuristics — no Claude call, no token
+ // spend. This is the default as of 0.8.8 (#138); users who want richer
+ // LLM-generated summaries set AGENTMEMORY_AUTO_COMPRESS=true.
+
+ function inferType(
+ toolName: string | undefined,
+ hookType: string,
+ ): ObservationType {
+ if (hookType === "post_tool_failure") return "error";
+ if (hookType === "prompt_submit") return "conversation";
+ if (hookType === "subagent_stop" || hookType === "task_completed")
+ return "subagent";
+ if (hookType === "notification") return "notification";
+
+ if (!toolName) return "other";
+ // Normalize camelCase and kebab-case into word chunks so we can match
+ // substrings like "WebFetch" -> "web" / "fetch".
+ const n = toolName
+ .replace(/([a-z])([A-Z])/g, "$1_$2")
+ .replace(/[-\s]+/g, "_")
+ .toLowerCase();
+ const hasWord = (word: string) =>
+ new RegExp(`(^|_)${word}(_|$)`).test(n) ||
+ n === word ||
+ n.endsWith(word) ||
+ n.startsWith(word);
+ if (["fetch", "http", "web"].some(hasWord)) return "web_fetch";
+ if (["grep", "search", "glob", "find"].some(hasWord)) return "search";
+ if (["bash", "shell", "exec", "run"].some(hasWord)) return "command_run";
+ if (["edit", "update", "patch", "replace"].some(hasWord)) return "file_edit";
+ if (["write", "create"].some(hasWord)) return "file_write";
+ if (["read", "view"].some(hasWord)) return "file_read";
+ if (["task", "agent"].some(hasWord)) return "subagent";
+ return "other";
+ }
+
+ function extractFiles(input: unknown): string[] {
+ if (!input || typeof input !== "object") return [];
+ const o = input as Record<string, unknown>;
+ const out = new Set<string>();
+ for (const key of [
+ "file_path",
+ "filepath",
+ "path",
+ "filePath",
+ "file",
+ "pattern",
+ ]) {
+ const v = o[key];
+ if (typeof v === "string" && v.length > 0 && v.length < 512) out.add(v);
+ }
+ return [...out];
+ }
+
+ function stringifyForNarrative(v: unknown): string {
+ if (v == null) return "";
+ if (typeof v === "string") return v;
+ try {
+ return JSON.stringify(v);
+ } catch {
+ return String(v);
+ }
+ }
+
+ function truncate(s: string, n: number): string {
+ return s.length > n ? s.slice(0, n - 1) + "\u2026" : s;
+ }
+
+ export function buildSyntheticCompression(
+ raw: RawObservation,
+ ): CompressedObservation {
+ const toolName = raw.toolName ?? raw.hookType;
+ const inputStr = stringifyForNarrative(raw.toolInput);
+ const outputStr = stringifyForNarrative(raw.toolOutput);
+ const promptStr = raw.userPrompt ?? "";
+
+ const narrativeParts = [promptStr, inputStr, outputStr].filter(
+ (s) => s.length > 0,
+ );
+
- return {
++ const result: CompressedObservation = {
+ id: raw.id,
+ sessionId: raw.sessionId,
+ timestamp: raw.timestamp,
+ type: inferType(toolName, raw.hookType),
+ title: truncate(toolName || "observation", 80),
+ subtitle: inputStr ? truncate(inputStr, 120) : undefined,
+ facts: [],
+ narrative: truncate(narrativeParts.join(" | "), 400),
+ concepts: [],
+ files: extractFiles(raw.toolInput),
+ importance: 5,
+ confidence: 0.3,
+ };
++ if (raw.modality) result.modality = raw.modality;
++ if (raw.imageData) result.imageData = raw.imageData;
++ return result;
+ }
diff --cc src/functions/compress.ts
index a2b84306,44562065..19a144cd
--- a/src/functions/compress.ts
+++ b/src/functions/compress.ts
@@@ -1,7 -1,4 +1,6 @@@
- import type { ISdk } from "iii-sdk";
- import { getContext } from "iii-sdk";
-import { TriggerAction, type ISdk } from "iii-sdk";
++import { TriggerAction, type ISdk, getContext } from "iii-sdk";
+import { readFileSync } from "node:fs";
+import { isManagedImagePath } from "../utils/image-store.js";
import type {
RawObservation,
CompressedObservation,
@@@ -80,39 -72,7 +76,38 @@@ export function registerCompressFunctio
sessionId: string;
raw: RawObservation;
}) => {
- const ctx = getContext();
const startMs = Date.now();
+
+ let imageDescription: string | undefined;
+ const hasImage = data.raw.modality === "image" || data.raw.modality === "mixed";
+
+ if (hasImage && data.raw.imageData && provider.describeImage) {
+ try {
+ let base64Data = data.raw.imageData;
+ let mimeType = "image/png";
+
+ if (!data.raw.imageData.startsWith("/9j/") && !data.raw.imageData.startsWith("iVBOR")) {
+ if (!isManagedImagePath(data.raw.imageData)) {
+ throw new Error(`Refusing to read image outside managed store: ${data.raw.imageData}`);
+ }
+ const fileBuffer = readFileSync(data.raw.imageData);
+ base64Data = fileBuffer.toString("base64");
+ if (data.raw.imageData.endsWith(".jpg") || data.raw.imageData.endsWith(".jpeg")) mimeType = "image/jpeg";
+ else if (data.raw.imageData.endsWith(".webp")) mimeType = "image/webp";
+ else if (data.raw.imageData.endsWith(".gif")) mimeType = "image/gif";
+ }
+
+ imageDescription = await provider.describeImage(base64Data, mimeType, VISION_DESCRIPTION_PROMPT);
- ctx.logger.info("Image described by vision model", { obsId: data.observationId });
++ logger.info("Image described by vision model", { obsId: data.observationId });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
- ctx.logger.warn("Vision model call failed, falling back to text-only compression", {
++ logger.warn("Vision model call failed, falling back to text-only compression", {
+ obsId: data.observationId,
+ error: msg,
+ });
+ }
+ }
+
const prompt = buildCompressionPrompt({
hookType: data.raw.hookType,
toolName: data.raw.toolName,
diff --cc src/functions/evict.ts
index dc9415a3,78579abd..46b1845a
--- a/src/functions/evict.ts
+++ b/src/functions/evict.ts
@@@ -35,15 -37,9 +37,10 @@@ interface EvictionStats
}
export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
- sdk.registerFunction(
- {
- id: "mem::evict",
- description: "Evict stale memories based on age and importance",
- },
+ sdk.registerFunction("mem::evict",
async (data: { dryRun?: boolean }): Promise<EvictionStats> => {
- const ctx = getContext();
const dryRun = data?.dryRun ?? false;
+ const { decrementImageRef } = await import("./image-refs.js");
const configOverride = await kv
.get<Partial<EvictionConfig>>(KV.config, "eviction")
@@@ -93,13 -105,27 +106,29 @@@
age > maxAge &&
(o.importance ?? 5) < cfg.lowImportanceThreshold
) {
- stats.lowImportanceObs++;
- if (!dryRun) {
+ if (dryRun) {
+ stats.lowImportanceObs++;
+ } else {
+ try {
+ await kv.delete(KV.observations(session.id), o.id);
+ stats.lowImportanceObs++;
+ } catch (err) {
+ logger.warn("Eviction delete failed", {
+ resource: "observation",
+ id: o.id,
+ sessionId: session.id,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ continue;
+ }
+ if (o.imageData) await decrementImageRef(kv, sdk, o.imageData);
+ if (o.imageRef && o.imageRef !== o.imageData) await decrementImageRef(kv, sdk, o.imageRef);
- await kv
- .delete(KV.observations(session.id), o.id)
- .catch(() => {});
+ await recordAudit(kv, "delete", "mem::evict", [o.id], {
+ resource: "observation",
+ reason: "low_importance_old_observation",
+ sessionId: session.id,
+ dryRun,
+ });
}
}
}
@@@ -119,14 -145,28 +148,30 @@@
0,
obs.length - cfg.maxObservationsPerProject,
);
- stats.capEvictions += toEvict.length;
- if (!dryRun) {
+ if (dryRun) {
+ stats.capEvictions += toEvict.length;
+ } else {
for (const o of toEvict) {
+ try {
+ await kv.delete(KV.observations(o.sessionId), o.id);
+ stats.capEvictions++;
+ } catch (err) {
+ logger.warn("Eviction delete failed", {
+ resource: "observation",
+ id: o.id,
+ sessionId: o.sessionId,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ continue;
+ }
+ if (o.imageData) await decrementImageRef(kv, sdk, o.imageData);
+ if (o.imageRef && o.imageRef !== o.imageData) await decrementImageRef(kv, sdk, o.imageRef);
- await kv
- .delete(KV.observations(o.sessionId), o.id)
- .catch(() => {});
+ await recordAudit(kv, "delete", "mem::evict", [o.id], {
+ resource: "observation",
+ reason: "project_observation_cap",
+ sessionId: o.sessionId,
+ dryRun,
+ });
}
}
}
@@@ -138,13 -178,29 +183,32 @@@
if (mem.forgetAfter) {
const expiry = new Date(mem.forgetAfter).getTime();
if (now > expiry) {
- stats.expiredMemories++;
- evictedMemIds.add(mem.id);
- if (!dryRun) {
+ if (dryRun) {
+ stats.expiredMemories++;
+ evictedMemIds.add(mem.id);
+ } else {
+ try {
+ await kv.delete(KV.memories, mem.id);
+ stats.expiredMemories++;
+ evictedMemIds.add(mem.id);
+ } catch (err) {
+ logger.warn("Eviction delete failed", {
+ resource: "memory",
+ id: mem.id,
+ reason: "expired_memory",
+ error: err instanceof Error ? err.message : String(err),
+ });
+ continue;
+ }
+ if (mem.imageRef) {
+ await decrementImageRef(kv, sdk, mem.imageRef);
+ }
- await kv.delete(KV.memories, mem.id).catch(() => {});
+ await recordAudit(kv, "delete", "mem::evict", [mem.id], {
+ resource: "memory",
+ reason: "expired_memory",
+ dryRun,
+ });
+ await deleteAccessLog(kv, mem.id);
}
}
}
@@@ -156,12 -212,27 +220,30 @@@
) {
const age = now - new Date(mem.createdAt).getTime();
if (age > cfg.lowImportanceMaxDays * MS_PER_DAY) {
- stats.nonLatestMemories++;
- if (!dryRun) {
+ if (dryRun) {
+ stats.nonLatestMemories++;
+ } else {
+ try {
+ await kv.delete(KV.memories, mem.id);
+ stats.nonLatestMemories++;
+ } catch (err) {
+ logger.warn("Eviction delete failed", {
+ resource: "memory",
+ id: mem.id,
+ reason: "old_non_latest_memory",
+ error: err instanceof Error ? err.message : String(err),
+ });
+ continue;
+ }
+ if (mem.imageRef) {
+ await decrementImageRef(kv, sdk, mem.imageRef);
+ }
- await kv.delete(KV.memories, mem.id).catch(() => {});
+ await recordAudit(kv, "delete", "mem::evict", [mem.id], {
+ resource: "memory",
+ reason: "old_non_latest_memory",
+ dryRun,
+ });
+ await deleteAccessLog(kv, mem.id);
}
}
}
diff --cc src/functions/observe.ts
index 261bca0f,2043647c..c13f7538
--- a/src/functions/observe.ts
+++ b/src/functions/observe.ts
@@@ -6,30 -5,11 +5,34 @@@ import { StateKV } from "../state/kv.js
import { stripPrivateData } from "./privacy.js";
import { DedupMap } from "./dedup.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
+ import { isAutoCompressEnabled } from "../config.js";
+ import { buildSyntheticCompression } from "./compress-synthetic.js";
+ import { getSearchIndex } from "./search.js";
+ import { logger } from "../logger.js";
+export function extractImage(d: unknown): string | undefined {
+ if (!d) return undefined;
+ if (typeof d === "string") {
+ if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) {
+ return d;
+ }
+ return undefined;
+ }
+ if (typeof d === "object" && d !== null) {
+ const obj = d as Record<string, unknown>;
+ if (typeof obj["image_data"] === "string") return obj["image_data"];
+ if (typeof obj["image_path"] === "string") return obj["image_path"];
+ if (typeof obj["imageBase64"] === "string") return obj["imageBase64"];
+ if (typeof obj["imagePath"] === "string") return obj["imagePath"];
+
+ for (const key of Object.keys(obj)) {
+ const match = extractImage(obj[key]);
+ if (match) return match;
+ }
+ }
+ return undefined;
+}
+
export function registerObserveFunction(
sdk: ISdk,
kv: StateKV,
diff --cc src/functions/remember.ts
index dced14e1,69579fe0..0fb389be
--- a/src/functions/remember.ts
+++ b/src/functions/remember.ts
@@@ -119,11 -122,15 +122,20 @@@ export function registerRememberFunctio
observationIds?: string[];
memoryId?: string;
}) => {
- const ctx = getContext();
let deleted = 0;
+ const deletedMemoryIds: string[] = [];
+ const deletedObservationIds: string[] = [];
+ let deletedSession = false;
++ const { decrementImageRef } = await import("./image-refs.js");
if (data.memoryId) {
++ const mem = await kv.get<Memory>(KV.memories, data.memoryId);
await kv.delete(KV.memories, data.memoryId);
++ if (mem?.imageRef) {
++ await decrementImageRef(kv, sdk, mem.imageRef);
++ }
+ await deleteAccessLog(kv, data.memoryId);
+ deletedMemoryIds.push(data.memoryId);
deleted++;
}
@@@ -133,7 -140,8 +145,16 @@@
data.observationIds.length > 0
) {
for (const obsId of data.observationIds) {
++ const obs = await kv.get<{ imageData?: string; imageRef?: string }>(
++ KV.observations(data.sessionId),
++ obsId,
++ );
await kv.delete(KV.observations(data.sessionId), obsId);
++ if (obs?.imageData) await decrementImageRef(kv, sdk, obs.imageData);
++ if (obs?.imageRef && obs.imageRef !== obs.imageData) {
++ await decrementImageRef(kv, sdk, obs.imageRef);
++ }
+ deletedObservationIds.push(obsId);
deleted++;
}
}
@@@ -143,11 -151,12 +164,16 @@@
(!data.observationIds || data.observationIds.length === 0) &&
!data.memoryId
) {
-- const observations = await kv.list<{ id: string }>(
++ const observations = await kv.list<{ id: string; imageData?: string; imageRef?: string }>(
KV.observations(data.sessionId),
);
for (const obs of observations) {
await kv.delete(KV.observations(data.sessionId), obs.id);
++ if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData);
++ if (obs.imageRef && obs.imageRef !== obs.imageData) {
++ await decrementImageRef(kv, sdk, obs.imageRef);
++ }
+ deletedObservationIds.push(obs.id);
deleted++;
}
await kv.delete(KV.sessions, data.sessionId);
diff --cc src/functions/retention.ts
index e031f260,3262c15d..c9c16573
--- a/src/functions/retention.ts
+++ b/src/functions/retention.ts
@@@ -191,10 -294,15 +294,16 @@@ export function registerRetentionFuncti
dryRun?: boolean;
maxEvict?: number;
}) => {
- const ctx = getContext();
- const threshold = data.threshold ?? DEFAULT_DECAY.tierThresholds.cold;
- const maxEvict = data.maxEvict ?? 50;
+ const threshold =
+ typeof data?.threshold === "number" && Number.isFinite(data.threshold)
+ ? data.threshold
+ : DEFAULT_DECAY.tierThresholds.cold;
+ const maxEvictRaw =
+ typeof data?.maxEvict === "number" && Number.isInteger(data.maxEvict)
+ ? data.maxEvict
+ : 50;
+ const maxEvict = Math.min(1000, Math.max(0, maxEvictRaw));
+ const { decrementImageRef } = await import("./image-refs.js");
const allScores = await kv.list<RetentionScore>(KV.retentionScores);
const candidates = allScores
@@@ -214,16 -322,53 +323,57 @@@
};
}
+ // Branch on source (#124). Pre-0.8.10 rows have no `source` field,
+ // and that includes semantic retention rows that were written by
+ // the old scorer — so we can't just default to episodic, that
+ // would silently no-op the delete and leave the stranded semantic
+ // memory alive (the exact bug #124 is about). When `source` is
+ // missing, probe both namespaces to find where the memoryId
+ // actually lives and route the delete there. After one re-score
+ // (mem::retention-score) every row will have the correct tag.
let evicted = 0;
+ let evictedEpisodic = 0;
+ let evictedSemantic = 0;
+ const evictedIds: string[] = [];
for (const candidate of candidates) {
try {
- const mem = await kv.get<Memory>(KV.memories, candidate.memoryId);
+ let scope: string | null = null;
+ let resolvedSource: "episodic" | "semantic" | null = null;
+ if (candidate.source === "semantic") {
+ scope = KV.semantic;
+ resolvedSource = "semantic";
+ } else if (candidate.source === "episodic") {
+ scope = KV.memories;
+ resolvedSource = "episodic";
+ } else {
+ const episodic = await kv.get(KV.memories, candidate.memoryId);
+ if (episodic !== null) {
+ scope = KV.memories;
+ resolvedSource = "episodic";
+ } else {
+ const semantic = await kv.get(KV.semantic, candidate.memoryId);
+ if (semantic !== null) {
+ scope = KV.semantic;
+ resolvedSource = "semantic";
+ }
+ }
+ }
+
+ if (!scope || !resolvedSource) {
+ continue;
+ }
+
++ const mem = await kv.get<Memory>(scope, candidate.memoryId);
+ if (mem && mem.imageRef) {
+ await decrementImageRef(kv, sdk, mem.imageRef);
+ }
- await kv.delete(KV.memories, candidate.memoryId);
+ await kv.delete(scope, candidate.memoryId);
await kv.delete(KV.retentionScores, candidate.memoryId);
+ await deleteAccessLog(kv, candidate.memoryId);
evicted++;
+ evictedIds.push(candidate.memoryId);
+ if (resolvedSource === "semantic") evictedSemantic++;
+ else evictedEpisodic++;
} catch {
continue;
}
diff --cc src/state/schema.ts
index 1781ac78,a39077aa..fb2c9fc3
--- a/src/state/schema.ts
+++ b/src/state/schema.ts
@@@ -40,8 -40,7 +40,8 @@@ export const KV =
enrichedChunks: (sessionId: string) => `mem:enriched:${sessionId}`,
latentEmbeddings: (obsId: string) => `mem:latent:${obsId}`,
retentionScores: "mem:retention",
+ accessLog: "mem:access",
+ imageRefs: "mem:image-refs",
- state: "mem:state",
} as const;
export const STREAM = {
diff --cc src/triggers/api.ts
index 636065fc,689a8f10..1db1daac
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@@ -1206,35 -1485,14 +1485,17 @@@ export function registerApiTriggers
config: { api_path: "/agentmemory/leases/renew", http_method: "POST" },
});
- sdk.registerFunction(
- { id: "api::routine-create" },
- async (
- req: ApiRequest<{
- name: string;
- description?: string;
- steps: unknown[];
- tags?: string[];
- frozen?: boolean;
- sourceProceduralIds?: string[];
- }>,
- ): Promise<Response> => {
- sdk.registerFunction("api::routine-create",
++ sdk.registerFunction("api::routine-create",
+ async (req: ApiRequest): Promise<Response> => {
const authErr = checkAuth(req, secret);
if (authErr) return authErr;
- if (!req.body?.name) {
- return { status_code: 400, body: { error: "name is required" } };
+ if (!req.body?.name || !req.body?.steps) {
+ return {
+ status_code: 400,
+ body: { error: "name and steps are required" },
+ };
}
- const payload = {
- name: req.body.name,
- description: req.body.description,
- steps: req.body.steps,
- tags: req.body.tags,
- frozen: req.body.frozen,
- sourceProceduralIds: req.body.sourceProceduralIds,
- };
- const result = await sdk.trigger("mem::routine-create", payload);
+ const result = await sdk.trigger({ function_id: "mem::routine-create", payload: req.body });
return { status_code: 201, body: result };
},
);
diff --cc test/multimodal.test.ts
index afcb3ed4,00000000..de694636
mode 100644,000000..100644
--- a/test/multimodal.test.ts
+++ b/test/multimodal.test.ts
@@@ -1,371 -1,0 +1,376 @@@
+import { describe, it, expect, vi, afterAll, beforeEach } from "vitest";
+import { existsSync, rmSync } from "node:fs";
+
- vi.mock("iii-sdk", () => ({
- getContext: () => ({
- logger: { info: vi.fn(), error: vi.fn(), warn: vi.fn() },
- }),
- }));
++vi.mock("iii-sdk", async (importOriginal) => {
++ const actual = await importOriginal<typeof import("iii-sdk")>();
++ return {
++ ...actual,
++ getContext: () => ({
++ logger: { info: vi.fn(), error: vi.fn(), warn: vi.fn() },
++ }),
++ };
++});
+
+vi.mock("../src/functions/search.js", () => ({
+ getSearchIndex: () => ({
+ add: vi.fn(),
+ }),
+}));
+
+const mockTriggerVoid = vi.fn();
- const mockSdk = { triggerVoid: mockTriggerVoid } as any;
++const mockTrigger = vi.fn().mockResolvedValue(undefined);
++const mockSdk = { triggerVoid: mockTriggerVoid, trigger: mockTrigger } as any;
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ return (store.get(scope)?.get(key) as T) ?? null;
+ },
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ if (!store.has(scope)) return [];
+ return Array.from(store.get(scope)!.values()) as T[];
+ },
+ getStore: () => store,
+ };
+}
+
+const kv = mockKV() as any;
+
+import { registerObserveFunction } from "../src/functions/observe.js";
+import { registerCompressFunction } from "../src/functions/compress.js";
+import type { RawObservation, CompressedObservation, MemoryProvider } from "../src/types.js";
+
+const VALID_COMPRESS_XML = `<type>image</type>
+<title>Screenshot of Red Dot</title>
+<subtitle>Test image observation</subtitle>
+<facts><fact>Image shows a single red pixel on white background</fact></facts>
+<narrative>A vision model described a screenshot showing a red dot on a white background</narrative>
+<concepts><concept>testing</concept><concept>screenshot</concept></concepts>
+<files></files>
+<importance>5</importance>`;
+
+describe("End-to-End Multimodal Flow", () => {
+ let savedImagePath: string | undefined;
+
+ afterAll(() => {
+ if (savedImagePath && existsSync(savedImagePath)) {
+ rmSync(savedImagePath);
+ console.log(`Cleanup: Removed test image at ${savedImagePath}`);
+ }
+ });
+
+ beforeEach(() => {
+ mockTriggerVoid.mockClear();
+ });
+
+ it("Step 1: Agent image should be successfully saved to hard drive", async () => {
+ let observeCallback: any = null;
- const sdkMocker = { ...mockSdk, registerFunction: vi.fn((config, cb) => { if (config.id === "mem::observe") observeCallback = cb; }) };
++ const sdkMocker = { ...mockSdk, registerFunction: vi.fn((id, cb) => { if (id === "mem::observe") observeCallback = cb; }) };
+ registerObserveFunction(sdkMocker, kv);
+
+ const fakeIncomingData = {
+ hookType: "post_tool_use",
+ sessionId: "test-session",
+ timestamp: new Date().toISOString(),
+ data: {
+ tool_name: "screenshot",
+ tool_output: {
+ image_data: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="
+ }
+ }
+ };
+
+ const res = await observeCallback(fakeIncomingData);
+ expect(res.observationId).toBeDefined();
+
+ const obsList = await kv.list("mem:obs:test-session");
+ expect(obsList.length).toBe(1);
+
+ const raw = obsList[0] as RawObservation;
+ expect(raw.modality).toBe("mixed");
+
+ expect(raw.imageData).toBeDefined();
+ expect(typeof raw.imageData).toBe("string");
+ expect(existsSync(raw.imageData!)).toBe(true);
+
+ savedImagePath = raw.imageData;
+
+ const deltaCalls = mockTriggerVoid.mock.calls.filter(
+ (c: any[]) => c[0] === "mem::disk-size-delta"
+ );
+ expect(deltaCalls.length).toBe(1);
+ expect(deltaCalls[0][1].deltaBytes).toBeGreaterThan(0);
+ });
+
+ it("Step 2 & 3: mem::compress should call the vision model and store compressed observation in KV", async () => {
+ const mockProvider: MemoryProvider = {
+ name: "mock-vision",
+ compress: async (_systemPrompt, userPrompt) => {
+ expect(userPrompt).toContain("TEST_VISION_RESULT: I see a red dot");
+ return VALID_COMPRESS_XML;
+ },
+ summarize: async () => "",
+ describeImage: async (_base64, _mimeType, _prompt) => {
+ return "TEST_VISION_RESULT: I see a red dot";
+ },
+ };
+
+ let compressCallback: any = null;
+ const sdkMocker = {
+ ...mockSdk,
- registerFunction: vi.fn((config, cb) => {
- if (config.id === "mem::compress") compressCallback = cb;
++ registerFunction: vi.fn((id, cb) => {
++ if (id === "mem::compress") compressCallback = cb;
+ }),
+ };
+ registerCompressFunction(sdkMocker, kv, mockProvider);
+
+ expect(compressCallback).not.toBeNull();
+
+ const rawObsList = await kv.list("mem:obs:test-session");
+ const raw = rawObsList[0] as RawObservation;
+
+ expect(raw.modality).toBeDefined();
+ expect(raw.imageData).toBe(savedImagePath);
+
+ const result = await compressCallback({
+ observationId: raw.id,
+ sessionId: raw.sessionId,
+ raw,
+ });
+
+ expect(result.success).toBe(true);
+ expect(result.compressed).toBeDefined();
+
+ const compressed = result.compressed as CompressedObservation;
+ expect(compressed.imageDescription).toBe("TEST_VISION_RESULT: I see a red dot");
+ expect(compressed.imageRef).toBe(savedImagePath);
+ expect(compressed.modality).toBe("mixed");
+ expect(compressed.title).toBe("Screenshot of Red Dot");
+ expect(compressed.narrative).toContain("red dot");
+
+ const stored = await kv.get("mem:obs:test-session", raw.id!) as CompressedObservation | null;
+ expect(stored).not.toBeNull();
+ expect(stored!.imageDescription).toBe("TEST_VISION_RESULT: I see a red dot");
+ expect(stored!.imageRef).toBe(savedImagePath);
+ });
+});
+
+describe("Disk Size Manager", () => {
+ it("should increment disk size and trigger cleanup when over quota", async () => {
+ const localKv = mockKV() as any;
+ const localTriggerVoid = vi.fn();
+ const localSdk = { triggerVoid: localTriggerVoid } as any;
+
+ let managerCallback: any = null;
+ const sdkMocker = {
+ ...localSdk,
+ registerFunction: vi.fn((config: any, cb: any) => {
+ if (config.id === "mem::disk-size-delta") managerCallback = cb;
+ }),
+ };
+
+ const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");
+ registerDiskSizeManager(sdkMocker, localKv);
+
+ expect(managerCallback).not.toBeNull();
+
+ const res1 = await managerCallback({ deltaBytes: 1000 });
+ expect(res1.success).toBe(true);
+ expect(res1.currentTotal).toBe(1000);
+
+ const res2 = await managerCallback({ deltaBytes: 2000 });
+ expect(res2.success).toBe(true);
+ expect(res2.currentTotal).toBe(3000);
+
+ expect(localTriggerVoid).not.toHaveBeenCalled();
+ });
+
+ it("should trigger cleanup when total exceeds max bytes", async () => {
+ const originalEnv = process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES;
+ try {
+ process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = "5000";
+
+ const localKv = mockKV() as any;
+ const localTriggerVoid = vi.fn();
+ const localSdk = { triggerVoid: localTriggerVoid } as any;
+
+ let managerCallback: any = null;
+ const sdkMocker = {
+ ...localSdk,
+ registerFunction: vi.fn((config: any, cb: any) => {
+ if (config.id === "mem::disk-size-delta") managerCallback = cb;
+ }),
+ };
+
+ const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");
+ registerDiskSizeManager(sdkMocker, localKv);
+
+ await managerCallback({ deltaBytes: 3000 });
+ expect(localTriggerVoid).not.toHaveBeenCalled();
+
+ await managerCallback({ deltaBytes: 3000 });
+ expect(localTriggerVoid).toHaveBeenCalledWith("mem::image-quota-cleanup", {});
+ } finally {
+ if (originalEnv === undefined) {
+ delete process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES;
+ } else {
+ process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES = originalEnv;
+ }
+ }
+ });
+
+ it("should clamp negative totals to zero", async () => {
+ const localKv = mockKV() as any;
+ const localSdk = { triggerVoid: vi.fn() } as any;
+
+ let managerCallback: any = null;
+ const sdkMocker = {
+ ...localSdk,
+ registerFunction: vi.fn((config: any, cb: any) => {
+ if (config.id === "mem::disk-size-delta") managerCallback = cb;
+ }),
+ };
+
+ const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");
+ registerDiskSizeManager(sdkMocker, localKv);
+
+ const res = await managerCallback({ deltaBytes: -99999 });
+ expect(res.currentTotal).toBe(0);
+ });
+
+ it("should reject invalid deltaBytes", async () => {
+ const localKv = mockKV() as any;
+ const localSdk = { triggerVoid: vi.fn() } as any;
+
+ let managerCallback: any = null;
+ const sdkMocker = {
+ ...localSdk,
+ registerFunction: vi.fn((config: any, cb: any) => {
+ if (config.id === "mem::disk-size-delta") managerCallback = cb;
+ }),
+ };
+
+ const { registerDiskSizeManager } = await import("../src/functions/disk-size-manager.js");
+ registerDiskSizeManager(sdkMocker, localKv);
+
+ const res = await managerCallback({ deltaBytes: "not-a-number" });
+ expect(res.success).toBe(false);
+ });
+});
+
+describe("Image Refs", () => {
+ it("should increment and decrement ref counts correctly with deletion parity", async () => {
+ const localKv = mockKV() as any;
+ const localTriggerVoid = vi.fn();
+ const localSdk = { triggerVoid: localTriggerVoid } as any;
+
+ const { saveImageToDisk } = await import("../src/utils/image-store.js");
+ const { incrementImageRef, decrementImageRef, getImageRefCount } = await import("../src/functions/image-refs.js");
+
+ const result = await saveImageToDisk(
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="
+ );
+ const testFile = result.filePath;
+
+ expect(existsSync(testFile)).toBe(true);
+
+ // Initial state
+ expect(await getImageRefCount(localKv, testFile)).toBe(0);
+
+ // Increment to 1
+ await incrementImageRef(localKv, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(1);
+
+ // Increment to 2 (shared image)
+ await incrementImageRef(localKv, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(2);
+
+ // Decrement from 2 to 1
+ await decrementImageRef(localKv, localSdk, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(1);
+
+ // (c) shared image with refcount >= 2 is NOT deleted when one decrements
+ expect(existsSync(testFile)).toBe(true);
+ expect(localTriggerVoid).not.toHaveBeenCalled();
+
+ // (a) decrementing to zero triggers image file deletion and negative delta
+ await decrementImageRef(localKv, localSdk, testFile);
+ expect(await getImageRefCount(localKv, testFile)).toBe(0);
+ expect(existsSync(testFile)).toBe(false);
+
+ const deltaCalls = localTriggerVoid.mock.calls.filter(
+ (c: any[]) => c[0] === "mem::disk-size-delta"
+ );
+ expect(deltaCalls.length).toBe(1);
+ expect(deltaCalls[0][1].deltaBytes).toBeLessThan(0);
+
+ // (b) decrementing an already-zero/unknown ref is a no-op
+ localTriggerVoid.mockClear();
+ await decrementImageRef(localKv, localSdk, "/fake/unknown/path.png");
+ expect(await getImageRefCount(localKv, "/fake/unknown/path.png")).toBe(0);
+ const noOpDeltaCalls = localTriggerVoid.mock.calls.filter(
+ (c: any[]) => c[0] === "mem::disk-size-delta"
+ );
+ expect(noOpDeltaCalls.length).toBe(0);
+ });
+});
+
+describe("Image Store", () => {
+ let testFilePath: string | undefined;
+
+ afterAll(() => {
+ if (testFilePath && existsSync(testFilePath)) {
+ rmSync(testFilePath);
+ }
+ });
+
+ it("saveImageToDisk should return filePath and bytesWritten", async () => {
+ const { saveImageToDisk } = await import("../src/utils/image-store.js");
+ const result = await saveImageToDisk(
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="
+ );
+
+ expect(result.filePath).toBeDefined();
+ expect(typeof result.filePath).toBe("string");
+ expect(result.filePath.length).toBeGreaterThan(0);
+ expect(result.bytesWritten).toBeGreaterThan(0);
+ expect(existsSync(result.filePath)).toBe(true);
+
+ testFilePath = result.filePath;
+ });
+
+ it("deleteImage should return deletedBytes after successful deletion", async () => {
+ const { deleteImage } = await import("../src/utils/image-store.js");
+
+ expect(testFilePath).toBeDefined();
+ expect(existsSync(testFilePath!)).toBe(true);
+
+ const result = await deleteImage(testFilePath!);
+ expect(result.deletedBytes).toBeGreaterThan(0);
+ expect(existsSync(testFilePath!)).toBe(false);
+
+ testFilePath = undefined;
+ });
+
+ it("deleteImage should return 0 for non-existent files", async () => {
+ const { deleteImage } = await import("../src/utils/image-store.js");
+ const result = await deleteImage("/non/existent/file.png");
+ expect(result.deletedBytes).toBe(0);
+ });
+
+ it("saveImageToDisk should return empty for empty input", async () => {
+ const { saveImageToDisk } = await import("../src/utils/image-store.js");
+ const result = await saveImageToDisk("");
+ expect(result.filePath).toBe("");
+ expect(result.bytesWritten).toBe(0);
+ });
+});