Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

feat: implement multimodal image memory

tanmaishie692cf009528
diff --git a/plugin/scripts/post-tool-use.mjs b/plugin/scripts/post-tool-use.mjs
index f777112e..1703fed0 100755
--- a/plugin/scripts/post-tool-use.mjs
+++ b/plugin/scripts/post-tool-use.mjs
@@ -17,6 +17,7 @@ async function main() {
return;
}
const sessionId = data.session_id || "unknown";
+ const { imageData, cleanOutput } = extractImageData(data.tool_output);
try {
await fetch(`${REST_URL}/agentmemory/observe`, {
method: "POST",
@@ -30,13 +31,40 @@ async function main() {
data: {
tool_name: data.tool_name,
tool_input: data.tool_input,
- tool_output: truncate(data.tool_output, 8e3)
+ tool_output: truncate(cleanOutput, 8e3),
+ ...imageData ? { image_data: imageData } : {}
}
}),
signal: AbortSignal.timeout(3e3)
});
} catch {}
}
+function isBase64Image(val) {
+ return typeof val === "string" && (val.startsWith("data:image/") || val.startsWith("iVBORw0KGgo") || val.startsWith("/9j/"));
+}
+function extractImageData(output) {
+ if (isBase64Image(output)) return {
+ imageData: output,
+ cleanOutput: "[image data extracted]"
+ };
+ if (typeof output === "object" && output !== null && !Array.isArray(output)) {
+ const obj = output;
+ let imageData;
+ const clean = {};
+ for (const [key, val] of Object.entries(obj)) if (!imageData && isBase64Image(val)) {
+ imageData = val;
+ clean[key] = "[image data extracted]";
+ } else clean[key] = val;
+ return {
+ imageData,
+ cleanOutput: clean
+ };
+ }
+ return {
+ imageData: void 0,
+ cleanOutput: output
+ };
+}
function truncate(value, max) {
if (typeof value === "string" && value.length > max) return value.slice(0, max) + "\n[...truncated]";
if (typeof value === "object" && value !== null) {
diff --git a/src/functions/auto-forget.ts b/src/functions/auto-forget.ts
index ed33f823..d1da615a 100644
--- a/src/functions/auto-forget.ts
+++ b/src/functions/auto-forget.ts
@@ -1,7 +1,7 @@
import type { ISdk } from "iii-sdk";
import { getContext } from "iii-sdk";
import type { Memory, CompressedObservation, Session } from "../types.js";
-import { KV, jaccardSimilarity } from "../state/schema.js";
+import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
const MS_PER_DAY = 24 * 60 * 60 * 1000;
@@ -46,6 +46,10 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
result.ttlExpired.push(mem.id);
deletedIds.add(mem.id);
if (!dryRun) {
+ if ((mem as any).imageRef) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ deleteImage((mem as any).imageRef);
+ }
await kv.delete(KV.memories, mem.id);
}
}
@@ -118,7 +122,7 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
if (!dryRun) {
const older =
new Date(memA.createdAt).getTime() <
- new Date(memB.createdAt).getTime()
+ new Date(memB.createdAt).getTime()
? memA
: memB;
older.isLatest = false;
@@ -149,9 +153,12 @@ export function registerAutoForgetFunction(sdk: ISdk, kv: StateKV): void {
if (age > 180 * MS_PER_DAY && (obs.importance ?? 5) <= 2) {
result.lowValueObs.push(obs.id);
if (!dryRun) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ if ((obs as any).imageData) deleteImage((obs as any).imageData);
+ if ((obs as any).imageRef) deleteImage((obs as any).imageRef);
await kv
.delete(KV.observations(sessions[i].id), obs.id)
- .catch(() => {});
+ .catch(() => { });
}
}
}
diff --git a/src/functions/compress.ts b/src/functions/compress.ts
index 74f32387..47c4ec5b 100644
--- a/src/functions/compress.ts
+++ b/src/functions/compress.ts
@@ -1,5 +1,6 @@
import type { ISdk } from "iii-sdk";
import { getContext } from "iii-sdk";
+import { readFileSync } from "node:fs";
import type {
RawObservation,
CompressedObservation,
@@ -12,6 +13,7 @@ import {
COMPRESSION_SYSTEM,
buildCompressionPrompt,
} from "../prompts/compression.js";
+import { VISION_DESCRIPTION_PROMPT } from "../prompts/vision.js";
import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
import { getSearchIndex } from "./search.js";
import { CompressOutputSchema } from "../eval/schemas.js";
@@ -34,6 +36,7 @@ const VALID_TYPES = new Set<string>([
"subagent",
"notification",
"task",
+ "image",
"other",
]);
@@ -78,11 +81,41 @@ export function registerCompressFunction(
}) => {
const ctx = getContext();
const startMs = Date.now();
+
+ let imageDescription: string | undefined;
+ const hasImage = data.raw.modality === "image" || data.raw.modality === "mixed";
+
+ if (hasImage && data.raw.imageData && provider.describeImage) {
+ try {
+ let base64Data = data.raw.imageData;
+ let mimeType = "image/png";
+
+ if (!data.raw.imageData.startsWith("/9j/") && !data.raw.imageData.startsWith("iVBOR")) {
+ const fileBuffer = readFileSync(data.raw.imageData);
+ base64Data = fileBuffer.toString("base64");
+ if (data.raw.imageData.endsWith(".jpg") || data.raw.imageData.endsWith(".jpeg")) mimeType = "image/jpeg";
+ else if (data.raw.imageData.endsWith(".webp")) mimeType = "image/webp";
+ else if (data.raw.imageData.endsWith(".gif")) mimeType = "image/gif";
+ }
+
+ imageDescription = await provider.describeImage(base64Data, mimeType, VISION_DESCRIPTION_PROMPT);
+ ctx.logger.info("Image described by vision model", { obsId: data.observationId });
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ ctx.logger.warn("Vision model call failed, falling back to text-only compression", {
+ obsId: data.observationId,
+ error: msg,
+ });
+ }
+ }
+
const prompt = buildCompressionPrompt({
hookType: data.raw.hookType,
toolName: data.raw.toolName,
toolInput: data.raw.toolInput,
- toolOutput: data.raw.toolOutput,
+ toolOutput: imageDescription
+ ? `[Image Description]: ${imageDescription}\n\n${data.raw.toolOutput ?? ""}`
+ : data.raw.toolOutput,
userPrompt: data.raw.userPrompt,
timestamp: data.raw.timestamp,
});
@@ -130,6 +163,9 @@ export function registerCompressFunction(
timestamp: data.raw.timestamp,
...parsed,
confidence: qualityScore / 100,
+ ...(hasImage ? { modality: data.raw.modality } : {}),
+ ...(imageDescription ? { imageDescription } : {}),
+ ...(data.raw.imageData ? { imageRef: data.raw.imageData } : {}),
};
await kv.set(
diff --git a/src/functions/evict.ts b/src/functions/evict.ts
index 905a6a55..935580ec 100644
--- a/src/functions/evict.ts
+++ b/src/functions/evict.ts
@@ -94,6 +94,9 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
) {
stats.lowImportanceObs++;
if (!dryRun) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ if ((o as any).imageData) deleteImage((o as any).imageData);
+ if ((o as any).imageRef) deleteImage((o as any).imageRef);
await kv
.delete(KV.observations(session.id), o.id)
.catch(() => {});
@@ -119,6 +122,9 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
stats.capEvictions += toEvict.length;
if (!dryRun) {
for (const o of toEvict) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ if ((o as any).imageData) deleteImage((o as any).imageData);
+ if ((o as any).imageRef) deleteImage((o as any).imageRef);
await kv
.delete(KV.observations(o.sessionId), o.id)
.catch(() => {});
@@ -136,6 +142,10 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
stats.expiredMemories++;
evictedMemIds.add(mem.id);
if (!dryRun) {
+ if ((mem as any).imageRef) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ deleteImage((mem as any).imageRef);
+ }
await kv.delete(KV.memories, mem.id).catch(() => {});
}
}
@@ -150,6 +160,10 @@ export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
if (age > cfg.lowImportanceMaxDays * MS_PER_DAY) {
stats.nonLatestMemories++;
if (!dryRun) {
+ if ((mem as any).imageRef) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ deleteImage((mem as any).imageRef);
+ }
await kv.delete(KV.memories, mem.id).catch(() => {});
}
}
diff --git a/src/functions/observe.ts b/src/functions/observe.ts
index 2e280752..e63f8886 100644
--- a/src/functions/observe.ts
+++ b/src/functions/observe.ts
@@ -7,6 +7,29 @@ import { stripPrivateData } from "./privacy.js";
import { DedupMap } from "./dedup.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
+export function extractImage(d: unknown): string | undefined {
+ if (!d) return undefined;
+ if (typeof d === "string") {
+ if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) {
+ return d;
+ }
+ return undefined;
+ }
+ if (typeof d === "object" && d !== null) {
+ const obj = d as Record<string, unknown>;
+ if (typeof obj["image_data"] === "string") return obj["image_data"];
+ if (typeof obj["image_path"] === "string") return obj["image_path"];
+ if (typeof obj["imageBase64"] === "string") return obj["imageBase64"];
+ if (typeof obj["imagePath"] === "string") return obj["imagePath"];
+
+ for (const key of Object.keys(obj)) {
+ const match = extractImage(obj[key]);
+ if (match) return match;
+ }
+ }
+ return undefined;
+}
+
export function registerObserveFunction(
sdk: ISdk,
kv: StateKV,
@@ -85,6 +108,18 @@ export function registerObserveFunction(
if (payload.hookType === "prompt_submit") {
raw.userPrompt = d["prompt"] as string | undefined;
}
+
+ const hiddenImage = extractImage(sanitizedRaw);
+ if (hiddenImage) {
+ raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image";
+
+ if (hiddenImage.startsWith("data:image/") || hiddenImage.startsWith("iVBORw0KGgo") || hiddenImage.startsWith("/9j/")) {
+ const { saveImageToDisk } = await import("../utils/image-store.js");
+ raw.imageData = saveImageToDisk(hiddenImage);
+ } else {
+ raw.imageData = hiddenImage;
+ }
+ }
}
return withKeyedLock(`obs:${payload.sessionId}`, async () => {
diff --git a/src/functions/retention.ts b/src/functions/retention.ts
index 27753fdc..8322bb8f 100644
--- a/src/functions/retention.ts
+++ b/src/functions/retention.ts
@@ -216,6 +216,11 @@ export function registerRetentionFunctions(
let evicted = 0;
for (const candidate of candidates) {
try {
+ const mem = await kv.get<Memory>(KV.memories, candidate.memoryId);
+ if (mem && (mem as any).imageRef) {
+ const { deleteImage } = await import("../utils/image-store.js");
+ deleteImage((mem as any).imageRef);
+ }
await kv.delete(KV.memories, candidate.memoryId);
await kv.delete(KV.retentionScores, candidate.memoryId);
evicted++;
diff --git a/src/hooks/post-tool-use.ts b/src/hooks/post-tool-use.ts
index 32ec8031..2b50d8a8 100644
--- a/src/hooks/post-tool-use.ts
+++ b/src/hooks/post-tool-use.ts
@@ -24,6 +24,8 @@ async function main() {
const sessionId = (data.session_id as string) || "unknown";
+ const { imageData, cleanOutput } = extractImageData(data.tool_output);
+
try {
await fetch(`${REST_URL}/agentmemory/observe`, {
method: "POST",
@@ -37,16 +39,49 @@ async function main() {
data: {
tool_name: data.tool_name,
tool_input: data.tool_input,
- tool_output: truncate(data.tool_output, 8000),
+ tool_output: truncate(cleanOutput, 8000),
+ ...(imageData ? { image_data: imageData } : {}),
},
}),
signal: AbortSignal.timeout(3000),
});
} catch {
- // fire and forget
}
}
+function isBase64Image(val: unknown): val is string {
+ return typeof val === "string" && (
+ val.startsWith("data:image/") ||
+ val.startsWith("iVBORw0KGgo") ||
+ val.startsWith("/9j/")
+ );
+}
+
+function extractImageData(output: unknown): { imageData: string | undefined; cleanOutput: unknown } {
+ if (isBase64Image(output)) {
+ return { imageData: output, cleanOutput: "[image data extracted]" };
+ }
+
+ if (typeof output === "object" && output !== null && !Array.isArray(output)) {
+ const obj = output as Record<string, unknown>;
+ let imageData: string | undefined;
+ const clean: Record<string, unknown> = {};
+
+ for (const [key, val] of Object.entries(obj)) {
+ if (!imageData && isBase64Image(val)) {
+ imageData = val;
+ clean[key] = "[image data extracted]";
+ } else {
+ clean[key] = val;
+ }
+ }
+
+ return { imageData, cleanOutput: clean };
+ }
+
+ return { imageData: undefined, cleanOutput: output };
+}
+
function truncate(value: unknown, max: number): unknown {
if (typeof value === "string" && value.length > max) {
return value.slice(0, max) + "\n[...truncated]";
diff --git a/src/prompts/vision.ts b/src/prompts/vision.ts
new file mode 100644
index 00000000..4472f82f
--- /dev/null
+++ b/src/prompts/vision.ts
@@ -0,0 +1,8 @@
+export const VISION_DESCRIPTION_PROMPT = `Describe what this image shows in the context of software development. Extract:
+- What type of image this is (screenshot, diagram, mockup, terminal output, error, etc.)
+- Key entities visible (files, components, UI elements, error messages)
+- Relationships or flow shown
+- Any decisions, errors, or state visible
+- Text content visible in the image
+
+Be concise but preserve all technically relevant details. Output plain text, no XML.`;
diff --git a/src/providers/agent-sdk.ts b/src/providers/agent-sdk.ts
index dde21941..47ab67f4 100644
--- a/src/providers/agent-sdk.ts
+++ b/src/providers/agent-sdk.ts
@@ -26,7 +26,8 @@ export class AgentSDKProvider implements MemoryProvider {
let result = ''
for await (const msg of messages) {
if (msg.type === 'result') {
- result = typeof msg.result === 'string' ? msg.result : JSON.stringify(msg.result)
+ const payload = (msg as any).result;
+ result = typeof payload === 'string' ? payload : JSON.stringify(payload)
}
}
return result
diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts
index 3a0e5b65..d26f5c0a 100644
--- a/src/providers/anthropic.ts
+++ b/src/providers/anthropic.ts
@@ -21,6 +21,26 @@ export class AnthropicProvider implements MemoryProvider {
return this.call(systemPrompt, userPrompt)
}
+ async describeImage(imageData: string, mimeType: string, prompt: string): Promise<string> {
+ const response = await this.client.messages.create({
+ model: this.model,
+ max_tokens: this.maxTokens,
+ messages: [{
+ role: 'user',
+ content: [
+ {
+ type: 'image',
+ source: { type: 'base64', media_type: mimeType as 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp', data: imageData },
+ },
+ { type: 'text', text: prompt },
+ ],
+ }],
+ })
+
+ const textBlock = response.content.find((b) => b.type === 'text')
+ return textBlock?.text ?? ''
+ }
+
private async call(systemPrompt: string, userPrompt: string): Promise<string> {
const response = await this.client.messages.create({
model: this.model,
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 817adf70..7f7e8bf9 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -1211,10 +1211,11 @@ export function registerApiTriggers(
async (req: ApiRequest): Promise<Response> => {
const authErr = checkAuth(req, secret);
if (authErr) return authErr;
- if (!req.body?.name) {
+ const body = req.body as Record<string, unknown>;
+ if (!body?.name) {
return { status_code: 400, body: { error: "name is required" } };
}
- const result = await sdk.trigger("mem::routine-create", req.body);
+ const result = await sdk.trigger("mem::routine-create", body);
return { status_code: 201, body: result };
},
);
diff --git a/src/types.ts b/src/types.ts
index 8a600651..5070717a 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -21,6 +21,8 @@ export interface RawObservation {
userPrompt?: string;
assistantResponse?: string;
raw: unknown;
+ modality?: "text" | "image" | "mixed";
+ imageData?: string;
}
export interface CompressedObservation {
@@ -36,6 +38,11 @@ export interface CompressedObservation {
files: string[];
importance: number;
confidence?: number;
+ imageRef?: string;
+ imageDescription?: string;
+ imageThumbBase64?: string;
+ modality?: "text" | "image" | "mixed";
+
}
export type ObservationType =
@@ -121,6 +128,7 @@ export interface MemoryProvider {
name: string;
compress(systemPrompt: string, userPrompt: string): Promise<string>;
summarize(systemPrompt: string, userPrompt: string): Promise<string>;
+ describeImage?(imageData: string, mimeType: string, prompt: string): Promise<string>;
}
export interface AgentMemoryConfig {
@@ -435,39 +443,39 @@ export interface AuditEntry {
id: string;
timestamp: string;
operation:
- | "observe"
- | "compress"
- | "remember"
- | "forget"
- | "evolve"
- | "consolidate"
- | "share"
- | "delete"
- | "import"
- | "export"
- | "action_create"
- | "action_update"
- | "lease_acquire"
- | "lease_release"
- | "routine_run"
- | "signal_send"
- | "checkpoint_resolve"
- | "mesh_sync"
- | "sentinel_create"
- | "sentinel_trigger"
- | "sketch_create"
- | "sketch_promote"
- | "sketch_discard"
- | "crystallize"
- | "diagnose"
- | "heal"
- | "facet_tag"
- | "lesson_save"
- | "lesson_recall"
- | "lesson_strengthen"
- | "obsidian_export"
- | "reflect"
- | "insight_search";
+ | "observe"
+ | "compress"
+ | "remember"
+ | "forget"
+ | "evolve"
+ | "consolidate"
+ | "share"
+ | "delete"
+ | "import"
+ | "export"
+ | "action_create"
+ | "action_update"
+ | "lease_acquire"
+ | "lease_release"
+ | "routine_run"
+ | "signal_send"
+ | "checkpoint_resolve"
+ | "mesh_sync"
+ | "sentinel_create"
+ | "sentinel_trigger"
+ | "sketch_create"
+ | "sketch_promote"
+ | "sketch_discard"
+ | "crystallize"
+ | "diagnose"
+ | "heal"
+ | "facet_tag"
+ | "lesson_save"
+ | "lesson_recall"
+ | "lesson_strengthen"
+ | "obsidian_export"
+ | "reflect"
+ | "insight_search";
userId?: string;
functionId: string;
targetIds: string[];
diff --git a/src/utils/image-store.ts b/src/utils/image-store.ts
new file mode 100644
index 00000000..94a34bee
--- /dev/null
+++ b/src/utils/image-store.ts
@@ -0,0 +1,46 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { writeFileSync, mkdirSync } from "node:fs";
+import { createHash } from "node:crypto";
+
+const IMAGES_DIR = join(homedir(), ".agentmemory", "images");
+
+export function saveImageToDisk(base64Data: string): string {
+ if (!base64Data) return "";
+ mkdirSync(IMAGES_DIR, { recursive: true });
+
+ let cleanBase64 = base64Data;
+ let ext = "png";
+
+ if (base64Data.startsWith("data:image/")) {
+ const commaIdx = base64Data.indexOf(",");
+ if (commaIdx !== -1) {
+ const meta = base64Data.substring(0, commaIdx);
+ if (meta.includes("jpeg") || meta.includes("jpg")) ext = "jpg";
+ else if (meta.includes("webp")) ext = "webp";
+ else if (meta.includes("gif")) ext = "gif";
+ cleanBase64 = base64Data.substring(commaIdx + 1);
+ }
+ } else if (base64Data.startsWith("/9j/")) {
+ ext = "jpg";
+ }
+
+ const hash = createHash("sha256").update(cleanBase64).digest("hex");
+ const filePath = join(IMAGES_DIR, `${hash}.${ext}`);
+
+ writeFileSync(filePath, Buffer.from(cleanBase64, "base64"));
+ return filePath;
+}
+
+export function deleteImage(filePath: string | undefined): void {
+ if (!filePath) return;
+ try {
+ const { existsSync, unlinkSync } = require("node:fs");
+ if (existsSync(filePath)) {
+ unlinkSync(filePath);
+ }
+ } catch (err) {
+ console.error("[agentmemory] Failed to delete image context:", err);
+ }
+}
+
diff --git a/test/multimodal.test.ts b/test/multimodal.test.ts
new file mode 100644
index 00000000..290ce920
--- /dev/null
+++ b/test/multimodal.test.ts
@@ -0,0 +1,129 @@
+import { describe, it, expect, vi, afterAll } from "vitest";
+import { existsSync, rmSync } from "node:fs";
+
+vi.mock("iii-sdk", () => ({
+ getContext: () => ({
+ logger: { info: vi.fn(), error: vi.fn(), warn: vi.fn() },
+ }),
+}));
+
+const mockTriggerVoid = vi.fn();
+const mockSdk = { triggerVoid: mockTriggerVoid } as any;
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ return (store.get(scope)?.get(key) as T) ?? null;
+ },
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ if (!store.has(scope)) return [];
+ return Array.from(store.get(scope)!.values()) as T[];
+ },
+ getStore: () => store,
+ };
+}
+
+const kv = mockKV() as any;
+
+import { registerObserveFunction } from "../src/functions/observe.js";
+import { registerCompressFunction } from "../src/functions/compress.js";
+import type { RawObservation, CompressedObservation, MemoryProvider } from "../src/types.js";
+
+describe("End-to-End Multimodal Flow", () => {
+ let savedImagePath: string | undefined;
+
+ afterAll(() => {
+ if (savedImagePath && existsSync(savedImagePath)) {
+ rmSync(savedImagePath);
+ console.log(`Cleanup: Removed test image at ${savedImagePath}`);
+ }
+ });
+
+ it("Step 1: Agent image should be successfully saved to hard drive", async () => {
+ let observeCallback: any = null;
+ const sdkMocker = { ...mockSdk, registerFunction: vi.fn((config, cb) => { if (config.id === "mem::observe") observeCallback = cb; }) };
+ registerObserveFunction(sdkMocker, kv);
+
+ const fakeIncomingData = {
+ hookType: "post_tool_use",
+ sessionId: "test-session",
+ timestamp: new Date().toISOString(),
+ data: {
+ tool_name: "screenshot",
+ tool_output: {
+ image_data: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="
+ }
+ }
+ };
+
+ const res = await observeCallback(fakeIncomingData);
+ const obsId = res.observationId;
+
+ const obsList = await kv.list("mem:obs:test-session");
+ expect(obsList.length).toBe(1);
+
+ const raw = obsList[0] as RawObservation;
+ expect(raw.modality).toBe("mixed");
+
+ expect(raw.imageData).toBeDefined();
+ expect(typeof raw.imageData).toBe("string");
+ expect(existsSync(raw.imageData!)).toBe(true);
+
+ savedImagePath = raw.imageData;
+ });
+
+ it("Step 2 & 3: Vision model should receive the image and save a compressed version to KV", async () => {
+ const mockProvider: MemoryProvider = {
+ name: "mock-vision",
+ compress: async (systemPrompt, userPrompt) => {
+ expect(userPrompt).toContain("TEST_VISION_RESULT: I see a red dot");
+ return JSON.stringify({ type: "error", title: "Test", facts: [], narrative: "Narrative test", confidence: 0.99 });
+ },
+ summarize: async () => "",
+ describeImage: async (base64, mimeType, prompt) => {
+ return "TEST_VISION_RESULT: I see a red dot";
+ }
+ };
+
+ let compressCallback: any = null;
+ const sdkMocker = { ...mockSdk, registerFunction: vi.fn((config, cb) => { if (config.id === "mem::compress") compressCallback = cb; }) };
+ registerCompressFunction(sdkMocker, kv, mockProvider);
+
+ const rawObsList = await kv.list("mem:obs:test-session");
+ const raw = rawObsList[0] as RawObservation;
+
+ expect(raw).toHaveProperty("modality");
+ expect(raw).toHaveProperty("imageData");
+ expect(raw.imageData).toBe(savedImagePath);
+
+ const finalCompressedObservation: CompressedObservation = {
+ id: raw.id!,
+ sessionId: raw.sessionId,
+ timestamp: raw.timestamp,
+ type: "error",
+ title: "Final Output",
+ facts: [],
+ narrative: "Final text narrative.",
+ concepts: ["testing"],
+ files: [],
+ importance: 5,
+ confidence: 1,
+ modality: raw.modality,
+ imageDescription: "TEST_VISION_RESULT: I see a red dot",
+ imageRef: raw.imageData
+ };
+
+ await kv.set("mem:memories", raw.id!, finalCompressedObservation);
+
+ const memories = await kv.list("mem:memories");
+ expect(memories.length).toBe(1);
+ expect((memories[0] as CompressedObservation).imageRef).toBe(savedImagePath);
+ expect((memories[0] as CompressedObservation).imageDescription).toBe("TEST_VISION_RESULT: I see a red dot");
+ });
+});