Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(vision): address CodeRabbit review on #179

Rohit Ghumare241be2669ce4
diff --git a/README.md b/README.md
index 8f3b8f2e..ed600d87 100644
--- a/README.md
+++ b/README.md
@@ -816,6 +816,17 @@ Create `~/.agentmemory/.env`:
# LLM provider to compress the
# observation — expect significant
# token spend on active sessions.
+# AGENTMEMORY_IMAGE_EMBEDDINGS=false # OFF by default. When on, loads
+ # CLIP ViT-B/32 (512d) via
+ # @xenova/transformers and embeds
+ # every captured screenshot into
+ # mem:image-embeddings. Enables
+ # memory_vision_search (cross-modal:
+ # text→image or image→image cosine).
+ # Model downloads (~350 MB) on first
+ # use — lazy. Observe pipeline fires
+ # mem::vision-embed via triggerVoid,
+ # so capture latency is unchanged.
# AGENTMEMORY_INJECT_CONTEXT=false # OFF by default (#143). When on:
# - SessionStart may inject ~1-2K
# chars of project context into
diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json
index 1168e765..059edc94 100644
--- a/plugin/.claude-plugin/plugin.json
+++ b/plugin/.claude-plugin/plugin.json
@@ -1,7 +1,7 @@
{
"name": "agentmemory",
"version": "0.9.1",
- "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 44 MCP tools, 4 skills, real-time viewer.",
+ "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 45 MCP tools, 4 skills, real-time viewer.",
"author": {
"name": "Rohit Ghumare",
"url": "https://github.com/rohitg00"
diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.ts
index fe7defe0..14336924 100644
--- a/src/functions/image-refs.ts
+++ b/src/functions/image-refs.ts
@@ -21,8 +21,8 @@ export async function decrementImageRef(kv: StateKV, sdk: ISdk, filePath: string
return withKeyedLock(`imgRef:${filePath}`, async () => {
const current = await getImageRefCount(kv, filePath);
if (current <= 1) {
+ await kv.delete(KV.imageEmbeddings, filePath);
await kv.delete(KV.imageRefs, filePath);
- await kv.delete(KV.imageEmbeddings, filePath).catch(() => {});
const { deletedBytes } = await deleteImage(filePath);
if (deletedBytes > 0) {
sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
diff --git a/src/functions/vision-search.ts b/src/functions/vision-search.ts
index e874cc38..3b5d54e9 100644
--- a/src/functions/vision-search.ts
+++ b/src/functions/vision-search.ts
@@ -2,6 +2,7 @@ import type { ISdk } from "iii-sdk";
import type { EmbeddingProvider } from "../types.js";
import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
+import { recordAudit } from "./audit.js";
import { logger } from "../logger.js";
interface StoredEmbedding {
@@ -44,6 +45,12 @@ export function registerVisionSearchFunctions(
observationId: data.observationId,
};
await kv.set(KV.imageEmbeddings, data.imageRef, stored);
+ await recordAudit(kv, "vision_embed", "mem::vision-embed", [data.imageRef], {
+ modelName: imageProvider.name,
+ dimensions: stored.dimensions,
+ sessionId: data.sessionId,
+ observationId: data.observationId,
+ });
return { success: true, imageRef: data.imageRef, dimensions: stored.dimensions };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
@@ -71,9 +78,13 @@ export function registerVisionSearchFunctions(
try {
if (data?.queryText) {
queryVec = await imageProvider.embed(data.queryText);
- } else if (data?.queryImageBase64 || data?.queryImageRef) {
- const src = data.queryImageBase64 || (data.queryImageRef as string);
- queryVec = await imageProvider.embedImage(src);
+ } else if (data?.queryImageBase64) {
+ const b64 = data.queryImageBase64.startsWith("data:")
+ ? data.queryImageBase64
+ : `data:image/png;base64,${data.queryImageBase64}`;
+ queryVec = await imageProvider.embedImage(b64);
+ } else if (data?.queryImageRef) {
+ queryVec = await imageProvider.embedImage(data.queryImageRef);
} else {
return { success: false, error: "queryText, queryImageRef, or queryImageBase64 required" };
}
diff --git a/src/providers/embedding/clip.ts b/src/providers/embedding/clip.ts
index 56ffe47a..2a0143ad 100644
--- a/src/providers/embedding/clip.ts
+++ b/src/providers/embedding/clip.ts
@@ -7,7 +7,6 @@ type TransformersModule = {
model: string,
) => Promise<ClipPipeline>;
RawImage: {
- read: (src: string | Blob | Buffer) => Promise<RawImageInstance>;
fromBlob: (blob: Blob) => Promise<RawImageInstance>;
};
};
@@ -45,13 +44,12 @@ export class ClipEmbeddingProvider implements EmbeddingProvider {
return output.tolist().map((v) => new Float32Array(v));
}
- async embedImage(imageBase64OrPath: string): Promise<Float32Array> {
+ async embedImage(src: string): Promise<Float32Array> {
const t = await this.getTransformers();
- const image = await loadImage(t, imageBase64OrPath);
+ const image = await loadImage(t, src);
const extractor = await this.getImageExtractor();
const output = await extractor(image);
- const raw = output.data ?? new Float32Array(output.tolist()[0] || []);
- const vec = new Float32Array(raw);
+ const vec = output.data ?? new Float32Array(output.tolist()[0] || []);
return normalize(vec);
}
@@ -93,11 +91,6 @@ async function loadImage(
const blob = new Blob([buf]);
return t.RawImage.fromBlob(blob);
}
- if (/^[A-Za-z0-9+/=]+$/.test(src.slice(0, 64)) && !src.includes("/")) {
- const buf = Buffer.from(src, "base64");
- const blob = new Blob([buf]);
- return t.RawImage.fromBlob(blob);
- }
const data = await readFile(src);
const blob = new Blob([data]);
return t.RawImage.fromBlob(blob);
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 26646489..a47f4996 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -1280,31 +1280,34 @@ export function registerApiTriggers(
});
sdk.registerFunction("api::vision-search",
- async (
- req: ApiRequest<{
- queryText?: string;
- queryImageRef?: string;
- queryImageBase64?: string;
- topK?: number;
- sessionId?: string;
- }>,
- ): Promise<Response> => {
+ async (req: ApiRequest): Promise<Response> => {
const authErr = checkAuth(req, secret);
if (authErr) return authErr;
- if (
- !req.body?.queryText &&
- !req.body?.queryImageRef &&
- !req.body?.queryImageBase64
- ) {
+ const body = (req.body ?? {}) as Record<string, unknown>;
+ const queryText = asNonEmptyString(body["queryText"]);
+ const queryImageRef = asNonEmptyString(body["queryImageRef"]);
+ const queryImageBase64 = asNonEmptyString(body["queryImageBase64"]);
+ const sessionId = asNonEmptyString(body["sessionId"]);
+ if (!queryText && !queryImageRef && !queryImageBase64) {
return {
status_code: 400,
body: { error: "queryText, queryImageRef, or queryImageBase64 required" },
};
}
- const result = await sdk.trigger({ function_id: "mem::vision-search", payload: req.body });
- const body = result as { success?: boolean; error?: string };
- if (body?.success === false) {
- return { status_code: body.error?.includes("disabled") ? 503 : 400, body };
+ const topKParsed = parseOptionalPositiveInt(body["topK"]);
+ if (topKParsed === null) {
+ return { status_code: 400, body: { error: "topK must be a positive integer" } };
+ }
+ const payload: Record<string, unknown> = {};
+ if (queryText) payload["queryText"] = queryText;
+ if (queryImageRef) payload["queryImageRef"] = queryImageRef;
+ if (queryImageBase64) payload["queryImageBase64"] = queryImageBase64;
+ if (sessionId) payload["sessionId"] = sessionId;
+ if (topKParsed !== undefined) payload["topK"] = Math.min(50, topKParsed);
+ const result = await sdk.trigger({ function_id: "mem::vision-search", payload });
+ const resp = result as { success?: boolean; error?: string };
+ if (resp?.success === false) {
+ return { status_code: resp.error?.includes("disabled") ? 503 : 400, body: resp };
}
return { status_code: 200, body: result };
},
@@ -1316,22 +1319,23 @@ export function registerApiTriggers(
});
sdk.registerFunction("api::vision-embed",
- async (
- req: ApiRequest<{
- imageRef: string;
- sessionId?: string;
- observationId?: string;
- }>,
- ): Promise<Response> => {
+ async (req: ApiRequest): Promise<Response> => {
const authErr = checkAuth(req, secret);
if (authErr) return authErr;
- if (!req.body?.imageRef) {
+ const body = (req.body ?? {}) as Record<string, unknown>;
+ const imageRef = asNonEmptyString(body["imageRef"]);
+ const sessionId = asNonEmptyString(body["sessionId"]);
+ const observationId = asNonEmptyString(body["observationId"]);
+ if (!imageRef) {
return { status_code: 400, body: { error: "imageRef is required" } };
}
- const result = await sdk.trigger({ function_id: "mem::vision-embed", payload: req.body });
- const body = result as { success?: boolean; error?: string };
- if (body?.success === false) {
- return { status_code: body.error?.includes("disabled") ? 503 : 400, body };
+ const payload: Record<string, unknown> = { imageRef };
+ if (sessionId) payload["sessionId"] = sessionId;
+ if (observationId) payload["observationId"] = observationId;
+ const result = await sdk.trigger({ function_id: "mem::vision-embed", payload });
+ const resp = result as { success?: boolean; error?: string };
+ if (resp?.success === false) {
+ return { status_code: resp.error?.includes("disabled") ? 503 : 400, body: resp };
}
return { status_code: 200, body: result };
},
diff --git a/src/types.ts b/src/types.ts
index 9cf0b41f..434778af 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -208,7 +208,7 @@ export interface EmbeddingProvider {
dimensions: number;
embed(text: string): Promise<Float32Array>;
embedBatch(texts: string[]): Promise<Float32Array[]>;
- embedImage?(imageBase64OrPath: string): Promise<Float32Array>;
+ embedImage?(src: string): Promise<Float32Array>;
}
export interface MemoryRelation {
@@ -502,7 +502,8 @@ export interface AuditEntry {
| "skill_extract"
| "core_add"
| "core_remove"
- | "auto_page";
+ | "auto_page"
+ | "vision_embed";
userId?: string;
functionId: string;
targetIds: string[];