#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix(vision): address CodeRabbit review on #179
Rohit Ghumare241be2669ce4
diff --git a/README.md b/README.mdindex 8f3b8f2e..ed600d87 100644--- a/README.md+++ b/README.md@@ -816,6 +816,17 @@ Create `~/.agentmemory/.env`:# LLM provider to compress the# observation — expect significant# token spend on active sessions.+# AGENTMEMORY_IMAGE_EMBEDDINGS=false # OFF by default. When on, loads+ # CLIP ViT-B/32 (512d) via+ # @xenova/transformers and embeds+ # every captured screenshot into+ # mem:image-embeddings. Enables+ # memory_vision_search (cross-modal:+ # text→image or image→image cosine).+ # Model downloads (~350 MB) on first+ # use — lazy. Observe pipeline fires+ # mem::vision-embed via triggerVoid,+ # so capture latency is unchanged.# AGENTMEMORY_INJECT_CONTEXT=false # OFF by default (#143). When on:# - SessionStart may inject ~1-2K# chars of project context intodiff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.jsonindex 1168e765..059edc94 100644--- a/plugin/.claude-plugin/plugin.json+++ b/plugin/.claude-plugin/plugin.json@@ -1,7 +1,7 @@{"name": "agentmemory","version": "0.9.1",- "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 44 MCP tools, 4 skills, real-time viewer.",+ "description": "Persistent memory for AI coding agents -- captures tool usage, compresses via LLM, injects context into future sessions. 12 hooks, 45 MCP tools, 4 skills, real-time viewer.","author": {"name": "Rohit Ghumare","url": "https://github.com/rohitg00"diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.tsindex fe7defe0..14336924 100644--- a/src/functions/image-refs.ts+++ b/src/functions/image-refs.ts@@ -21,8 +21,8 @@ export async function decrementImageRef(kv: StateKV, sdk: ISdk, filePath: stringreturn withKeyedLock(`imgRef:${filePath}`, async () => {const current = await getImageRefCount(kv, filePath);if (current <= 1) {+ await kv.delete(KV.imageEmbeddings, filePath);await kv.delete(KV.imageRefs, filePath);- await kv.delete(KV.imageEmbeddings, filePath).catch(() => {});const { deletedBytes } = await deleteImage(filePath);if (deletedBytes > 0) {sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });diff --git a/src/functions/vision-search.ts b/src/functions/vision-search.tsindex e874cc38..3b5d54e9 100644--- a/src/functions/vision-search.ts+++ b/src/functions/vision-search.ts@@ -2,6 +2,7 @@ import type { ISdk } from "iii-sdk";import type { EmbeddingProvider } from "../types.js";import { KV } from "../state/schema.js";import { StateKV } from "../state/kv.js";+import { recordAudit } from "./audit.js";import { logger } from "../logger.js";interface StoredEmbedding {@@ -44,6 +45,12 @@ export function registerVisionSearchFunctions(observationId: data.observationId,};await kv.set(KV.imageEmbeddings, data.imageRef, stored);+ await recordAudit(kv, "vision_embed", "mem::vision-embed", [data.imageRef], {+ modelName: imageProvider.name,+ dimensions: stored.dimensions,+ sessionId: data.sessionId,+ observationId: data.observationId,+ });return { success: true, imageRef: data.imageRef, dimensions: stored.dimensions };} catch (err) {const msg = err instanceof Error ? err.message : String(err);@@ -71,9 +78,13 @@ export function registerVisionSearchFunctions(try {if (data?.queryText) {queryVec = await imageProvider.embed(data.queryText);- } else if (data?.queryImageBase64 || data?.queryImageRef) {- const src = data.queryImageBase64 || (data.queryImageRef as string);- queryVec = await imageProvider.embedImage(src);+ } else if (data?.queryImageBase64) {+ const b64 = data.queryImageBase64.startsWith("data:")+ ? data.queryImageBase64+ : `data:image/png;base64,${data.queryImageBase64}`;+ queryVec = await imageProvider.embedImage(b64);+ } else if (data?.queryImageRef) {+ queryVec = await imageProvider.embedImage(data.queryImageRef);} else {return { success: false, error: "queryText, queryImageRef, or queryImageBase64 required" };}diff --git a/src/providers/embedding/clip.ts b/src/providers/embedding/clip.tsindex 56ffe47a..2a0143ad 100644--- a/src/providers/embedding/clip.ts+++ b/src/providers/embedding/clip.ts@@ -7,7 +7,6 @@ type TransformersModule = {model: string,) => Promise<ClipPipeline>;RawImage: {- read: (src: string | Blob | Buffer) => Promise<RawImageInstance>;fromBlob: (blob: Blob) => Promise<RawImageInstance>;};};@@ -45,13 +44,12 @@ export class ClipEmbeddingProvider implements EmbeddingProvider {return output.tolist().map((v) => new Float32Array(v));}- async embedImage(imageBase64OrPath: string): Promise<Float32Array> {+ async embedImage(src: string): Promise<Float32Array> {const t = await this.getTransformers();- const image = await loadImage(t, imageBase64OrPath);+ const image = await loadImage(t, src);const extractor = await this.getImageExtractor();const output = await extractor(image);- const raw = output.data ?? new Float32Array(output.tolist()[0] || []);- const vec = new Float32Array(raw);+ const vec = output.data ?? new Float32Array(output.tolist()[0] || []);return normalize(vec);}@@ -93,11 +91,6 @@ async function loadImage(const blob = new Blob([buf]);return t.RawImage.fromBlob(blob);}- if (/^[A-Za-z0-9+/=]+$/.test(src.slice(0, 64)) && !src.includes("/")) {- const buf = Buffer.from(src, "base64");- const blob = new Blob([buf]);- return t.RawImage.fromBlob(blob);- }const data = await readFile(src);const blob = new Blob([data]);return t.RawImage.fromBlob(blob);diff --git a/src/triggers/api.ts b/src/triggers/api.tsindex 26646489..a47f4996 100644--- a/src/triggers/api.ts+++ b/src/triggers/api.ts@@ -1280,31 +1280,34 @@ export function registerApiTriggers(});sdk.registerFunction("api::vision-search",- async (- req: ApiRequest<{- queryText?: string;- queryImageRef?: string;- queryImageBase64?: string;- topK?: number;- sessionId?: string;- }>,- ): Promise<Response> => {+ async (req: ApiRequest): Promise<Response> => {const authErr = checkAuth(req, secret);if (authErr) return authErr;- if (- !req.body?.queryText &&- !req.body?.queryImageRef &&- !req.body?.queryImageBase64- ) {+ const body = (req.body ?? {}) as Record<string, unknown>;+ const queryText = asNonEmptyString(body["queryText"]);+ const queryImageRef = asNonEmptyString(body["queryImageRef"]);+ const queryImageBase64 = asNonEmptyString(body["queryImageBase64"]);+ const sessionId = asNonEmptyString(body["sessionId"]);+ if (!queryText && !queryImageRef && !queryImageBase64) {return {status_code: 400,body: { error: "queryText, queryImageRef, or queryImageBase64 required" },};}- const result = await sdk.trigger({ function_id: "mem::vision-search", payload: req.body });- const body = result as { success?: boolean; error?: string };- if (body?.success === false) {- return { status_code: body.error?.includes("disabled") ? 503 : 400, body };+ const topKParsed = parseOptionalPositiveInt(body["topK"]);+ if (topKParsed === null) {+ return { status_code: 400, body: { error: "topK must be a positive integer" } };+ }+ const payload: Record<string, unknown> = {};+ if (queryText) payload["queryText"] = queryText;+ if (queryImageRef) payload["queryImageRef"] = queryImageRef;+ if (queryImageBase64) payload["queryImageBase64"] = queryImageBase64;+ if (sessionId) payload["sessionId"] = sessionId;+ if (topKParsed !== undefined) payload["topK"] = Math.min(50, topKParsed);+ const result = await sdk.trigger({ function_id: "mem::vision-search", payload });+ const resp = result as { success?: boolean; error?: string };+ if (resp?.success === false) {+ return { status_code: resp.error?.includes("disabled") ? 503 : 400, body: resp };}return { status_code: 200, body: result };},@@ -1316,22 +1319,23 @@ export function registerApiTriggers(});sdk.registerFunction("api::vision-embed",- async (- req: ApiRequest<{- imageRef: string;- sessionId?: string;- observationId?: string;- }>,- ): Promise<Response> => {+ async (req: ApiRequest): Promise<Response> => {const authErr = checkAuth(req, secret);if (authErr) return authErr;- if (!req.body?.imageRef) {+ const body = (req.body ?? {}) as Record<string, unknown>;+ const imageRef = asNonEmptyString(body["imageRef"]);+ const sessionId = asNonEmptyString(body["sessionId"]);+ const observationId = asNonEmptyString(body["observationId"]);+ if (!imageRef) {return { status_code: 400, body: { error: "imageRef is required" } };}- const result = await sdk.trigger({ function_id: "mem::vision-embed", payload: req.body });- const body = result as { success?: boolean; error?: string };- if (body?.success === false) {- return { status_code: body.error?.includes("disabled") ? 503 : 400, body };+ const payload: Record<string, unknown> = { imageRef };+ if (sessionId) payload["sessionId"] = sessionId;+ if (observationId) payload["observationId"] = observationId;+ const result = await sdk.trigger({ function_id: "mem::vision-embed", payload });+ const resp = result as { success?: boolean; error?: string };+ if (resp?.success === false) {+ return { status_code: resp.error?.includes("disabled") ? 503 : 400, body: resp };}return { status_code: 200, body: result };},diff --git a/src/types.ts b/src/types.tsindex 9cf0b41f..434778af 100644--- a/src/types.ts+++ b/src/types.ts@@ -208,7 +208,7 @@ export interface EmbeddingProvider {dimensions: number;embed(text: string): Promise<Float32Array>;embedBatch(texts: string[]): Promise<Float32Array[]>;- embedImage?(imageBase64OrPath: string): Promise<Float32Array>;+ embedImage?(src: string): Promise<Float32Array>;}export interface MemoryRelation {@@ -502,7 +502,8 @@ export interface AuditEntry {| "skill_extract"| "core_add"| "core_remove"- | "auto_page";+ | "auto_page"+ | "vision_embed";userId?: string;functionId: string;targetIds: string[];