#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
feat(multimodal): Phase 1 — CLIP visual embeddings + vision-search
Rohit Ghumarebf6f33f48dbc
diff --git a/README.md b/README.mdindex b400c8f7..8f3b8f2e 100644--- a/README.md+++ b/README.md@@ -25,7 +25,7 @@<p align="center"><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-recall.svg"><img src="assets/tags/stat-recall.svg" alt="95.2% retrieval R@5" height="38" /></picture><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tokens.svg"><img src="assets/tags/stat-tokens.svg" alt="92% fewer tokens" height="38" /></picture>- <picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tools.svg"><img src="assets/tags/stat-tools.svg" alt="44 MCP tools" height="38" /></picture>+ <picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tools.svg"><img src="assets/tags/stat-tools.svg" alt="45 MCP tools" height="38" /></picture><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-hooks.svg"><img src="assets/tags/stat-hooks.svg" alt="12 auto hooks" height="38" /></picture><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-deps.svg"><img src="assets/tags/stat-deps.svg" alt="0 external DBs" height="38" /></picture><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tests.svg"><img src="assets/tags/stat-tests.svg" alt="654 tests passing" height="38" /></picture>@@ -334,7 +334,7 @@ Implementation details live in `src/cli.ts` (see `runUpgrade` around the `src/cl### Claude Code (one block, paste it)```-Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 4 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 44 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113.+Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 4 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 45 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113.```<details>@@ -593,7 +593,7 @@ npm install @xenova/transformers<h2 id="mcp-server"><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/section-mcp.svg"><img src="assets/tags/section-mcp.svg" alt="MCP Server" height="32" /></picture></h2>-44 tools, 6 resources, 3 prompts, and 4 skills — the most comprehensive MCP memory toolkit for any agent.+45 tools, 6 resources, 3 prompts, and 4 skills — the most comprehensive MCP memory toolkit for any agent.### 44 Tools@@ -843,7 +843,7 @@ Create `~/.agentmemory/.env`:# USER_ID=# TEAM_MODE=private-# Tool visibility: "core" (8 tools) or "all" (44 tools)+# Tool visibility: "core" (8 tools) or "all" (45 tools)# AGENTMEMORY_TOOLS=core```diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.tsindex 857bd1c5..fe7defe0 100644--- a/src/functions/image-refs.ts+++ b/src/functions/image-refs.ts@@ -22,6 +22,7 @@ export async function decrementImageRef(kv: StateKV, sdk: ISdk, filePath: stringconst current = await getImageRefCount(kv, filePath);if (current <= 1) {await kv.delete(KV.imageRefs, filePath);+ await kv.delete(KV.imageEmbeddings, filePath).catch(() => {});const { deletedBytes } = await deleteImage(filePath);if (deletedBytes > 0) {sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });diff --git a/src/functions/observe.ts b/src/functions/observe.tsindex c13f7538..a3a748cc 100644--- a/src/functions/observe.ts+++ b/src/functions/observe.ts@@ -140,6 +140,13 @@ export function registerObserveFunction(const { incrementImageRef } = await import("./image-refs.js");await incrementImageRef(kv, filePath);sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: bytesWritten });+ if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") {+ sdk.triggerVoid("mem::vision-embed", {+ imageRef: filePath,+ sessionId: payload.sessionId,+ observationId: obsId,+ });+ }}try {diff --git a/src/functions/vision-search.ts b/src/functions/vision-search.tsnew file mode 100644index 00000000..e874cc38--- /dev/null+++ b/src/functions/vision-search.ts@@ -0,0 +1,117 @@+import type { ISdk } from "iii-sdk";+import type { EmbeddingProvider } from "../types.js";+import { KV } from "../state/schema.js";+import { StateKV } from "../state/kv.js";+import { logger } from "../logger.js";++interface StoredEmbedding {+ imageRef: string;+ vector: number[];+ modelName: string;+ dimensions: number;+ updatedAt: string;+ sessionId?: string;+ observationId?: string;+}++export function registerVisionSearchFunctions(+ sdk: ISdk,+ kv: StateKV,+ imageProvider: EmbeddingProvider | null,+): void {+ sdk.registerFunction(+ "mem::vision-embed",+ async (data: {+ imageRef: string;+ sessionId?: string;+ observationId?: string;+ }) => {+ if (!imageProvider?.embedImage) {+ return { success: false, error: "image embeddings disabled (set AGENTMEMORY_IMAGE_EMBEDDINGS=true)" };+ }+ if (!data?.imageRef || typeof data.imageRef !== "string") {+ return { success: false, error: "imageRef required" };+ }+ try {+ const vec = await imageProvider.embedImage(data.imageRef);+ const stored: StoredEmbedding = {+ imageRef: data.imageRef,+ vector: Array.from(vec),+ modelName: imageProvider.name,+ dimensions: imageProvider.dimensions,+ updatedAt: new Date().toISOString(),+ sessionId: data.sessionId,+ observationId: data.observationId,+ };+ await kv.set(KV.imageEmbeddings, data.imageRef, stored);+ return { success: true, imageRef: data.imageRef, dimensions: stored.dimensions };+ } catch (err) {+ const msg = err instanceof Error ? err.message : String(err);+ logger.warn("vision-embed failed", { imageRef: data.imageRef, error: msg });+ return { success: false, error: msg };+ }+ },+ );++ sdk.registerFunction(+ "mem::vision-search",+ async (data: {+ queryText?: string;+ queryImageRef?: string;+ queryImageBase64?: string;+ topK?: number;+ sessionId?: string;+ }) => {+ if (!imageProvider?.embedImage) {+ return { success: false, error: "image embeddings disabled (set AGENTMEMORY_IMAGE_EMBEDDINGS=true)" };+ }+ const topK = Math.min(50, Math.max(1, data?.topK ?? 10));++ let queryVec: Float32Array | null = null;+ try {+ if (data?.queryText) {+ queryVec = await imageProvider.embed(data.queryText);+ } else if (data?.queryImageBase64 || data?.queryImageRef) {+ const src = data.queryImageBase64 || (data.queryImageRef as string);+ queryVec = await imageProvider.embedImage(src);+ } else {+ return { success: false, error: "queryText, queryImageRef, or queryImageBase64 required" };+ }+ } catch (err) {+ const msg = err instanceof Error ? err.message : String(err);+ return { success: false, error: `query embed failed: ${msg}` };+ }++ if (!queryVec) return { success: false, error: "failed to build query vector" };++ const stored = await kv.list<StoredEmbedding>(KV.imageEmbeddings);+ const filtered = data?.sessionId+ ? stored.filter((s) => s.sessionId === data.sessionId)+ : stored;++ const scored = filtered.map((s) => ({+ imageRef: s.imageRef,+ score: cosine(queryVec!, s.vector),+ sessionId: s.sessionId,+ observationId: s.observationId,+ updatedAt: s.updatedAt,+ }));+ scored.sort((a, b) => b.score - a.score);+ return { success: true, results: scored.slice(0, topK), total: scored.length };+ },+ );+}++function cosine(a: Float32Array, b: number[]): number {+ if (a.length !== b.length) return 0;+ let dot = 0;+ let normA = 0;+ let normB = 0;+ for (let i = 0; i < a.length; i++) {+ dot += a[i] * b[i];+ normA += a[i] * a[i];+ normB += b[i] * b[i];+ }+ const denom = Math.sqrt(normA) * Math.sqrt(normB);+ return denom === 0 ? 0 : dot / denom;+}diff --git a/src/index.ts b/src/index.tsindex ee30c44d..580ee547 100644--- a/src/index.ts+++ b/src/index.ts@@ -16,6 +16,7 @@ import {createProvider,createFallbackProvider,createEmbeddingProvider,+ createImageEmbeddingProvider,} from "./providers/index.js";import { StateKV } from "./state/kv.js";import { VectorIndex } from "./state/vector-index.js";@@ -24,6 +25,7 @@ import { IndexPersistence } from "./state/index-persistence.js";import { registerPrivacyFunction } from "./functions/privacy.js";import { registerObserveFunction } from "./functions/observe.js";import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js";+import { registerVisionSearchFunctions } from "./functions/vision-search.js";import { registerDiskSizeManager } from "./functions/disk-size-manager.js";import { registerCompressFunction } from "./functions/compress.js";import {@@ -111,6 +113,7 @@ async function main() {: createProvider(config.provider);const embeddingProvider = createEmbeddingProvider();+ const imageEmbeddingProvider = createImageEmbeddingProvider();console.log(`[agentmemory] Starting worker v${VERSION}...`);console.log(`[agentmemory] Engine: ${config.engineUrl}`);@@ -124,6 +127,11 @@ async function main() {} else {console.log(`[agentmemory] Embedding provider: none (BM25-only mode)`);}+ if (imageEmbeddingProvider) {+ console.log(+ `[agentmemory] Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`,+ );+ }console.log(`[agentmemory] REST API: http://localhost:${config.restPort}/agentmemory/*`,);@@ -154,6 +162,7 @@ async function main() {registerPrivacyFunction(sdk);registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession);registerImageQuotaCleanup(sdk, kv);+ registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider);registerDiskSizeManager(sdk, kv);registerCompressFunction(sdk, kv, provider, metricsStore);registerSearchFunction(sdk, kv);diff --git a/src/mcp/server.ts b/src/mcp/server.tsindex 82283e90..69539a2f 100644--- a/src/mcp/server.ts+++ b/src/mcp/server.ts@@ -275,6 +275,32 @@ export function registerMcpEndpoints(};}+ case "memory_vision_search": {+ const queryText = typeof args.queryText === "string" ? args.queryText : undefined;+ const queryImageRef = typeof args.queryImageRef === "string" ? args.queryImageRef : undefined;+ const queryImageBase64 = typeof args.queryImageBase64 === "string" ? args.queryImageBase64 : undefined;+ if (!queryText && !queryImageRef && !queryImageBase64) {+ return {+ status_code: 400,+ body: { error: "queryText, queryImageRef, or queryImageBase64 required" },+ };+ }+ const topK = Math.max(1, Math.min(50, asNumber(args.topK, 10) ?? 10));+ const sessionId = typeof args.sessionId === "string" ? args.sessionId : undefined;+ const result = await sdk.trigger({+ function_id: "mem::vision-search",+ payload: { queryText, queryImageRef, queryImageBase64, topK, sessionId },+ });+ return {+ status_code: 200,+ body: {+ content: [+ { type: "text", text: JSON.stringify(result, null, 2) },+ ],+ },+ };+ }+case "memory_timeline": {if (typeof args.anchor !== "string" || !args.anchor.trim()) {return {diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.tsindex 4297e459..679e5288 100644--- a/src/mcp/tools-registry.ts+++ b/src/mcp/tools-registry.ts@@ -126,6 +126,21 @@ export const CORE_TOOLS: McpToolDef[] = [required: ["query"],},},+ {+ name: "memory_vision_search",+ description:+ "Cross-modal image search via CLIP embeddings. Pass queryText to find screenshots matching a description, or queryImageBase64/queryImageRef to find similar images. Requires AGENTMEMORY_IMAGE_EMBEDDINGS=true.",+ inputSchema: {+ type: "object",+ properties: {+ queryText: { type: "string", description: "Text query (e.g. 'login form with error banner')" },+ queryImageRef: { type: "string", description: "Absolute path to a stored image to match against" },+ queryImageBase64: { type: "string", description: "Raw base64 image bytes or data URL" },+ topK: { type: "number", description: "Max results (default 10, max 50)" },+ sessionId: { type: "string", description: "Filter to a single session" },+ },+ },+ },{name: "memory_timeline",description: "Chronological observations around an anchor point.",diff --git a/src/providers/embedding/clip.ts b/src/providers/embedding/clip.tsnew file mode 100644index 00000000..56ffe47a--- /dev/null+++ b/src/providers/embedding/clip.ts@@ -0,0 +1,114 @@+import { readFile } from "node:fs/promises";+import type { EmbeddingProvider } from "../../types.js";++type TransformersModule = {+ pipeline: (+ task: string,+ model: string,+ ) => Promise<ClipPipeline>;+ RawImage: {+ read: (src: string | Blob | Buffer) => Promise<RawImageInstance>;+ fromBlob: (blob: Blob) => Promise<RawImageInstance>;+ };+};++type RawImageInstance = unknown;++type ClipPipeline = (+ input: string[] | RawImageInstance | RawImageInstance[],+ options?: { pooling?: string; normalize?: boolean },+) => Promise<{ tolist: () => number[][]; data: Float32Array }>;++const DEFAULT_MODEL = "Xenova/clip-vit-base-patch32";+const DIMENSIONS = 512;++export class ClipEmbeddingProvider implements EmbeddingProvider {+ readonly name = "clip";+ readonly dimensions = DIMENSIONS;+ private textExtractor: ClipPipeline | null = null;+ private imageExtractor: ClipPipeline | null = null;+ private transformers: TransformersModule | null = null;+ private readonly modelId: string;++ constructor(modelId: string = DEFAULT_MODEL) {+ this.modelId = modelId;+ }++ async embed(text: string): Promise<Float32Array> {+ const [vec] = await this.embedBatch([text]);+ return vec;+ }++ async embedBatch(texts: string[]): Promise<Float32Array[]> {+ const extractor = await this.getTextExtractor();+ const output = await extractor(texts, { pooling: "mean", normalize: true });+ return output.tolist().map((v) => new Float32Array(v));+ }++ async embedImage(imageBase64OrPath: string): Promise<Float32Array> {+ const t = await this.getTransformers();+ const image = await loadImage(t, imageBase64OrPath);+ const extractor = await this.getImageExtractor();+ const output = await extractor(image);+ const raw = output.data ?? new Float32Array(output.tolist()[0] || []);+ const vec = new Float32Array(raw);+ return normalize(vec);+ }++ private async getTransformers(): Promise<TransformersModule> {+ if (this.transformers) return this.transformers;+ try {+ this.transformers = (await import("@xenova/transformers")) as unknown as TransformersModule;+ } catch {+ throw new Error(+ "Install @xenova/transformers for CLIP image embeddings: npm install @xenova/transformers",+ );+ }+ return this.transformers;+ }++ private async getTextExtractor(): Promise<ClipPipeline> {+ if (this.textExtractor) return this.textExtractor;+ const t = await this.getTransformers();+ this.textExtractor = await t.pipeline("feature-extraction", this.modelId);+ return this.textExtractor;+ }++ private async getImageExtractor(): Promise<ClipPipeline> {+ if (this.imageExtractor) return this.imageExtractor;+ const t = await this.getTransformers();+ this.imageExtractor = await t.pipeline("image-feature-extraction", this.modelId);+ return this.imageExtractor;+ }+}++async function loadImage(+ t: TransformersModule,+ src: string,+): Promise<RawImageInstance> {+ if (src.startsWith("data:")) {+ const comma = src.indexOf(",");+ const b64 = comma >= 0 ? src.slice(comma + 1) : src;+ const buf = Buffer.from(b64, "base64");+ const blob = new Blob([buf]);+ return t.RawImage.fromBlob(blob);+ }+ if (/^[A-Za-z0-9+/=]+$/.test(src.slice(0, 64)) && !src.includes("/")) {+ const buf = Buffer.from(src, "base64");+ const blob = new Blob([buf]);+ return t.RawImage.fromBlob(blob);+ }+ const data = await readFile(src);+ const blob = new Blob([data]);+ return t.RawImage.fromBlob(blob);+}++function normalize(vec: Float32Array): Float32Array {+ let sum = 0;+ for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];+ const norm = Math.sqrt(sum);+ if (norm === 0) return vec;+ const out = new Float32Array(vec.length);+ for (let i = 0; i < vec.length; i++) out[i] = vec[i] / norm;+ return out;+}diff --git a/src/providers/embedding/index.ts b/src/providers/embedding/index.tsindex 0f52514e..4580232c 100644--- a/src/providers/embedding/index.ts+++ b/src/providers/embedding/index.ts@@ -6,6 +6,7 @@ import { VoyageEmbeddingProvider } from "./voyage.js";import { CohereEmbeddingProvider } from "./cohere.js";import { OpenRouterEmbeddingProvider } from "./openrouter.js";import { LocalEmbeddingProvider } from "./local.js";+import { ClipEmbeddingProvider } from "./clip.js";export {GeminiEmbeddingProvider,@@ -14,8 +15,18 @@ export {CohereEmbeddingProvider,OpenRouterEmbeddingProvider,LocalEmbeddingProvider,+ ClipEmbeddingProvider,};+let imageEmbeddingProvider: EmbeddingProvider | null = null;++export function createImageEmbeddingProvider(): EmbeddingProvider | null {+ if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] !== "true") return null;+ if (imageEmbeddingProvider) return imageEmbeddingProvider;+ imageEmbeddingProvider = new ClipEmbeddingProvider();+ return imageEmbeddingProvider;+}+export function createEmbeddingProvider(): EmbeddingProvider | null {const detected = detectEmbeddingProvider();if (!detected) return null;diff --git a/src/providers/index.ts b/src/providers/index.tsindex 3e3978b6..c9e09b49 100644--- a/src/providers/index.ts+++ b/src/providers/index.ts@@ -11,7 +11,7 @@ import { ResilientProvider } from "./resilient.js";import { FallbackChainProvider } from "./fallback-chain.js";import { getEnvVar } from "../config.js";-export { createEmbeddingProvider } from "./embedding/index.js";+export { createEmbeddingProvider, createImageEmbeddingProvider } from "./embedding/index.js";function requireEnvVar(key: string): string {const value = getEnvVar(key);diff --git a/src/state/schema.ts b/src/state/schema.tsindex fb2c9fc3..b3c8bac5 100644--- a/src/state/schema.ts+++ b/src/state/schema.ts@@ -42,6 +42,7 @@ export const KV = {retentionScores: "mem:retention",accessLog: "mem:access",imageRefs: "mem:image-refs",+ imageEmbeddings: "mem:image-embeddings",} as const;export const STREAM = {diff --git a/src/triggers/api.ts b/src/triggers/api.tsindex 1db1daac..26646489 100644--- a/src/triggers/api.ts+++ b/src/triggers/api.ts@@ -1279,6 +1279,69 @@ export function registerApiTriggers(config: { api_path: "/agentmemory/relations", http_method: "GET" },});+ sdk.registerFunction("api::vision-search",+ async (+ req: ApiRequest<{+ queryText?: string;+ queryImageRef?: string;+ queryImageBase64?: string;+ topK?: number;+ sessionId?: string;+ }>,+ ): Promise<Response> => {+ const authErr = checkAuth(req, secret);+ if (authErr) return authErr;+ if (+ !req.body?.queryText &&+ !req.body?.queryImageRef &&+ !req.body?.queryImageBase64+ ) {+ return {+ status_code: 400,+ body: { error: "queryText, queryImageRef, or queryImageBase64 required" },+ };+ }+ const result = await sdk.trigger({ function_id: "mem::vision-search", payload: req.body });+ const body = result as { success?: boolean; error?: string };+ if (body?.success === false) {+ return { status_code: body.error?.includes("disabled") ? 503 : 400, body };+ }+ return { status_code: 200, body: result };+ },+ );+ sdk.registerTrigger({+ type: "http",+ function_id: "api::vision-search",+ config: { api_path: "/agentmemory/vision-search", http_method: "POST" },+ });++ sdk.registerFunction("api::vision-embed",+ async (+ req: ApiRequest<{+ imageRef: string;+ sessionId?: string;+ observationId?: string;+ }>,+ ): Promise<Response> => {+ const authErr = checkAuth(req, secret);+ if (authErr) return authErr;+ if (!req.body?.imageRef) {+ return { status_code: 400, body: { error: "imageRef is required" } };+ }+ const result = await sdk.trigger({ function_id: "mem::vision-embed", payload: req.body });+ const body = result as { success?: boolean; error?: string };+ if (body?.success === false) {+ return { status_code: body.error?.includes("disabled") ? 503 : 400, body };+ }+ return { status_code: 200, body: result };+ },+ );+ sdk.registerTrigger({+ type: "http",+ function_id: "api::vision-embed",+ config: { api_path: "/agentmemory/vision-embed", http_method: "POST" },+ });+sdk.registerFunction("api::action-create",async (req: ApiRequest<{diff --git a/src/types.ts b/src/types.tsindex 3f40c7c5..9cf0b41f 100644--- a/src/types.ts+++ b/src/types.ts@@ -208,6 +208,7 @@ export interface EmbeddingProvider {dimensions: number;embed(text: string): Promise<Float32Array>;embedBatch(texts: string[]): Promise<Float32Array[]>;+ embedImage?(imageBase64OrPath: string): Promise<Float32Array>;}export interface MemoryRelation {diff --git a/test/mcp-standalone.test.ts b/test/mcp-standalone.test.tsindex def82928..59bd985e 100644--- a/test/mcp-standalone.test.ts+++ b/test/mcp-standalone.test.ts@@ -46,8 +46,8 @@ describe("Tools Registry", () => {}});- it("CORE_TOOLS has 11 items", () => {- expect(CORE_TOOLS.length).toBe(11);+ it("CORE_TOOLS has 12 items", () => {+ expect(CORE_TOOLS.length).toBe(12);});it("V040_TOOLS has 8 items", () => {diff --git a/test/vision-search.test.ts b/test/vision-search.test.tsnew file mode 100644index 00000000..55cf1306--- /dev/null+++ b/test/vision-search.test.ts@@ -0,0 +1,135 @@+import { describe, it, expect, beforeEach, vi } from "vitest";+import { registerVisionSearchFunctions } from "../src/functions/vision-search.js";+import type { EmbeddingProvider } from "../src/types.js";+import { KV } from "../src/state/schema.js";++function mockKV() {+ const store = new Map<string, Map<string, unknown>>();+ return {+ get: async <T>(scope: string, key: string): Promise<T | null> => {+ return (store.get(scope)?.get(key) as T) ?? null;+ },+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, data);+ return data;+ },+ delete: async (scope: string, key: string): Promise<void> => {+ store.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> => {+ if (!store.has(scope)) return [];+ return Array.from(store.get(scope)!.values()) as T[];+ },+ };+}++function unit(v: number[]): Float32Array {+ let norm = 0;+ for (const x of v) norm += x * x;+ norm = Math.sqrt(norm);+ return new Float32Array(v.map((x) => x / (norm || 1)));+}++describe("vision-search", () => {+ let visionSearch: (data: Record<string, unknown>) => Promise<Record<string, unknown>>;+ let visionEmbed: (data: Record<string, unknown>) => Promise<Record<string, unknown>>;+ let kv: ReturnType<typeof mockKV>;++ const fakeProvider: EmbeddingProvider = {+ name: "fake-clip",+ dimensions: 3,+ embed: async (text: string) => {+ if (text.includes("login")) return unit([1, 0, 0]);+ if (text.includes("dashboard")) return unit([0, 1, 0]);+ return unit([0, 0, 1]);+ },+ embedBatch: async (texts: string[]) => Promise.all(texts.map((t) => fakeProvider.embed(t))),+ embedImage: async (ref: string) => {+ if (ref.endsWith("login.png")) return unit([1, 0.1, 0]);+ if (ref.endsWith("dashboard.png")) return unit([0, 1, 0.1]);+ return unit([0, 0.1, 1]);+ },+ };++ beforeEach(() => {+ kv = mockKV();+ const handlers: Record<string, (data: Record<string, unknown>) => Promise<Record<string, unknown>>> = {};+ const sdk = {+ registerFunction: vi.fn((id: string, cb) => {+ handlers[id] = cb;+ }),+ } as unknown as import("iii-sdk").ISdk;+ registerVisionSearchFunctions(sdk, kv as never, fakeProvider);+ visionSearch = handlers["mem::vision-search"]!;+ visionEmbed = handlers["mem::vision-embed"]!;+ });++ it("embeds and stores image vectors in KV.imageEmbeddings", async () => {+ const res = (await visionEmbed({ imageRef: "/tmp/login.png" })) as { success: boolean; dimensions: number };+ expect(res.success).toBe(true);+ expect(res.dimensions).toBe(3);+ const stored = await kv.list(KV.imageEmbeddings);+ expect(stored.length).toBe(1);+ });++ it("text query ranks the matching image first", async () => {+ await visionEmbed({ imageRef: "/tmp/login.png" });+ await visionEmbed({ imageRef: "/tmp/dashboard.png" });+ await visionEmbed({ imageRef: "/tmp/other.png" });++ const res = (await visionSearch({ queryText: "the login form", topK: 3 })) as {+ success: boolean;+ results: Array<{ imageRef: string; score: number }>;+ };+ expect(res.success).toBe(true);+ expect(res.results[0].imageRef).toBe("/tmp/login.png");+ expect(res.results[0].score).toBeGreaterThan(res.results[1].score);+ });++ it("image-to-image query finds the same image first", async () => {+ await visionEmbed({ imageRef: "/tmp/login.png" });+ await visionEmbed({ imageRef: "/tmp/dashboard.png" });++ const res = (await visionSearch({ queryImageRef: "/tmp/login.png", topK: 2 })) as {+ success: boolean;+ results: Array<{ imageRef: string; score: number }>;+ };+ expect(res.results[0].imageRef).toBe("/tmp/login.png");+ expect(res.results[0].score).toBeGreaterThan(0.9);+ });++ it("sessionId filters out embeddings from other sessions", async () => {+ await visionEmbed({ imageRef: "/tmp/login.png", sessionId: "sess_a" });+ await visionEmbed({ imageRef: "/tmp/dashboard.png", sessionId: "sess_b" });++ const res = (await visionSearch({ queryText: "anything", sessionId: "sess_a", topK: 5 })) as {+ success: boolean;+ results: Array<{ sessionId?: string }>;+ };+ expect(res.results.every((r) => r.sessionId === "sess_a")).toBe(true);+ expect(res.results.length).toBe(1);+ });++ it("returns 503-equivalent error when provider is absent", async () => {+ const handlers: Record<string, (data: Record<string, unknown>) => Promise<Record<string, unknown>>> = {};+ const sdk = {+ registerFunction: vi.fn((id: string, cb) => {+ handlers[id] = cb;+ }),+ } as unknown as import("iii-sdk").ISdk;+ registerVisionSearchFunctions(sdk, kv as never, null);+ const res = (await handlers["mem::vision-search"]!({ queryText: "login" })) as {+ success: boolean;+ error: string;+ };+ expect(res.success).toBe(false);+ expect(res.error).toMatch(/disabled/);+ });++ it("rejects missing query", async () => {+ const res = (await visionSearch({})) as { success: boolean; error: string };+ expect(res.success).toBe(false);+ expect(res.error).toMatch(/required/);+ });+});