Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

feat(multimodal): Phase 1 — CLIP visual embeddings + vision-search

Rohit Ghumarebf6f33f48dbc
diff --git a/README.md b/README.md
index b400c8f7..8f3b8f2e 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@
<p align="center">
<picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-recall.svg"><img src="assets/tags/stat-recall.svg" alt="95.2% retrieval R@5" height="38" /></picture>
<picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tokens.svg"><img src="assets/tags/stat-tokens.svg" alt="92% fewer tokens" height="38" /></picture>
- <picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tools.svg"><img src="assets/tags/stat-tools.svg" alt="44 MCP tools" height="38" /></picture>
+ <picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tools.svg"><img src="assets/tags/stat-tools.svg" alt="45 MCP tools" height="38" /></picture>
<picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-hooks.svg"><img src="assets/tags/stat-hooks.svg" alt="12 auto hooks" height="38" /></picture>
<picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-deps.svg"><img src="assets/tags/stat-deps.svg" alt="0 external DBs" height="38" /></picture>
<picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/stat-tests.svg"><img src="assets/tags/stat-tests.svg" alt="654 tests passing" height="38" /></picture>
@@ -334,7 +334,7 @@ Implementation details live in `src/cli.ts` (see `runUpgrade` around the `src/cl
### Claude Code (one block, paste it)
```
-Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 4 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 44 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113.
+Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 4 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 45 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113.
```
<details>
@@ -593,7 +593,7 @@ npm install @xenova/transformers
<h2 id="mcp-server"><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/section-mcp.svg"><img src="assets/tags/section-mcp.svg" alt="MCP Server" height="32" /></picture></h2>
-44 tools, 6 resources, 3 prompts, and 4 skills — the most comprehensive MCP memory toolkit for any agent.
+45 tools, 6 resources, 3 prompts, and 4 skills — the most comprehensive MCP memory toolkit for any agent.
### 44 Tools
@@ -843,7 +843,7 @@ Create `~/.agentmemory/.env`:
# USER_ID=
# TEAM_MODE=private
-# Tool visibility: "core" (8 tools) or "all" (44 tools)
+# Tool visibility: "core" (8 tools) or "all" (45 tools)
# AGENTMEMORY_TOOLS=core
```
diff --git a/src/functions/image-refs.ts b/src/functions/image-refs.ts
index 857bd1c5..fe7defe0 100644
--- a/src/functions/image-refs.ts
+++ b/src/functions/image-refs.ts
@@ -22,6 +22,7 @@ export async function decrementImageRef(kv: StateKV, sdk: ISdk, filePath: string
const current = await getImageRefCount(kv, filePath);
if (current <= 1) {
await kv.delete(KV.imageRefs, filePath);
+ await kv.delete(KV.imageEmbeddings, filePath).catch(() => {});
const { deletedBytes } = await deleteImage(filePath);
if (deletedBytes > 0) {
sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
diff --git a/src/functions/observe.ts b/src/functions/observe.ts
index c13f7538..a3a748cc 100644
--- a/src/functions/observe.ts
+++ b/src/functions/observe.ts
@@ -140,6 +140,13 @@ export function registerObserveFunction(
const { incrementImageRef } = await import("./image-refs.js");
await incrementImageRef(kv, filePath);
sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: bytesWritten });
+ if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") {
+ sdk.triggerVoid("mem::vision-embed", {
+ imageRef: filePath,
+ sessionId: payload.sessionId,
+ observationId: obsId,
+ });
+ }
}
try {
diff --git a/src/functions/vision-search.ts b/src/functions/vision-search.ts
new file mode 100644
index 00000000..e874cc38
--- /dev/null
+++ b/src/functions/vision-search.ts
@@ -0,0 +1,117 @@
+import type { ISdk } from "iii-sdk";
+import type { EmbeddingProvider } from "../types.js";
+import { KV } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+import { logger } from "../logger.js";
+
+interface StoredEmbedding {
+ imageRef: string;
+ vector: number[];
+ modelName: string;
+ dimensions: number;
+ updatedAt: string;
+ sessionId?: string;
+ observationId?: string;
+}
+
+export function registerVisionSearchFunctions(
+ sdk: ISdk,
+ kv: StateKV,
+ imageProvider: EmbeddingProvider | null,
+): void {
+ sdk.registerFunction(
+ "mem::vision-embed",
+ async (data: {
+ imageRef: string;
+ sessionId?: string;
+ observationId?: string;
+ }) => {
+ if (!imageProvider?.embedImage) {
+ return { success: false, error: "image embeddings disabled (set AGENTMEMORY_IMAGE_EMBEDDINGS=true)" };
+ }
+ if (!data?.imageRef || typeof data.imageRef !== "string") {
+ return { success: false, error: "imageRef required" };
+ }
+ try {
+ const vec = await imageProvider.embedImage(data.imageRef);
+ const stored: StoredEmbedding = {
+ imageRef: data.imageRef,
+ vector: Array.from(vec),
+ modelName: imageProvider.name,
+ dimensions: imageProvider.dimensions,
+ updatedAt: new Date().toISOString(),
+ sessionId: data.sessionId,
+ observationId: data.observationId,
+ };
+ await kv.set(KV.imageEmbeddings, data.imageRef, stored);
+ return { success: true, imageRef: data.imageRef, dimensions: stored.dimensions };
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ logger.warn("vision-embed failed", { imageRef: data.imageRef, error: msg });
+ return { success: false, error: msg };
+ }
+ },
+ );
+
+ sdk.registerFunction(
+ "mem::vision-search",
+ async (data: {
+ queryText?: string;
+ queryImageRef?: string;
+ queryImageBase64?: string;
+ topK?: number;
+ sessionId?: string;
+ }) => {
+ if (!imageProvider?.embedImage) {
+ return { success: false, error: "image embeddings disabled (set AGENTMEMORY_IMAGE_EMBEDDINGS=true)" };
+ }
+ const topK = Math.min(50, Math.max(1, data?.topK ?? 10));
+
+ let queryVec: Float32Array | null = null;
+ try {
+ if (data?.queryText) {
+ queryVec = await imageProvider.embed(data.queryText);
+ } else if (data?.queryImageBase64 || data?.queryImageRef) {
+ const src = data.queryImageBase64 || (data.queryImageRef as string);
+ queryVec = await imageProvider.embedImage(src);
+ } else {
+ return { success: false, error: "queryText, queryImageRef, or queryImageBase64 required" };
+ }
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ return { success: false, error: `query embed failed: ${msg}` };
+ }
+
+ if (!queryVec) return { success: false, error: "failed to build query vector" };
+
+ const stored = await kv.list<StoredEmbedding>(KV.imageEmbeddings);
+ const filtered = data?.sessionId
+ ? stored.filter((s) => s.sessionId === data.sessionId)
+ : stored;
+
+ const scored = filtered.map((s) => ({
+ imageRef: s.imageRef,
+ score: cosine(queryVec!, s.vector),
+ sessionId: s.sessionId,
+ observationId: s.observationId,
+ updatedAt: s.updatedAt,
+ }));
+ scored.sort((a, b) => b.score - a.score);
+ return { success: true, results: scored.slice(0, topK), total: scored.length };
+ },
+ );
+}
+
+function cosine(a: Float32Array, b: number[]): number {
+ if (a.length !== b.length) return 0;
+ let dot = 0;
+ let normA = 0;
+ let normB = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i] * b[i];
+ normA += a[i] * a[i];
+ normB += b[i] * b[i];
+ }
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
+ return denom === 0 ? 0 : dot / denom;
+}
diff --git a/src/index.ts b/src/index.ts
index ee30c44d..580ee547 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -16,6 +16,7 @@ import {
createProvider,
createFallbackProvider,
createEmbeddingProvider,
+ createImageEmbeddingProvider,
} from "./providers/index.js";
import { StateKV } from "./state/kv.js";
import { VectorIndex } from "./state/vector-index.js";
@@ -24,6 +25,7 @@ import { IndexPersistence } from "./state/index-persistence.js";
import { registerPrivacyFunction } from "./functions/privacy.js";
import { registerObserveFunction } from "./functions/observe.js";
import { registerImageQuotaCleanup } from "./functions/image-quota-cleanup.js";
+import { registerVisionSearchFunctions } from "./functions/vision-search.js";
import { registerDiskSizeManager } from "./functions/disk-size-manager.js";
import { registerCompressFunction } from "./functions/compress.js";
import {
@@ -111,6 +113,7 @@ async function main() {
: createProvider(config.provider);
const embeddingProvider = createEmbeddingProvider();
+ const imageEmbeddingProvider = createImageEmbeddingProvider();
console.log(`[agentmemory] Starting worker v${VERSION}...`);
console.log(`[agentmemory] Engine: ${config.engineUrl}`);
@@ -124,6 +127,11 @@ async function main() {
} else {
console.log(`[agentmemory] Embedding provider: none (BM25-only mode)`);
}
+ if (imageEmbeddingProvider) {
+ console.log(
+ `[agentmemory] Image embedding provider: ${imageEmbeddingProvider.name} (${imageEmbeddingProvider.dimensions} dims) — vision-search active`,
+ );
+ }
console.log(
`[agentmemory] REST API: http://localhost:${config.restPort}/agentmemory/*`,
);
@@ -154,6 +162,7 @@ async function main() {
registerPrivacyFunction(sdk);
registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession);
registerImageQuotaCleanup(sdk, kv);
+ registerVisionSearchFunctions(sdk, kv, imageEmbeddingProvider);
registerDiskSizeManager(sdk, kv);
registerCompressFunction(sdk, kv, provider, metricsStore);
registerSearchFunction(sdk, kv);
diff --git a/src/mcp/server.ts b/src/mcp/server.ts
index 82283e90..69539a2f 100644
--- a/src/mcp/server.ts
+++ b/src/mcp/server.ts
@@ -275,6 +275,32 @@ export function registerMcpEndpoints(
};
}
+ case "memory_vision_search": {
+ const queryText = typeof args.queryText === "string" ? args.queryText : undefined;
+ const queryImageRef = typeof args.queryImageRef === "string" ? args.queryImageRef : undefined;
+ const queryImageBase64 = typeof args.queryImageBase64 === "string" ? args.queryImageBase64 : undefined;
+ if (!queryText && !queryImageRef && !queryImageBase64) {
+ return {
+ status_code: 400,
+ body: { error: "queryText, queryImageRef, or queryImageBase64 required" },
+ };
+ }
+ const topK = Math.max(1, Math.min(50, asNumber(args.topK, 10) ?? 10));
+ const sessionId = typeof args.sessionId === "string" ? args.sessionId : undefined;
+ const result = await sdk.trigger({
+ function_id: "mem::vision-search",
+ payload: { queryText, queryImageRef, queryImageBase64, topK, sessionId },
+ });
+ return {
+ status_code: 200,
+ body: {
+ content: [
+ { type: "text", text: JSON.stringify(result, null, 2) },
+ ],
+ },
+ };
+ }
+
case "memory_timeline": {
if (typeof args.anchor !== "string" || !args.anchor.trim()) {
return {
diff --git a/src/mcp/tools-registry.ts b/src/mcp/tools-registry.ts
index 4297e459..679e5288 100644
--- a/src/mcp/tools-registry.ts
+++ b/src/mcp/tools-registry.ts
@@ -126,6 +126,21 @@ export const CORE_TOOLS: McpToolDef[] = [
required: ["query"],
},
},
+ {
+ name: "memory_vision_search",
+ description:
+ "Cross-modal image search via CLIP embeddings. Pass queryText to find screenshots matching a description, or queryImageBase64/queryImageRef to find similar images. Requires AGENTMEMORY_IMAGE_EMBEDDINGS=true.",
+ inputSchema: {
+ type: "object",
+ properties: {
+ queryText: { type: "string", description: "Text query (e.g. 'login form with error banner')" },
+ queryImageRef: { type: "string", description: "Absolute path to a stored image to match against" },
+ queryImageBase64: { type: "string", description: "Raw base64 image bytes or data URL" },
+ topK: { type: "number", description: "Max results (default 10, max 50)" },
+ sessionId: { type: "string", description: "Filter to a single session" },
+ },
+ },
+ },
{
name: "memory_timeline",
description: "Chronological observations around an anchor point.",
diff --git a/src/providers/embedding/clip.ts b/src/providers/embedding/clip.ts
new file mode 100644
index 00000000..56ffe47a
--- /dev/null
+++ b/src/providers/embedding/clip.ts
@@ -0,0 +1,114 @@
+import { readFile } from "node:fs/promises";
+import type { EmbeddingProvider } from "../../types.js";
+
+type TransformersModule = {
+ pipeline: (
+ task: string,
+ model: string,
+ ) => Promise<ClipPipeline>;
+ RawImage: {
+ read: (src: string | Blob | Buffer) => Promise<RawImageInstance>;
+ fromBlob: (blob: Blob) => Promise<RawImageInstance>;
+ };
+};
+
+type RawImageInstance = unknown;
+
+type ClipPipeline = (
+ input: string[] | RawImageInstance | RawImageInstance[],
+ options?: { pooling?: string; normalize?: boolean },
+) => Promise<{ tolist: () => number[][]; data: Float32Array }>;
+
+const DEFAULT_MODEL = "Xenova/clip-vit-base-patch32";
+const DIMENSIONS = 512;
+
+export class ClipEmbeddingProvider implements EmbeddingProvider {
+ readonly name = "clip";
+ readonly dimensions = DIMENSIONS;
+ private textExtractor: ClipPipeline | null = null;
+ private imageExtractor: ClipPipeline | null = null;
+ private transformers: TransformersModule | null = null;
+ private readonly modelId: string;
+
+ constructor(modelId: string = DEFAULT_MODEL) {
+ this.modelId = modelId;
+ }
+
+ async embed(text: string): Promise<Float32Array> {
+ const [vec] = await this.embedBatch([text]);
+ return vec;
+ }
+
+ async embedBatch(texts: string[]): Promise<Float32Array[]> {
+ const extractor = await this.getTextExtractor();
+ const output = await extractor(texts, { pooling: "mean", normalize: true });
+ return output.tolist().map((v) => new Float32Array(v));
+ }
+
+ async embedImage(imageBase64OrPath: string): Promise<Float32Array> {
+ const t = await this.getTransformers();
+ const image = await loadImage(t, imageBase64OrPath);
+ const extractor = await this.getImageExtractor();
+ const output = await extractor(image);
+ const raw = output.data ?? new Float32Array(output.tolist()[0] || []);
+ const vec = new Float32Array(raw);
+ return normalize(vec);
+ }
+
+ private async getTransformers(): Promise<TransformersModule> {
+ if (this.transformers) return this.transformers;
+ try {
+ this.transformers = (await import("@xenova/transformers")) as unknown as TransformersModule;
+ } catch {
+ throw new Error(
+ "Install @xenova/transformers for CLIP image embeddings: npm install @xenova/transformers",
+ );
+ }
+ return this.transformers;
+ }
+
+ private async getTextExtractor(): Promise<ClipPipeline> {
+ if (this.textExtractor) return this.textExtractor;
+ const t = await this.getTransformers();
+ this.textExtractor = await t.pipeline("feature-extraction", this.modelId);
+ return this.textExtractor;
+ }
+
+ private async getImageExtractor(): Promise<ClipPipeline> {
+ if (this.imageExtractor) return this.imageExtractor;
+ const t = await this.getTransformers();
+ this.imageExtractor = await t.pipeline("image-feature-extraction", this.modelId);
+ return this.imageExtractor;
+ }
+}
+
+async function loadImage(
+ t: TransformersModule,
+ src: string,
+): Promise<RawImageInstance> {
+ if (src.startsWith("data:")) {
+ const comma = src.indexOf(",");
+ const b64 = comma >= 0 ? src.slice(comma + 1) : src;
+ const buf = Buffer.from(b64, "base64");
+ const blob = new Blob([buf]);
+ return t.RawImage.fromBlob(blob);
+ }
+ if (/^[A-Za-z0-9+/=]+$/.test(src.slice(0, 64)) && !src.includes("/")) {
+ const buf = Buffer.from(src, "base64");
+ const blob = new Blob([buf]);
+ return t.RawImage.fromBlob(blob);
+ }
+ const data = await readFile(src);
+ const blob = new Blob([data]);
+ return t.RawImage.fromBlob(blob);
+}
+
+function normalize(vec: Float32Array): Float32Array {
+ let sum = 0;
+ for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];
+ const norm = Math.sqrt(sum);
+ if (norm === 0) return vec;
+ const out = new Float32Array(vec.length);
+ for (let i = 0; i < vec.length; i++) out[i] = vec[i] / norm;
+ return out;
+}
diff --git a/src/providers/embedding/index.ts b/src/providers/embedding/index.ts
index 0f52514e..4580232c 100644
--- a/src/providers/embedding/index.ts
+++ b/src/providers/embedding/index.ts
@@ -6,6 +6,7 @@ import { VoyageEmbeddingProvider } from "./voyage.js";
import { CohereEmbeddingProvider } from "./cohere.js";
import { OpenRouterEmbeddingProvider } from "./openrouter.js";
import { LocalEmbeddingProvider } from "./local.js";
+import { ClipEmbeddingProvider } from "./clip.js";
export {
GeminiEmbeddingProvider,
@@ -14,8 +15,18 @@ export {
CohereEmbeddingProvider,
OpenRouterEmbeddingProvider,
LocalEmbeddingProvider,
+ ClipEmbeddingProvider,
};
+let imageEmbeddingProvider: EmbeddingProvider | null = null;
+
+export function createImageEmbeddingProvider(): EmbeddingProvider | null {
+ if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] !== "true") return null;
+ if (imageEmbeddingProvider) return imageEmbeddingProvider;
+ imageEmbeddingProvider = new ClipEmbeddingProvider();
+ return imageEmbeddingProvider;
+}
+
export function createEmbeddingProvider(): EmbeddingProvider | null {
const detected = detectEmbeddingProvider();
if (!detected) return null;
diff --git a/src/providers/index.ts b/src/providers/index.ts
index 3e3978b6..c9e09b49 100644
--- a/src/providers/index.ts
+++ b/src/providers/index.ts
@@ -11,7 +11,7 @@ import { ResilientProvider } from "./resilient.js";
import { FallbackChainProvider } from "./fallback-chain.js";
import { getEnvVar } from "../config.js";
-export { createEmbeddingProvider } from "./embedding/index.js";
+export { createEmbeddingProvider, createImageEmbeddingProvider } from "./embedding/index.js";
function requireEnvVar(key: string): string {
const value = getEnvVar(key);
diff --git a/src/state/schema.ts b/src/state/schema.ts
index fb2c9fc3..b3c8bac5 100644
--- a/src/state/schema.ts
+++ b/src/state/schema.ts
@@ -42,6 +42,7 @@ export const KV = {
retentionScores: "mem:retention",
accessLog: "mem:access",
imageRefs: "mem:image-refs",
+ imageEmbeddings: "mem:image-embeddings",
} as const;
export const STREAM = {
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 1db1daac..26646489 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -1279,6 +1279,69 @@ export function registerApiTriggers(
config: { api_path: "/agentmemory/relations", http_method: "GET" },
});
+ sdk.registerFunction("api::vision-search",
+ async (
+ req: ApiRequest<{
+ queryText?: string;
+ queryImageRef?: string;
+ queryImageBase64?: string;
+ topK?: number;
+ sessionId?: string;
+ }>,
+ ): Promise<Response> => {
+ const authErr = checkAuth(req, secret);
+ if (authErr) return authErr;
+ if (
+ !req.body?.queryText &&
+ !req.body?.queryImageRef &&
+ !req.body?.queryImageBase64
+ ) {
+ return {
+ status_code: 400,
+ body: { error: "queryText, queryImageRef, or queryImageBase64 required" },
+ };
+ }
+ const result = await sdk.trigger({ function_id: "mem::vision-search", payload: req.body });
+ const body = result as { success?: boolean; error?: string };
+ if (body?.success === false) {
+ return { status_code: body.error?.includes("disabled") ? 503 : 400, body };
+ }
+ return { status_code: 200, body: result };
+ },
+ );
+ sdk.registerTrigger({
+ type: "http",
+ function_id: "api::vision-search",
+ config: { api_path: "/agentmemory/vision-search", http_method: "POST" },
+ });
+
+ sdk.registerFunction("api::vision-embed",
+ async (
+ req: ApiRequest<{
+ imageRef: string;
+ sessionId?: string;
+ observationId?: string;
+ }>,
+ ): Promise<Response> => {
+ const authErr = checkAuth(req, secret);
+ if (authErr) return authErr;
+ if (!req.body?.imageRef) {
+ return { status_code: 400, body: { error: "imageRef is required" } };
+ }
+ const result = await sdk.trigger({ function_id: "mem::vision-embed", payload: req.body });
+ const body = result as { success?: boolean; error?: string };
+ if (body?.success === false) {
+ return { status_code: body.error?.includes("disabled") ? 503 : 400, body };
+ }
+ return { status_code: 200, body: result };
+ },
+ );
+ sdk.registerTrigger({
+ type: "http",
+ function_id: "api::vision-embed",
+ config: { api_path: "/agentmemory/vision-embed", http_method: "POST" },
+ });
+
sdk.registerFunction("api::action-create",
async (
req: ApiRequest<{
diff --git a/src/types.ts b/src/types.ts
index 3f40c7c5..9cf0b41f 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -208,6 +208,7 @@ export interface EmbeddingProvider {
dimensions: number;
embed(text: string): Promise<Float32Array>;
embedBatch(texts: string[]): Promise<Float32Array[]>;
+ embedImage?(imageBase64OrPath: string): Promise<Float32Array>;
}
export interface MemoryRelation {
diff --git a/test/mcp-standalone.test.ts b/test/mcp-standalone.test.ts
index def82928..59bd985e 100644
--- a/test/mcp-standalone.test.ts
+++ b/test/mcp-standalone.test.ts
@@ -46,8 +46,8 @@ describe("Tools Registry", () => {
}
});
- it("CORE_TOOLS has 11 items", () => {
- expect(CORE_TOOLS.length).toBe(11);
+ it("CORE_TOOLS has 12 items", () => {
+ expect(CORE_TOOLS.length).toBe(12);
});
it("V040_TOOLS has 8 items", () => {
diff --git a/test/vision-search.test.ts b/test/vision-search.test.ts
new file mode 100644
index 00000000..55cf1306
--- /dev/null
+++ b/test/vision-search.test.ts
@@ -0,0 +1,135 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { registerVisionSearchFunctions } from "../src/functions/vision-search.js";
+import type { EmbeddingProvider } from "../src/types.js";
+import { KV } from "../src/state/schema.js";
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ return (store.get(scope)?.get(key) as T) ?? null;
+ },
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ if (!store.has(scope)) return [];
+ return Array.from(store.get(scope)!.values()) as T[];
+ },
+ };
+}
+
+function unit(v: number[]): Float32Array {
+ let norm = 0;
+ for (const x of v) norm += x * x;
+ norm = Math.sqrt(norm);
+ return new Float32Array(v.map((x) => x / (norm || 1)));
+}
+
+describe("vision-search", () => {
+ let visionSearch: (data: Record<string, unknown>) => Promise<Record<string, unknown>>;
+ let visionEmbed: (data: Record<string, unknown>) => Promise<Record<string, unknown>>;
+ let kv: ReturnType<typeof mockKV>;
+
+ const fakeProvider: EmbeddingProvider = {
+ name: "fake-clip",
+ dimensions: 3,
+ embed: async (text: string) => {
+ if (text.includes("login")) return unit([1, 0, 0]);
+ if (text.includes("dashboard")) return unit([0, 1, 0]);
+ return unit([0, 0, 1]);
+ },
+ embedBatch: async (texts: string[]) => Promise.all(texts.map((t) => fakeProvider.embed(t))),
+ embedImage: async (ref: string) => {
+ if (ref.endsWith("login.png")) return unit([1, 0.1, 0]);
+ if (ref.endsWith("dashboard.png")) return unit([0, 1, 0.1]);
+ return unit([0, 0.1, 1]);
+ },
+ };
+
+ beforeEach(() => {
+ kv = mockKV();
+ const handlers: Record<string, (data: Record<string, unknown>) => Promise<Record<string, unknown>>> = {};
+ const sdk = {
+ registerFunction: vi.fn((id: string, cb) => {
+ handlers[id] = cb;
+ }),
+ } as unknown as import("iii-sdk").ISdk;
+ registerVisionSearchFunctions(sdk, kv as never, fakeProvider);
+ visionSearch = handlers["mem::vision-search"]!;
+ visionEmbed = handlers["mem::vision-embed"]!;
+ });
+
+ it("embeds and stores image vectors in KV.imageEmbeddings", async () => {
+ const res = (await visionEmbed({ imageRef: "/tmp/login.png" })) as { success: boolean; dimensions: number };
+ expect(res.success).toBe(true);
+ expect(res.dimensions).toBe(3);
+ const stored = await kv.list(KV.imageEmbeddings);
+ expect(stored.length).toBe(1);
+ });
+
+ it("text query ranks the matching image first", async () => {
+ await visionEmbed({ imageRef: "/tmp/login.png" });
+ await visionEmbed({ imageRef: "/tmp/dashboard.png" });
+ await visionEmbed({ imageRef: "/tmp/other.png" });
+
+ const res = (await visionSearch({ queryText: "the login form", topK: 3 })) as {
+ success: boolean;
+ results: Array<{ imageRef: string; score: number }>;
+ };
+ expect(res.success).toBe(true);
+ expect(res.results[0].imageRef).toBe("/tmp/login.png");
+ expect(res.results[0].score).toBeGreaterThan(res.results[1].score);
+ });
+
+ it("image-to-image query finds the same image first", async () => {
+ await visionEmbed({ imageRef: "/tmp/login.png" });
+ await visionEmbed({ imageRef: "/tmp/dashboard.png" });
+
+ const res = (await visionSearch({ queryImageRef: "/tmp/login.png", topK: 2 })) as {
+ success: boolean;
+ results: Array<{ imageRef: string; score: number }>;
+ };
+ expect(res.results[0].imageRef).toBe("/tmp/login.png");
+ expect(res.results[0].score).toBeGreaterThan(0.9);
+ });
+
+ it("sessionId filters out embeddings from other sessions", async () => {
+ await visionEmbed({ imageRef: "/tmp/login.png", sessionId: "sess_a" });
+ await visionEmbed({ imageRef: "/tmp/dashboard.png", sessionId: "sess_b" });
+
+ const res = (await visionSearch({ queryText: "anything", sessionId: "sess_a", topK: 5 })) as {
+ success: boolean;
+ results: Array<{ sessionId?: string }>;
+ };
+ expect(res.results.every((r) => r.sessionId === "sess_a")).toBe(true);
+ expect(res.results.length).toBe(1);
+ });
+
+ it("returns 503-equivalent error when provider is absent", async () => {
+ const handlers: Record<string, (data: Record<string, unknown>) => Promise<Record<string, unknown>>> = {};
+ const sdk = {
+ registerFunction: vi.fn((id: string, cb) => {
+ handlers[id] = cb;
+ }),
+ } as unknown as import("iii-sdk").ISdk;
+ registerVisionSearchFunctions(sdk, kv as never, null);
+ const res = (await handlers["mem::vision-search"]!({ queryText: "login" })) as {
+ success: boolean;
+ error: string;
+ };
+ expect(res.success).toBe(false);
+ expect(res.error).toMatch(/disabled/);
+ });
+
+ it("rejects missing query", async () => {
+ const res = (await visionSearch({})) as { success: boolean; error: string };
+ expect(res.success).toBe(false);
+ expect(res.error).toMatch(/required/);
+ });
+});