Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

5.3 KB
Raw Blame
import type { IIIClient } from "iii-sdk";
import type { EmbeddingProvider } from "../types.js";
import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { isManagedImagePath } from "../utils/image-store.js";
import { recordAudit } from "./audit.js";
import { logger } from "../logger.js";

interface StoredEmbedding {
  imageRef: string;
  vector: number[];
  modelName: string;
  dimensions: number;
  updatedAt: string;
  sessionId?: string;
  observationId?: string;
}

export function registerVisionSearchFunctions(
  sdk: IIIClient,
  kv: StateKV,
  imageProvider: EmbeddingProvider | null,
): void {
  sdk.registerFunction(
    "mem::vision-embed",
    async (data: {
      imageRef: string;
      sessionId?: string;
      observationId?: string;
    }) => {
      if (!imageProvider?.embedImage) {
        return { success: false, error: "image embeddings disabled (set AGENTMEMORY_IMAGE_EMBEDDINGS=true)" };
      }
      if (!data?.imageRef || typeof data.imageRef !== "string") {
        return { success: false, error: "imageRef required" };
      }
      if (!isManagedImagePath(data.imageRef)) {
        return { success: false, error: "imageRef must point to a file under the managed image store" };
      }
      const refCount = await kv.get<number>(KV.imageRefs, data.imageRef);
      if (!refCount || Number(refCount) < 1) {
        return { success: false, error: "imageRef not registered in mem:image-refs" };
      }
      try {
        const vec = await imageProvider.embedImage(data.imageRef);
        const stored: StoredEmbedding = {
          imageRef: data.imageRef,
          vector: Array.from(vec),
          modelName: imageProvider.name,
          dimensions: imageProvider.dimensions,
          updatedAt: new Date().toISOString(),
          sessionId: data.sessionId,
          observationId: data.observationId,
        };
        await kv.set(KV.imageEmbeddings, data.imageRef, stored);
        await recordAudit(kv, "vision_embed", "mem::vision-embed", [data.imageRef], {
          modelName: imageProvider.name,
          dimensions: stored.dimensions,
          sessionId: data.sessionId,
          observationId: data.observationId,
        });
        return { success: true, imageRef: data.imageRef, dimensions: stored.dimensions };
      } catch (err) {
        const msg = err instanceof Error ? err.message : String(err);
        logger.warn("vision-embed failed", { imageRef: data.imageRef, error: msg });
        return { success: false, error: msg };
      }
    },
  );

  sdk.registerFunction(
    "mem::vision-search",
    async (data: {
      queryText?: string;
      queryImageRef?: string;
      queryImageBase64?: string;
      topK?: number;
      sessionId?: string;
    }) => {
      if (!imageProvider?.embedImage) {
        return { success: false, error: "image embeddings disabled (set AGENTMEMORY_IMAGE_EMBEDDINGS=true)" };
      }
      const requestedTopK =
        typeof data?.topK === "number" && Number.isFinite(data.topK)
          ? Math.trunc(data.topK)
          : 10;
      const topK = Math.min(50, Math.max(1, requestedTopK));

      let queryVec: Float32Array | null = null;
      try {
        if (data?.queryText) {
          queryVec = await imageProvider.embed(data.queryText);
        } else if (data?.queryImageBase64) {
          const b64 = data.queryImageBase64.startsWith("data:")
            ? data.queryImageBase64
            : `data:image/png;base64,${data.queryImageBase64}`;
          queryVec = await imageProvider.embedImage(b64);
        } else if (data?.queryImageRef) {
          if (!isManagedImagePath(data.queryImageRef)) {
            return { success: false, error: "queryImageRef must point to a file under the managed image store" };
          }
          const refCount = await kv.get<number>(KV.imageRefs, data.queryImageRef);
          if (!refCount || Number(refCount) < 1) {
            return { success: false, error: "queryImageRef not registered in mem:image-refs" };
          }
          queryVec = await imageProvider.embedImage(data.queryImageRef);
        } else {
          return { success: false, error: "queryText, queryImageRef, or queryImageBase64 required" };
        }
      } catch (err) {
        const msg = err instanceof Error ? err.message : String(err);
        return { success: false, error: `query embed failed: ${msg}` };
      }

      if (!queryVec) return { success: false, error: "failed to build query vector" };

      const stored = await kv.list<StoredEmbedding>(KV.imageEmbeddings);
      const filtered = data?.sessionId
        ? stored.filter((s) => s.sessionId === data.sessionId)
        : stored;

      const scored = filtered.map((s) => ({
        imageRef: s.imageRef,
        score: cosine(queryVec!, s.vector),
        sessionId: s.sessionId,
        observationId: s.observationId,
        updatedAt: s.updatedAt,
      }));
      scored.sort((a, b) => b.score - a.score);
      return { success: true, results: scored.slice(0, topK), total: scored.length };
    },
  );
}

function cosine(a: Float32Array, b: number[]): number {
  if (a.length !== b.length) return 0;
  let dot = 0;
  let normA = 0;
  let normB = 0;
  for (let i = 0; i < a.length; i++) {
    dot += a[i] * b[i];
    normA += a[i] * a[i];
    normB += b[i] * b[i];
  }
  const denom = Math.sqrt(normA) * Math.sqrt(normB);
  return denom === 0 ? 0 : dot / denom;
}