Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

15.5 KB
Raw Blame
import { describe, it, expect, beforeEach, vi } from "vitest";

vi.mock("../src/logger.js", () => ({
  logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));

import { registerSmartSearchFunction } from "../src/functions/smart-search.js";
import type {
  CompressedObservation,
  HybridSearchResult,
  CompactSearchResult,
  Memory,
  Session,
} from "../src/types.js";

function mockKV() {
  const store = new Map<string, Map<string, unknown>>();
  return {
    get: async <T>(scope: string, key: string): Promise<T | null> => {
      return (store.get(scope)?.get(key) as T) ?? null;
    },
    set: async <T>(scope: string, key: string, data: T): Promise<T> => {
      if (!store.has(scope)) store.set(scope, new Map());
      store.get(scope)!.set(key, data);
      return data;
    },
    delete: async (scope: string, key: string): Promise<void> => {
      store.get(scope)?.delete(key);
    },
    list: async <T>(scope: string): Promise<T[]> => {
      const entries = store.get(scope);
      return entries ? (Array.from(entries.values()) as T[]) : [];
    },
  };
}

function mockSdk() {
  const functions = new Map<string, Function>();
  return {
    registerFunction: (idOrOpts: string | { id: string }, handler: Function) => {
      const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id;
      functions.set(id, handler);
    },
    registerTrigger: () => {},
    trigger: async (idOrInput: string | { function_id: string; payload: unknown }, data?: unknown) => {
      const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id;
      const payload = typeof idOrInput === "string" ? data : idOrInput.payload;
      const fn = functions.get(id);
      if (!fn) throw new Error(`No function: ${id}`);
      return fn(payload);
    },
  };
}

function makeObs(
  overrides: Partial<CompressedObservation> = {},
): CompressedObservation {
  return {
    id: "obs_1",
    sessionId: "ses_1",
    timestamp: "2026-02-01T10:00:00Z",
    type: "file_edit",
    title: "Edit auth handler",
    facts: [],
    narrative: "Modified auth",
    concepts: ["auth"],
    files: ["src/auth.ts"],
    importance: 7,
    ...overrides,
  };
}

describe("Smart Search Function", () => {
  let sdk: ReturnType<typeof mockSdk>;
  let kv: ReturnType<typeof mockKV>;
  let searchResults: HybridSearchResult[];

  beforeEach(async () => {
    sdk = mockSdk();
    kv = mockKV();

    const obs1 = makeObs({ id: "obs_1", sessionId: "ses_1", title: "Auth handler" });
    const obs2 = makeObs({ id: "obs_2", sessionId: "ses_1", title: "Database setup" });

    searchResults = [
      {
        observation: obs1,
        bm25Score: 0.8,
        vectorScore: 0,
        combinedScore: 0.8,
        sessionId: "ses_1",
      },
      {
        observation: obs2,
        bm25Score: 0.3,
        vectorScore: 0,
        combinedScore: 0.3,
        sessionId: "ses_1",
      },
    ];

    const session: Session = {
      id: "ses_1",
      project: "my-project",
      cwd: "/tmp",
      startedAt: "2026-02-01T00:00:00Z",
      status: "completed",
      observationCount: 2,
    };
    await kv.set("mem:sessions", "ses_1", session);
    await kv.set("mem:obs:ses_1", "obs_1", obs1);
    await kv.set("mem:obs:ses_1", "obs_2", obs2);

    const searchFn = async (_query: string, _limit: number) => searchResults;
    registerSmartSearchFunction(sdk as never, kv as never, searchFn);
  });

  it("compact mode returns CompactSearchResult array", async () => {
    const result = (await sdk.trigger("mem::smart-search", {
      query: "auth",
    })) as { mode: string; results: CompactSearchResult[] };

    expect(result.mode).toBe("compact");
    expect(result.results.length).toBe(2);
    expect(result.results[0]).toHaveProperty("obsId");
    expect(result.results[0]).toHaveProperty("title");
    expect(result.results[0]).toHaveProperty("type");
    expect(result.results[0]).toHaveProperty("score");
    expect(result.results[0]).toHaveProperty("timestamp");
    expect(result.results[0]).not.toHaveProperty("narrative");
  });

  it("expand mode returns full observations for given IDs", async () => {
    const result = (await sdk.trigger("mem::smart-search", {
      expandIds: ["obs_1"],
    })) as { mode: string; results: Array<{ obsId: string; observation: CompressedObservation }> };

    expect(result.mode).toBe("expanded");
    expect(result.results.length).toBe(1);
    expect(result.results[0].observation.title).toBe("Auth handler");
  });

  it("expand mode returns full durable memories", async () => {
    const memory: Memory = {
      id: "mem_wiki",
      createdAt: "2026-07-19T00:00:00Z",
      updatedAt: "2026-07-19T00:00:00Z",
      type: "architecture",
      title: "AgentMemory knowledge backend",
      content: "Full durable wiki content that must not be truncated.",
      concepts: ["agentmemory"],
      files: ["wiki/agentmemory.md"],
      sessionIds: [],
      strength: 7,
      version: 1,
      supersedes: [],
      sourceObservationIds: [],
      isLatest: true,
      project: "obsidian-vault",
    };
    await kv.set("mem:memories", memory.id, memory);

    const result = (await sdk.trigger("mem::smart-search", {
      expandIds: [memory.id],
    })) as {
      mode: string;
      results: Array<{ obsId: string; observation: CompressedObservation }>;
    };

    expect(result.results).toHaveLength(1);
    expect(result.results[0].observation.narrative).toBe(memory.content);
  });

  it("filters expanded durable memories to the requested project", async () => {
    const memory: Memory = {
      id: "mem_other_project",
      createdAt: "2026-07-19T00:00:00Z",
      updatedAt: "2026-07-19T00:00:00Z",
      type: "architecture",
      title: "Other project memory",
      content: "This memory must not cross the project boundary.",
      concepts: [],
      files: [],
      sessionIds: [],
      strength: 5,
      version: 1,
      supersedes: [],
      sourceObservationIds: [],
      isLatest: true,
      project: "other-project",
    };
    await kv.set("mem:memories", memory.id, memory);

    const result = (await sdk.trigger("mem::smart-search", {
      expandIds: [memory.id],
      project: "my-project",
    })) as { results: Array<{ obsId: string }> };

    expect(result.results).toEqual([]);
  });

  it("preserves durable memory agent scope during expansion", async () => {
    const memory: Memory = {
      id: "mem_scoped",
      createdAt: "2026-07-19T00:00:00Z",
      updatedAt: "2026-07-19T00:00:00Z",
      type: "architecture",
      title: "Scoped memory",
      content: "Only the owning agent may expand this memory.",
      concepts: [],
      files: [],
      sessionIds: [],
      strength: 5,
      version: 1,
      supersedes: [],
      sourceObservationIds: [],
      isLatest: true,
      project: "my-project",
      agentId: "agent-owner",
    };
    await kv.set("mem:memories", memory.id, memory);

    const result = (await sdk.trigger("mem::smart-search", {
      expandIds: [memory.id],
      agentId: "agent-owner",
    })) as { results: Array<{ obsId: string }> };

    expect(result.results.map((item) => item.obsId)).toEqual([memory.id]);
  });

  describe.each(["compact", "expanded"])("%s durable memory project scope", (mode) => {
    it.each([
      ["uses the linked session when the memory has no project", undefined, "my-project", true],
      ["excludes a memory linked to another project", undefined, "other-project", false],
      ["excludes a memory whose linked session is missing", undefined, undefined, false],
      ["keeps a conflicting memory project authoritative", "other-project", "my-project", false],
      ["keeps a matching memory project authoritative", "my-project", "other-project", true],
    ] as const)("%s", async (_name, memoryProject, sessionProject, included) => {
      const sessionId = "ses_linked";
      const memory: Memory = {
        id: "mem_linked",
        createdAt: "2026-07-19T00:00:00Z",
        updatedAt: "2026-07-19T00:00:00Z",
        type: "fact",
        title: "Linked project policy",
        content: "The policy belongs to the linked session's project.",
        concepts: [],
        files: [],
        sessionIds: [sessionId],
        strength: 5,
        version: 1,
        isLatest: true,
      };
      if (memoryProject !== undefined) memory.project = memoryProject;
      await kv.set("mem:memories", memory.id, memory);
      if (sessionProject !== undefined) {
        const session: Session = {
          id: sessionId,
          project: sessionProject,
          cwd: "/linked",
          startedAt: "2026-07-19T00:00:00Z",
          status: "completed",
          observationCount: 1,
        };
        await kv.set("mem:sessions", sessionId, session);
      }
      searchResults = [{
        observation: makeObs({ id: memory.id, sessionId, narrative: memory.content }),
        sessionId,
        bm25Score: 1,
        vectorScore: 0,
        combinedScore: 1,
      }];

      const request = mode === "compact"
        ? { query: "policy" }
        : { expandIds: [memory.id] };
      const result = (await sdk.trigger("mem::smart-search", {
        ...request,
        project: "my-project",
        includeLessons: false,
      })) as { mode: string; results: Array<{ obsId: string }> };

      expect(result.mode).toBe(mode);
      expect(result.results.map((item) => item.obsId)).toEqual(included ? [memory.id] : []);
    });
  });

  it("filters compact candidates to the requested project", async () => {
    const other = makeObs({
      id: "obs_other",
      sessionId: "ses_other",
      title: "Unrelated project",
    });
    await kv.set("mem:sessions", "ses_other", {
      id: "ses_other",
      project: "other-project",
      cwd: "/other",
      startedAt: "2026-02-01T00:00:00Z",
      status: "completed",
      observationCount: 1,
    });
    await kv.set("mem:obs:ses_other", other.id, other);
    searchResults.unshift({
      observation: other,
      bm25Score: 1,
      vectorScore: 1,
      combinedScore: 1,
      sessionId: "ses_other",
    });

    const result = (await sdk.trigger("mem::smart-search", {
      query: "auth",
      project: "my-project",
    })) as { results: CompactSearchResult[] };

    expect(result.results.map((item) => item.obsId)).toEqual(["obs_1", "obs_2"]);
  });

  it("returns error when query is missing and no expandIds", async () => {
    const result = (await sdk.trigger("mem::smart-search", {})) as {
      mode: string;
      error: string;
    };

    expect(result.mode).toBe("compact");
    expect(result.error).toBe("query is required");
    expect((result as { results: unknown[] }).results).toEqual([]);
  });

  it("respects limit parameter in compact mode", async () => {
    const result = (await sdk.trigger("mem::smart-search", {
      query: "auth",
      limit: 1,
    })) as { mode: string; results: CompactSearchResult[] };

    expect(result.results.length).toBeLessThanOrEqual(2);
  });

  it("expand returns empty for nonexistent observation IDs", async () => {
    const result = (await sdk.trigger("mem::smart-search", {
      expandIds: ["obs_nonexistent_ses_xxx"],
    })) as { mode: string; results: unknown[] };

    expect(result.mode).toBe("expanded");
    expect(result.results.length).toBe(0);
  });

  it("compact mode records access for every returned observation id", async () => {
    await sdk.trigger("mem::smart-search", { query: "auth" });
    // recordAccessBatch is fire-and-forget — let the microtask queue drain.
    await new Promise((r) => setImmediate(r));

    const log1 = (await kv.get("mem:access", "obs_1")) as {
      count: number;
    } | null;
    const log2 = (await kv.get("mem:access", "obs_2")) as {
      count: number;
    } | null;

    expect(log1?.count).toBe(1);
    expect(log2?.count).toBe(1);
  });

  it("expand mode records access for expanded observation ids", async () => {
    await sdk.trigger("mem::smart-search", { expandIds: ["obs_1"] });
    await new Promise((r) => setImmediate(r));

    const log = (await kv.get("mem:access", "obs_1")) as {
      count: number;
    } | null;
    expect(log?.count).toBe(1);
  });

  describe("lesson inclusion (#lesson-visibility)", () => {
    it("compact mode returns lessons array alongside observation results", async () => {
      sdk.registerFunction("mem::lesson-recall", async (payload: any) => ({
        success: true,
        lessons: [
          { id: "lsn_a", content: "always rebase before push", confidence: 0.9, createdAt: "2026-04-01T00:00:00Z", project: "p", tags: ["git"], score: 0.81 },
          { id: "lsn_b", content: "never force-push to main", confidence: 0.95, createdAt: "2026-04-02T00:00:00Z", project: "p", tags: ["git"], score: 0.76 },
        ],
      }));

      const result = (await sdk.trigger("mem::smart-search", {
        query: "rebase",
      })) as { mode: string; results: CompactSearchResult[]; lessons?: any[] };

      expect(result.mode).toBe("compact");
      expect(result.results.length).toBe(2); // observations unchanged
      expect(result.lessons).toBeDefined();
      expect(result.lessons!.length).toBe(2);
      expect(result.lessons![0]).toMatchObject({
        lessonId: "lsn_a",
        confidence: 0.9,
        score: 0.81,
      });
      expect(result.lessons![0].tags).toEqual(["git"]);
    });

    it("compact mode truncates long lesson content for preview", async () => {
      const long = "x".repeat(500);
      sdk.registerFunction("mem::lesson-recall", async () => ({
        success: true,
        lessons: [{ id: "lsn_long", content: long, confidence: 0.5, createdAt: "", tags: [], score: 0.4 }],
      }));

      const result = (await sdk.trigger("mem::smart-search", {
        query: "x",
      })) as { lessons: any[] };

      expect(result.lessons[0].content.length).toBeLessThan(long.length);
      expect(result.lessons[0].content).toMatch(/…$/);
    });

    it("includeLessons:false omits the lessons array entirely", async () => {
      // No lesson-recall handler registered — would throw if invoked.
      const result = (await sdk.trigger("mem::smart-search", {
        query: "auth",
        includeLessons: false,
      })) as { mode: string; results: CompactSearchResult[]; lessons?: unknown };

      expect(result.results.length).toBe(2);
      expect(result.lessons).toBeUndefined();
    });

    it("forwards project filter to mem::lesson-recall", async () => {
      let receivedPayload: any = null;
      sdk.registerFunction("mem::lesson-recall", async (payload: any) => {
        receivedPayload = payload;
        return { success: true, lessons: [] };
      });

      await sdk.trigger("mem::smart-search", {
        query: "rebase",
        project: "gitops-assistant",
      });

      expect(receivedPayload).toMatchObject({
        query: "rebase",
        project: "gitops-assistant",
      });
    });

    it("tolerates mem::lesson-recall failure: returns empty lessons, observations unchanged", async () => {
      sdk.registerFunction("mem::lesson-recall", async () => {
        throw new Error("lessons store unavailable");
      });

      const result = (await sdk.trigger("mem::smart-search", {
        query: "auth",
      })) as { results: CompactSearchResult[]; lessons: any[] };

      expect(result.results.length).toBe(2);
      expect(result.lessons).toEqual([]);
    });

    it("tolerates non-success lesson-recall response shape", async () => {
      sdk.registerFunction("mem::lesson-recall", async () => ({
        success: false,
        error: "query is required",
      }));

      const result = (await sdk.trigger("mem::smart-search", {
        query: "auth",
      })) as { results: CompactSearchResult[]; lessons: any[] };

      expect(result.results.length).toBe(2);
      expect(result.lessons).toEqual([]);
    });
  });
});