Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.8 KB
Raw Blame
import { test, expect, beforeEach, afterEach } from "bun:test";
import { Database } from "bun:sqlite";

import { indexVault } from "../../../src/core/search/indexer.ts";
import { search } from "../../../src/core/search/search.ts";
import { parseStructuredRecallQueryDocument } from "../../../src/core/search/structured-query.ts";
import { SearchError } from "../../../src/core/search/types.ts";
import { FTS_MATCH_MODE, type FtsMatchMode } from "../../../src/core/search/fts-match-mode.ts";
import { createTempVault, makeConfig, writeMd } from "../../helpers/search-fixtures.ts";
import { startFakeHttp, type FakeHttp } from "../../helpers/fake-http.ts";
import { sqliteVecLoadable } from "../../helpers/sqlite-vec.ts";
import { FAKE_PROVIDER_KEY } from "../../helpers/fake-credentials.ts";

let vault: string;
let dbPath: string;
let cleanup: () => void;
let server: FakeHttp;

beforeEach(async () => {
  const v = createTempVault("search");
  vault = v.vault;
  dbPath = v.dbPath;
  cleanup = v.cleanup;
  server = await startFakeHttp();
});

afterEach(async () => {
  cleanup();
  await server.close();
});

function semanticConfig() {
  return makeConfig({
    vault,
    dbPath,
    semantic: {
      enabled: true,
      provider: "openai-compat",
      baseUrl: server.url,
      model: "fake-model",
      apiKey: FAKE_PROVIDER_KEY,
      dimension: 4,
      timeoutMs: 5_000,
      concurrency: 2,
      batchSize: 8,
      costGateUsd: 0,
      maxRetries: 3,
    },
  });
}

async function seedKeyword() {
  writeMd(
    vault,
    "Notes/foo.md",
    "# Foo\n\nThe quick brown fox jumps over the lazy dog. Foxes are interesting.",
  );
  writeMd(vault, "Other/bar.md", "# Bar\n\nA different note about cats and turtles.");
  writeMd(vault, "Notes/ru.md", "# Ru\n\nКириллический текст про брожение тестового зерна.");
  return makeConfig({ vault, dbPath });
}

function cacheRowCount(): number {
  const db = new Database(dbPath);
  try {
    return db.query<{ c: number }, []>("SELECT count(*) AS c FROM query_cache").get()?.c ?? 0;
  } finally {
    db.close();
  }
}

test("query cache (opt-in) stores a row and serves an identical request", async () => {
  await seedKeyword();
  const cfg = makeConfig({ vault, dbPath, cacheEnabled: true });
  await indexVault(cfg);
  expect(cacheRowCount()).toBe(0);

  const r1 = await search(cfg, { query: "fox", limit: 5 });
  expect(cacheRowCount()).toBeGreaterThan(0);

  const r2 = await search(cfg, { query: "fox", limit: 5 });
  expect(r2.results.map((x) => x.path)).toEqual(r1.results.map((x) => x.path));
});

test("query cache is invalidated by a reindex (corpus-generation bump)", async () => {
  await seedKeyword();
  const cfg = makeConfig({ vault, dbPath, cacheEnabled: true });
  await indexVault(cfg);

  const r1 = await search(cfg, { query: "fox", limit: 10 });
  expect(r1.results.map((x) => x.path)).not.toContain("Notes/fox2.md");

  // A new matching doc + reindex bumps the index revision -> generation
  // changes -> the cached row is not served and the result reflects it.
  writeMd(vault, "Notes/fox2.md", "# Fox2\n\nAnother fox appears here.");
  await indexVault(cfg);
  const r2 = await search(cfg, { query: "fox", limit: 10 });
  expect(r2.results.map((x) => x.path)).toContain("Notes/fox2.md");
});

test("no query_cache rows are written when the cache is disabled", async () => {
  await seedKeyword();
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);
  await search(cfg, { query: "fox", limit: 5 });
  expect(cacheRowCount()).toBe(0);
});

async function seedExpansion() {
  // d1 + d2 both match "alpha" and co-mention "beta" -> "beta" is a
  // co-occurrence expansion term. d3 has "beta" but not "alpha".
  writeMd(vault, "Notes/d1.md", "# D1\n\nalpha beta gamma notes.");
  writeMd(vault, "Notes/d2.md", "# D2\n\nalpha beta delta notes.");
  writeMd(vault, "Notes/d3.md", "# D3\n\nbeta epsilon unrelated content.");
}

test("synonym expansion (opt-in) broadens recall to a co-occurring-term doc", async () => {
  await seedExpansion();
  const off = makeConfig({ vault, dbPath });
  await indexVault(off);

  const baseline = await search(off, { query: "alpha", limit: 10 });
  expect(baseline.results.map((r) => r.path)).not.toContain("Notes/d3.md");

  const on = makeConfig({ vault, dbPath, synonymEnabled: true });
  const expanded = await search(on, { query: "alpha", limit: 10 });
  // "beta" co-occurs in d1+d2, so expansion pulls in d3 via the OR.
  expect(expanded.results.map((r) => r.path)).toContain("Notes/d3.md");
});

test("synonym expansion is suppressed for an exact-intent (quoted) query", async () => {
  await seedExpansion();
  const on = makeConfig({ vault, dbPath, synonymEnabled: true });
  await indexVault(on);
  const res = await search(on, { query: '"alpha"', limit: 10 });
  // Quoted -> exact intent -> no expansion -> d3 (no "alpha") stays out.
  expect(res.results.map((r) => r.path)).not.toContain("Notes/d3.md");
});

test("search returns keyword-only results when semantic is disabled", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);

  const out = await search(cfg, { query: "fox", limit: 5 });
  expect(out.results.length).toBe(1);
  expect(out.results[0]?.path).toBe("Notes/foo.md");
  expect(out.results[0]?.searchType).toBe("keyword");
  expect(out.warnings).toEqual([]);
});

test("search warns when keyword retrieval rebuilds a desynced FTS table", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  const db = new Database(dbPath);
  try {
    db.run("DELETE FROM chunk_fts");
  } finally {
    db.close();
  }

  const out = await search(cfg, { query: "fox", limit: 5 });

  expect(out.results.map((r) => r.path)).toContain("Notes/foo.md");
  expect(out.warnings.some((w) => w.includes("rebuilt FTS"))).toBe(true);
});

test("search accepts structured lexical lanes with safe exclusions", async () => {
  writeMd(vault, "Notes/final.md", "# Final\n\nrelease notes mention recall diagnostics.");
  writeMd(vault, "Notes/draft.md", "# Draft\n\ndraft release notes mention recall diagnostics.");
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);
  const structuredQuery = parseStructuredRecallQueryDocument('lex: "release notes" -draft');

  const out = await search(cfg, {
    query: "release notes",
    structuredQuery,
    limit: 10,
  });

  expect(out.results.map((r) => r.path)).toContain("Notes/final.md");
  expect(out.results.map((r) => r.path)).not.toContain("Notes/draft.md");
  expect(out.results[0]?.reasons.some((reason) => reason.includes("lane:lex/fts5"))).toBe(true);
});

test("search backfills structured lexical exclusions before applying limit", async () => {
  writeMd(vault, "Notes/draft-a.md", "# Draft A\n\nalpha alpha alpha draft.");
  writeMd(vault, "Notes/draft-b.md", "# Draft B\n\nalpha alpha draft.");
  writeMd(vault, "Notes/final-a.md", "# Final A\n\nalpha current.");
  writeMd(vault, "Notes/final-b.md", "# Final B\n\nalpha stable.");
  const cfg = makeConfig({ vault, dbPath, mmrLambda: 1, maxHops: 0 });
  await indexVault(cfg);
  const structuredQuery = parseStructuredRecallQueryDocument("lex: alpha -draft");

  const out = await search(cfg, {
    query: "alpha",
    structuredQuery,
    limit: 2,
    mmrLambda: 1,
    maxHops: 0,
  });

  expect(out.results.map((r) => r.path).toSorted()).toEqual([
    "Notes/final-a.md",
    "Notes/final-b.md",
  ]);
});

test("search degrades semantic structured lanes when semantic search is disabled", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  const structuredQuery = parseStructuredRecallQueryDocument("vec: fuzzy fox memory");

  const out = await search(cfg, { query: "fox", structuredQuery, limit: 5 });

  expect(out.results.map((r) => r.path)).toContain("Notes/foo.md");
  expect(
    out.warnings.some((warning) => warning.includes("semantic structured lanes skipped")),
  ).toBe(true);
});

test("search can return an evidence pack and downrank terminal-state support", async () => {
  // Terminal state is declared via the controlled frontmatter `status:`
  // field, not inferred from prose - language-agnostic downranking.
  writeMd(
    vault,
    "Notes/terminal.md",
    "---\nstatus: superseded\n---\n\n# Terminal\n\nalpha beta old support.",
  );
  writeMd(vault, "Notes/active.md", "# Active\n\nalpha beta current support.");
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);

  const structuredQuery = parseStructuredRecallQueryDocument('lex: "alpha beta"');
  const out = await search(cfg, {
    query: "alpha beta gamma",
    structuredQuery,
    limit: 5,
    evidencePack: true,
  });

  expect(out.results[0]?.path).toBe("Notes/active.md");
  expect(out.evidencePack?.missingTerms).toContain("gamma");
  expect(
    out.evidencePack?.records.find((record) => record.path === "Notes/terminal.md")?.terminalState,
  ).toBe(true);
});
test("search returns empty (no error) when no results match", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  const out = await search(cfg, { query: "nonexistentterm", limit: 5 });
  expect(out.results).toEqual([]);
  expect(out.total).toBe(0);
});

test("search supports Cyrillic query against Cyrillic content", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  const out = await search(cfg, { query: "брожение", limit: 5 });
  expect(out.results.length).toBe(1);
  expect(out.results[0]?.path).toBe("Notes/ru.md");
});

test("search supports unspaced CJK query segments without polluting result content", async () => {
  writeMd(vault, "Notes/cjk.md", "# CJK\n\n我喜欢苹果派,也喜欢机器学习。");
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);

  const out = await search(cfg, { query: "苹果", limit: 5 });

  expect(out.results.map((r) => r.path)).toContain("Notes/cjk.md");
  const hit = out.results.find((r) => r.path === "Notes/cjk.md");
  expect(hit?.content).toContain("我喜欢苹果派");
  expect(hit?.content).not.toContain("苹果 果派");
});

test("path_prefix filter scopes results", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  const all = await search(cfg, { query: "quick", limit: 5 });
  expect(all.results.length).toBe(1);
  const scoped = await search(cfg, {
    query: "quick",
    limit: 5,
    pathPrefix: "Other/",
  });
  expect(scoped.results.length).toBe(0);
});

test("path_prefix escaping is rejected with INVALID_INPUT", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  let err: SearchError | null = null;
  try {
    await search(cfg, { query: "x", pathPrefix: "../etc/" });
  } catch (e) {
    err = e as SearchError;
  }
  expect(err?.code).toBe("INVALID_INPUT");
});

test("empty query throws INVALID_INPUT", async () => {
  const cfg = await seedKeyword();
  await indexVault(cfg);
  let err: SearchError | null = null;
  try {
    await search(cfg, { query: "   " });
  } catch (e) {
    err = e as SearchError;
  }
  expect(err?.code).toBe("INVALID_INPUT");
});

test("missing index self-heals: search builds it on first use", async () => {
  // A missing index used to throw INDEX_MISSING; the read path now rebuilds
  // once and returns results so an upgrade never needs a manual `o2b search
  // index`. (Dedicated coverage in self-heal.test.ts.)
  const cfg = await seedKeyword();
  // skip indexVault — the search must build the index itself
  const out = await search(cfg, { query: "fox" });
  expect(out.results.length).toBeGreaterThan(0);
});

test("hybrid search combines keyword and semantic when both contribute", async () => {
  if (!sqliteVecLoadable()) return;
  writeMd(vault, "A/foo.md", "alpha beta gamma");
  writeMd(vault, "A/bar.md", "delta epsilon zeta");
  const cfg = semanticConfig();
  await indexVault(cfg, { embeddings: true });

  const out = await search(cfg, { query: "alpha", limit: 5, semantic: true });
  expect(out.results.length).toBeGreaterThan(0);
  // The keyword hit should be in the top result; searchType may be hybrid.
  const first = out.results[0]!;
  expect(["keyword", "hybrid", "semantic"]).toContain(first.searchType);
});

test("explicit --semantic without vec extension throws (configured-down case)", async () => {
  await seedKeyword();
  const cfg = makeConfig({
    vault,
    dbPath,
    semantic: {
      enabled: true,
      provider: "openai-compat",
      baseUrl: server.url,
      model: "fake-model",
      apiKey: "k",
      dimension: 4,
      timeoutMs: 5_000,
      concurrency: 1,
      batchSize: 8,
      costGateUsd: 0,
      maxRetries: 3,
    },
  });
  await indexVault(cfg);
  // Now disable vec by closing the index and reopening through a config
  // that uses a fresh dbPath without vec. We can't unload the extension —
  // so emulate by stubbing config.semantic.enabled=true but no embeddings
  // recorded. The store loads vec OK but counts.embeddings===0, which is
  // the data-state warning case (warn + skip even when explicit).
  const out = await search(cfg, { query: "fox", limit: 5, semantic: true });
  expect(out.warnings.some((w) => w.includes("no compatible embeddings"))).toBe(true);
});

test("implicit semantic + no embeddings → keyword-only + warning, no throw", async () => {
  await seedKeyword();
  const cfg = semanticConfig();
  await indexVault(cfg); // no --embeddings
  const out = await search(cfg, { query: "fox", limit: 5 });
  expect(out.warnings.some((w) => w.includes("no compatible embeddings"))).toBe(true);
  expect(out.results.length).toBeGreaterThan(0);
});

test("limit truncates results", async () => {
  for (let i = 0; i < 20; i++) {
    writeMd(vault, `notes/n${i}.md`, `# Note ${i}\n\nrepeated word repeated word repeated word`);
  }
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);
  const out = await search(cfg, { query: "repeated", limit: 5 });
  expect(out.results.length).toBe(5);
});

test("matchMode any widens the keyword lane; the default all stays exact (t_c5326ece)", async () => {
  writeMd(vault, "Notes/both.md", "# Both\n\nchimera basilisk together in one lair.");
  writeMd(vault, "Notes/one.md", "# One\n\nA lone chimera wanders here.");
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);
  const strict = await search(cfg, { query: "chimera basilisk", limit: 10 });
  const widened = await search(cfg, {
    query: "chimera basilisk",
    limit: 10,
    matchMode: FTS_MATCH_MODE.any,
  });
  expect(strict.results.some((h) => h.path.includes("one.md"))).toBe(false);
  expect(widened.results.some((h) => h.path.includes("one.md"))).toBe(true);
  expect(widened.results.some((h) => h.path.includes("both.md"))).toBe(true);
});

test("an unknown matchMode is refused, not treated as the default", async () => {
  writeMd(vault, "Notes/both.md", "# Both\n\nchimera basilisk together in one lair.");
  const cfg = makeConfig({ vault, dbPath });
  await indexVault(cfg);
  expect(
    search(cfg, { query: "chimera", matchMode: "sometimes" as unknown as FtsMatchMode }),
  ).rejects.toThrow(SearchError);
});