Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
8.9 KB
Raw Blame
import { test, expect, beforeEach, afterEach } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { Store } from "../../../src/core/search/store.ts";
import { DEFAULT_FTS_TOKENIZE } from "../../../src/core/search/schema.ts";
import type {
  ResolvedSearchConfig,
  ResolvedEmbeddingConfig,
} from "../../../src/core/search/types.ts";
import { sqliteVecLoadable } from "../../helpers/sqlite-vec.ts";

let tmp: string;
let dbPath: string;

function semanticConfig(
  model: string,
  dim: number,
  overrides?: Partial<ResolvedSearchConfig>,
): ResolvedSearchConfig {
  const semantic: ResolvedEmbeddingConfig = Object.freeze({
    enabled: true,
    provider: "openai-compat",
    baseUrl: "https://x/v1",
    model,
    apiKey: "k",
    dimension: dim,
    timeoutMs: 10_000,
    concurrency: 4,
    batchSize: 32,
    costGateUsd: 0,
    maxRetries: 3,
  });
  return Object.freeze({
    vault: tmp,
    dbPath,
    scopeRules: Object.freeze({
      ignore: Object.freeze([{ raw: ".git", kind: "name" as const }]),
      include: null,
    }),
    chunkSize: 800,
    chunkOverlap: 100,
    chunkMinSize: 100,
    keywordWeight: 0.6,
    semanticWeight: 0.4,
    fusionMode: "linear" as const,
    rrfK: 60,
    semantic,
    recall: Object.freeze({
      mmrLambda: 0.7,
      maxHops: 1,
      hopDecay: 0.5,
      maxExpansionPerHit: 3,
      recencyShape: 0.8,
      recencyScale: 30,
      recencyAmplitude: 0.05,
      intentEnabled: true,
      synonymEnabled: false,
      synonymMaxTerms: 3,
      cacheEnabled: false,
      cacheTtlSeconds: 300,
      relationPolarityEnabled: true,
      relationalArmEnabled: false,
      retrievalTrustGateEnabled: false,
      supersedeFadeEnabled: false,
      learnedWeightsEnabled: false,
      activationEnabled: true,
      twoPassEnabled: true,
      poolMultiplier: 3,
      selfTuningEnabled: false,
      chainStopEnabled: false,
      chainStopScore: 0.8,
      trigramPrefilterEnabled: false,
      trigramPrefilterMinChunks: 5000,
      trigramPrefilterMaxSelectivity: 0.5,
    }),
    rerank: Object.freeze({
      enabled: false,
      kind: "openai-compat",
      baseUrl: null,
      model: null,
      envKey: null,
      apiKey: null,
      topK: 20,
      minScore: 0,
    }),
    shutdownGraceMs: 5_000,
    resumeReindex: false,
    ftsTokenize: DEFAULT_FTS_TOKENIZE,
    ...overrides,
  });
}

beforeEach(() => {
  tmp = mkdtempSync(join(tmpdir(), "osb-vec-"));
  dbPath = join(tmp, "brain.sqlite");
});

afterEach(() => {
  rmSync(tmp, { recursive: true, force: true });
});

function unit(values: number[]): number[] {
  const norm = Math.hypot(...values);
  if (norm === 0) return values;
  return values.map((v) => v / norm);
}

test("vec round-trip when sqlite-vec is loaded", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });
  expect(store.vecLoaded()).toBe(true);

  const docId = store.upsertDocument({
    path: "vec/a.md",
    title: null,
    contentHash: "h",
    mtime: 0,
    size: 1,
  });
  const [c0, c1] = store.replaceChunks(docId, [
    {
      chunkIndex: 0,
      content: "alpha",
      contentHash: "h0",
      startLine: 1,
      endLine: 1,
      tokenCount: 1,
    },
    {
      chunkIndex: 1,
      content: "bravo",
      contentHash: "h1",
      startLine: 2,
      endLine: 2,
      tokenCount: 1,
    },
  ]);

  store.vecUpsert(c0!, unit([1, 0, 0, 0]), "m1", 4, "eh0");
  store.vecUpsert(c1!, unit([0, 1, 0, 0]), "m1", 4, "eh1");

  // Query close to first vector → first chunk is nearest.
  const hits = store.semanticTopK(unit([0.9, 0.1, 0, 0]), { limit: 5 });
  expect(hits.length).toBe(2);
  expect(hits[0]?.chunkId).toBe(c0!);
  expect(hits[1]?.chunkId).toBe(c1!);
  expect(hits[0]!.distance).toBeLessThan(hits[1]!.distance);
  await store.close();
});

test("deleting a document leaves zero rows in chunk_vec/chunk_vec_map", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });
  const docId = store.upsertDocument({
    path: "vec/b.md",
    title: null,
    contentHash: "h",
    mtime: 0,
    size: 1,
  });
  const [c0, c1] = store.replaceChunks(docId, [
    {
      chunkIndex: 0,
      content: "x",
      contentHash: "h0",
      startLine: 1,
      endLine: 1,
      tokenCount: 1,
    },
    {
      chunkIndex: 1,
      content: "y",
      contentHash: "h1",
      startLine: 2,
      endLine: 2,
      tokenCount: 1,
    },
  ]);
  store.vecUpsert(c0!, unit([1, 0, 0, 0]), "m1", 4, "eh0");
  store.vecUpsert(c1!, unit([0, 1, 0, 0]), "m1", 4, "eh1");

  const beforeVec = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM chunk_vec");
  const beforeMap = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM chunk_vec_map");
  expect(beforeVec[0]?.c).toBe(2);
  expect(beforeMap[0]?.c).toBe(2);

  store.deleteDocument("vec/b.md");

  const afterVec = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM chunk_vec");
  const afterMap = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM chunk_vec_map");
  expect(afterVec[0]?.c).toBe(0);
  expect(afterMap[0]?.c).toBe(0);
  await store.close();
});

test("ensureEmbeddingModel drops chunk_vec when dimension changes", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });

  // Touch the vec table to confirm it exists.
  let tables = store.rawQuery<{ name: string }>(
    "SELECT name FROM sqlite_master WHERE type='table' AND name='chunk_vec'",
  );
  expect(tables.length).toBe(1);

  // Change to a different model+dim.
  const change = store.ensureEmbeddingModel("m2", 8);
  expect(change.wasChanged).toBe(true);
  expect(change.previousModel).toBe("m1");
  expect(change.currentModel).toBe("m2");

  // Vec table recreated at new dim, embeddings empty.
  tables = store.rawQuery<{ name: string }>(
    "SELECT name FROM sqlite_master WHERE type='table' AND name='chunk_vec'",
  );
  expect(tables.length).toBe(1);
  const emb = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM embeddings");
  expect(emb[0]?.c).toBe(0);
  await store.close();
});

test("a NaN vector cannot reach the vec table through vecUpsert", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });
  const docId = store.upsertDocument({
    path: "vec/nan.md",
    title: null,
    contentHash: "h",
    mtime: 0,
    size: 1,
  });
  const [c0] = store.replaceChunks(docId, [
    {
      chunkIndex: 0,
      content: "x",
      contentHash: "h0",
      startLine: 1,
      endLine: 1,
      tokenCount: 1,
    },
  ]);
  expect(() => store.vecUpsert(c0!, [Number.NaN, 0, 0, 0], "m1", 4, "eh")).toThrow(
    /EMBEDDING_INVALID_VECTOR|vecUpsert/,
  );
  const rows = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM chunk_vec");
  expect(rows[0]?.c).toBe(0);
  await store.close();
});

test("an all-zero vector cannot reach the vec table through vecUpsert", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });
  const docId = store.upsertDocument({
    path: "vec/zero.md",
    title: null,
    contentHash: "h",
    mtime: 0,
    size: 1,
  });
  const [c0] = store.replaceChunks(docId, [
    {
      chunkIndex: 0,
      content: "x",
      contentHash: "h0",
      startLine: 1,
      endLine: 1,
      tokenCount: 1,
    },
  ]);
  expect(() => store.vecUpsert(c0!, [0, 0, 0, 0], "m1", 4, "eh")).toThrow(/zero/i);
  const rows = store.rawQuery<{ c: number }>("SELECT count(*) AS c FROM chunk_vec");
  expect(rows[0]?.c).toBe(0);
  await store.close();
});

test("semanticTopK rejects a non-finite query vector before querying", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });
  expect(() => store.semanticTopK([Number.POSITIVE_INFINITY, 0, 0, 0], { limit: 5 })).toThrow(
    /semanticTopK|EMBEDDING_INVALID_VECTOR/,
  );
  // An all-zero query vector has no direction for cosine similarity and must
  // be rejected before querying, exactly as vecUpsert rejects it.
  expect(() => store.semanticTopK([0, 0, 0, 0], { limit: 5 })).toThrow(
    /zero|semanticTopK|EMBEDDING_INVALID_VECTOR/,
  );
  await store.close();
});

test("dimension mismatch is rejected on vecUpsert", async () => {
  if (!sqliteVecLoadable()) return;
  const store = await Store.open(semanticConfig("m1", 4), { mode: "write" });
  const docId = store.upsertDocument({
    path: "z.md",
    title: null,
    contentHash: "h",
    mtime: 0,
    size: 1,
  });
  const [c0] = store.replaceChunks(docId, [
    {
      chunkIndex: 0,
      content: "x",
      contentHash: "h",
      startLine: 1,
      endLine: 1,
      tokenCount: 1,
    },
  ]);
  expect(() => store.vecUpsert(c0!, [0.1, 0.2, 0.3], "m1", 4, "eh")).toThrow(/dimension/);
  await store.close();
});