Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
3.7 KB
Raw Blame
/**
 * Task Q2 (t_618f7211): configurable FTS tokenizer language and
 * diacritic rules.
 *
 * Acceptance coverage:
 *   - with no config the generated schema keeps
 *     `unicode61 remove_diacritics 2` byte-identically;
 *   - a valid config changes the tokenizer clause on the materialized
 *     chunk_fts table (after reindex);
 *   - invalid tokenizer options reject with a typed error listing the
 *     allowed values;
 *   - the CJK trigram path is untouched (chunk_trigram stays `trigram`);
 *   - no implicit reindex (buildFtsTokenize is pure; changing config
 *     alone rewrites nothing).
 */

import { describe, expect, test } from "bun:test";
import { Database } from "bun:sqlite";

import {
  applyMigrations,
  buildFtsTokenize,
  DEFAULT_FTS_TOKENIZE,
} from "../../../src/core/search/schema.ts";
import { SearchError } from "../../../src/core/search/types.ts";

function ftsSql(db: Database, table: string): string {
  const row = db
    .query<{ sql: string }, [string]>("SELECT sql FROM sqlite_master WHERE name = ?")
    .get(table);
  return row?.sql ?? "";
}

describe("buildFtsTokenize", () => {
  test("no config yields the byte-identical default clause", () => {
    expect(buildFtsTokenize({})).toBe("unicode61 remove_diacritics 2");
    expect(buildFtsTokenize({})).toBe(DEFAULT_FTS_TOKENIZE);
    expect(buildFtsTokenize({ diacritics: null, stemmer: null })).toBe(DEFAULT_FTS_TOKENIZE);
  });

  test("diacritics option changes the remove_diacritics rule", () => {
    expect(buildFtsTokenize({ diacritics: "0" })).toBe("unicode61 remove_diacritics 0");
    expect(buildFtsTokenize({ diacritics: "1" })).toBe("unicode61 remove_diacritics 1");
  });

  test("porter stemmer wraps unicode61 (language stemming)", () => {
    expect(buildFtsTokenize({ stemmer: "porter" })).toBe("porter unicode61 remove_diacritics 2");
    expect(buildFtsTokenize({ stemmer: "porter", diacritics: "0" })).toBe(
      "porter unicode61 remove_diacritics 0",
    );
  });

  test("invalid diacritics rejects with a typed error listing allowed values", () => {
    let err: unknown;
    try {
      buildFtsTokenize({ diacritics: "3" });
    } catch (e) {
      err = e;
    }
    expect(err).toBeInstanceOf(SearchError);
    expect((err as SearchError).code).toBe("INVALID_INPUT");
    expect((err as SearchError).message).toMatch(/0.*1.*2/);
  });

  test("invalid stemmer rejects with a typed error listing allowed values", () => {
    let err: unknown;
    try {
      buildFtsTokenize({ stemmer: "snowball" });
    } catch (e) {
      err = e;
    }
    expect(err).toBeInstanceOf(SearchError);
    expect((err as SearchError).code).toBe("INVALID_INPUT");
    expect((err as SearchError).message).toMatch(/none.*porter|porter.*none/);
  });
});

describe("applyMigrations tokenizer wiring", () => {
  test("default schema keeps unicode61 remove_diacritics 2 byte-identically", () => {
    const db = new Database(":memory:");
    applyMigrations(db);
    expect(ftsSql(db, "chunk_fts")).toContain("tokenize='unicode61 remove_diacritics 2'");
    db.close();
  });

  test("a configured clause is applied to the materialized chunk_fts table", () => {
    const db = new Database(":memory:");
    applyMigrations(db, { ftsTokenize: buildFtsTokenize({ stemmer: "porter" }) });
    expect(ftsSql(db, "chunk_fts")).toContain("tokenize='porter unicode61 remove_diacritics 2'");
    db.close();
  });

  test("the CJK trigram path stays trigram regardless of the tokenizer config", () => {
    const db = new Database(":memory:");
    applyMigrations(db, { ftsTokenize: buildFtsTokenize({ stemmer: "porter", diacritics: "0" }) });
    expect(ftsSql(db, "chunk_trigram")).toContain("tokenize='trigram'");
    expect(ftsSql(db, "chunk_trigram")).not.toContain("porter");
    db.close();
  });
});