Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
16.5 KB
Raw Blame
/**
 * Sargable path-prefix predicates (recall-quality, task C1).
 *
 * Every prefix predicate the store ran was `substr(d.path, 1, length(?)) = ?`,
 * which cannot use `idx_documents_path`: SQLite must evaluate the expression
 * per row. The range form `d.path >= ?1 AND d.path < ?2` is exact for the
 * same set of rows - `documents.path` is vault-relative POSIX + NFC under
 * BINARY collation, so strings compare as UTF-8 bytes - and the b-tree
 * answers it as a range scan.
 *
 * The bound (recall-quality review): `?2` is the prefix with its LAST code
 * point incremented by one, so every string starting with the prefix holds
 * the old code point where the bound holds the incremented one and sorts
 * strictly below it, while the bound itself does not start with the prefix
 * (tight). A `prefix || U+FFFF` bound is WRONG under byte ordering: an
 * astral continuation (emoji filenames, U+1F300+) encodes with a leading
 * byte above U+FFFF's 0xEF and sorts ABOVE the bound, dropping rows the
 * substr form matched. When the last code point is U+10FFFF and cannot
 * increment, the bound is null and the read runs lower-bound-only.
 *
 * This file pins three things:
 *
 *   1. the four prefixed reads (keyword lane, semantic lane, the prefix
 *      document census, the pending-vector census) return exactly the rows
 *      the historical `substr` form returns on the same fixture store -
 *      including astral-plane continuations and the no-increment fallback;
 *   2. the caller prefix passes through `canonicalNotePath`, so a win32
 *      backslash prefix now matches the stored POSIX paths it names;
 *   3. `assertSafePathPrefix` refusals are unchanged - the range form
 *      never widens what a caller may ask for.
 */

import { Database } from "bun:sqlite";
import { afterEach, beforeEach, describe, expect, test } from "bun:test";

import { sqliteVecLoadable } from "../../helpers/sqlite-vec.ts";
import { canonicalNotePath } from "../../../src/core/path-safety.ts";
import { applyMigrations, ensureVecTable } from "../../../src/core/search/schema.ts";
import { assertSafePathPrefix } from "../../../src/core/search/pipeline/request.ts";
import { countDocumentsUnderPrefix } from "../../../src/core/search/store/documents.ts";
import { pathPrefixRange } from "../../../src/core/search/store/path-prefix.ts";
import { keywordTopK, keywordTopKSql } from "../../../src/core/search/store/keyword.ts";
import { trigramCandidates } from "../../../src/core/search/store/trigram.ts";
import {
  countChunksWithoutEmbeddings,
  findChunksWithoutEmbeddings,
} from "../../../src/core/search/store/chunks.ts";
import { semanticTopK, vecUpsert } from "../../../src/core/search/store/vectors.ts";
import { SearchError } from "../../../src/core/search/types.ts";

const VEC_AVAILABLE = sqliteVecLoadable();

const NOW = "2026-10-11T00:00:00.000Z";

interface SeedDoc {
  readonly path: string;
  readonly chunkText: string;
}

const DOCS: ReadonlyArray<SeedDoc> = [
  { path: "notes/alpha.md", chunkText: "apple alpha rind" },
  { path: "notes/sub/beta.md", chunkText: "apple beta rind" },
  { path: "notes/alphabetic.md", chunkText: "apple alphabetic rind" },
  { path: "other/gamma.md", chunkText: "apple gamma rind" },
  // Astral-plane continuations (recall-quality review): an emoji directly
  // after the prefix sorts above any `prefix || U+FFFF` bound byte-wise,
  // and an emoji after an emoji does too. The U+10FFFF sentinel is the
  // no-increment case: its prefix has no incremented bound at all.
  { path: "notes/\u{1F680}-launch.md", chunkText: "apple launch rind" },
  { path: "notes/\u{1F680}\u{1F680}.md", chunkText: "apple twin rind" },
  { path: "notes/launch\u{10FFFF}.md", chunkText: "apple sentinel rind" },
];

let db: Database;

beforeEach(() => {
  db = new Database(":memory:");
  applyMigrations(db);
  if (VEC_AVAILABLE) {
    // eslint-disable-next-line @typescript-eslint/no-require-imports
    const vec = require("sqlite-vec") as { getLoadablePath(): string };
    db.loadExtension(vec.getLoadablePath());
    ensureVecTable(db, 4);
  }
  let chunkId = 0;
  for (const doc of DOCS) {
    const docId = db
      .query<{ id: number }, [string, string, number, number, string, string, string]>(
        "INSERT INTO documents(path, content_hash, mtime, size, created_at, updated_at, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING id",
      )
      .get(doc.path, `hash-${doc.path}`, 0, 0, NOW, NOW, NOW)!.id;
    const row = db
      .query<{ id: number }, [number, string, string, string, string, string]>(
        "INSERT INTO chunks(document_id, chunk_index, content, fts_content, content_hash, start_line, end_line, token_count, created_at, updated_at) " +
          "VALUES (?, 0, ?, ?, ?, 1, 1, 3, ?, ?) RETURNING id",
      )
      .get(docId, doc.chunkText, doc.chunkText, `chash-${doc.path}`, NOW, NOW)!;
    if (VEC_AVAILABLE && doc.path !== "notes/sub/beta.md") {
      // beta stays pending, so the scoped pending-vector census below has
      // a nonzero population to scope.
      vecUpsert(
        db,
        true,
        row.id,
        unitVector(0.1 * (chunkId + 1)),
        "test-model",
        4,
        `ehash-${chunkId}`,
      );
    }
    chunkId++;
  }
});

afterEach(() => {
  db.close();
});

/** The keyword hit paths under the historical `substr` predicate, in lane order. */
function keywordPathsUnderSubstr(prefix: string): ReadonlyArray<string> {
  const rows = db
    .query<{ path: string }, [string, string, string, number]>(
      "SELECT d.path AS path FROM chunk_fts " +
        "JOIN chunks c ON c.id = chunk_fts.rowid " +
        "JOIN documents d ON d.id = c.document_id " +
        "WHERE chunk_fts MATCH ? AND substr(d.path, 1, length(?)) = ? " +
        "ORDER BY bm25(chunk_fts, 1.0, 0.3) ASC, c.id ASC LIMIT ?",
    )
    .all('"apple"', prefix, prefix, 50);
  return rows.map((r) => r.path);
}

function keywordPaths(prefix: string | null): ReadonlyArray<string> {
  return keywordTopK(db, '"apple"', { limit: 50, pathPrefix: prefix }).map((h) => {
    const path = db
      .query<{ path: string }, [number]>("SELECT path FROM documents WHERE id = ?")
      .get(h.documentId);
    return path!.path;
  });
}

describe("keyword lane prefix pushdown", () => {
  test("range form returns exactly the substr form's rows", () => {
    for (const prefix of ["notes", "notes/", "notes/sub", "notes/alpha.md", "other", "zzz"]) {
      expect(keywordPaths(prefix)).toEqual(keywordPathsUnderSubstr(prefix));
    }
  });

  test("a prefix equal to a full path matches that document only", () => {
    expect(keywordPaths("notes/alpha.md")).toEqual(["notes/alpha.md"]);
  });

  test("an emoji continuation is not dropped (differential vs substr)", () => {
    // The defect: `notes/` + U+FFFF sorts BELOW `notes/<emoji>` under
    // BINARY (UTF-8 byte) collation, so the launch note vanished while the
    // substr form matched it. Byte-identical row sets, and the row is there.
    for (const prefix of ["notes/", "notes/\u{1F680}", "notes/\u{1F680}-launch.md"]) {
      expect(keywordPaths(prefix)).toEqual(keywordPathsUnderSubstr(prefix));
    }
    expect(keywordPaths("notes/")).toContain("notes/\u{1F680}-launch.md");
  });

  test("an astral last-char prefix with an astral continuation (differential vs substr)", () => {
    // Prefix `notes/<emoji>`: the increment bound moves the emoji itself
    // (U+1F680 -> U+1F681), so the twin-emoji note - which the U+FFFF form
    // dropped - stays strictly below the bound.
    for (const prefix of ["notes/\u{1F680}", "notes/\u{1F680}\u{1F680}"]) {
      expect(keywordPaths(prefix)).toEqual(keywordPathsUnderSubstr(prefix));
    }
    expect(keywordPaths("notes/\u{1F680}")).toContain("notes/\u{1F680}\u{1F680}.md");
  });

  test("a U+10FFFF-final prefix bounds through the last incrementable code point", () => {
    // U+10FFFF cannot increment, but nothing sorts above it either, so the
    // bound steps back to the last incrementable code point and stays exact
    // AND tight - byte-identical to the substr form, not a wide scan.
    const prefix = "notes/launch\u{10FFFF}";
    expect(pathPrefixRange(prefix)).toEqual({ lo: prefix, hi: "notes/launci" });
    expect(keywordTopKSql({ pathPrefix: prefix })).toContain("d.path < ?");
    expect(keywordPaths(prefix)).toEqual(keywordPathsUnderSubstr(prefix));
    expect(keywordPaths(prefix)).toEqual(["notes/launch\u{10FFFF}.md"]);
  });

  test("an all-U+10FFFF prefix has no bound and runs lower-bound-only", () => {
    const prefix = "\u{10FFFF}\u{10FFFF}";
    expect(pathPrefixRange(prefix).hi).toBeNull();
    expect(keywordTopKSql({ pathPrefix: prefix })).not.toContain("d.path < ?");
    expect(keywordTopKSql({ pathPrefix: prefix })).toContain("d.path >= ?");
  });

  test("a win32 backslash prefix is canonicalized and matches", () => {
    expect(keywordPaths("notes\\sub")).toEqual(["notes/sub/beta.md"]);
  });

  test("absent prefix scans the whole vault as before", () => {
    expect(keywordPaths(null).length).toBe(DOCS.length);
  });
});

describe("trigram lane prefix pushdown", () => {
  test("an all-U+10FFFF prefix runs the lane lower-bound-only", () => {
    // Seed a document under the boundless prefix directly: its row is only
    // reachable through the lower-bound scan the null bound demands.
    const docId = db
      .query<{ id: number }, [string, string, number, number, string, string, string]>(
        "INSERT INTO documents(path, content_hash, mtime, size, created_at, updated_at, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING id",
      )
      .get("\u{10FFFF}\u{10FFFF}sentinel.md", "hash-sentinel", 0, 0, NOW, NOW, NOW)!.id;
    db.query(
      "INSERT INTO chunks(document_id, chunk_index, content, fts_content, content_hash, start_line, end_line, token_count, created_at, updated_at) " +
        "VALUES (?, 0, 'apple sentinel rind', 'apple sentinel rind', 'chash-sentinel', 1, 1, 3, ?, ?)",
    ).run(docId, NOW, NOW);

    expect(pathPrefixRange("\u{10FFFF}\u{10FFFF}").hi).toBeNull();
    const outcome = trigramCandidates(db, '"apple"', {
      limit: 50,
      pathPrefix: "\u{10FFFF}\u{10FFFF}",
    });
    // The defect: the lane bound `d.path < ?` unconditionally, so the
    // boundless prefix's null bound went into the scan as NULL and the
    // read answered an empty candidate list.
    expect(outcome.warnings).toEqual([]);
    expect(outcome.hits.length).toBe(1);
    expect(outcome.hits[0]!.documentId).toBe(docId);
  });

  test("a bounded prefix still binds both ends of the trigram range", () => {
    const seeded = keywordPaths("notes");
    expect(seeded.length).toBeGreaterThan(0);
    const outcome = trigramCandidates(db, '"apple"', { limit: 50, pathPrefix: "notes" });
    expect(outcome.warnings).toEqual([]);
    expect(outcome.hits.length).toBe(seeded.length);
  });
});

describe("prefix document census", () => {
  test("countDocumentsUnderPrefix agrees with the substr form", () => {
    const substrCount = (prefix: string): number =>
      db
        .query<{ c: number }, [string, string]>(
          "SELECT count(*) AS c FROM documents WHERE substr(path, 1, length(?)) = ?",
        )
        .get(prefix, prefix)!.c;
    for (const prefix of ["notes", "notes/sub", "other", "zzz"]) {
      expect(countDocumentsUnderPrefix(db, prefix)).toBe(substrCount(prefix));
    }
  });

  test("astral prefixes count the same rows the substr form counts", () => {
    const substrCount = (prefix: string): number =>
      db
        .query<{ c: number }, [string, string]>(
          "SELECT count(*) AS c FROM documents WHERE substr(path, 1, length(?)) = ?",
        )
        .get(prefix, prefix)!.c;
    for (const prefix of [
      "notes/",
      "notes/\u{1F680}",
      "notes/\u{1F680}\u{1F680}.md",
      "notes/launch\u{10FFFF}",
    ]) {
      expect(countDocumentsUnderPrefix(db, prefix)).toBe(substrCount(prefix));
    }
    expect(countDocumentsUnderPrefix(db, "notes/\u{1F680}")).toBe(2);
  });

  test("a win32 backslash prefix is canonicalized and matches", () => {
    expect(countDocumentsUnderPrefix(db, "notes\\sub")).toBe(1);
  });
});

describe("pending-vector census prefix scope", () => {
  test("the scoped anti-join agrees with the substr form", () => {
    const substrPending = (prefix: string): number =>
      db
        .query<{ n: number }, [string, string]>(
          "SELECT COUNT(*) AS n FROM chunks c JOIN documents d ON d.id = c.document_id " +
            "LEFT JOIN embeddings e ON e.chunk_id = c.id " +
            "WHERE e.chunk_id IS NULL AND substr(d.path, 1, length(?)) = ?",
        )
        .get(prefix, prefix)!.n;
    for (const prefix of ["notes", "notes/sub", "other", "notes/\u{1F680}"]) {
      expect(countChunksWithoutEmbeddings(db, { pathPrefixes: [prefix] })).toBe(
        substrPending(prefix),
      );
      expect(findChunksWithoutEmbeddings(db, { pathPrefixes: [prefix] }).length).toBe(
        substrPending(prefix),
      );
    }
  });

  test("a win32 backslash prefix is canonicalized and matches", () => {
    expect(countChunksWithoutEmbeddings(db, { pathPrefixes: ["notes\\sub"] })).toBe(1);
  });

  test("an empty scope stays vault-wide", () => {
    // With sqlite-vec loaded every chunk but beta carries an embedding, so
    // exactly one is pending; without it, every chunk is.
    expect(countChunksWithoutEmbeddings(db)).toBe(VEC_AVAILABLE ? 1 : DOCS.length);
    expect(countChunksWithoutEmbeddings(db, { pathPrefixes: [] })).toBe(
      VEC_AVAILABLE ? 1 : DOCS.length,
    );
  });
});

describe.skipIf(!VEC_AVAILABLE)("semantic lane prefix pushdown", () => {
  test("range form returns exactly the substr form's rows", () => {
    const vector = unitVector(0.5);
    const substrPaths = semanticPathsUnderSubstr(vector, "notes");
    expect(substrPaths.length).toBeGreaterThan(0);
    expect(semanticPaths(vector, "notes")).toEqual(substrPaths);
    // beta carries no embedding (it keeps the pending census populated),
    // so the win32 case targets the alph* pair instead.
    expect(semanticPaths(vector, "notes\\alph")).toEqual(["notes/alphabetic.md", "notes/alpha.md"]);
  });

  test("an emoji continuation survives the prefixed KNN (differential vs substr)", () => {
    const vector = unitVector(0.5);
    expect(semanticPaths(vector, "notes/")).toEqual(semanticPathsUnderSubstr(vector, "notes/"));
    expect(semanticPaths(vector, "notes/")).toContain("notes/\u{1F680}-launch.md");
  });
});

function semanticPaths(vector: Float32Array, prefix: string): ReadonlyArray<string> {
  return semanticTopK(db, true, vector, { limit: 10, pathPrefix: prefix }).map((h) => {
    const path = db
      .query<{ path: string }, [number]>("SELECT path FROM documents WHERE id = ?")
      .get(h.documentId);
    return path!.path;
  });
}

function semanticPathsUnderSubstr(vector: Float32Array, prefix: string): ReadonlyArray<string> {
  const rows = db
    .query<{ path: string }, [Uint8Array, string, string]>(
      "SELECT d.path AS path FROM chunk_vec v " +
        "JOIN chunk_vec_map m ON m.vec_rowid = v.rowid " +
        "JOIN chunks c ON c.id = m.chunk_id " +
        "JOIN documents d ON d.id = c.document_id " +
        "WHERE v.embedding MATCH ? AND k = 40 AND substr(d.path, 1, length(?)) = ? " +
        "ORDER BY v.distance ASC, m.chunk_id ASC",
    )
    .all(float32Blob(vector), prefix, prefix);
  return rows.map((r) => r.path);
}

describe("assertSafePathPrefix refusals are unchanged", () => {
  test("vault escapes are still refused verbatim", () => {
    for (const prefix of ["../escape", "notes/../..", "/absolute", "C:\\notes", "c:notes"]) {
      expect(() => assertSafePathPrefix(prefix)).toThrow(SearchError);
      expect(() => assertSafePathPrefix(prefix)).toThrow(
        new RegExp(
          `escapes vault: ${JSON.stringify(prefix).replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`,
        ),
      );
    }
  });

  test("a safe prefix passes through untouched", () => {
    expect(assertSafePathPrefix("notes/sub")).toBe("notes/sub");
    expect(assertSafePathPrefix(undefined)).toBeUndefined();
  });
});

function unitVector(first: number): Float32Array {
  const v = new Float32Array(4).fill(0.1);
  v[0] = first;
  return v;
}

function float32Blob(values: Float32Array): Uint8Array {
  return new Uint8Array(
    values.buffer.slice(values.byteOffset, values.byteOffset + values.byteLength),
  );
}

// The NFC half of the canonicalization this feature relies on, pinned here
// so a change to `canonicalNotePath` cannot silently unmatch the cases
// above. The separator half is exercised through the store functions, whose
// prefix bound normalizes the caller's win32 separator convention before
// the canonical form.
test("canonicalNotePath normalizes NFC", () => {
  expect(canonicalNotePath("caf\u0065\u0301/notes")).toBe("caf\u00e9/notes");
});