Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.6 KB
Raw Blame
/**
 * Score stability over the committed benchmark corpus
 * (a-label-is-not-a-boundary, U4).
 *
 * U4 moved four absolute thresholds off the result score and onto
 * IDF-weighted coverage, and wired the tier producer that `tierByDoc`
 * never had. Both claims include "no search score moves", and a claim of
 * byte-identity needs an instrument that can see a moved byte.
 * `recall-benchmark.test.ts` cannot: it pins hit@5 / MRR / containment
 * floors, and every score in the corpus could shift while all three stay
 * exactly where they are.
 *
 * So this pins the ordered `(path, score)` window itself. The corpus is
 * committed, the embedding provider is the deterministic local one, and
 * the ranker resolves its clock from the request, so the list is
 * reproducible run to run.
 *
 * The corpus stamps no `tier:` on any note, which is the condition under
 * which the tier producer must be inert: `supporting` is the identity
 * weight, so an untagged vault emits no tier map at all. A tier layer that
 * multiplied by one instead of being absent would move nothing here
 * either - `tier-producer.test.ts` is what separates those two.
 *
 * Re-pinning protocol, same as the benchmark gate's: a number that moves
 * is a finding first. Re-measure only after deciding the move is intended,
 * and say in the commit message which change moved it. Never re-pin to
 * make a red suite green.
 */

import { test, expect, beforeAll, afterAll, describe } from "bun:test";
import { cpSync, mkdtempSync, readdirSync, readFileSync, rmSync, utimesSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { indexVault } from "../../../src/core/search/indexer.ts";
import { search } from "../../../src/core/search/search.ts";
import { parseRecallBenchmarkDataset } from "../../../src/core/search/benchmark.ts";
import { makeConfig } from "../../helpers/search-fixtures.ts";
import type { ResolvedSearchConfig } from "../../../src/core/search/types.ts";

const FIXTURE = join(import.meta.dir, "..", "..", "fixtures", "recall-benchmark");

/** The window each query is pinned over; the benchmark gate's own k. */
const WINDOW = 5;

/**
 * Decimals the comparison keeps.
 *
 * Four, not more, and the reason is a property of this corpus worth
 * stating: no note in it declares an `authored_at`, so the freshness
 * prior falls back to storage mtime - which is the instant the fixture
 * was copied into a temp vault. Two runs seconds apart therefore differ
 * in the sixth decimal on the rows freshness touches. Four decimals is
 * reproducible run to run (verified by measuring twice) and still an
 * order of magnitude finer than any ranking change this pin exists to
 * catch.
 */
const SCORE_PRECISION = 4;

/**
 * Measured 2026-08-16 on the committed fixture vault with the local
 * 256-dimension embedding provider. Keyed by the dataset's own query ids.
 *
 * The rows are NOT score-descending, and that is the shipped order, not a
 * transcription slip: MMR and the relevance rerank re-order a ranked
 * window by diversity, so position and score genuinely disagree. Pinning
 * the emitted order rather than a sorted copy is the point - a change that
 * reshuffled the window while preserving every score would otherwise pass.
 */
const PINNED: Readonly<Record<string, ReadonlyArray<readonly [string, number]>>> = Object.freeze({
  canary: [
    ["deploy-canary.md", 0.8716],
    ["link-source.md", 0.1343],
    ["deploy-bluegreen.md", 0.1353],
    ["style-guide.md", 0.1256],
    ["incident-2026-05.md", 0.0986],
  ],
  bluegreen: [
    ["deploy-bluegreen.md", 0.907],
    ["search-tuning.md", 0.1711],
    ["deploy-canary.md", 0.1487],
    ["incident-2026-05.md", 0.1471],
    ["db-backups.md", 0.1503],
  ],
  oauth: [
    ["auth-oauth.md", 0.9139],
    ["incident-2026-05.md", 0.1388],
    ["search-tuning.md", 0.1385],
    ["style-guide.md", 0.1154],
    ["deploy-canary.md", 0.1323],
  ],
  passwords: [
    ["auth-passwords.md", 0.9393],
    ["recipe-borscht.md", 0.1765],
    ["embeddings-pipeline.md", 0.1671],
    ["deploy-bluegreen.md", 0.1573],
    ["db-indexing.md", 0.1626],
  ],
  backups: [
    ["db-backups.md", 0.8881],
    ["style-guide.md", 0.1376],
    ["deploy-canary.md", 0.1543],
    ["notes-standup.md", 0.1294],
    ["project-alpha.md", 0.15],
  ],
  indexing: [
    ["db-indexing.md", 0.8823],
    ["project-alpha.md", 0.1683],
    ["db-backups.md", 0.1714],
    ["style-guide.md", 0.1434],
    ["embeddings-pipeline.md", 0.1423],
  ],
  incident: [
    ["incident-2026-05.md", 0.8523],
    ["deploy-bluegreen.md", 0.1556],
    ["style-guide.md", 0.1257],
    ["deploy-canary.md", 0.1433],
    ["link-source.md", 0.1096],
  ],
  "alias-hop": [
    ["link-source.md", 0.8768],
    ["project-alpha.md", 0.2478],
    ["auth-passwords.md", 0.166],
    ["notes-standup.md", 0.1761],
    ["project-alpha.md", 0.2065],
  ],
  embeddings: [
    ["embeddings-pipeline.md", 0.9405],
    ["style-guide.md", 0.1943],
    ["incident-2026-05.md", 0.1909],
    ["project-alpha.md", 0.1744],
    ["vacation-policy.md", 0.15],
  ],
  tuning: [
    ["search-tuning.md", 0.2675],
    ["project-alpha.md", 0.201],
    ["deploy-bluegreen.md", 0.1724],
    ["notes-standup.md", 0.1538],
    ["deploy-canary.md", 0.1565],
  ],
  standup: [
    ["notes-standup.md", 0.8499],
    ["db-indexing.md", 0.1761],
    ["incident-2026-05.md", 0.1626],
    ["project-alpha.md", 0.1705],
    ["style-guide.md", 0.136],
  ],
  style: [
    ["style-guide.md", 0.8835],
    ["incident-2026-05.md", 0.2101],
    ["auth-oauth.md", 0.1788],
    ["db-backups.md", 0.1871],
    ["vacation-policy.md", 0.1718],
  ],
});

let vault: string;
let config: ResolvedSearchConfig;

beforeAll(async () => {
  vault = mkdtempSync(join(tmpdir(), "o2b-score-pin-"));
  cpSync(join(FIXTURE, "vault"), vault, { recursive: true });
  // The recency layer reads mtime. On POSIX `cpSync` stamps the copies
  // "now"; on Windows it keeps the checkout's timestamps (CopyFile carries
  // them over), which ages every note and shifts every pinned score. Stamp
  // them explicitly so the corpus is equally fresh on every host.
  const now = new Date();
  for (const entry of readdirSync(vault, { recursive: true, withFileTypes: true })) {
    if (entry.isFile()) utimesSync(join(entry.parentPath, entry.name), now, now);
  }
  config = makeConfig({
    vault,
    dbPath: join(vault, "index.sqlite"),
    semantic: { enabled: true, provider: "local", dimension: 256 },
  });
  await indexVault(config, { embeddings: true });
});

afterAll(() => {
  rmSync(vault, { recursive: true, force: true });
});

function loadDataset() {
  return parseRecallBenchmarkDataset(
    JSON.parse(readFileSync(join(FIXTURE, "dataset.json"), "utf8")),
  );
}

describe("the ranked window over the committed corpus", () => {
  test("every dataset query is pinned, so a new query cannot slip in unpinned", () => {
    const ids = loadDataset()
      .queries.map((q) => q.id)
      .toSorted();
    expect(ids).toEqual(Object.keys(PINNED).toSorted());
  });

  test("the ordered (path, score) window is unchanged", async () => {
    const dataset = loadDataset();
    const actual: Record<string, Array<readonly [string, number]>> = {};
    for (const query of dataset.queries) {
      // eslint-disable-next-line no-await-in-loop -- one indexed corpus, sequential by design
      const outcome = await search(config, { query: query.query, limit: WINDOW });
      actual[query.id] = outcome.results.map(
        (r) => [r.path, Number(r.score.toFixed(SCORE_PRECISION))] as const,
      );
    }
    // One comparison over the whole corpus: a per-query loop of assertions
    // stops at the first mismatch and hides how wide a ranking change was.
    expect(actual).toEqual(PINNED as unknown as typeof actual);
  });
});