Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.0 KB
Raw Blame
/**
 * Retrieval feedback loop (recall-trust-suite, Feature B).
 *
 * Explicit per-result recall feedback lands as one JSON file per event
 * under `Brain/search/feedback/` (the conflict-free inbox pattern), and
 * a deterministic bounded fold derives per-layer learned multipliers
 * into `Brain/search/learned-weights.json`. Search applies them only
 * when the config opt-in is on, tags affected results with a
 * `learned_weights:` reason, and folds the weights state into the
 * query-cache key.
 */

import { join } from "node:path";
import { test, expect, beforeEach, afterEach, describe } from "bun:test";
import { existsSync, mkdirSync, readdirSync } from "node:fs";

import {
  computeLearnedWeights,
  contributionsFromResult,
  feedbackDir,
  learnedWeightsPath,
  learnedWeightsFingerprint,
  loadFeedbackEvents,
  readLearnedWeights,
  recordRecallFeedback,
  resetLearnedWeights,
  LEARNED_WEIGHT_MIN,
  LEARNED_WEIGHT_MAX,
  type RecallFeedbackEvent,
  writeLearnedWeights,
} from "../../../src/core/search/feedback.ts";
import { indexVault } from "../../../src/core/search/indexer.ts";
import { search } from "../../../src/core/search/search.ts";
import type { BrainSearchResult } from "../../../src/core/search/types.ts";
import { createTempVault, makeConfig, writeMd } from "../../helpers/search-fixtures.ts";

let vault: string;
let dbPath: string;
let cleanup: () => void;

beforeEach(() => {
  ({ vault, dbPath, cleanup } = createTempVault("feedback"));
  // A legacy Brain/ tree: an unswitched machine (a vault without one is store-only).
  mkdirSync(join(vault, "Brain"), { recursive: true });
});
afterEach(() => cleanup());

function event(over: Partial<RecallFeedbackEvent>): RecallFeedbackEvent {
  return {
    ts: 1_700_000_000_000,
    queryHash: "abcd1234",
    resultPath: "note.md",
    verdict: "up",
    contributions: { keyword: 0.8, semantic: 0, entity: 0, recency: 0.05 },
    ...over,
  };
}

describe("feedback event store", () => {
  test("recordRecallFeedback writes one JSON file per event and round-trips", () => {
    const a = event({ ts: 1_700_000_000_000 });
    const b = event({ ts: 1_700_000_000_001, verdict: "down" });
    recordRecallFeedback(vault, a);
    recordRecallFeedback(vault, b);
    const files = readdirSync(feedbackDir(vault));
    expect(files).toHaveLength(2);
    expect(files.every((f) => f.endsWith(".json"))).toBe(true);
    const events = loadFeedbackEvents(vault);
    expect(events).toHaveLength(2);
    expect(events[0]!.ts).toBe(a.ts);
    expect(events[1]!.verdict).toBe("down");
  });

  test("recording the same event twice is idempotent (deterministic filename)", () => {
    const a = event({});
    recordRecallFeedback(vault, a);
    recordRecallFeedback(vault, a);
    expect(readdirSync(feedbackDir(vault))).toHaveLength(1);
  });

  test("loadFeedbackEvents on a vault without feedback returns empty", () => {
    expect(loadFeedbackEvents(vault)).toEqual([]);
  });
});

describe("learned-weight fold", () => {
  test("no events → neutral multipliers", () => {
    const w = computeLearnedWeights([]);
    expect(w.keywordMul).toBe(1);
    expect(w.semanticMul).toBe(1);
    expect(w.entityMul).toBe(1);
    expect(w.recencyMul).toBe(1);
    expect(w.events).toBe(0);
  });

  test("up-votes on keyword-driven results raise keywordMul; down-votes lower it", () => {
    const ups = [
      event({ ts: 1, contributions: { keyword: 1, semantic: 0, entity: 0, recency: 0 } }),
      event({ ts: 2, contributions: { keyword: 1, semantic: 0, entity: 0, recency: 0 } }),
    ];
    const up = computeLearnedWeights(ups);
    expect(up.keywordMul).toBeGreaterThan(1);
    expect(up.semanticMul).toBe(1);

    const downs = ups.map((e, i) => ({ ...e, ts: 10 + i, verdict: "down" as const }));
    const down = computeLearnedWeights(downs);
    expect(down.keywordMul).toBeLessThan(1);
  });

  test("multipliers never escape the documented bounds", () => {
    const extreme = Array.from({ length: 50 }, (_, i) =>
      event({
        ts: i,
        verdict: "down",
        contributions: { keyword: 1, semantic: 1, entity: 1, recency: 1 },
      }),
    );
    const w = computeLearnedWeights(extreme);
    for (const mul of [w.keywordMul, w.semanticMul, w.entityMul, w.recencyMul]) {
      expect(mul).toBeGreaterThanOrEqual(LEARNED_WEIGHT_MIN);
      expect(mul).toBeLessThanOrEqual(LEARNED_WEIGHT_MAX);
    }
  });

  test("the fold is deterministic and order-insensitive for the same event set", () => {
    const events = [
      event({ ts: 1, contributions: { keyword: 0.9, semantic: 0.1, entity: 0, recency: 0 } }),
      event({
        ts: 2,
        verdict: "down",
        contributions: { keyword: 0.1, semantic: 0.8, entity: 0, recency: 0 },
      }),
      event({ ts: 3, contributions: { keyword: 0, semantic: 0, entity: 0.5, recency: 0.5 } }),
    ];
    const a = computeLearnedWeights(events);
    const b = computeLearnedWeights(events.toReversed());
    expect(a).toEqual(b);
  });

  test("contributionsFromResult reads the scoring layers off a search result", () => {
    const result = {
      documentId: 1,
      chunkId: 10,
      path: "x.md",
      title: null,
      content: "",
      startLine: 1,
      endLine: 1,
      score: 0.7,
      keywordScore: 0.6,
      semanticScore: 0.2,
      linkBoost: 0,
      recencyBoost: 0.05,
      searchType: "hybrid",
      reasons: ["fts5_bm25: 0.600", "semantic_cos: 0.200", "entity_match: 0.020"],
      breakdown: {
        keyword: 0.6,
        semantic: 0.2,
        rrf: 0,
        entity: 0.02,
        activation: 0,
        coAccess: 0,
        link: 0,
        recency: 0.05,
        tier: 1,
        trend: 1,
        sessionFocus: 0,
      },
    } as unknown as BrainSearchResult;
    const c = contributionsFromResult(result);
    expect(c.keyword).toBeCloseTo(0.6, 5);
    expect(c.semantic).toBeCloseTo(0.2, 5);
    expect(c.entity).toBeCloseTo(0.02, 5);
    expect(c.recency).toBeCloseTo(0.05, 5);
  });

  test("contributionsFromResult reports zero entity when no breakdown is present", () => {
    // A synthetic result (e.g. a traversal expansion) carries no breakdown;
    // the entity layer is honestly zero, not a parse artifact.
    const result = {
      documentId: 1,
      chunkId: 11,
      path: "y.md",
      title: null,
      content: "",
      startLine: 1,
      endLine: 1,
      score: 0.4,
      keywordScore: 0.4,
      semanticScore: 0,
      linkBoost: 0,
      recencyBoost: 0,
      searchType: "keyword",
      reasons: ["fts5_bm25: 0.400"],
    } as unknown as BrainSearchResult;
    const c = contributionsFromResult(result);
    expect(c.entity).toBe(0);
    expect(c.keyword).toBeCloseTo(0.4, 5);
  });
});

describe("derived weights file", () => {
  test("record + recompute persists learned weights; reset removes them but keeps events", () => {
    recordRecallFeedback(vault, event({}));
    const w = computeLearnedWeights(loadFeedbackEvents(vault));
    expect(existsSync(learnedWeightsPath(vault))).toBe(true); // record() recomputes
    expect(readLearnedWeights(vault)?.events).toBe(w.events);

    resetLearnedWeights(vault);
    expect(existsSync(learnedWeightsPath(vault))).toBe(false);
    expect(loadFeedbackEvents(vault)).toHaveLength(1);
  });

  test("a hand-edited multiplier outside the fold's own bounds is refused", () => {
    // The fold clamps every multiplier into [LEARNED_WEIGHT_MIN,
    // LEARNED_WEIGHT_MAX], so a value outside that range is one
    // computeLearnedWeights could never have produced. Under rank fusion a
    // zero deletes a lane and a negative inverts its rank order, so the
    // reader refuses the file rather than composing with it.
    const sound = computeLearnedWeights([event({})]);
    writeLearnedWeights(vault, sound);
    expect(readLearnedWeights(vault)).not.toBeNull();

    for (const bad of [0, -1, LEARNED_WEIGHT_MAX + 0.1, LEARNED_WEIGHT_MIN - 0.1]) {
      writeLearnedWeights(vault, { ...sound, keywordMul: bad });
      expect(readLearnedWeights(vault)).toBeNull();
    }

    // The bound is inclusive at both ends - the fold can emit exactly these.
    for (const edge of [LEARNED_WEIGHT_MIN, LEARNED_WEIGHT_MAX]) {
      writeLearnedWeights(vault, { ...sound, keywordMul: edge });
      expect(readLearnedWeights(vault)).not.toBeNull();
    }
  });

  test("fingerprint changes when weights change and is stable otherwise", () => {
    const before = learnedWeightsFingerprint(vault);
    expect(learnedWeightsFingerprint(vault)).toBe(before);
    recordRecallFeedback(vault, event({}));
    expect(learnedWeightsFingerprint(vault)).not.toBe(before);
  });
});

describe("search integration", () => {
  test("learned weights shift scores only when the config opt-in is on", async () => {
    writeMd(vault, "doc.md", "# Doc\n\nthe migration ledger covers warehouse sync");
    // Two keyword-down events: learned keywordMul < 1.
    recordRecallFeedback(
      vault,
      event({
        ts: 1,
        verdict: "down",
        contributions: { keyword: 1, semantic: 0, entity: 0, recency: 0 },
      }),
    );
    recordRecallFeedback(
      vault,
      event({
        ts: 2,
        verdict: "down",
        contributions: { keyword: 1, semantic: 0, entity: 0, recency: 0 },
      }),
    );

    const off = makeConfig({ vault, dbPath });
    const on = makeConfig({ vault, dbPath, learnedWeightsEnabled: true });
    await indexVault(off);

    const plain = await search(off, { query: "migration ledger", limit: 3 });
    const learned = await search(on, { query: "migration ledger", limit: 3 });
    expect(plain.results).toHaveLength(1);
    expect(learned.results).toHaveLength(1);
    expect(learned.results[0]!.score).toBeLessThan(plain.results[0]!.score);
    expect(learned.results[0]!.reasons.some((r) => r.startsWith("learned_weights:"))).toBe(true);
    expect(plain.results[0]!.reasons.some((r) => r.startsWith("learned_weights:"))).toBe(false);
  });

  test("neutral learned weights leave results untagged", async () => {
    writeMd(vault, "doc.md", "# Doc\n\nplain note about ledger sync");
    const on = makeConfig({ vault, dbPath, learnedWeightsEnabled: true });
    await indexVault(on);
    const out = await search(on, { query: "ledger sync", limit: 3 });
    expect(out.results[0]!.reasons.some((r) => r.startsWith("learned_weights:"))).toBe(false);
  });
});

describe("malformed event hardening", () => {
  test("an event with non-numeric contributions is skipped by the fold", () => {
    recordRecallFeedback(vault, event({ ts: 1 }));
    // Hand-write a malformed event file alongside the valid one.
    const dir = feedbackDir(vault);
    const malformed = JSON.stringify({
      ts: 2,
      queryHash: "ffff0000",
      resultPath: "x.md",
      verdict: "down",
      contributions: { keyword: "high", semantic: null },
    });
    require("node:fs").writeFileSync(`${dir}/2-malformed.json`, malformed);
    const events = loadFeedbackEvents(vault);
    expect(events).toHaveLength(1); // the malformed file never enters the fold
    const w = computeLearnedWeights(events);
    for (const mul of [w.keywordMul, w.semanticMul, w.entityMul, w.recencyMul]) {
      expect(Number.isFinite(mul)).toBe(true);
    }
  });
});