Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.1 KB
Raw Blame
/**
 * The `standing_instruction` bench category (recall-lane-latency):
 * the standing-instruction facet lane measured end to end over a
 * disposable fixture vault.
 *
 * The category's retrieve branch drives the shipped `standing_query`
 * attention action with the facet option on - the provenance join, not
 * the search pipeline, is the seam under test - and `evaluateQuestion`
 * asserts the preference id plus its origin turn. Reports without the
 * category serialize byte-identically, the default knob pin is
 * untouched, and the facet-lane pin marker is part of the run identity:
 * a resumed run whose pin changed fails the guard instead of mixing
 * identities.
 *
 * The fixture vault files below hand-write the same facts the live
 * writers emit, in a YAML form `parsePreference`/`parseSignal` and the
 * attention-flow reader `listAttentionFlows` read to identical values -
 * the bytes are not the writers' own output (the writers quote
 * `#`-bearing scalars and emit inline arrays where the fixture writes
 * unquoted scalars and block lists). The bench materializes its vault
 * from raw fixture notes, so the parsers are the seam under
 * measurement; the producer formats themselves are pinned at the unit
 * layer (tests/core/brain/standing-instruction-facets.test.ts)
 * against the live writers.
 */

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { parseBenchFixture } from "../../../src/core/bench/fixture.ts";
import { runMemoryBench } from "../../../src/core/bench/phases.ts";
import {
  DEFAULT_BENCH_KNOB_PIN,
  type BenchFixture,
  type BenchPinnedKnobs,
} from "../../../src/core/bench/types.ts";

let runsDir: string;

beforeEach(() => {
  runsDir = mkdtempSync(join(tmpdir(), "o2b-bench-standing-"));
});

afterEach(() => {
  rmSync(runsDir, { recursive: true, force: true });
});

/**
 * The vault files the surfacing lane reads, in the shapes the live
 * writers emit. The import-session signal carries the `<basename>#<turn>`
 * ref (kind `turn`); the auto-extract signal carries the bare
 * `session:<id>` ref that resolves to nothing.
 */
const FLOW_MD = [
  "---",
  "kind: brain-attention-flow",
  "id: open-loops",
  "title: Open loops",
  "actions:",
  "  - standing_query",
  "standing_queries:",
  "  - testing",
  "status: active",
  "---",
  "",
  "# Open loops",
  "",
].join("\n");

function preferenceMd(slug: string, topic: string, principle: string, evidence: string): string {
  return [
    "---",
    "kind: brain-preference",
    `id: pref-${slug}`,
    "created_at: 2026-06-01T00:00:00Z",
    "_confirmed_at: 2026-06-02T00:00:00Z",
    "unconfirmed_until: 2026-07-01T00:00:00Z",
    "tags:",
    "  - bench",
    `topic: ${topic}`,
    "_status: confirmed",
    `principle: ${principle}`,
    "_evidenced_by:",
    `  - "[[${evidence}]]"`,
    "_applied_count: 0",
    "_violated_count: 0",
    "_last_evidence_at: null",
    "_confidence: low",
    "_confidence_value: null",
    "pinned: false",
    "scope: testing",
    "---",
    "",
    principle,
    "",
  ].join("\n");
}

function signalMd(slug: string, topic: string, principle: string, sessionRef: string): string {
  return [
    "---",
    "kind: brain-signal",
    `id: ${slug}`,
    "created_at: 2026-06-01T00:00:00Z",
    "tags:",
    "  - bench",
    `topic: ${topic}`,
    "signal: positive",
    "agent: bench",
    `principle: ${principle}`,
    "scope: testing",
    `session_ref: ${sessionRef}`,
    "---",
    "",
    "recorded preference signal",
    "",
  ].join("\n");
}

/** Built per test, so a fixture parse refusal fails one test, not the file. */
function standingFixture(): BenchFixture {
  return parseBenchFixture({
    name: "standing-instruction-bench",
    notes: STANDING_NOTES,
    continuity: [],
    questions: [
      {
        id: "q-standing",
        category: "standing_instruction",
        // The facet key (the preference id) that must be delivered ...
        expected_ids: ["pref:pref-tabs"],
        // ... and the full annotation that must close that preference's
        // item: compared exactly, closing bracket included, so a
        // near-miss origin (turn-2 vs turn-20) and a session-annotated
        // origin both fail by name.
        expected_text: "earliest cited turn bench-session.json#turn-2]",
      },
      {
        // The auto-extract shape resolves to no origin, so its facet is
        // honestly absent and the expectation fails by name.
        id: "q-standing-unresolved",
        category: "standing_instruction",
        expected_ids: ["pref:pref-commits"],
      },
    ],
  });
}

/** The standing category's fixture vault files, shared with the mixed fixture. */
const STANDING_NOTES: BenchFixture["notes"] = [
  {
    path: "Brain/attention/flows/open-loops.md",
    body: FLOW_MD,
  },
  {
    path: "Brain/preferences/pref-tabs.md",
    body: preferenceMd("tabs", "tabs", "Use tabs for indentation", "sig-bench-turn"),
  },
  {
    path: "Brain/preferences/pref-commits.md",
    body: preferenceMd("commits", "commits", "Prefer small commits", "sig-bench-bare"),
  },
  {
    path: "Brain/inbox/sig-bench-turn.md",
    body: signalMd(
      "sig-bench-turn",
      "tabs",
      "use tabs for indentation",
      "bench-session.json#turn-2",
    ),
  },
  {
    path: "Brain/inbox/sig-bench-bare.md",
    body: signalMd("sig-bench-bare", "commits", "prefer small commits", "session:conv-9"),
  },
];

/**
 * A retrieval question alongside the facet category: the control that
 * proves the headline counts the retrieval question alone.
 */
function mixedFixture(): BenchFixture {
  return parseBenchFixture({
    name: "standing-headline-mixed",
    notes: [
      {
        path: "Brain/notes/coffee.md",
        body: "# Coffee\n\nThe coffee preference is a flat white every morning.\n",
      },
      ...STANDING_NOTES,
    ],
    continuity: [],
    questions: [
      {
        id: "q-coffee",
        category: "single_hop",
        query: "coffee preference",
        top_k: 5,
        expected_paths: ["Brain/notes/coffee.md"],
      },
      {
        id: "q-standing",
        category: "standing_instruction",
        // A deliberately wrong origin: the question fails, and the
        // headline must not move because of it.
        expected_ids: ["pref:pref-tabs"],
        expected_text: "earliest cited turn nowhere.json#turn-9]",
      },
      {
        id: "q-standing-unresolved",
        category: "standing_instruction",
        expected_ids: ["pref:pref-commits"],
      },
    ],
  });
}

/** A keyword-only fixture: no standing_instruction questions anywhere. */
function keywordFixture(): BenchFixture {
  return parseBenchFixture({
    name: "standing-keyword-control",
    notes: [
      {
        path: "Brain/notes/coffee.md",
        body: "# Coffee\n\nThe coffee preference is a flat white every morning.\n",
      },
    ],
    continuity: [],
    questions: [
      {
        id: "q-coffee",
        category: "single_hop",
        query: "coffee preference",
        top_k: 5,
        expected_paths: ["Brain/notes/coffee.md"],
      },
    ],
  });
}

describe("bench standing_instruction category", () => {
  test("reports without the new category serialize byte-identically", async () => {
    const report = await runMemoryBench({ fixture: keywordFixture(), runsDir });
    const serialized = JSON.stringify(report, null, 2);
    // No standing_instruction key anywhere: the category is absent from
    // the quality buckets, the questions and every other family, so a
    // pre-category consumer reads the same bytes.
    expect(serialized).not.toContain("standing_instruction");
    expect(Object.keys(report)).toEqual([
      "schema",
      "run_id",
      "fixture",
      "fixture_hash",
      "created_at",
      "quality",
      "latency_ms",
      "context_cost",
      "failure_modes",
      "judge",
      "questions",
    ]);
    expect(Object.keys(report.quality.by_category)).toEqual(["single_hop"]);
    expect(report.quality.passed).toBe(report.quality.total);
  }, 30_000);

  test("the standing_instruction question asserts the preference id plus origin turn", async () => {
    const report = await runMemoryBench({ fixture: standingFixture(), runsDir });
    expect(report.quality.by_category["standing_instruction"]).toEqual({ passed: 1, total: 2 });
    const passing = report.questions.find((question) => question.id === "q-standing")!;
    expect(passing.category).toBe("standing_instruction");
    expect(passing.pass).toBe(true);
    // A standing result adds no metric fields: its shape is the shared
    // id/category/latency/pass core, nothing else.
    expect(Object.keys(passing).toSorted()).toEqual(["category", "id", "latency_ms", "pass"]);
  }, 30_000);

  test("an unresolved preference fails its question by name", async () => {
    const report = await runMemoryBench({ fixture: standingFixture(), runsDir });
    const unresolved = report.questions.find(
      (question) => question.id === "q-standing-unresolved",
    )!;
    expect(unresolved.pass).toBe(false);
    // The bare auto-extract ref resolves to no origin, so the facet is
    // absent - a named failure, never an implied provenance.
    expect(unresolved.failure).toContain("pref:pref-commits");
  }, 30_000);

  test("a near-miss origin never satisfies the expectation, closing bracket included", async () => {
    // The surfaced annotation ends ...#t10]; the expectation names
    // ...#t1 - a substring the old includes() check accepted as a false
    // positive. Tied to the expected preference's item and compared
    // exactly against the annotation with its closing bracket, it fails
    // by name instead.
    const nearMiss = parseBenchFixture({
      name: "standing-near-miss",
      notes: [
        ...STANDING_NOTES.filter((note) => note.path !== "Brain/inbox/sig-bench-turn.md"),
        {
          path: "Brain/inbox/sig-bench-turn.md",
          body: signalMd(
            "sig-bench-turn",
            "tabs",
            "use tabs for indentation",
            "bench-session.json#t10",
          ),
        },
      ],
      continuity: [],
      questions: [
        {
          id: "q-standing",
          category: "standing_instruction",
          expected_ids: ["pref:pref-tabs"],
          expected_text: "earliest cited turn bench-session.json#t1",
        },
      ],
    });
    const report = await runMemoryBench({ fixture: nearMiss, runsDir });
    const failing = report.questions.find((question) => question.id === "q-standing")!;
    expect(failing.pass).toBe(false);
    expect(failing.failure).toContain("earliest cited turn bench-session.json#t1");
  }, 30_000);

  test("the default knob pin serializes byte-for-byte unchanged", () => {
    expect(JSON.stringify(DEFAULT_BENCH_KNOB_PIN)).toBe('{"semantic":{"enabled":false}}');
  });

  test("standing_instruction questions never shift the headline pass rate", async () => {
    const report = await runMemoryBench({ fixture: mixedFixture(), runsDir });
    // The retrieval question alone is the headline: both facet questions
    // fail, yet passed/total/pass_rate count only q-coffee. Adding facet
    // questions to a fixture cannot move the rate a baseline compares.
    expect(report.quality.passed).toBe(1);
    expect(report.quality.total).toBe(1);
    expect(report.quality.pass_rate).toBe(1);
    // The category keeps its failures where they belong: its own named
    // bucket, which the human line prints.
    expect(report.quality.by_category["single_hop"]).toEqual({ passed: 1, total: 1 });
    expect(report.quality.by_category["standing_instruction"]).toEqual({ passed: 0, total: 2 });
    const failing = report.questions.find((question) => question.id === "q-standing")!;
    expect(failing.pass).toBe(false);
  }, 30_000);

  test("a pin of false runs the lane off and the facet expectations fail by name", async () => {
    const laneOffPin: BenchPinnedKnobs = {
      semantic: { enabled: false },
      standingInstructionFacets: false,
    };
    const report = await runMemoryBench({
      fixture: standingFixture(),
      runsDir,
      pinnedKnobs: laneOffPin,
    });
    // The plain lane emitted only the legacy strings, so both facet-key
    // expectations fail, naming the missing facet keys.
    expect(report.quality.by_category["standing_instruction"]).toEqual({ passed: 0, total: 2 });
    const failing = report.questions.find((question) => question.id === "q-standing")!;
    expect(failing.pass).toBe(false);
    expect(failing.failure).toContain("pref:pref-tabs");
    // The persisted raw record proves the off arm ran: plain legacy
    // strings, no annotations, and no facet ids at all.
    const raw = JSON.parse(
      readFileSync(join(runsDir, report.run_id, "results", "q-standing.json"), "utf8"),
    ) as { standing_items?: string[]; facet_ids?: string[] };
    expect(raw.standing_items).toEqual(["pref-commits (testing)", "pref-tabs (testing)"]);
    expect(raw.facet_ids).toEqual([]);
  }, 30_000);

  test("the facet-lane pin marker is part of the run identity", async () => {
    const laneOnPin: BenchPinnedKnobs = {
      semantic: { enabled: false },
      standingInstructionFacets: true,
    };
    const first = await runMemoryBench({
      fixture: standingFixture(),
      runsDir,
      pinnedKnobs: laneOnPin,
    });
    // The marker rides the checkpoint verbatim ...
    const checkpoint = JSON.parse(
      readFileSync(join(runsDir, first.run_id, "checkpoint.json"), "utf8"),
    ) as { pinned_knobs?: Record<string, unknown> };
    expect(checkpoint.pinned_knobs).toEqual({
      semantic: { enabled: false },
      standingInstructionFacets: true,
    });
    // ... so a resumed run under the plain pin fails the identity guard ...
    await expect(
      runMemoryBench({ fixture: standingFixture(), runsDir, resume: first.run_id }),
    ).rejects.toThrow(/run identity mismatch/);
    // ... and resuming under the same marker reuses the run.
    const resumed = await runMemoryBench({
      fixture: standingFixture(),
      runsDir,
      resume: first.run_id,
      pinnedKnobs: laneOnPin,
    });
    expect(resumed.run_id).toBe(first.run_id);
    expect(resumed.quality).toEqual(first.quality);
  }, 30_000);
});