Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
20.7 KB
Raw Blame
/**
 * `o2b brain bench memory` (Memory Observability Suite, t_882c396a):
 * memory quality benchmark over a disposable fixture vault.
 *
 * The harness NEVER touches the operator's configured vault: it
 * materializes the fixture into `<runs-dir>/<run-id>/vault` and runs
 * the public search/context-pack APIs there. Deterministic and
 * network-free by default; `bench_judge_cmd` arms the optional
 * external judge. Quality, latency, and context cost stay separate
 * report families (the MemScore lesson).
 */

import { existsSync } from "node:fs";
import { join, resolve } from "node:path";

import { loadBenchFixture } from "../../../core/bench/fixture.ts";
import { BenchRefusalError } from "../../../core/bench/guards.ts";
import { LONGMEMEVAL_SUITE_NAME, loadLongMemEvalFixture } from "../../../core/bench/longmemeval.ts";
import { loadLocomoFixture } from "../../../core/bench/locomo.ts";
import { loadReplayBenchFixture } from "../../../core/bench/replay.ts";
import { runMemoryBench } from "../../../core/bench/phases.ts";
import type {
  BenchPinnedKnobs,
  BenchReport,
  BenchSampleRequest,
  BenchSuiteLoad,
  BenchSuiteRegistration,
} from "../../../core/bench/types.ts";
import { defaultConfigPath, resolveBenchJudgeCmd } from "../../../core/config.ts";
import { BENCH_RUNS_REL } from "../../../core/brain/path-constants.ts";
import { CliError, fail, parse } from "../helpers.ts";

const USAGE =
  "usage: o2b brain bench memory --fixture <name|path> [--suite <name>] [--pin <json>] " +
  "[--recall-floor <n>] [--sample-seed <value>] [--sample-cap <n>] " +
  "[--resume <run-id>] [--runs-dir <dir>] [--json]";

// Named in `path-constants.ts` as a member of the derived store, not as
// a path of its own: the runs dir is gitignored only once brain init has
// seeded the vault's .gitignore guard, whose single vault-root-anchored
// `/.open-second-brain/` entry covers the whole store, bench-runs
// included - until that seeding, nothing here is ignored.
const DEFAULT_RUNS_DIR = BENCH_RUNS_REL;
const REPO_FIXTURE_DIR = join("tests", "fixtures", "bench");

/**
 * The overlapped-lane human line (recall-lane-latency), printed only when
 * the aggregate latency family carries the figure - a keyword-only run
 * keeps its stdout byte-identical.
 */
const OVERLAPPED_LINE_PREFIX = "overlapped_ms";

/**
 * The standing-instruction human line (recall-lane-latency), printed only
 * when the report's quality buckets carry the category - a run without
 * standing_instruction questions keeps its stdout byte-identical. The
 * headline passed/total/pass_rate excludes the category (buildReport), so
 * the category rates stay this named line of their own, never a fold into
 * the overall pass rate.
 */
const STANDING_INSTRUCTION_LINE_PREFIX = "standing_instruction";

/** A suite name must be safe to echo into errors, reports and run ids. */
const SUITE_NAME_PATTERN = /^[a-z0-9][a-z0-9_-]*$/;

/**
 * The run's all-pass gate. The headline excludes the standing_instruction
 * category (its rate is the named line of its own, never a fold into the
 * overall pass rate), so the gate reads the category's own bucket beside
 * the headline: a failing facet question still exits nonzero, while a
 * passing one never shifts the headline rate a baseline compares.
 */
function runAllPass(report: BenchReport): boolean {
  const standing = report.quality.by_category["standing_instruction"];
  return (
    report.quality.passed === report.quality.total &&
    (standing === undefined || standing.passed === standing.total)
  );
}

/**
 * The path with no `--suite`: a native fixture loads directly. Not a
 * named suite - it is the byte-identical default the registry must
 * preserve.
 */
const DEFAULT_SUITE_REGISTRATION: BenchSuiteRegistration = Object.freeze({
  loader: (fixturePath: string): BenchSuiteLoad => ({ fixture: loadBenchFixture(fixturePath) }),
  capabilities: Object.freeze({ embeddings: false }),
});

/**
 * Suite registry: maps a `--suite` name to its loader and declared
 * capabilities. This map is the appendable registration point - suite
 * modules add their own entries here (or through
 * {@link registerBenchSuite}), never a new branch in the verb body.
 */
const BENCH_SUITES = new Map<string, BenchSuiteRegistration>([
  [
    "locomo",
    Object.freeze({
      loader: (fixturePath: string): BenchSuiteLoad => ({
        fixture: loadLocomoFixture(fixturePath),
      }),
      capabilities: Object.freeze({ embeddings: false }),
    }),
  ],
  [
    LONGMEMEVAL_SUITE_NAME,
    Object.freeze({
      loader: loadLongMemEvalFixture,
      capabilities: Object.freeze({ embeddings: true, sampling: true }),
    }),
  ],
  [
    "surfacing-replay",
    Object.freeze({
      loader: (fixturePath: string): BenchSuiteLoad => ({
        fixture: loadReplayBenchFixture(fixturePath),
      }),
      capabilities: Object.freeze({ embeddings: false }),
    }),
  ],
]);

/** Register a bench suite. Refuses an invalid or already-taken name. */
export function registerBenchSuite(name: string, registration: BenchSuiteRegistration): void {
  if (!SUITE_NAME_PATTERN.test(name)) {
    throw new Error(`bench suite name is invalid: '${name}'`);
  }
  if (BENCH_SUITES.has(name)) {
    throw new Error(`bench suite already registered: ${name}`);
  }
  BENCH_SUITES.set(name, registration);
}

/** Look up a registered suite by name. */
export function benchSuite(name: string): BenchSuiteRegistration | undefined {
  return BENCH_SUITES.get(name);
}

/** Registered suite names, sorted for stable listings. */
export function benchSuiteNames(): ReadonlyArray<string> {
  return [...BENCH_SUITES.keys()].toSorted();
}

/**
 * The knob names a `--pin` may set. Listed by hand against
 * {@link BenchPinnedKnobs} - the `satisfies` bounds make a renamed or
 * removed member a compile error - because TypeScript types have no
 * runtime keys to reflect on. A NEW member compiles silently until it is
 * listed here; a pin that names it is then refused, which is the loud
 * maintenance path, not a silent one.
 */
const BENCH_PIN_KNOBS = [
  "chunkSize",
  "chunkOverlap",
  "chunkMinSize",
  "keywordWeight",
  "semanticWeight",
  "fusionMode",
  "rrfK",
  "recall",
  "shutdownGraceMs",
  "hybridDeadlineMs",
  "resumeReindex",
  "ftsTokenize",
  "freshen",
  "scopeRules",
  "semantic",
  "rerank",
  // The standing-instruction facet lane's arm (recall-lane-latency):
  // listing it is what makes the lane-off arm reachable from the CLI
  // (`--pin '{"standingInstructionFacets":false}'`), not only the API.
  "standingInstructionFacets",
] as const satisfies ReadonlyArray<keyof BenchPinnedKnobs>;

/**
 * The knobs the harness itself owns: the run's vault identity. A pin may
 * never set them - pointing the bench at the operator's configured vault
 * is the one thing the harness exists to never do.
 */
const BENCH_PIN_HARNESS_KEYS = ["configPath", "dbPath", "vault"] as const satisfies ReadonlyArray<
  keyof BenchPinnedKnobs
>;

/** Member names of the pin's nested knob families, same listing rule. */
const BENCH_PIN_NESTED_KEYS: Readonly<Record<string, ReadonlyArray<string>>> = {
  semantic: [
    "enabled",
    "provider",
    "baseUrl",
    "model",
    "apiKey",
    "apiKeys",
    "allowInsecureHttp",
    "dimension",
    "timeoutMs",
    "concurrency",
    "batchSize",
    "batchTokens",
    "inputWindowTokens",
    "maxRetries",
    "costGateUsd",
    "priceOverride",
    "queryPrefix",
    "passagePrefix",
    "extraBody",
  ] as const satisfies ReadonlyArray<keyof NonNullable<BenchPinnedKnobs["semantic"]>>,
  rerank: [
    "enabled",
    "kind",
    "baseUrl",
    "model",
    "envKey",
    "apiKey",
    "allowInsecureHttp",
    "topK",
    "minScore",
    "relationalRerankPin",
    "decisionModel",
  ] as const satisfies ReadonlyArray<keyof NonNullable<BenchPinnedKnobs["rerank"]>>,
  recall: [
    "mmrLambda",
    "maxHops",
    "hopDecay",
    "maxExpansionPerHit",
    "recencyShape",
    "recencyScale",
    "recencyAmplitude",
    "intentEnabled",
    "synonymEnabled",
    "synonymMaxTerms",
    "cacheEnabled",
    "cacheTtlSeconds",
    "relationPolarityEnabled",
    "relationalArmEnabled",
    "searchTrustGateMode",
    "supersedeFadeEnabled",
    "learnedWeightsEnabled",
    "activationEnabled",
    "twoPassEnabled",
    "iterativeRecallEnabled",
    "iterativeRecallMaxRounds",
    "poolMultiplier",
    "poolMaxPerSession",
    "selfTuningEnabled",
    "chainStopEnabled",
    "chainStopScore",
    "trigramPrefilterEnabled",
    "trigramPrefilterMinChunks",
    "trigramPrefilterMaxSelectivity",
    "metadataBoostGateEnabled",
  ] as const satisfies ReadonlyArray<keyof NonNullable<BenchPinnedKnobs["recall"]>>,
  freshen: ["intervalSeconds", "embeddings", "configPath"] as const satisfies ReadonlyArray<
    keyof NonNullable<BenchPinnedKnobs["freshen"]>
  >,
  scopeRules: ["ignore", "include"] as const satisfies ReadonlyArray<
    keyof NonNullable<BenchPinnedKnobs["scopeRules"]>
  >,
};

/**
 * Parse the `--pin` value into a {@link BenchPinnedKnobs} object.
 *
 * Garbage is refused by name: non-JSON, a non-object, a knob the harness
 * owns, an unknown knob, and a nested family that is not an object or
 * names an unknown member. VALUE semantics stay with
 * `resolveSearchConfig`, which applies the pin last and reports its own
 * named errors for a bad provider name or an out-of-range number - this
 * validation refuses shape garbage the resolver would silently ignore,
 * because an ignored override would pin nothing and report a run under a
 * configuration it did not run.
 *
 * An explicit pin REPLACES the default keyword-only pin
 * (`DEFAULT_BENCH_KNOB_PIN`) wholesale: `--pin '{"keywordWeight":0.7}'`
 * alone leaves the semantic lane env-driven, and the run then fails the
 * embeddings-capability gate loudly unless the suite declared it or the
 * pin also carries `"semantic":{"enabled":false}`.
 */
export function parseBenchPinnedKnobs(raw: string): BenchPinnedKnobs {
  let parsed: unknown;
  try {
    parsed = JSON.parse(raw);
  } catch (e) {
    throw new CliError(
      `brain bench memory: --pin is not valid JSON: ${e instanceof Error ? e.message : String(e)}`,
    );
  }
  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
    throw new CliError("brain bench memory: --pin must be a JSON object of search knobs");
  }
  const pin = parsed as Record<string, unknown>;
  for (const key of Object.keys(pin)) {
    if ((BENCH_PIN_HARNESS_KEYS as ReadonlyArray<string>).includes(key)) {
      throw new CliError(
        `brain bench memory: --pin may not set '${key}' - the harness owns the run's ${key}`,
      );
    }
    if (!(BENCH_PIN_KNOBS as ReadonlyArray<string>).includes(key)) {
      throw new CliError(
        `brain bench memory: --pin has an unknown knob '${key}' (known knobs: ${BENCH_PIN_KNOBS.join(", ")})`,
      );
    }
    const nestedKeys = BENCH_PIN_NESTED_KEYS[key];
    const value = pin[key];
    if (nestedKeys === undefined) continue;
    if (value === null || typeof value !== "object" || Array.isArray(value)) {
      throw new CliError(`brain bench memory: --pin '${key}' must be a JSON object`);
    }
    for (const nested of Object.keys(value as Record<string, unknown>)) {
      if (!(nestedKeys as ReadonlyArray<string>).includes(nested)) {
        throw new CliError(
          `brain bench memory: --pin '${key}.${nested}' is not a known knob (known: ${nestedKeys.join(", ")})`,
        );
      }
    }
  }
  return pin as BenchPinnedKnobs;
}

/**
 * Parse the `--recall-floor` value: absent keeps the shipped admission
 * floor; present it must be a positive number, and it becomes part of the
 * run identity (a resumed run whose floor changed fails the guard).
 */
export function parseRecallFloorFlag(raw: string | undefined): number | undefined {
  if (raw === undefined) return undefined;
  const value = Number(raw);
  if (raw.trim() === "" || !Number.isFinite(value) || value <= 0) {
    throw new CliError(
      `brain bench memory: --recall-floor must be a positive number, got '${raw}'`,
    );
  }
  return value;
}

/**
 * Parse the `--sample-seed` value: absent keeps the whole-dataset load;
 * present it must be a non-empty string and pairs with `--sample-cap`.
 * The seed is opaque - the sampler hashes it, so any string works and
 * none is ever interpolated into a run id. The value is passed through
 * UNTRIMMED on purpose: the sampler trims the seed in one place before
 * hashing, so the CLI and the API path hash the same value for the same
 * input. This surface only refuses a blank value by name, before any
 * run state is created.
 */
export function parseSampleSeedFlag(raw: string | undefined): string | undefined {
  if (raw === undefined) return undefined;
  if (raw.trim() === "") {
    throw new CliError(
      `brain bench memory: --sample-seed must be a non-empty string, got '${raw}'`,
    );
  }
  return raw;
}

/**
 * Parse the `--sample-cap` value: absent keeps the whole-dataset load;
 * present it must be a positive integer - the bound on selected
 * questions - and pairs with `--sample-seed`.
 */
export function parseSampleCapFlag(raw: string | undefined): number | undefined {
  if (raw === undefined) return undefined;
  const value = Number(raw);
  if (raw.trim() === "" || !Number.isInteger(value) || value <= 0) {
    throw new CliError(`brain bench memory: --sample-cap must be a positive integer, got '${raw}'`);
  }
  return value;
}

export async function cmdBrainBench(argv: string[]): Promise<number> {
  if (argv[0] !== "memory") return fail(USAGE);
  const { flags } = parse(argv.slice(1), {
    fixture: { type: "string" },
    suite: { type: "string" },
    pin: { type: "string" },
    "recall-floor": { type: "string" },
    "sample-seed": { type: "string" },
    "sample-cap": { type: "string" },
    resume: { type: "string" },
    "runs-dir": { type: "string" },
    json: { type: "boolean" },
  });
  const fixtureFlag = typeof flags["fixture"] === "string" ? flags["fixture"].trim() : "";
  if (fixtureFlag === "") return fail(USAGE);
  const suite = typeof flags["suite"] === "string" ? flags["suite"].trim() : "";
  let registration = DEFAULT_SUITE_REGISTRATION;
  if (suite !== "") {
    const known = BENCH_SUITES.get(suite);
    if (known === undefined) {
      return fail(
        `brain bench memory: unknown --suite '${suite}' (supported: ${benchSuiteNames().join(", ")})`,
      );
    }
    registration = known;
  }
  const fixturePath = resolveFixturePath(fixtureFlag);
  if (fixturePath === null) {
    return fail(
      `brain bench memory: fixture not found: ${fixtureFlag} (tried the literal path and ${REPO_FIXTURE_DIR}/${fixtureFlag}.json)`,
    );
  }

  // Sampling flags are LOADER options: validated (pair, values,
  // capability) before the registry dispatch, so a refused request never
  // reaches a loader and a sampling-capable loader receives
  // `BenchSuiteLoadOptions { sample }`.
  const sampleSeed = parseSampleSeedFlag(
    typeof flags["sample-seed"] === "string" ? flags["sample-seed"] : undefined,
  );
  const sampleCap = parseSampleCapFlag(
    typeof flags["sample-cap"] === "string" ? flags["sample-cap"] : undefined,
  );
  if ((sampleSeed === undefined) !== (sampleCap === undefined)) {
    const loneFlag = sampleSeed !== undefined ? "--sample-seed" : "--sample-cap";
    return fail(
      "brain bench memory: --sample-seed and --sample-cap must be given together " +
        `(sampling takes a seed and a cap; got only ${loneFlag})`,
    );
  }
  const sample: BenchSampleRequest | undefined =
    sampleSeed !== undefined && sampleCap !== undefined
      ? { seed: sampleSeed, cap: sampleCap }
      : undefined;
  if (sample !== undefined && registration.capabilities.sampling !== true) {
    return fail(
      "brain bench memory: --sample-seed/--sample-cap need a suite that declares the " +
        "sampling capability; " +
        (suite === "" ? "no --suite was given" : `suite '${suite}' does not declare it`),
    );
  }

  // One dispatch for every suite: the registry entry owns loading and
  // declares what the suite may do; the capabilities and the loader's
  // conversion receipt ride BenchRunOptions.
  const loaded = registration.loader(fixturePath, sample === undefined ? {} : { sample });
  const fixture = loaded.fixture;
  const runsDir = resolve(
    typeof flags["runs-dir"] === "string" && flags["runs-dir"].trim() !== ""
      ? flags["runs-dir"].trim()
      : DEFAULT_RUNS_DIR,
  );
  const resume = typeof flags["resume"] === "string" ? flags["resume"].trim() : undefined;
  const judgeCmd = resolveBenchJudgeCmd(defaultConfigPath());
  // Refusals throw CliError with the offending value named; the
  // dispatcher prints them and exits non-zero.
  const pinFlag = typeof flags["pin"] === "string" ? flags["pin"] : undefined;
  const pinnedKnobs: BenchPinnedKnobs | undefined =
    pinFlag === undefined ? undefined : parseBenchPinnedKnobs(pinFlag);
  const recallFloor = parseRecallFloorFlag(
    typeof flags["recall-floor"] === "string" ? flags["recall-floor"] : undefined,
  );

  // Refusals render the guard contract on stderr and exit nonzero: the
  // BenchRefusalError message IS the render - `refused (<kind>): <detail>`
  // - and every detail names the repair (--resume, a fresh run, a
  // writable --runs-dir). Any other error keeps propagating to the
  // dispatcher untouched.
  let report: BenchReport;
  try {
    report = await runMemoryBench({
      fixture,
      runsDir,
      ...(suite !== "" ? { suite } : {}),
      capabilities: registration.capabilities,
      ...(pinnedKnobs !== null ? { pinnedKnobs } : {}),
      ...(recallFloor !== undefined ? { recallFloor } : {}),
      ...(sample !== undefined ? { sample } : {}),
      ...(loaded.receipt !== undefined ? { suiteReceipt: loaded.receipt } : {}),
      ...(resume !== undefined && resume !== "" ? { resume } : {}),
      ...(judgeCmd !== undefined ? { judgeCmd } : {}),
    });
  } catch (e) {
    if (e instanceof BenchRefusalError) {
      return fail(`brain bench memory: ${e.message}`);
    }
    throw e;
  }

  if (flags["json"] === true) {
    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
    return runAllPass(report) ? 0 : 1;
  }
  process.stdout.write(`bench ${report.fixture} run ${report.run_id}\n`);
  process.stdout.write(
    `quality: ${report.quality.passed}/${report.quality.total} (pass rate ${report.quality.pass_rate})\n`,
  );
  process.stdout.write(`latency_ms: avg ${report.latency_ms.avg} max ${report.latency_ms.max}\n`);
  if (report.latency_ms.overlapped_ms !== undefined) {
    process.stdout.write(`${OVERLAPPED_LINE_PREFIX}: avg ${report.latency_ms.overlapped_ms}\n`);
  }
  process.stdout.write(
    `context_cost: avg_chars ${report.context_cost.avg_chars} avg_injected_tokens ${report.context_cost.avg_injected_tokens}\n`,
  );
  process.stdout.write(
    `failure_modes: know_to_ask ${report.failure_modes.know_to_ask_failure_rate} ` +
      `false_fire ${report.failure_modes.false_fire_rate} ` +
      `isolation_violations ${report.failure_modes.source_isolation_violations}\n`,
  );
  if (report.longmemeval !== undefined) {
    process.stdout.write(
      `longmemeval: recall_all@5 ${report.longmemeval.recall_all_at_5} ` +
        `abstention_excluded ${report.longmemeval.abstention_excluded}` +
        // Present only on a sampled run, so the whole-dataset line is
        // byte-identical to before. The population rides beside the
        // selected count, so the line shows the share of the dataset
        // the run covers (both fields are written together by the one
        // spread site that emits the sampled family).
        (report.longmemeval.sample_selected !== undefined
          ? ` sample_population ${report.longmemeval.sample_population} sample_selected ${report.longmemeval.sample_selected}`
          : "") +
        "\n",
    );
  }
  if (report.quality.by_category["standing_instruction"] !== undefined) {
    const standing = report.quality.by_category["standing_instruction"];
    process.stdout.write(
      `${STANDING_INSTRUCTION_LINE_PREFIX}: ${standing.passed}/${standing.total} ` +
        `(pass rate ${round3(standing.passed / standing.total)})\n`,
    );
  }
  process.stdout.write(`judge: ${report.judge.status}\n`);
  for (const question of report.questions) {
    process.stdout.write(
      `  ${question.pass ? "PASS" : "FAIL"}  ${question.id} (${question.category})${question.failure ? ` - ${question.failure}` : ""}\n`,
    );
  }
  return runAllPass(report) ? 0 : 1;
}

function resolveFixturePath(flag: string): string | null {
  if (existsSync(flag)) return flag;
  const repoLocal = join(REPO_FIXTURE_DIR, `${flag}.json`);
  if (existsSync(repoLocal)) return repoLocal;
  return null;
}

/**
 * Round to three decimals, the same figure the core bench reports carry
 * (`round3` in core/bench/phases.ts); the human standing_instruction line
 * prints the category rate at the same precision as the overall pass rate.
 */
function round3(value: number): number {
  return Math.round(value * 1000) / 1000;
}