Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.5 KB
Raw Blame
/**
 * Graph-efficacy holdout harness (G1, t_6832aac6).
 *
 * A graph-neighbor holdout is an (anchor, target) pair where the anchor is a
 * note assumed to be directly recalled and the target is a note reached by
 * following graph edges. The harness measures GRAPH LIFT (targets reachable
 * only through the graph, not as a direct neighbor of the anchor) separately
 * from DIRECT RECALL (targets that are already a direct neighbor), so the
 * value the graph edges add is visible on its own.
 *
 * Every target must resolve to durable memory (the note exists) and hydrate
 * into bounded evidence (a non-empty body, capped at a named constant). A
 * dangling edge - a target that resolves to nothing - fails the gate, because
 * a graph that points at absent memory cannot lift recall.
 *
 * Reads only link structure and note bodies; no natural-language word list.
 */

import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";

import { ensureInsideVault } from "../../path-safety.ts";
import { EXCLUDED_DIRS, extractWikilinks, listVaultPages, parseFrontmatter } from "../../vault.ts";
import { ENTITY_STATUS_SCOPE, vaultPageInStatusScope } from "../entities/page-scope.ts";
import { canonicalCoOccurrenceKey } from "./co-occurrence.ts";
import { MAINTENANCE_LANE_REACH } from "../../graph/transport-reach.ts";

/** Upper bound on the evidence hydrated from a target note. */
export const HOLDOUT_EVIDENCE_MAX_CHARS = 2000;

/** One graph-neighbor holdout: an anchor and the target reached from it. */
export interface GraphHoldout {
  /** Vault-relative path of the directly-recalled anchor note. */
  readonly anchor: string;
  /** Vault-relative path of the graph-neighbor target under test. */
  readonly target: string;
}

export interface HoldoutResolution {
  readonly holdout: GraphHoldout;
  /** The target resolves to a durable-memory note. */
  readonly resolved: boolean;
  /** The target hydrates into non-empty bounded evidence. */
  readonly hydrated: boolean;
  /** Length of the hydrated evidence, capped at {@link HOLDOUT_EVIDENCE_MAX_CHARS}. */
  readonly evidenceChars: number;
  /** The edge points at absent memory (not resolved). */
  readonly dangling: boolean;
  /** The target is a direct (1-hop) neighbor of the anchor. */
  readonly directNeighbor: boolean;
  /** The target is reachable only through the graph (2-hop, not 1-hop). */
  readonly graphReachable: boolean;
}

export interface HoldoutGateResult {
  /** True only when every target both resolves and hydrates into bounded evidence. */
  readonly passed: boolean;
  readonly total: number;
  readonly resolvedCount: number;
  /** Targets that resolve to nothing (absent memory). */
  readonly danglingCount: number;
  /** Targets that resolve to durable memory but hydrate into empty evidence. */
  readonly unhydratedCount: number;
  /** Holdouts whose target is a direct neighbor of the anchor. */
  readonly directRecall: number;
  /** Holdouts whose target is reachable only through the graph. */
  readonly graphLift: number;
  readonly resolutions: readonly HoldoutResolution[];
}

/**
 * Canonical-key adjacency over the vault's resolved wikilink structure.
 *
 * The walk covers `Brain/`, so entity pages are in it. A quarantined
 * record's links were authored by the untrusted source that put it aside,
 * and admitting them would let that source shape the graph this lane
 * measures recall over.
 */
function buildKeyAdjacency(vault: string): Map<string, Set<string>> {
  const adjacency = new Map<string, Set<string>>();
  for (const page of listVaultPages(vault, {
    skipDirs: [...EXCLUDED_DIRS],
    reach: MAINTENANCE_LANE_REACH,
  })) {
    if (!vaultPageInStatusScope(page.metadata, ENTITY_STATUS_SCOPE.readable)) continue;
    const sourceKey = canonicalCoOccurrenceKey(page.path);
    if (sourceKey === null) continue;
    let body = "";
    try {
      [, body] = parseFrontmatter(page.path);
    } catch {
      body = "";
    }
    const neighbours = adjacency.get(sourceKey) ?? new Set<string>();
    for (const raw of extractWikilinks(body)) {
      const targetKey = canonicalCoOccurrenceKey(raw);
      if (targetKey !== null && targetKey !== sourceKey) neighbours.add(targetKey);
    }
    adjacency.set(sourceKey, neighbours);
  }
  return adjacency;
}

/** Two-hop neighbour keys of `key`, excluding the key itself and its 1-hop set. */
function twoHopKeys(adjacency: ReadonlyMap<string, ReadonlySet<string>>, key: string): Set<string> {
  const direct = adjacency.get(key) ?? new Set<string>();
  const twoHop = new Set<string>();
  for (const mid of direct) {
    for (const far of adjacency.get(mid) ?? new Set<string>()) {
      if (far !== key && !direct.has(far)) twoHop.add(far);
    }
  }
  return twoHop;
}

/** Read a target note's body as bounded evidence. Empty when unresolved. */
function hydrateEvidence(vault: string, target: string): { resolved: boolean; text: string } {
  let abs: string;
  try {
    abs = ensureInsideVault(join(vault, target), vault);
  } catch {
    // A target that escapes the vault resolves to no durable memory: treat it
    // as unresolved (dangling) so the gate fails gracefully per item instead
    // of throwing and aborting the whole evaluation.
    return { resolved: false, text: "" };
  }
  if (!existsSync(abs)) return { resolved: false, text: "" };
  try {
    const [, body] = parseFrontmatter(abs);
    return { resolved: true, text: body.trim().slice(0, HOLDOUT_EVIDENCE_MAX_CHARS) };
  } catch {
    try {
      const raw = readFileSync(abs, "utf8");
      return { resolved: true, text: raw.trim().slice(0, HOLDOUT_EVIDENCE_MAX_CHARS) };
    } catch {
      return { resolved: false, text: "" };
    }
  }
}

/**
 * Evaluate graph-neighbor holdouts, reporting graph lift separately from
 * direct recall and failing the gate on any target that does not both resolve
 * to durable memory and hydrate into bounded evidence.
 */
export function evaluateGraphHoldouts(
  vault: string,
  holdouts: readonly GraphHoldout[],
): HoldoutGateResult {
  const adjacency = buildKeyAdjacency(vault);
  const resolutions: HoldoutResolution[] = [];
  let resolvedCount = 0;
  let danglingCount = 0;
  let unhydratedCount = 0;
  let directRecall = 0;
  let graphLift = 0;

  for (const holdout of holdouts) {
    const anchorKey = canonicalCoOccurrenceKey(holdout.anchor);
    const targetKey = canonicalCoOccurrenceKey(holdout.target);
    const direct =
      anchorKey !== null ? (adjacency.get(anchorKey) ?? new Set<string>()) : new Set<string>();
    const twoHop = anchorKey !== null ? twoHopKeys(adjacency, anchorKey) : new Set<string>();

    const directNeighbor = targetKey !== null && direct.has(targetKey);
    const graphReachable = targetKey !== null && !directNeighbor && twoHop.has(targetKey);

    const evidence = hydrateEvidence(vault, holdout.target);
    const resolved = evidence.resolved;
    const hydrated = resolved && evidence.text.length > 0;
    const dangling = !resolved;

    if (resolved) resolvedCount += 1;
    if (dangling) danglingCount += 1;
    if (resolved && !hydrated) unhydratedCount += 1;
    if (directNeighbor) directRecall += 1;
    if (graphReachable) graphLift += 1;

    resolutions.push({
      holdout,
      resolved,
      hydrated,
      evidenceChars: evidence.text.length,
      dangling,
      directNeighbor,
      graphReachable,
    });
  }

  return {
    passed: danglingCount === 0 && unhydratedCount === 0,
    total: holdouts.length,
    resolvedCount,
    danglingCount,
    unhydratedCount,
    directRecall,
    graphLift,
    resolutions: Object.freeze(resolutions),
  };
}