Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
16.1 KB
Raw Blame
/**
 * Payload registry, end to end (t_35440e83): session import externalizes
 * oversized content before the continuity row is written, the exact
 * stored bytes page back, redaction and private regions never reach a
 * payload file, the inventory finds orphans and missing refs, the gc
 * removes only what nothing references behind a recovery point that
 * holds the store, and the doctor reports all three conditions.
 */

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  readdirSync,
  readFileSync,
  rmSync,
  statSync,
  utimesSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { DEFAULT_BRAIN_CONFIG_YAML } from "../../../src/core/brain/config-template.ts";
import { appendContinuityRecord } from "../../../src/core/brain/continuity/store.ts";
import { runDoctor } from "../../../src/core/brain/doctor.ts";
import { brainDirs, payloadPath, payloadsDir } from "../../../src/core/brain/paths.ts";
import {
  buildPayloadInventory,
  collectPayloadGarbage,
} from "../../../src/core/brain/payload-inventory.ts";
import {
  PAYLOAD_GC_GRACE_MS,
  PayloadNotFoundError,
  PayloadRefError,
  PayloadRegistry,
} from "../../../src/core/brain/payload-registry.ts";
import {
  importSessionRecall,
  searchSessionRecall,
} from "../../../src/core/brain/session-recall.ts";
import { exportBankBundle } from "../../../src/core/brain/portability/bundle.ts";
import { buildOkfBundle } from "../../../src/core/brain/portability/okf.ts";
import { importSession } from "../../../src/core/brain/sessions/import.ts";
import { admitToIndex } from "../../../src/core/vault-scope/index-admission.ts";
import { listSnapshotArchive } from "../../helpers/snapshot-archive.ts";

let vault: string;

/** 2,000 chars of the base64 alphabet: well past the 512-char default. */
const BLOB = "QUJD".repeat(500);

beforeEach(() => {
  vault = mkdtempSync(join(tmpdir(), "o2b-payload-registry-"));
  const dirs = brainDirs(vault);
  for (const dir of [dirs.brain, dirs.inbox, dirs.processed, dirs.preferences, dirs.log]) {
    mkdirSync(dir, { recursive: true });
  }
  writeFileSync(join(dirs.brain, "_brain.yaml"), DEFAULT_BRAIN_CONFIG_YAML);
});

afterEach(() => {
  rmSync(vault, { recursive: true, force: true });
});

/** Every continuity shard, concatenated as it sits on disk. */
function continuityText(): string {
  const dir = join(vault, "Brain", "log", "continuity");
  return readdirSync(dir)
    .filter((name) => name.endsWith(".jsonl"))
    .map((name) => readFileSync(join(dir, name), "utf8"))
    .join("");
}

function continuityRows(): Array<Record<string, unknown>> {
  return continuityText()
    .split("\n")
    .filter((line) => line.trim() !== "")
    .map((line) => JSON.parse(line) as Record<string, unknown>);
}

function refsIn(text: string): string[] {
  return [...text.matchAll(/osb-payload:\/\/[a-f0-9]{64}/g)].map((m) => m[0]);
}

/** Read a payload back page by page and join the pages. */
function readAll(ref: string, pageChars: number): string {
  const registry = new PayloadRegistry({ vault, maxInlineChars: 1 });
  let out = "";
  let offset: number | null = 0;
  while (offset !== null) {
    const page = registry.get(ref, { offset, limit: pageChars });
    out += page.content;
    offset = page.nextOffset;
  }
  return out;
}

function withSessionConfig(block: string): void {
  writeFileSync(
    join(vault, "Brain", "_brain.yaml"),
    `${DEFAULT_BRAIN_CONFIG_YAML}\nsessions:\n${block}`,
  );
}

describe("session import externalizes before the row is written", () => {
  test("a long base64 run becomes a placeholder in continuity and in recall search", async () => {
    const transcript = join(vault, "transcript.jsonl");
    writeFileSync(
      transcript,
      `${JSON.stringify({
        parentUuid: null,
        isSidechain: false,
        type: "user",
        uuid: "turn-1",
        timestamp: "2026-09-20T10:00:00.000Z",
        sessionId: "s-1",
        message: { role: "user", content: `screenshot follows ${BLOB} end of paste` },
      })}\n`,
    );

    const result = await importSession(vault, transcript, {
      agent: "test",
      recall: true,
      recallSessionId: "payload-session",
      format: "claude",
    });
    expect(result.recall_turns_imported).toBe(1);

    const ledger = continuityText();
    expect(ledger).not.toContain(BLOB);
    expect(ledger).toContain("[payload: osb-payload://");

    const hits = searchSessionRecall(vault, { query: "screenshot follows" }).hits;
    expect(hits.length).toBeGreaterThan(0);
    for (const hit of hits) expect(hit.snippet).not.toContain(BLOB.slice(0, 200));
    expect(searchSessionRecall(vault, { query: BLOB.slice(0, 120) }).hits).toHaveLength(0);

    const [ref] = refsIn(ledger);
    expect(readAll(ref!, 333)).toBe(BLOB);
  });

  test("an oversized plain tool output keeps a bounded preview and pages back exactly", () => {
    withSessionConfig("  payload_max_text_chars: 4000\n");
    const lines = Array.from({ length: 2000 }, (_, i) => `build step ${i} ok`);
    const output = `${lines.join("\n")}\n${BLOB}\n`;

    importSessionRecall(vault, {
      sessionId: "tool-session",
      createdAt: "2026-09-20T10:00:00.000Z",
      turns: [{ turnId: "t1", role: "tool", timestamp: "2026-09-20T10:00:01.000Z", text: output }],
    });

    const row = continuityRows().find((r) => r["kind"] === "session_turn")!;
    const text = (row["payload"] as Record<string, unknown>)["text"] as string;
    expect(text.length).toBeLessThan(1_200);
    expect(text.startsWith("build step 0 ok")).toBe(true);
    const refs = (row["payload"] as Record<string, unknown>)["payload_refs"] as string[];
    // The blob moved first, then the remaining text as a whole.
    expect(refs).toHaveLength(2);
    const whole = readAll(refs[1]!, 1_000);
    expect(whole).toContain("build step 1999 ok");
    expect(whole).not.toContain(BLOB);
    expect(whole).toContain(refs[0]!);
    expect(readAll(refs[0]!, 500)).toBe(BLOB);
    // The blob is live through the whole-text payload that names it.
    expect(buildPayloadInventory(vault).orphans).toHaveLength(0);
  });

  test("an ordinary turn is stored byte-identically, with no payload", () => {
    importSessionRecall(vault, {
      sessionId: "plain",
      createdAt: "2026-09-20T10:00:00.000Z",
      turns: [{ turnId: "t1", role: "user", timestamp: "2026-09-20T10:00:01.000Z", text: "hi" }],
    });
    const row = continuityRows().find((r) => r["kind"] === "session_turn")!;
    expect((row["payload"] as Record<string, unknown>)["text"]).toBe("hi");
    expect((row["payload"] as Record<string, unknown>)["payload_refs"]).toBeUndefined();
    expect(buildPayloadInventory(vault).stored).toHaveLength(0);
  });

  test("secrets are redacted and private regions stripped before a payload file is written", () => {
    withSessionConfig("  payload_max_text_chars: 1000\n");
    const text =
      `${"plain words ".repeat(100)}\npassword=hunter2secret\n` +
      "<private>my home address</private>\ntrailing";

    importSessionRecall(vault, {
      sessionId: "secret-session",
      createdAt: "2026-09-20T10:00:00.000Z",
      turns: [{ turnId: "t1", role: "tool", timestamp: "2026-09-20T10:00:01.000Z", text }],
    });

    const stored = readdirSync(payloadsDir(vault)).map((name) =>
      readFileSync(join(payloadsDir(vault), name), "utf8"),
    );
    expect(stored).toHaveLength(1);
    expect(stored[0]).not.toContain("hunter2secret");
    expect(stored[0]).not.toContain("my home address");
    expect(stored[0]).toContain("***REDACTED***");
    const row = continuityRows().find((r) => r["kind"] === "session_turn")!;
    expect(row["private"]).toBe(true);
  });
});

describe("refs are validated before any path is built", () => {
  test("traversal and malformed refs are refused", () => {
    const registry = new PayloadRegistry({ vault, maxInlineChars: 1 });
    for (const ref of [
      "osb-payload://../../etc/passwd",
      `osb-payload://${"A".repeat(64)}`,
      `osb-payload://${"a".repeat(63)}`,
      `file:///${"a".repeat(64)}`,
    ]) {
      expect(() => registry.get(ref, { offset: 0, limit: 10 })).toThrow(PayloadRefError);
    }
    expect(() => payloadPath(vault, "../escape")).toThrow();
    expect(() => registry.get(`osb-payload://${"a".repeat(64)}`, { offset: 0, limit: 1 })).toThrow(
      PayloadNotFoundError,
    );
  });

  test("the store is refused by the index admission predicate", () => {
    expect(admitToIndex(`Brain/.payloads/${"a".repeat(64)}.txt`)).toEqual({
      admit: false,
      reason: "payload-store",
    });
    expect(admitToIndex("Brain/payloads-notes.md").admit).toBe(true);
  });
});

describe("inventory, gc and doctor", () => {
  /** One live payload (imported), one orphan, one missing ref in a note. */
  function seed(): { live: string; orphan: string; missing: string } {
    importSessionRecall(vault, {
      sessionId: "gc-session",
      createdAt: "2026-09-20T10:00:00.000Z",
      turns: [
        { turnId: "t1", role: "user", timestamp: "2026-09-20T10:00:01.000Z", text: `a ${BLOB}` },
      ],
    });
    const live = refsIn(continuityText())[0]!;
    // Externalized but never written anywhere: nothing references it.
    const orphan = new PayloadRegistry({ vault, maxInlineChars: 10 }).externalizeOversized(
      `b ${"QkNE".repeat(300)}`,
    ).payloads[0]!.ref;
    const missing = `osb-payload://${"c".repeat(64)}`;
    writeFileSync(join(vault, "note.md"), `# Note\n\nSee [payload: ${missing} chars=5]\n`);
    return { live, orphan, missing };
  }

  test("the inventory separates live, orphaned and missing payloads", () => {
    const { live, orphan, missing } = seed();
    const inventory = buildPayloadInventory(vault);
    expect(inventory.stored.map((e) => e.ref).toSorted()).toEqual([live, orphan].toSorted());
    expect(inventory.orphans.map((e) => e.ref)).toEqual([orphan]);
    expect(inventory.missing.map((e) => e.ref)).toEqual([missing]);
    expect(inventory.missing[0]!.referrers.map((r) => r.path)).toEqual(["note.md"]);
  });

  test("gc is a dry run by default and removes only orphans behind a snapshot on apply", () => {
    const { live, orphan } = seed();
    const orphanFile = payloadPath(vault, orphan.slice("osb-payload://".length));
    const liveFile = payloadPath(vault, live.slice("osb-payload://".length));

    ageFile(orphanFile);

    const dry = collectPayloadGarbage(vault, { apply: false });
    expect(dry.applied).toBe(false);
    expect(dry.orphans.map((e) => e.ref)).toEqual([orphan]);
    expect(dry.snapshot).toBeNull();
    expect(readdirSync(payloadsDir(vault))).toHaveLength(2);

    const applied = collectPayloadGarbage(vault, { apply: true });
    expect(applied.removed).toEqual([
      `Brain/.payloads/${orphan.slice("osb-payload://".length)}.txt`,
    ]);
    expect(applied.snapshot?.runId.startsWith("payload-gc-")).toBe(true);
    expect(() => readFileSync(orphanFile)).toThrow();
    expect(readFileSync(liveFile, "utf8")).toBe(BLOB);
    // The recovery point holds the store, so a restore resolves both refs.
    const listing = listSnapshotArchive(applied.snapshot!.path);
    expect(listing).toContain(`.payloads/${orphan.slice("osb-payload://".length)}.txt`);
    expect(listing).toContain(`.payloads/${live.slice("osb-payload://".length)}.txt`);
  });

  test("gc leaves an orphan inside the grace period for a later pass", () => {
    const { orphan } = seed();
    const orphanFile = payloadPath(vault, orphan.slice("osb-payload://".length));
    const now = new Date("2026-09-26T12:00:00.000Z");
    setMtime(orphanFile, now.getTime() - PAYLOAD_GC_GRACE_MS + 60_000);

    const young = collectPayloadGarbage(vault, { apply: true, now });
    expect(young.orphans).toEqual([]);
    expect(young.deferred.map((e) => e.ref)).toEqual([orphan]);
    expect(young.removed).toEqual([]);
    expect(existsSync(orphanFile)).toBe(true);

    setMtime(orphanFile, now.getTime() - PAYLOAD_GC_GRACE_MS - 1);
    const old = collectPayloadGarbage(vault, { apply: true, now });
    expect(old.removed).toEqual([`Brain/.payloads/${orphan.slice("osb-payload://".length)}.txt`]);
    expect(existsSync(orphanFile)).toBe(false);
  });

  test("a put that finds the payload already stored makes it young again", () => {
    const registry = new PayloadRegistry({ vault, maxInlineChars: 10 });
    const blob = `b ${"QkNE".repeat(300)}`;
    const ref = registry.externalizeOversized(blob).payloads[0]!.ref;
    const file = payloadPath(vault, ref.slice("osb-payload://".length));
    ageFile(file);
    const before = Date.now();
    registry.externalizeOversized(blob);
    expect(lstatMtime(file)).toBeGreaterThanOrEqual(before - 1000);
    expect(collectPayloadGarbage(vault, { apply: false }).deferred.map((e) => e.ref)).toEqual([
      ref,
    ]);
  });

  test(
    "gc apply and an externalizing import both wait on the payload store lock",
    () => {
      const { orphan } = seed();
      const orphanFile = payloadPath(vault, orphan.slice("osb-payload://".length));
      ageFile(orphanFile);
      const lockFile = `${payloadsDir(vault)}.lock`;
      const turn = (turnId: string, text: string) => ({
        sessionId: "lock-session",
        createdAt: "2026-09-20T10:00:00.000Z",
        turns: [{ turnId, role: "user" as const, timestamp: "2026-09-20T10:00:01.000Z", text }],
      });

      // Another holder has the store: the gc's removal pass and an import
      // with a payload to write both refuse once their wait runs out, and
      // nothing is removed or half-imported.
      writeFileSync(lockFile, "held by another writer\n");
      try {
        expect(() => collectPayloadGarbage(vault, { apply: true })).toThrow(/lock busy/);
        expect(existsSync(orphanFile)).toBe(true);
        expect(() => importSessionRecall(vault, turn("t9", `c ${"QUJE".repeat(400)}`))).toThrow(
          /lock busy/,
        );
        // A turn with nothing to externalize never touches the lock.
        expect(importSessionRecall(vault, turn("t10", "short")).rawTurns).toHaveLength(1);
      } finally {
        rmSync(lockFile, { force: true });
      }

      // Released, both run, and neither leaves the lock behind.
      expect(
        importSessionRecall(vault, turn("t9", `c ${"QUJE".repeat(400)}`)).rawTurns,
      ).toHaveLength(1);
      expect(collectPayloadGarbage(vault, { apply: true }).removed).toHaveLength(1);
      expect(existsSync(lockFile)).toBe(false);
    },
    { timeout: 20_000 },
  );

  test("doctor reports orphans, missing payloads and oversized continuity rows", () => {
    const { orphan, missing } = seed();
    // A row written before the registry bounded session text.
    appendContinuityRecord(vault, {
      kind: "session_turn",
      createdAt: "2026-09-20T10:00:00.000Z",
      payload: { session_id: "old", turn_id: "t0", text: "y".repeat(80_000) },
    });

    const issues = runDoctor(vault).warnings;
    const byCode = (code: string) => issues.filter((issue) => issue.code === code);
    expect(
      byCode("payload-orphan")
        .map((i) => i.message)
        .join("\n"),
    ).toContain(orphan);
    expect(
      byCode("payload-missing")
        .map((i) => i.message)
        .join("\n"),
    ).toContain(missing);
    expect(byCode("continuity-row-oversized")).toHaveLength(1);
  });
});

describe("export policy", () => {
  test("bank and OKF exports carry the placeholder, never the payload bytes", () => {
    const registry = new PayloadRegistry({ vault, maxInlineChars: 10 });
    const { text } = registry.externalizeOversized(`attachment ${BLOB}`);
    mkdirSync(join(vault, "Brain", "reports"), { recursive: true });
    writeFileSync(
      join(vault, "Brain", "reports", "2026-09-20-attachment.md"),
      `---\ntitle: Attachment\n---\n\n${text}\n`,
    );

    for (const exported of [
      JSON.stringify(exportBankBundle(vault)),
      JSON.stringify(buildOkfBundle(vault)),
    ]) {
      expect(exported).not.toContain(BLOB.slice(0, 200));
    }
    expect(JSON.stringify(buildOkfBundle(vault))).toContain("osb-payload://");
  });
});

function setMtime(file: string, ms: number): void {
  const at = new Date(ms);
  utimesSync(file, at, at);
}

/** Put `file` well past the gc grace period. */
function ageFile(file: string): void {
  setMtime(file, Date.now() - PAYLOAD_GC_GRACE_MS - 60_000);
}

function lstatMtime(file: string): number {
  return statSync(file).mtimeMs;
}