Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* What the payload store holds, who points at it, and what may go
* (t_35440e83).
*
* The registry (`payload-registry.ts`) writes and reads single payloads.
* This module answers the questions about the store as a whole, which
* the list verb, the doctor, the gc and the remote read gate all ask:
*
* - which payload files exist (`Brain/.payloads/<sha256>.txt`);
* - which refs the vault holds, and in which files;
* - which stored payloads nothing references (orphans) and which
* referenced payloads have no file (missing);
* - whether a payload is reachable from something a REMOTE caller may
* read.
*
* ## "Referenced anywhere in the vault"
*
* Every text-shaped file (`.md`, `.jsonl`, `.json`, `.txt`, `.yaml`,
* `.yml`, `.canvas`, `.base`) under the vault is scanned for
* `osb-payload://<sha256>`, except:
*
* - `Brain/.payloads/` itself, which is followed TRANSITIVELY instead:
* a whole-turn payload can hold the placeholders of the blobs that
* were externalized out of it first, and those stay live exactly as
* long as the payload holding them does;
* - `Brain/.snapshots/`, whose archives carry their own copy of the
* store, so a restore brings back every payload the restored rows
* name;
* - `.git`, `node_modules` and `.stversions` (Syncthing's own version
* history) - tooling state, not vault content. The derived-store
* directory is NOT skipped by name: its databases are not text files
* and never match the extension filter, and a JSON ledger inside it
* that ever held a ref would be a reference like any other.
*
* A ref anywhere else keeps its payload, including in `.trash` and in a
* hand-written note: the gc is dry-run by default and removes only what
* this scan cannot find a single reference to.
*
* ## Remote readability
*
* Payloads are session data, and only the session turns they were cut
* out of may grant a remote read. A continuity row carries a `private`
* flag (set when a `<private>` region was stripped from it; the read
* model drops such rows from every export). A payload is readable at
* remote reach only when a non-private `session_turn` row references it,
* or a payload that is itself remotely readable does.
*
* Everything else keeps a payload live for the gc and grants nothing:
*
* - a summary node names a payload only because it copied the text of
* the turns it summarizes, and carries no `private` flag of its own -
* counting it would let every private turn's payload through its
* summary;
* - a page, whatever its visibility, because pages (the Brain log
* included) are written by tools a session turn is not - a ref pasted
* into a note must not turn a private payload public;
* - any other file (a JSON ledger, a YAML config), and an orphan: a
* payload nothing references is not something a remote caller can
* have been shown a ref for.
*
* Because only the continuity ledger and the store itself can grant, the
* remote check ({@link isPayloadRemotelyReadable}) reads just those two
* directories rather than walking the vault.
*/
import { existsSync, lstatSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { join, posix } from "node:path";
import { continuityLogDir } from "./continuity/store.ts";
import {
BRAIN_LOG_REL,
BRAIN_PAYLOADS_REL,
BRAIN_SNAPSHOTS_REL,
payloadPath,
payloadsDir,
vaultRelative,
} from "./paths.ts";
import {
PAYLOAD_GC_GRACE_MS,
PAYLOAD_REF_RE,
payloadRefFor,
withPayloadStoreLock,
} from "./payload-registry.ts";
import { withDestructiveSnapshot, type DestructiveSnapshot } from "./snapshot-gate.ts";
import { LOCK_WAIT_INTERACTIVE_MS } from "./sync-lockfile.ts";
import { BRAIN_SNAPSHOT_REASON } from "./types.ts";
import { assertVaultIdentityForWrite } from "./vault-identity.ts";
/** One file (and, for a ledger, one line) that names a payload. */
export interface PayloadReferrer {
/** Vault-relative POSIX path of the referring file. */
readonly path: string;
/** 1-based line, for a continuity ledger row. */
readonly line?: number;
/** Whether a remote caller may read this referrer. */
readonly remoteReadable: boolean;
}
/** One stored payload file. */
export interface StoredPayload {
readonly sha256: string;
readonly ref: string;
/** Vault-relative POSIX path of the file. */
readonly path: string;
readonly bytes: number;
/** Distinct referring files/rows, transitive through other payloads. */
readonly referrers: number;
readonly remoteReadable: boolean;
}
/** A ref some file holds whose payload file is gone. */
export interface MissingPayload {
readonly sha256: string;
readonly ref: string;
readonly referrers: ReadonlyArray<PayloadReferrer>;
}
export interface PayloadInventory {
readonly stored: ReadonlyArray<StoredPayload>;
readonly orphans: ReadonlyArray<StoredPayload>;
readonly missing: ReadonlyArray<MissingPayload>;
}
/** What a gc pass found and, with `apply`, removed. */
export interface PayloadGcResult {
readonly applied: boolean;
/** Unreferenced payloads past the grace period: removed on apply. */
readonly orphans: ReadonlyArray<StoredPayload>;
/**
* Unreferenced payloads younger than {@link PAYLOAD_GC_GRACE_MS}, left
* for a later pass: an import may be about to name them.
*/
readonly deferred: ReadonlyArray<StoredPayload>;
readonly removed: ReadonlyArray<string>;
readonly bytes: number;
/** The recovery point taken before anything was removed. */
readonly snapshot: DestructiveSnapshot | null;
}
const PAYLOAD_FILE_RE = /^([a-f0-9]{64})\.txt$/;
const TEXT_EXTENSIONS: ReadonlySet<string> = new Set([
".md",
".jsonl",
".json",
".txt",
".yaml",
".yml",
".canvas",
".base",
]);
const SKIPPED_DIR_NAMES: ReadonlySet<string> = new Set([".git", "node_modules", ".stversions"]);
const SKIPPED_PATHS: ReadonlySet<string> = new Set([BRAIN_PAYLOADS_REL, BRAIN_SNAPSHOTS_REL]);
const CONTINUITY_PREFIX = `${BRAIN_LOG_REL}/continuity/`;
/** Every payload file in the store, sorted by digest. */
export function listStoredPayloadFiles(
vault: string,
): ReadonlyArray<{ sha256: string; bytes: number }> {
const dir = payloadsDir(vault);
if (!existsSync(dir)) return Object.freeze([]);
const out: Array<{ sha256: string; bytes: number }> = [];
for (const name of readdirSync(dir)) {
const match = PAYLOAD_FILE_RE.exec(name);
if (match === null) continue;
const path = payloadPath(vault, match[1]!);
let size: number;
try {
const stat = lstatSync(path);
if (!stat.isFile()) continue;
size = stat.size;
} catch {
continue;
}
out.push({ sha256: match[1]!, bytes: size });
}
out.sort((a, b) => (a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0));
return Object.freeze(out);
}
/**
* Every payload ref the vault holds, keyed by digest, with the files
* (and ledger rows) that hold it - transitive through the store.
*/
export function scanPayloadReferences(vault: string): ReadonlyMap<string, PayloadReferrer[]> {
const refs = new Map<string, PayloadReferrer[]>();
const add = (sha256: string, referrer: PayloadReferrer): void => {
const list = refs.get(sha256);
if (list === undefined) refs.set(sha256, [referrer]);
else list.push(referrer);
};
for (const rel of walkTextFiles(vault)) scanFile(vault, rel, add);
// Transitive pass: a payload reachable from the vault keeps the
// payloads its own text names. Remote readability is resolved after,
// because a payload's readability depends on its referrers'.
const expanded = new Set<string>();
const queue = [...refs.keys()];
const contained = new Map<string, ReadonlyArray<string>>();
while (queue.length > 0) {
const sha256 = queue.shift()!;
if (expanded.has(sha256)) continue;
expanded.add(sha256);
const inner = payloadRefsInside(vault, sha256);
contained.set(sha256, inner);
for (const child of inner) if (!expanded.has(child)) queue.push(child);
}
const remote = new Set<string>();
for (const [sha256, list] of refs) if (list.some((r) => r.remoteReadable)) remote.add(sha256);
let grew = true;
while (grew) {
grew = false;
for (const [sha256, inner] of contained) {
if (!remote.has(sha256)) continue;
for (const child of inner) {
if (!remote.has(child)) {
remote.add(child);
grew = true;
}
}
}
}
for (const [sha256, inner] of contained) {
const path = posix.join(BRAIN_PAYLOADS_REL, `${sha256}.txt`);
for (const child of inner) add(child, { path, remoteReadable: remote.has(sha256) });
}
return refs;
}
/** Stored, orphaned and missing payloads in one pass. */
export function buildPayloadInventory(vault: string): PayloadInventory {
const refs = scanPayloadReferences(vault);
const files = listStoredPayloadFiles(vault);
const storedSet = new Set(files.map((file) => file.sha256));
const stored = files.map((file) => {
const referrers = refs.get(file.sha256) ?? [];
return Object.freeze({
sha256: file.sha256,
ref: payloadRefFor(file.sha256),
path: posix.join(BRAIN_PAYLOADS_REL, `${file.sha256}.txt`),
bytes: file.bytes,
referrers: referrers.length,
remoteReadable: referrers.some((r) => r.remoteReadable),
});
});
const missing = [...refs.entries()]
.filter(([sha256]) => !storedSet.has(sha256))
.map(([sha256, referrers]) =>
Object.freeze({ sha256, ref: payloadRefFor(sha256), referrers: Object.freeze(referrers) }),
)
.toSorted((a, b) => (a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0));
return Object.freeze({
stored: Object.freeze(stored),
orphans: Object.freeze(stored.filter((entry) => entry.referrers === 0)),
missing: Object.freeze(missing),
});
}
/**
* May a caller at remote reach read the payload `sha256`?
*
* Only when a non-private session turn references it, directly or through
* a chain of payloads that starts at one - see the module docblock. Reads
* the continuity ledger and the payload store only.
*/
export function isPayloadRemotelyReadable(vault: string, sha256: string): boolean {
const granted = new Set<string>();
const queue: string[] = [];
const dir = continuityLogDir(vault);
let names: string[] = [];
try {
names = readdirSync(dir);
} catch {
names = [];
}
for (const name of names) {
if (!name.endsWith(".jsonl")) continue;
let text: string;
try {
if (!lstatSync(join(dir, name)).isFile()) continue;
text = readFileSync(join(dir, name), "utf8");
} catch {
continue;
}
if (!text.includes("osb-payload://")) continue;
for (const line of text.split("\n")) {
if (!line.includes("osb-payload://") || !continuityRowIsShareable(line)) continue;
for (const ref of distinctRefs(line)) {
if (!granted.has(ref)) {
granted.add(ref);
queue.push(ref);
}
}
}
}
while (queue.length > 0) {
if (granted.has(sha256)) return true;
for (const child of payloadRefsInside(vault, queue.shift()!)) {
if (!granted.has(child)) {
granted.add(child);
queue.push(child);
}
}
}
return granted.has(sha256);
}
/**
* Remove every payload file nothing in the vault references and that is
* older than {@link PAYLOAD_GC_GRACE_MS}.
*
* Dry-run unless `apply`: the plan is the same scan the apply performs.
* The apply re-plans INSIDE the recovery point and under the payload
* store lock, so a payload an import referenced between the dry run and
* the apply is never removed, an import cannot land a row naming a
* payload while it is being removed, and every removal sits behind a
* snapshot that still holds the file.
*/
export function collectPayloadGarbage(
vault: string,
opts: { readonly apply: boolean; readonly now?: Date },
): PayloadGcResult {
const nowMs = (opts.now ?? new Date()).getTime();
if (!opts.apply) {
const split = splitByGrace(vault, buildPayloadInventory(vault).orphans, nowMs);
return Object.freeze({
applied: false,
orphans: split.removable,
deferred: split.deferred,
removed: Object.freeze([]),
bytes: sumBytes(split.removable),
snapshot: null,
});
}
assertVaultIdentityForWrite(vault);
const planned = splitByGrace(vault, buildPayloadInventory(vault).orphans, nowMs);
if (planned.removable.length === 0) {
return Object.freeze({
applied: true,
orphans: planned.removable,
deferred: planned.deferred,
removed: Object.freeze([]),
bytes: 0,
snapshot: null,
});
}
const gated = withDestructiveSnapshot(
vault,
BRAIN_SNAPSHOT_REASON.payloadGc,
() =>
withPayloadStoreLock(
vault,
() => {
const split = splitByGrace(vault, buildPayloadInventory(vault).orphans, nowMs);
const removed: string[] = [];
for (const orphan of split.removable) {
rmSync(payloadPath(vault, orphan.sha256), { force: true });
removed.push(orphan.path);
}
return { ...split, removed };
},
// Operator-run: a busy import answers with a named ELOCKED after a
// short wait rather than a frozen terminal.
LOCK_WAIT_INTERACTIVE_MS,
),
opts.now !== undefined ? { now: opts.now } : {},
);
return Object.freeze({
applied: true,
orphans: gated.result.removable,
deferred: gated.result.deferred,
removed: Object.freeze(gated.result.removed),
bytes: sumBytes(gated.result.removable),
snapshot: gated.snapshot,
});
}
/**
* Orphans past the grace period, and the ones still inside it. A file
* whose age cannot be read is deferred: not knowing it is old is not a
* reason to remove it.
*/
function splitByGrace(
vault: string,
orphans: ReadonlyArray<StoredPayload>,
nowMs: number,
): { removable: ReadonlyArray<StoredPayload>; deferred: ReadonlyArray<StoredPayload> } {
const removable: StoredPayload[] = [];
const deferred: StoredPayload[] = [];
for (const orphan of orphans) {
let mtimeMs: number;
try {
mtimeMs = lstatSync(payloadPath(vault, orphan.sha256)).mtimeMs;
} catch {
deferred.push(orphan);
continue;
}
(nowMs - mtimeMs >= PAYLOAD_GC_GRACE_MS ? removable : deferred).push(orphan);
}
return { removable: Object.freeze(removable), deferred: Object.freeze(deferred) };
}
/** One continuity row longer than the bound, reported per shard. */
export interface OversizedContinuityShard {
/** Vault-relative POSIX path of the shard. */
readonly path: string;
readonly rows: number;
readonly largestChars: number;
}
/**
* Continuity shards holding rows longer than `maxRowChars`.
*
* A row written through the registry carries at most the text bound plus
* its envelope, so a longer row predates the registry or came from a
* writer that bypassed it. Reported per shard rather than per row: a
* pre-registry vault can hold thousands, and the answer an operator needs
* is where they are and how bad the worst one is.
*/
export function findOversizedContinuityRows(
vault: string,
maxRowChars: number,
): ReadonlyArray<OversizedContinuityShard> {
const dir = continuityLogDir(vault);
if (!existsSync(dir)) return Object.freeze([]);
const out: OversizedContinuityShard[] = [];
for (const name of readdirSync(dir).toSorted()) {
if (!name.endsWith(".jsonl")) continue;
const abs = join(dir, name);
let text: string;
try {
text = readFileSync(abs, "utf8");
} catch {
continue;
}
let rows = 0;
let largest = 0;
for (const line of text.split("\n")) {
if (line.length <= maxRowChars) continue;
rows++;
largest = Math.max(largest, line.length);
}
if (rows > 0) out.push({ path: vaultRelative(abs, vault), rows, largestChars: largest });
}
return Object.freeze(out);
}
function sumBytes(entries: ReadonlyArray<StoredPayload>): number {
return entries.reduce((total, entry) => total + entry.bytes, 0);
}
function payloadRefsInside(vault: string, sha256: string): ReadonlyArray<string> {
let text: string;
try {
text = readFileSync(payloadPath(vault, sha256), "utf8");
} catch {
return [];
}
const out = new Set<string>();
for (const match of text.matchAll(PAYLOAD_REF_RE)) {
if (match[1] !== sha256) out.add(match[1]!);
}
return [...out];
}
function* walkTextFiles(vault: string): Generator<string> {
const stack: string[] = [""];
while (stack.length > 0) {
const relDir = stack.pop()!;
let entries: string[];
try {
entries = readdirSync(relDir === "" ? vault : join(vault, relDir));
} catch {
continue;
}
for (const name of entries) {
const rel = relDir === "" ? name : `${relDir}/${name}`;
let stat;
try {
// lstat: a symlink is never followed, so the scan cannot leave
// the vault or loop through a link back into it.
stat = lstatSync(join(vault, rel));
} catch {
continue;
}
if (stat.isDirectory()) {
if (SKIPPED_DIR_NAMES.has(name) || SKIPPED_PATHS.has(rel)) continue;
stack.push(rel);
continue;
}
if (!stat.isFile()) continue;
const dot = name.lastIndexOf(".");
if (dot < 0 || !TEXT_EXTENSIONS.has(name.slice(dot).toLowerCase())) continue;
yield rel;
}
}
}
function scanFile(
vault: string,
rel: string,
add: (sha256: string, referrer: PayloadReferrer) => void,
): void {
let text: string;
try {
text = readFileSync(join(vault, rel), "utf8");
} catch {
return;
}
if (!text.includes("osb-payload://")) return;
if (rel.startsWith(CONTINUITY_PREFIX) && rel.endsWith(".jsonl")) {
const lines = text.split("\n");
for (let i = 0; i < lines.length; i++) {
const line = lines[i]!;
if (!line.includes("osb-payload://")) continue;
const remoteReadable = continuityRowIsShareable(line);
for (const sha256 of distinctRefs(line)) {
add(sha256, { path: rel, line: i + 1, remoteReadable });
}
}
return;
}
// Pages and every other file keep a payload live; none grants a remote
// read (see the module docblock).
for (const sha256 of distinctRefs(text)) add(sha256, { path: rel, remoteReadable: false });
}
function distinctRefs(text: string): ReadonlySet<string> {
return new Set([...text.matchAll(PAYLOAD_REF_RE)].map((match) => match[1]!));
}
/** A ledger row that grants remote reads: a parseable, non-private session turn. */
function continuityRowIsShareable(line: string): boolean {
try {
const row = JSON.parse(line) as { private?: unknown; kind?: unknown };
return row.kind === "session_turn" && row.private !== true;
} catch {
return false;
}
}