Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The durable raw-source archive (answer trust, t_9b835c8f).
*
* Provenance rots because intake throws the bytes away: a summary page
* records a digest of the source it was read from, and when the file behind
* that digest decays nothing can re-verify the page against what it claimed.
* This store keeps the bytes. The ingest trusted-local branch persists its
* source here - ONCE, redacted, content-addressed - and stamps the entry's
* digest on the summary page, so the page carries a pointer to evidence that
* outlives the source file.
*
* The contract, each clause load-bearing:
*
* - **Content-addressed.** An entry is `Brain/.archives/<sha256>.txt`,
* the digest of the persisted bytes. Identical bytes persist once -
* the address IS the dedup - and a re-ingest of an unchanged source
* re-lands the same entry it already has.
* - **Redacted BEFORE persist.** The bytes run through `redactRawOutput`
* before anything touches disk, so a credential captured in the source
* never lands in a second file after v1.78-v1.79 taught the rest of the
* tree to hold credentials in one custody store. The digest is computed
* over the REDACTED bytes - the address names exactly what is on disk,
* which is what lets {@link verifySourceArchive} re-hash the file it
* finds.
* - **Refused, never truncated.** A past-ceiling input raises
* {@link SourceArchiveError} and writes nothing: a truncated archive
* would no longer be the source's bytes, which is the one property it
* exists to keep (the `CaptureExcerptError` posture beside this module).
* - **Evidence, not scratch.** No TTL, no prune - the deliberate contrast
* to the artifact store. Retention is explicit operator pruning, and
* delete-by-source cleanup drops an entry only when its LAST citing page
* goes with it.
* - **Never wider than the page.** The stamp is written only on success,
* so an unstamped page means exactly "not captured" - absence is a
* state, never a failure. A source whose bytes are refused to remote
* readers is never archived at all; the visibility rule the digest
* already obeys is the one the archive inherits.
*
* Paths are built from the {@link BRAIN_ARCHIVES_REL} name constant and the
* entry filename is a validated 64-hex digest before any join, so a
* caller-supplied hash cannot traverse out of the store.
*/
import { createHash } from "node:crypto";
import { FileAlreadyExistsError } from "../../fs-atomic.ts";
import { join, posix, relative } from "node:path";
import { ensureInsideVault } from "../../path-safety.ts";
import { redactRawOutput } from "../../redactor.ts";
import { listVaultNotePaths } from "../../vault.ts";
import type { FrontmatterMap } from "../../types.ts";
import { BRAIN_ARCHIVES_REL, BRAIN_PENDING_REL, BRAIN_SOURCES_REL } from "../path-constants.ts";
import {
lanePageExists,
lanePagesInStore,
listLaneFiles,
listLaneTree,
parseLanePage,
readLaneText,
writeLaneText,
laneFileBytes,
removeLanePage,
} from "../../store/lane-pages.ts";
/**
* Largest input an archive entry will hold, in UTF-8 bytes: 1 MiB, the
* `PRE_EXTRACT_MAX_SOURCE_BYTES` ceiling. An archive is a re-verification
* copy of a source an agent read into a bounded context, not a mirror of
* the vault; a larger input is refused by {@link SourceArchiveError} rather
* than truncated, because a truncated archive would no longer be the bytes
* its address names.
*/
export const ARCHIVE_MAX_BYTES = 1_048_576;
/**
* An input the store must not hold. The message carries identifiers and
* integers only - the byte counts and the ceiling - never input text.
*/
export class SourceArchiveError extends Error {
constructor(message: string) {
super(message);
this.name = "SourceArchiveError";
}
}
/** Shape of an archive entry's filename stem: the digest of its own bytes. */
const ARCHIVE_HASH_RE = /^[0-9a-f]{64}$/;
/** Extension every archive entry carries. */
const ARCHIVE_ENTRY_EXT = ".txt";
/** Is this string shaped like the entry digest the store mints? */
export function isArchiveHash(value: unknown): value is string {
return typeof value === "string" && ARCHIVE_HASH_RE.test(value);
}
/**
* The closed vocabulary of an archive verification: what comparing a
* stamped digest against the entry it names concluded. `missing` and
* `mismatched` are separate members because "the evidence is gone" and
* "the evidence disagrees" are different repairs, and a boolean would
* collapse them. The values cross to the wire - they ride the
* capture-scope detector's finding evidence and the `archive_get` result -
* so a member must survive the round trip through a string.
*/
export const ARCHIVE_VERIFICATION = Object.freeze({
/** The entry is there and re-hashes to exactly the stamped digest. */
verified: "verified",
/** The entry is there but its bytes no longer hash to the stamp. */
mismatched: "mismatched",
/** No entry answers to the stamp. */
missing: "missing",
} as const);
/** Closed union over {@link ARCHIVE_VERIFICATION}. */
export type ArchiveVerification = (typeof ARCHIVE_VERIFICATION)[keyof typeof ARCHIVE_VERIFICATION];
/** Membership list, best-news-first. */
export const ARCHIVE_VERIFICATIONS: ReadonlyArray<ArchiveVerification> = Object.freeze(
Object.values(ARCHIVE_VERIFICATION),
);
/** Narrow a verification read back off a wire payload. */
export function isArchiveVerification(value: unknown): value is ArchiveVerification {
return (
typeof value === "string" && (ARCHIVE_VERIFICATIONS as ReadonlyArray<string>).includes(value)
);
}
/** Frontmatter key of the archive digest stamped on a summary page. */
export const SOURCE_ARCHIVE_HASH_KEY = "source_archive_hash";
/** Frontmatter key of the archived entry's byte count, beside the digest. */
export const SOURCE_ARCHIVE_BYTES_KEY = "source_archive_bytes";
/** The store root: `<vault>/Brain/.archives/`, contained. */
export function sourceArchivesDir(vault: string): string {
return ensureInsideVault(join(vault, BRAIN_ARCHIVES_REL), vault);
}
/**
* Vault-relative path of one entry, from its digest - the spelling every
* wire surface reports, so no host path and no second suffix spelling can
* drift. The digest is validated before the join.
*/
export function sourceArchiveEntryRel(hash: string): string {
if (!isArchiveHash(hash)) {
// The negative branch of the guard narrows a `string` parameter to
// `never`; widen before quoting so the refusal can still name the
// argument, truncated, without echoing arbitrary length.
const shown = String(hash);
throw new SourceArchiveError(
`archive entry refused: ${JSON.stringify(shown.slice(0, 72))} is not a 64-hex sha-256 digest`,
);
}
return posix.join(BRAIN_ARCHIVES_REL, `${hash}${ARCHIVE_ENTRY_EXT}`);
}
/**
* One entry's path. The digest is validated BEFORE the join, so a
* caller-supplied hash cannot traverse out of the store the way an
* unvalidated filename could.
*/
export function sourceArchivePath(vault: string, hash: string): string {
return ensureInsideVault(join(vault, sourceArchiveEntryRel(hash)), vault);
}
/** What {@link putSourceArchive} persisted. */
export interface ArchivePutResult {
/** Digest of the REDACTED bytes on disk - the entry's address. */
readonly hash: string;
/** UTF-8 byte count of the persisted (redacted) entry. */
readonly bytes: number;
}
/**
* Redact, then persist, then return the address. Idempotent by
* construction: the address is the digest of the persisted bytes, so the
* second put of the same bytes finds its own entry already there and
* leaves it untouched when it still verifies - one file per distinct
* source text, however many pages cite it. An entry that no longer
* verifies (truncated or edited on disk, the state the hygiene scan
* reports as `mismatched`) is rewritten with the same atomic write: the
* re-ingest of the source is the natural repair, and skipping the write
* behind an existing entry would leave the mismatch standing forever.
*
* The redaction is the artifact-store posture: the whole input scanned
* with no scan-window cap (the archive IS the recoverable bytes), and the
* infra-topology pass on, because a bare internal hostname in a captured
* source is a likelier leak than a key=value token.
*/
export function putSourceArchive(vault: string, bytes: Uint8Array | string): ArchivePutResult {
const raw = typeof bytes === "string" ? bytes : Buffer.from(bytes).toString("utf8");
const redacted = redactRawOutput(raw, { maxInput: Number.POSITIVE_INFINITY, redactInfra: true });
const persisted = Buffer.from(redacted, "utf8");
if (persisted.length > ARCHIVE_MAX_BYTES) {
throw new SourceArchiveError(
`source archive refused: the redacted source is ${persisted.length} bytes, past the ` +
`${ARCHIVE_MAX_BYTES}-byte ceiling; nothing was written`,
);
}
const hash = createHash("sha256").update(redacted, "utf8").digest("hex");
const path = sourceArchivePath(vault, hash);
if (!lanePageExists(vault, path)) {
// Identical bytes persist once: the write happens only when no entry
// with this address is already there, so a re-ingest never churns the
// entry's mtime. The create is exclusive, so of two concurrent puts of
// one content one writes and the other finds it persisted (a store
// record is never doubled; the address names the same bytes).
try {
writeLaneText(vault, path, redacted, { exclusive: true });
} catch (err) {
if (!(err instanceof FileAlreadyExistsError)) throw err;
}
} else if (entryMismatched(vault, path, hash)) {
// The repair arm: same address, disagreeing bytes. Written with the
// store's atomic write, so the entry is never observed half-replaced.
writeLaneText(vault, path, redacted);
}
return { hash, bytes: persisted.length };
}
/**
* Does the entry file at `path` still hash to the digest it is addressed
* by? A file that vanished mid-check is no mismatch - there is nothing
* here to repair, and the next put will find the store empty.
*/
function entryMismatched(vault: string, path: string, hash: string): boolean {
const text = readLaneText(vault, path);
if (text === null) return false;
return createHash("sha256").update(text, "utf8").digest("hex") !== hash;
}
/** What {@link verifySourceArchive} concluded. */
export interface ArchiveVerificationResult {
readonly state: ArchiveVerification;
/** The digest the caller asked about, verbatim. */
readonly hash: string;
}
/**
* Re-hash the entry a stamp names and compare. A malformed stamp names no
* entry the store could hold, so it answers {@link ARCHIVE_VERIFICATION.missing}
* - the stamp cannot be true - rather than an error; the boundaries that
* take a hash from a CALLER (not off a page) validate the argument first
* and refuse it as their own concern.
*/
export function verifySourceArchive(vault: string, hash: string): ArchiveVerificationResult {
if (!isArchiveHash(hash)) return { state: ARCHIVE_VERIFICATION.missing, hash };
const entry = readSourceArchiveEntry(vault, hash);
return entry === null
? { state: ARCHIVE_VERIFICATION.missing, hash }
: { state: entry.state, hash };
}
/** One entry read back: its verification against its own address, and its bytes. */
export interface SourceArchiveEntry {
/** `verified` or `mismatched` - a read entry is never `missing`. */
readonly state: ArchiveVerification;
/** The digest the entry was addressed by, verbatim. */
readonly hash: string;
/** UTF-8 byte count of the stored text. */
readonly bytes: number;
/** The stored (redacted) text. */
readonly text: string;
}
/**
* Read the entry addressed by `hash`: `null` when no such entry exists or
* the stamp is malformed (the caller decides what a malformed argument
* deserves), otherwise the stored text with its re-hash verdict attached.
*/
export function readSourceArchiveEntry(vault: string, hash: string): SourceArchiveEntry | null {
const path = sourceArchivePath(vault, hash);
const text = readLaneText(vault, path);
if (text === null) return null;
const actual = createHash("sha256").update(text, "utf8").digest("hex");
return {
state: actual === hash ? ARCHIVE_VERIFICATION.verified : ARCHIVE_VERIFICATION.mismatched,
hash,
bytes: Buffer.byteLength(text, "utf8"),
text,
};
}
/** What {@link sourceArchiveStats} measured about the store. */
export interface ArchiveStats {
/** Entries held: `<64-hex>.txt` regular files, however stale. */
readonly entries: number;
/** Their combined size in bytes - the growth number the doctor reports. */
readonly totalBytes: number;
}
/**
* Measure the store. Never throws for an absent store - a vault that has
* never captured a source has an empty one - and counts nothing it does
* not recognize (a stray non-entry file is not an archive entry).
*/
export function sourceArchiveStats(vault: string): ArchiveStats {
const root = sourceArchivesDir(vault);
if (!lanePageExists(vault, root)) return { entries: 0, totalBytes: 0 };
let entries = 0;
let totalBytes = 0;
for (const name of listLaneFiles(vault, root, "").toSorted()) {
if (!name.endsWith(ARCHIVE_ENTRY_EXT)) continue;
if (!isArchiveHash(name.slice(0, -ARCHIVE_ENTRY_EXT.length))) continue;
let size: number;
try {
size = laneFileBytes(vault, join(root, name));
} catch {
continue; // Vanished or unreadable mid-walk: not an entry this run counted.
}
entries += 1;
totalBytes += size;
}
return { entries, totalBytes };
}
/**
* Remove the entry addressed by `hash`, when there is one and nothing else
* has decided to keep it - the caller owns the last-citing-page question.
* False when there was nothing to remove; true only when bytes went away.
*/
export function discardSourceArchive(vault: string, hash: string): boolean {
const path = sourceArchivePath(vault, hash);
if (!lanePageExists(vault, path)) return false;
removeLanePage(vault, path);
return true;
}
/**
* The only pages a stamp is written on: `Brain/sources/` and its staged
* counterparts under `Brain/pending/`. The citation probe walks exactly
* these, which makes the one-writer assumption load-bearing: a stamped
* page anywhere else would cite an entry the probe cannot see. That
* assumption is architecture, not convention - it is pinned by the
* source-archive stamp census (`tests/core/architecture/source-archive-
* stamp-census.test.ts`), which fails by name when a second writer of
* the `source_archive_hash` key appears outside the ingest lane.
*/
const STAMP_ROOTS: ReadonlyArray<string> = Object.freeze([BRAIN_SOURCES_REL, BRAIN_PENDING_REL]);
/** Visit every stampable page's frontmatter, once per page. */
function walkStampedPages(
vault: string,
visit: (root: string, rel: string, meta: FrontmatterMap) => void,
): void {
for (const root of STAMP_ROOTS) {
const dir = join(vault, root);
// A store-only machine holds these pages as lane records, not files:
// the walk reads the lane fold there, the tree on an unswitched one.
const rels = stampRootRels(vault, dir);
for (const rel of rels) {
const abs = join(dir, rel);
// `parseFrontmatter` never throws (it reads inside its own try and
// answers an empty map for a failed read), so a page that will not
// parse falls out as citing nothing - the fail-safe answer for an
// entry-removal decision.
const [meta] = parseLanePage(vault, abs);
visit(root, rel, meta);
}
}
}
/**
* Every ingest summary page stamping `hash` as its archive digest,
* vault-relative and sorted. The inverse index of the stamp, for the
* delete path that must remove an entry only when its LAST citing page
* goes. A staged page counts as citing: rejecting it removes the citation
* with the page, but publishing it would revive a stamp whose entry must
* still be there.
*/
export function sourceArchiveCitingPages(vault: string, hash: string): ReadonlyArray<string> {
const pages: string[] = [];
walkStampedPages(vault, (root, rel, meta) => {
if (meta[SOURCE_ARCHIVE_HASH_KEY] === hash) {
pages.push(posix.join(root, rel.split("\\").join("/")));
}
});
return pages.toSorted();
}
/**
* The digests of every entry the store holds, sorted - the entry stems of
* the `<64-hex>.txt` files {@link sourceArchiveStats} counts, so a stray
* file is never mistaken for an entry and a vanished one is never named.
*/
export function sourceArchiveEntryHashes(vault: string): ReadonlyArray<string> {
const root = sourceArchivesDir(vault);
if (!lanePageExists(vault, root)) return [];
const hashes: string[] = [];
for (const name of listLaneFiles(vault, root, "")) {
if (!name.endsWith(ARCHIVE_ENTRY_EXT)) continue;
const stem = name.slice(0, -ARCHIVE_ENTRY_EXT.length);
if (!isArchiveHash(stem)) continue;
try {
laneFileBytes(vault, join(root, name));
} catch {
continue; // Vanished or unreadable mid-walk: not an entry this run named.
}
hashes.push(stem);
}
return hashes.toSorted();
}
/**
* The store's entries that NO page cites - the leftovers the citation
* probe cannot reach through a page: a re-stamped page orphans the entry
* the old stamp named, and a staged ingest rejected by review leaves its
* entry behind. One walk over the stamp roots builds the set of stamps
* any page carries, so the verdict is the exact answer the per-entry
* probe ({@link sourceArchiveCitingPages} empty) would give, without
* re-walking the pages once per entry. Sorted, so audit output is
* deterministic.
*/
export function sourceArchiveUncitedHashes(vault: string): ReadonlyArray<string> {
const cited = new Set<string>();
walkStampedPages(vault, (_root, _rel, meta) => {
const value = meta[SOURCE_ARCHIVE_HASH_KEY];
if (typeof value === "string") cited.add(value);
});
return sourceArchiveEntryHashes(vault).filter((hash) => !cited.has(hash));
}
/** The pages below one stamp root, relative to it: the lane fold once switched, the tree before. */
function stampRootRels(vault: string, dir: string): string[] {
if (lanePagesInStore(vault)) return listLaneTree(vault, dir).map((abs) => relative(dir, abs));
if (lanePageExists(vault, dir)) return listVaultNotePaths(dir);
return [];
}