Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Document retirement applier core (storage lifecycle, Lane 4; plan
* task 4.3).
*
* The consumer of the `document-dedup` detector's findings: for each
* near-duplicate copy a plan selects, retire the copy and keep the
* canonical page -
*
* 1. retarget every inbound `[[stale-copy]]` wikilink to the
* canonical spelling (the report is carried, never swallowed);
* a write that could not land refuses the copy before the
* removal, so no retired page strands the link it could not
* repoint (review M7);
* 2. remove the stale page (per-file removal via the shared
* `removeFile`, so a failure is one copy's failure);
* 3. MARK the copy's source manifest entry `retired_into` the
* canonical page, round-tripping the contract the manifest was
* read under - the same t_586d5d8b rule deleteBySource applies: a
* retirement never re-stamps the contract the surviving entries
* were recorded under. The marker is the resurrect guard (review
* H3): dropping the entry would classify the unchanged source
* `new` again and the next batch plan would re-ingest it, quietly
* undoing the retirement. With the marker, the unchanged source
* classifies `retired` and no plan queues it; a real byte change
* re-admits it as `modified`, and a deliberate re-ingest re-records
* the entry without the marker;
* 4. append one `source_invalidation` continuity record PER COPY
* naming canonical id, retired id, marked manifest key and the
* retained source-archive hash - the audit the page-dedup merge
* never needed, because a merge keeps both pages and this
* operation deletes one.
*
* The whole span sits inside `withDestructiveSnapshot`. The snapshot
* reason is `deleteBySource`'s: a retired summary page is a source's
* derived material, which is exactly what that reason documents, and
* the reason vocabulary lives in a file this lane does not own. The
* source archive (`Brain/.archives/<hash>.txt`) is deliberately NOT
* removed - it is evidence under a no-TTL contract, and the audit
* names its digest so the provenance chain survives the page.
*
* Failure posture is per-copy fail-soft: one copy's failure is
* collected as `{path, message}`, and the remaining copies still
* retire. The message says WHERE the copy ended up: a failure BEFORE
* the page's own removal (the existence, frontmatter or canonical
* checks, the ambiguous-basename refusal, an inbound-link rewrite that
* could not land) leaves the copy on disk; a failure AFTER it - the
* manifest marker, the continuity record - reports the page as REMOVED,
* because it is, and names the step that lost it. Copy-level input
* mistakes (a path equal to its own canonical) are caller bugs and
* refuse loudly before any work.
*
* Dry runs never reach this module: the hygiene apply plan previews
* retire findings without executing them, and the surfaces' count
* guard runs on the selected findings before this applier is called.
*/
import { existsSync, statSync } from "node:fs";
import { join } from "node:path";
import { parseFrontmatter } from "../vault.ts";
import { ensureInsideVault } from "../path-safety.ts";
import { isoSecond } from "./time.ts";
import { markManifestRetired } from "./ingest/content-manifest.ts";
import { appendContinuitySourceInvalidation } from "./continuity/store.ts";
import { removeFile } from "./source-cleanup.ts";
import { basenameCollides, retargetWikilinks } from "./page-dedup.ts";
import type { WikilinkRetargetReport } from "./page-dedup.ts";
import { BRAIN_SNAPSHOT_REASON } from "./types.ts";
import { withDestructiveSnapshot } from "./snapshot-gate.ts";
import { REVIEW_LANE } from "./write-gate.ts";
import {
classifyRecoverability,
type DestructiveBlastRadius,
type RecoverabilityVerdict,
} from "./gates/recoverability.ts";
import { assertVaultIdentityForWrite } from "./vault-identity.ts";
import { refuseDocumentDeny, type WriteSubject } from "./write-disposition.ts";
import type { ResolveWriteDispositionOptions } from "./write-disposition.ts";
import { BRAIN_SOURCE_KIND } from "./ingest/ingest.ts";
import type { SourceCleanupFailure } from "./source-cleanup.ts";
/** The agent named on the audit record when the caller names none. */
const DEFAULT_AGENT = "document_retire";
/** Basename without the `.md` extension - the wikilink spelling of a page. */
function linkTarget(pagePath: string): string {
const base = pagePath.split("/").at(-1) ?? pagePath;
return base.replace(/\.md$/, "");
}
/**
* Why a copy whose inbound-link pass cannot run unambiguously is REFUSED
* (storage lifecycle, PR #257 review L5): another page in the vault
* carries the retired copy's basename, so the bare `[[basename]]`
* spelling resolves to more than one page and a retarget keyed on the
* spelling alone would repoint links that meant the OTHER page - edits
* to pages outside the detector's near-duplicate scope, made on a guess.
* Removing the page anyway is worse than skipping the rewrite: every
* leftover `[[basename]]` would then RESOLVE to the namesake, silently
* re-pointed by a deletion instead of left visibly stale. So the copy is
* refused and stays on disk; the finding comes back to the operator.
*/
export const RETIRE_BASENAME_COLLISION_REFUSAL =
"another page carries the same basename, so the bare-basename link spelling is ambiguous";
/** Frontmatter key carrying the digest of the archived source bytes. */
const SOURCE_ARCHIVE_HASH_KEY = "source_archive_hash";
/** Frontmatter key carrying the canonical source path a page derives from. */
const SOURCE_PATH_KEY = "source_path";
/** One near-duplicate copy to retire, with the page that survives it. */
export interface RetireDocumentCopyInput {
/** Vault-relative path of the near-duplicate summary page to retire. */
readonly path: string;
/** Vault-relative path of the canonical page that survives. */
readonly canonical: string;
}
export interface RetireDocumentsOptions {
/** Named on the audit record. Defaults to {@link DEFAULT_AGENT}. */
readonly agent?: string;
/** Injected clock for the audit stamps. */
readonly now?: Date;
/**
* Credential subject for the permissions-document deny consult, which
* runs per copy before anything is read for deletion (the ingest
* lane, matching deleteBySource). With no permissions document this
* is a no-op.
*/
readonly subject?: WriteSubject;
}
/** One copy that retired cleanly, with everything the run did to it. */
export interface RetiredDocumentCopy {
readonly path: string;
readonly canonical: string;
/** The source path whose manifest entry was targeted, or `null`. */
readonly manifestKey: string | null;
/** True when the manifest was rewritten to carry the `retired_into` marker. */
readonly manifestRetired: boolean;
/**
* The retired page's size in bytes, measured BEFORE its removal - the
* freed half of the run's disk accounting (review M6).
*/
readonly pageBytes: number;
/**
* The full retarget report for this copy's inbound wikilinks. A copy
* whose pass could not run unambiguously - the basename collision - is
* refused before any write and lands in {@link RetireDocumentsResult.failed}
* instead, so a report row always carries a pass that ran.
*/
readonly wikilinks: WikilinkRetargetReport;
/** Digest named in the audit; from the page's frontmatter, or `null`. */
readonly sourceArchiveHash: string | null;
/** The id of the `source_invalidation` continuity record this retirement appended. */
readonly auditRecordId: string;
}
/**
* The run's disk accounting (review M6, retire half): the destructive span
* archives `Brain/` into `Brain/.snapshots/` - a surface the storage census
* counts - BEFORE it frees anything, so removal counts alone can read as a
* shrink on a run that actually grew the vault. The result carries both
* sides and the net between them.
*/
export interface RetireDiskAccounting {
/**
* Bytes of the snapshot archive the gate wrote for this run, or `null`
* when no snapshot was taken (a run with no copies).
*/
readonly snapshotBytes: number | null;
/** Summed bytes of the pages the run removed, measured before removal. */
readonly removedBytes: number;
/** `snapshotBytes - removedBytes`; `null` beside a `null` snapshot size. */
readonly netBytes: number | null;
}
export interface RetireDocumentsResult {
readonly retired: ReadonlyArray<RetiredDocumentCopy>;
/**
* Per-copy failures, collected. A failure BEFORE the page's own
* removal leaves the copy on disk; a failure after it says so in its
* message - the page is gone, and the message names the step that
* lost it.
*/
readonly failed: ReadonlyArray<SourceCleanupFailure>;
/**
* The run's disk accounting (review M6): both sides and the net
* between them, as the module header promises. Per-copy bytes ride
* {@link RetiredDocumentCopy.pageBytes}; this carries the run totals.
*/
readonly disk: RetireDiskAccounting;
readonly snapshotRunId: string | null;
readonly snapshotPath: string | null;
readonly recoverability: RecoverabilityVerdict;
}
/** What one copy's retirement read off the page before acting. */
interface CopyFacts {
readonly sourcePath: string | null;
readonly archiveHash: string | null;
readonly kind: string | null;
}
function readCopyFacts(vault: string, pagePath: string): CopyFacts | null {
let meta: Record<string, unknown>;
try {
[meta] = parseFrontmatter(join(vault, pagePath));
} catch {
return null;
}
const sourcePath = meta[SOURCE_PATH_KEY];
const archiveHash = meta[SOURCE_ARCHIVE_HASH_KEY];
return {
kind: typeof meta["kind"] === "string" ? meta["kind"] : null,
sourcePath: typeof sourcePath === "string" && sourcePath.length > 0 ? sourcePath : null,
archiveHash: typeof archiveHash === "string" && archiveHash.length > 0 ? archiveHash : null,
};
}
/**
* Retire the given near-duplicate copies. Writes by contract: the
* vault-identity write guard runs at the entry point, before any other
* work, per the one rule every applier shares.
*/
export function retireDocumentCopies(
vault: string,
copies: ReadonlyArray<RetireDocumentCopyInput>,
opts: RetireDocumentsOptions = {},
): RetireDocumentsResult {
// Write guard first - see the module header. This applier always
// writes when called; the dry-run boundary lives in the hygiene plan.
assertVaultIdentityForWrite(vault);
for (const copy of copies) {
if (copy.path === copy.canonical) {
throw new Error(
`retireDocumentCopies: ${copy.path} is named as both the retired copy and the canonical`,
);
}
}
if (copies.length === 0) {
return Object.freeze({
retired: Object.freeze([]),
failed: Object.freeze([]),
disk: Object.freeze({ snapshotBytes: null, removedBytes: 0, netBytes: null }),
snapshotRunId: null,
snapshotPath: null,
// Nothing to destroy, so the classifier says so - neither the
// alarm of `unproven` nor the false comfort of `covered`.
recoverability: classifyRecoverability({ recoveryPoint: false, blastRadius: {} }),
});
}
// Deny-consulted evidence (write-side trust, Task 12), per copy and
// BEFORE any read-for-deletion, one ledger row behind a deny - the
// same ladder deleteBySource climbs. A deny throws here, so nothing
// has been snapshotted or removed when it lands.
const denyOpts: ResolveWriteDispositionOptions = opts.now !== undefined ? { now: opts.now } : {};
for (const copy of copies) {
refuseDocumentDeny(vault, REVIEW_LANE.ingest, opts.subject, {
target: copy.path,
...denyOpts,
});
}
const retired: RetiredDocumentCopy[] = [];
const failed: SourceCleanupFailure[] = [];
const runRetirement = (): void => {
for (const copy of copies) {
try {
retired.push(retireOneCopy(vault, copy, opts));
} catch (error) {
failed.push({
path: copy.path,
message: error instanceof Error ? error.message : String(error),
});
}
}
};
// Only snapshot when there is something to remove; a caller that
// passes no work gets no archive churn.
const blastRadius: DestructiveBlastRadius = { brainTopLevel: true, outsideBrainRoot: false };
const gated = withDestructiveSnapshot(
vault,
BRAIN_SNAPSHOT_REASON.deleteBySource,
runRetirement,
{
...(opts.now !== undefined ? { now: opts.now } : {}),
blastRadius,
},
);
// The disk accounting (review M6, retire half): the archive sits in a
// counted surface, so the result states what the snapshot cost beside
// what the removals freed.
const removedBytes = retired.reduce((sum, copy) => sum + copy.pageBytes, 0);
const snapshotBytes = existsSync(gated.snapshot.path) ? statSync(gated.snapshot.path).size : null;
const disk: RetireDiskAccounting = Object.freeze({
snapshotBytes,
removedBytes,
netBytes: snapshotBytes === null ? null : snapshotBytes - removedBytes,
});
return Object.freeze({
retired: Object.freeze(retired),
failed: Object.freeze(failed),
disk,
snapshotRunId: gated.snapshot.runId,
snapshotPath: gated.snapshot.path,
recoverability: gated.recoverability,
});
}
/** Retire one copy. Throws to the caller's per-copy collector. */
function retireOneCopy(
vault: string,
copy: RetireDocumentCopyInput,
opts: RetireDocumentsOptions,
): RetiredDocumentCopy {
const abs = join(vault, copy.path);
// Containment before existence: removeFile answers a vault-escaping path
// with the same `false` a vanished page gets, and the message below would
// then claim a copy vanished that was never inside the vault to begin
// with. Named here for what it is, so the per-copy failure says the path
// escapes the vault rather than inventing a disappearance.
try {
ensureInsideVault(abs, vault);
} catch {
throw new Error(`retired copy path escapes the vault: ${copy.path}`);
}
if (!existsSync(abs)) {
throw new Error(`retired copy not found on disk: ${copy.path}`);
}
// Measured before anything acts: the freed half of the run's disk
// accounting (review M6).
const pageBytes = statSync(abs).size;
const facts = readCopyFacts(vault, copy.path);
if (facts === null) {
throw new Error(`retired copy frontmatter does not parse: ${copy.path}`);
}
if (facts.kind !== BRAIN_SOURCE_KIND) {
throw new Error(`retired copy is not an ingest summary page (kind ${String(facts.kind)})`);
}
if (!existsSync(join(vault, copy.canonical))) {
throw new Error(`canonical page not found on disk: ${copy.canonical}`);
}
// 1. Links first, so no reader between steps ever sees a stranded
// `[[stale-copy]]`. The pass keys on the bare basename, so when another
// page carries the same basename the spelling is ambiguous: rewriting it
// would repoint links that meant the other page, and removing the page
// anyway would leave every leftover spelling resolving to the namesake
// silently (review L5). The copy is refused BEFORE any write and stays
// on disk - a refusal is one copy's collected failure.
if (basenameCollides(vault, linkTarget(copy.path), copy.path)) {
throw new Error(
`retirement refused, the copy stays in place: ${RETIRE_BASENAME_COLLISION_REFUSAL}: ${copy.path}`,
);
}
// Failure to write a file is collected in the report, not thrown - the
// pass has already rewritten the files before it (the retargetWikilinks
// argument, verbatim).
const wikilinks = retargetWikilinks(vault, [
{ from: linkTarget(copy.path), to: linkTarget(copy.canonical) },
]);
// A retarget write that failed refuses the copy (review M7): retiring
// the page over an unreplicable link would strand exactly the inbound
// reference the pass could not repoint - the design's "a copy whose
// retarget fails stays in place". The refusal is BEFORE the removal, so
// the copy stays on disk as one collected per-copy failure; the files
// the pass DID rewrite now point at the canonical, which exists, so no
// link is stranded by the partial rewrite either.
if (wikilinks.failed.length > 0) {
throw new Error(
`inbound wikilink retarget left ${wikilinks.failed.length} file(s) unwritten; ` +
`the copy stays in place: ` +
wikilinks.failed.map((f) => `${f.path} (${f.reason})`).join("; "),
);
}
// 2. Remove the stale page. `false` here means the page vanished
// between the existence check and the unlink - that is a failure to
// report, not a success to claim.
if (!removeFile(vault, copy.path)) {
throw new Error(`retired copy vanished before removal: ${copy.path}`);
}
// Everything below runs AFTER the page is gone. A failure here cannot
// leave the copy on disk, so it is rethrown AS that fact - the report
// must never send an operator looking for a page that is not there.
try {
// 3. Mark the manifest entry for the copy's source `retired_into` the
// canonical page, round-tripping the read contract (t_586d5d8b): the
// surviving entries keep the fingerprint they were recorded under, so
// the owed one-time reprocess is never forfeited by a retirement. The
// per-copy reason is worded from what actually happened - a mark, a
// refusal, an entry already marked, an entry absent, or no key
// recorded at all - never from the key's existence alone.
let manifestRetired = false;
let manifestClause: string;
if (facts.sourcePath === null) {
manifestClause = "no manifest key recorded";
} else {
// One locked read-modify-write, so a concurrent ingest's manifest
// write can neither be lost to this one nor drop the marker.
const mark = markManifestRetired(vault, facts.sourcePath, copy.canonical);
if (mark === "no-manifest") {
manifestClause = `manifest absent, nothing to mark for ${facts.sourcePath}`;
} else if (mark === "no-entry") {
manifestClause = `manifest key ${facts.sourcePath} absent, nothing to mark`;
} else if (mark === "already-marked") {
// The entry already carries THIS retirement's marker: the clause
// names that success-shaped fact - a concurrent retirer got there
// first.
manifestClause = `manifest key ${facts.sourcePath} already marked retired`;
} else {
manifestRetired = true;
manifestClause = `marked manifest key ${facts.sourcePath} retired into ${copy.canonical}`;
}
}
// 4. The provenance record: one per copy, naming what survives, what
// went, what was marked and what evidence is retained.
const agent = opts.agent?.trim() ? opts.agent.trim() : DEFAULT_AGENT;
const record = appendContinuitySourceInvalidation(vault, {
createdAt: isoSecond(opts.now ?? new Date()),
source: {
id: facts.sourcePath ?? copy.path,
...(facts.sourcePath !== null ? { path: facts.sourcePath } : {}),
kind: "source",
},
reason:
`${agent}: retired near-duplicate copy ${linkTarget(copy.path)} into canonical ` +
`${linkTarget(copy.canonical)}; ${manifestClause}; ` +
`source archive ${facts.archiveHash ?? "none"} retained`,
});
return Object.freeze({
path: copy.path,
canonical: copy.canonical,
manifestKey: facts.sourcePath,
manifestRetired,
pageBytes,
wikilinks,
sourceArchiveHash: facts.archiveHash,
auditRecordId: record.id,
});
} catch (error) {
const detail = error instanceof Error ? error.message : String(error);
throw new Error(
`retirement removed ${copy.path}, then a post-removal step failed ` +
`(the copy is not on disk): ${detail}`,
{ cause: error },
);
}
}