Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Delete and search by exact source file (C6 / t_edde2198).
*
* When a benchmark file, log, or accidental import pollutes a Brain, the
* operator needs to (a) find every derived entry that traces back to one
* EXACT source path and (b) surgically remove those entries — including
* the ingest index artifacts (the per-source summary page and the
* content-manifest entry) — without re-mining the vault or hand-chasing
* summary pages.
*
* Three surfaces, one shared tracer:
* - {@link searchBySourceFile} — read-only. Every Brain page that traces
* to the exact source.
* - {@link deleteBySource} — DRY-RUN BY DEFAULT. Reports the blast radius
* and deletes nothing. With `confirm`, removes the derived entries and
* the index artifacts; original user notes are removed ONLY with an
* explicit `includeOriginals`. Every confirmed cleanup writes a
* `source_invalidation` continuity record (auditable).
* - {@link traceNoteDerivations} — read-only. The same classification
* run for a NOTE path rather than an imported source, so
* `noteLifecycle`'s `--delete-linked` cascade decides "derived solely
* from this" with this module's rule instead of a second one of its
* own. It removes nothing; the caller owns the ladder.
*
* Provenance a page can carry back to a source:
* - frontmatter `source_path` (the ingest summary page, kind
* `brain-source`),
* - frontmatter `session_ref` (a session-derived signal),
* - a `[[source]]` wikilink in the frontmatter `source` array or in the
* page body's `## Sources` provenance section,
* - a preference `evidenced_by` link to a signal that itself derives
* from the source (transitive fold).
*
* Blast-radius safety. A page is only auto-DELETED when it is a
* single-purpose derived file that traces SOLELY to this source:
* - a per-item derivation page (summary / signal / preference / entity /
* retired), AND
* - it co-references no OTHER source (a signal citing a second source, or
* a preference folded from a second signal, is REPORTED, never
* deleted).
* Everything else that merely mentions the source — daily logs, research
* reports, MOCs, any aggregate surface — is reported as a protected
* mention so the operator can hand-edit it. This is the "never remove more
* than intended" guarantee the dry-run report exists to make good on.
*/
import { existsSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs";
import { join, sep } from "node:path";
import type { FrontmatterMap } from "../types.ts";
import { canonicalNotePath, ensureInsideVault, vaultRelative } from "../path-safety.ts";
import { formatFrontmatter, parseFrontmatter } from "../vault.ts";
import { AUTHORITY_FIELD, PREFERENCE_REMOVE_KIND } from "../store/constants.ts";
import {
assertLaneWrite,
lanePageExists,
listLaneTree,
parseLanePage,
readLaneText,
removeLanePage,
} from "../store/lane-pages.ts";
import { withOperationPrivacy } from "../store/operation-scope.ts";
import { isStoreOnly } from "../store/store-only.ts";
import { assertNoteVaultForWrite } from "../vault-adapter/index.ts";
import { stripBrainIdPrefix } from "./wikilink.ts";
import { authorityFromOrigin } from "../store/dual-write.ts";
import { storeWrite } from "../store/store-write.ts";
import { removeStoreSignal } from "./signal-state.ts";
import { storeMemoryPageAt, storeRecordPages } from "./memory-pages.ts";
import {
BRAIN_ENTITIES_REL,
BRAIN_INBOX_REL,
BRAIN_PREFERENCES_REL,
BRAIN_REPORTS_REL,
BRAIN_RETIRED_REL,
BRAIN_SOURCES_REL,
brainDirs,
} from "./paths.ts";
import { BRAIN_SOURCE_KIND } from "./ingest/ingest.ts";
import { isSourceHidden } from "./intake/source-trust.ts";
import { manifestPath, readManifest, writeManifestAtomic } from "./ingest/content-manifest.ts";
import { appendContinuitySourceInvalidation } from "./continuity/store.ts";
import { isoSecond } from "./time.ts";
import {
classifyRecoverability,
type DestructiveBlastRadius,
type RecoverabilityVerdict,
} from "./gates/recoverability.ts";
import { withDestructiveSnapshot } from "./snapshot-gate.ts";
import { stringArrayField, wikilinkTarget } from "./source-links.ts";
import { BRAIN_SNAPSHOT_REASON } from "./types.ts";
import { refuseDocumentDeny, type WriteSubject } from "./write-disposition.ts";
import { REVIEW_LANE } from "./write-gate.ts";
/** How a page was found to trace back to the source. */
export type SourceCleanupMatch = "source_path" | "session_ref" | "wikilink" | "evidenced_by";
/** The shared empty failure list a dry run (and every clean run) reports. */
const NO_FAILURES: ReadonlyArray<SourceCleanupFailure> = Object.freeze([]);
/** Coarse classification of the referencing page, by its home directory. */
export type SourceCleanupKind =
| "summary"
| "signal"
| "preference"
| "entity"
| "retired"
| "report"
| "log"
| "other";
export interface SourceCleanupEntry {
/** Vault-relative POSIX path of the referencing page. */
readonly path: string;
/** Frontmatter `id` when present, else the file basename (no extension). */
readonly id: string;
readonly kind: SourceCleanupKind;
readonly match: SourceCleanupMatch;
/** True for the ingest summary page — an index artifact, always purged. */
readonly isIndexArtifact: boolean;
/**
* True when the page is a single-purpose derived file tracing SOLELY to
* this source and is therefore safe to auto-delete on `confirm`. False
* for aggregate surfaces and shared-derivation pages (reported only).
*/
readonly deletable: boolean;
}
/**
* A traced entry plus the one thing a NOTE delete needs and a source
* delete does not.
*
* The extra field is on this type rather than on
* {@link SourceCleanupEntry} deliberately: `deleteBySource`'s payload is
* pinned byte-for-byte by
* `tests/core/brain/gates/recoverability.test.ts`, and its rule is
* long-settled. Only {@link traceNoteDerivations} reports it.
*/
export interface NoteDerivationEntry extends SourceCleanupEntry {
/**
* True when the page names the subject in STRUCTURED provenance - a
* `source_path`, a `session_ref`, a `source:` link, or the
* `evidenced_by` fold - rather than only mentioning it in prose.
*
* {@link tracesSolelyToSource} is vacuously true for a page carrying no
* `source:` array at all, so `deletable` alone cannot tell a derivation
* from a paragraph that happens to write `[[Note]]`. The
* `--delete-linked` cascade requires this as well before it will put a
* Brain page in a deletion set.
*/
readonly structuredProvenance: boolean;
}
/**
* One per-entry removal failure: the vault-relative path that stayed on disk
* and the named fs error, so a retry knows exactly what is left.
*/
export interface SourceCleanupFailure {
readonly path: string;
readonly message: string;
/** The errno code the removal failed with, when the error carried one. */
readonly code?: string;
}
export interface SourceCleanupPlan {
/** Canonical form of the queried source. */
readonly source: string;
/** False on a dry run (the default); true only when `confirm` was set. */
readonly confirmed: boolean;
readonly includeOriginals: boolean;
/** Single-purpose derived entries removed (or that would be) on confirm. */
readonly derived: ReadonlyArray<SourceCleanupEntry>;
/** Referencing pages reported but NOT auto-deleted (aggregates / shared). */
readonly mentions: ReadonlyArray<SourceCleanupEntry>;
/** Original source file(s) outside `Brain/`; removed only with includeOriginals. */
readonly originals: ReadonlyArray<string>;
/** Manifest key for the source, when the content manifest tracks it. */
readonly manifestEntry: string | null;
/** Paths actually removed this run (empty on a dry run). */
readonly deleted: ReadonlyArray<string>;
/**
* Per-entry removal failures this run, with the named fs error (empty on a
* dry run and on a clean run). One blocked file never aborts the cleanup:
* every remaining entry is still attempted, the failures are reported here
* with the failed paths left on disk, and the audit record counts both
* sides honestly.
*/
readonly failed: ReadonlyArray<SourceCleanupFailure>;
readonly manifestEntryRemoved: boolean;
/** Id of the audit continuity record, when one was written. */
readonly auditRecordId: string | null;
/**
* Run id of the pre-deletion snapshot taken behind the D1 gate, or
* null when none was taken (a dry run, or a confirmed run with nothing
* to delete). This is the recovery point the operator can roll back to
* FOR THE PART OF THE DELETION IT COVERS - see {@link recoverability}.
*/
readonly snapshotRunId: string | null;
/** Absolute path of the snapshot archive, paired with {@link snapshotRunId}. */
readonly snapshotPath: string | null;
/**
* What that recovery point is actually worth for this run.
*
* `--include-originals` is the reason this field exists. An original is
* only ever returned when it lies OUTSIDE `Brain/` (see
* {@link findOriginals}), and the snapshot archive holds top-level
* entries under `Brain/` and nothing else. So a confirmed run with that
* flag deletes files no archive has ever held while reporting a
* `snapshotRunId` beside them - a response that reads as fully
* reversible and is not. The verdict names the uncovered lane instead
* of leaving the caller to infer it.
*/
readonly recoverability: RecoverabilityVerdict;
/** Total referencing pages + originals (the reported blast radius). */
readonly blastRadius: number;
}
export interface DeleteBySourceOptions {
/** Required true to actually delete; absent/false = dry run. */
readonly confirm?: boolean;
/** Also remove the original source file(s) outside `Brain/`. */
readonly includeOriginals?: boolean;
/** Injected clock for the audit record's timestamp. */
readonly now?: Date;
/** Agent identity recorded in the audit reason. */
readonly agent?: string;
/**
* May the caller read the page or the original at this vault-relative
* path? Asked before anything is stat-ed or deleted, so a derived page,
* a mention, an original or a manifest entry whose source it may not
* read is planned, counted and left on disk exactly as an absent one
* would be. Absent includes everything (the CLI, which reads the vault
* directly).
*/
readonly include?: (rel: string) => boolean;
/**
* The credential-minted subject of the caller (write-side-trust, Task
* 7), when the transport resolved one. It WINS over `agent` as the
* identity the permissions document consult answers - a caller-supplied
* agent is content attribution, never the identity a document's rules
* answer. Absent (stdio, the CLI) the consult falls back to the config
* identity, exactly like every other lane.
*/
readonly subject?: WriteSubject;
}
/**
* The spellings that name ONE subject, and the canonical form they all
* reduce to.
*
* `deleteBySource` has exactly one spelling - the canonical path of the
* imported file, which is how the ingest pipeline stamps `source_path`
* and how its provenance wikilinks are written. A NOTE is the same
* subject reached from the other side and Obsidian writes it two ways,
* `[[Projects/Note.md]]` and `[[Projects/Note]]`, so a tracer that knew
* only the first would leave a solely-derived signal unmatched and report
* it as a protected mention - a safe answer, but the wrong one.
*
* The bare `[[Note]]` spelling is deliberately NOT a member of any
* identity built here. It resolves to one note only when exactly one note
* in the vault carries the basename (`store/links.ts:66-69`), so
* attributing it would let a page derived from a DIFFERENT note of the
* same name into a deletion set. The caller still reports it: the
* note-delete surface's own inbound probe reads all three spellings.
*/
interface SourceIdentity {
/** Canonical form of the subject's own path. */
readonly canonical: string;
/** The subject as the caller spelled it, for `session_ref` equality. */
readonly sourceFile: string;
/** Every wikilink / `source_path` spelling that names the subject. */
readonly canonicalSpellings: ReadonlySet<string>;
/** The same spellings in match order, for the wikilink probes. */
readonly spellings: ReadonlyArray<string>;
}
/** One canonical path, spelled one way: `deleteBySource`'s subject. */
function exactSourceIdentity(sourceFile: string): SourceIdentity {
const canonical = canonicalNotePath(sourceFile);
return Object.freeze({
canonical,
sourceFile,
canonicalSpellings: new Set([canonical]),
spellings: Object.freeze([canonical]),
});
}
/** The `.md` suffix, matched case-insensitively as the note surfaces do. */
const MARKDOWN_SUFFIX = ".md";
/** One note path, spelled with and without its `.md` suffix. */
function noteIdentity(notePath: string): SourceIdentity {
const canonical = canonicalNotePath(notePath);
const bare = canonical.toLowerCase().endsWith(MARKDOWN_SUFFIX)
? canonical.slice(0, -MARKDOWN_SUFFIX.length)
: canonical;
const spellings = bare === canonical ? [canonical] : [canonical, bare];
return Object.freeze({
canonical,
sourceFile: notePath,
canonicalSpellings: new Set(spellings),
spellings: Object.freeze(spellings),
});
}
/** Does `target` - any spelling - name this subject? */
function namesSubject(identity: SourceIdentity, target: string): boolean {
return identity.canonicalSpellings.has(canonicalNotePath(target));
}
/** Directories whose pages are per-item, single-source derived artifacts. */
const DERIVATION_DIRS: ReadonlyArray<string> = Object.freeze([
BRAIN_SOURCES_REL,
BRAIN_INBOX_REL,
BRAIN_PREFERENCES_REL,
BRAIN_ENTITIES_REL,
BRAIN_RETIRED_REL,
]);
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
/**
* A wikilink to exactly `target`: `[[target]]`, `[[target|alias]]`, or
* `[[target#heading]]`. Anchored on the `[[` boundary so `[[foo]]` never
* matches `[[foobar]]`.
*/
function wikilinkRegExp(target: string): RegExp {
return new RegExp(`\\[\\[${escapeRegExp(target)}(?:[|#][^\\]]*)?\\]\\]`);
}
function stringField(meta: FrontmatterMap, key: string): string | null {
const value = meta[key];
return typeof value === "string" && value.length > 0 ? value : null;
}
/**
* A preference's evidence links. The writer stores the field under the
* managed `_evidenced_by` key (with a plain `evidenced_by` on legacy
* pages), so consult both.
*/
function evidencedByLinks(meta: FrontmatterMap): ReadonlyArray<string> {
const managed = stringArrayField(meta, "_evidenced_by");
return managed.length > 0 ? managed : stringArrayField(meta, "evidenced_by");
}
function classifyKind(vault: string, absPath: string): SourceCleanupKind {
const dirs = brainDirs(vault);
if (insideDir(absPath, join(vault, BRAIN_SOURCES_REL))) return "summary";
if (insideDir(absPath, dirs.retired)) return "retired";
if (insideDir(absPath, dirs.preferences)) return "preference";
if (insideDir(absPath, dirs.inbox)) return "signal";
if (insideDir(absPath, dirs.entities)) return "entity";
if (insideDir(absPath, join(vault, BRAIN_REPORTS_REL))) return "report";
if (insideDir(absPath, dirs.log)) return "log";
return "other";
}
function insideDir(path: string, dir: string): boolean {
return path === dir || path.startsWith(`${dir}${sep}`);
}
function inDerivationDir(vault: string, absPath: string): boolean {
return DERIVATION_DIRS.some((rel) => insideDir(absPath, join(vault, rel)));
}
function readId(meta: FrontmatterMap, absPath: string): string {
const id = stringField(meta, "id") ?? stringField(meta, "entity_id");
if (id) return id;
const base = absPath.slice(absPath.lastIndexOf(sep) + 1);
return base.endsWith(".md") ? base.slice(0, -3) : base;
}
interface RawMatch {
readonly absPath: string;
readonly path: string;
readonly id: string;
readonly kind: SourceCleanupKind;
readonly match: SourceCleanupMatch;
readonly isIndexArtifact: boolean;
/** Structured source links this page carries (signal `source` array). */
readonly sourceLinks: ReadonlyArray<string>;
/** Structured evidence links this page carries (preference `evidenced_by`). */
readonly evidencedBy: ReadonlyArray<string>;
}
/**
* Classify one Brain page against the canonical source. Returns null when
* the page does not reference the source at all. Only DIRECT references are
* detected here (source_path / session_ref / `[[source]]` wikilink); the
* transitive `evidenced_by` fold is resolved in a second pass once the set
* of directly-derived signals is known.
*/
function directMatch(
vault: string,
absPath: string,
identity: SourceIdentity,
storeOnly: boolean,
): RawMatch | null {
const page = brainPageAt(vault, absPath, storeOnly);
if (page === null) return null;
const text = page.text;
// Unit F: `parseFrontmatter` cannot throw (it reads inside its own try;
// everything after is string work), so the `catch { meta = {} }` that
// stood here was unreachable - and its intent is what the parser
// already does, returning an empty map for a failed read, so a page
// whose frontmatter is unavailable is still scanned as raw text.
// Dropped lines are reported centrally; see the "Why most readers keep
// the two-tuple form" section in src/core/vault.ts.
const meta = page.meta;
const kind = classifyKind(vault, absPath);
const sourceLinks = stringArrayField(meta, "source");
// Managed `_evidenced_by` links take precedence over the legacy plain key
// (see evidencedByLinks). Reading only the plain key here let a preference
// whose evidence lives in `_evidenced_by` slip past the foreign-evidence
// guard in computeDeletable and be auto-deleted as a first-pass wikilink
// match, even when it is a shared fold.
const evidencedBy = evidencedByLinks(meta);
const base = {
absPath,
path: vaultRelative(absPath, vault),
id: readId(meta, absPath),
kind,
sourceLinks,
evidencedBy,
};
const sourcePath = stringField(meta, "source_path");
if (sourcePath !== null && namesSubject(identity, sourcePath)) {
return {
...base,
match: "source_path",
isIndexArtifact: meta["kind"] === BRAIN_SOURCE_KIND,
};
}
const sessionRef = stringField(meta, "session_ref");
if (
sessionRef !== null &&
(sessionRef === identity.canonical || sessionRef === identity.sourceFile)
) {
return { ...base, match: "session_ref", isIndexArtifact: false };
}
if (identity.spellings.some((spelling) => wikilinkRegExp(spelling).test(text))) {
return { ...base, match: "wikilink", isIndexArtifact: false };
}
return null;
}
/**
* A directly-derived page traces SOLELY to this source when it declares no
* OTHER structured source link. A signal whose `source` array names a
* second source is a shared observation — reported, never auto-deleted.
*/
function tracesSolelyToSource(raw: RawMatch, identity: SourceIdentity): boolean {
for (const link of raw.sourceLinks) {
if (!namesSubject(identity, wikilinkTarget(link))) return false;
}
return true;
}
/**
* Does the page declare the subject as its PROVENANCE, or does it merely
* mention it?
*
* Every match except `wikilink` is structured by construction: a
* `source_path`, a `session_ref` and the `evidenced_by` fold are all
* frontmatter this project writes. A wikilink match is structured only
* when one of the page's own `source:` links names the subject - the same
* spelling in a paragraph of prose is a reference, not a derivation, and
* {@link tracesSolelyToSource} cannot tell the two apart because a page
* with no `source:` array satisfies it vacuously.
*/
function hasStructuredProvenance(raw: RawMatch, identity: SourceIdentity): boolean {
if (raw.match !== "wikilink") return true;
return raw.sourceLinks.some((link) => namesSubject(identity, wikilinkTarget(link)));
}
interface Traced {
readonly derived: ReadonlyArray<SourceCleanupEntry>;
readonly mentions: ReadonlyArray<SourceCleanupEntry>;
/**
* Vault-relative paths of the pages that named the subject in
* structured provenance. Carried beside the two lists rather than on
* the entries so `deleteBySource`'s payload keeps the exact shape it
* shipped with; {@link traceNoteDerivations} is the one reader.
*/
readonly structured: ReadonlySet<string>;
}
/**
* Walk `Brain/` and classify every page referencing the source into the
* deletable-derived set and the reported-only mentions set. Deterministic:
* both lists are sorted by vault-relative path.
*/
function traceReferences(vault: string, identity: SourceIdentity): Traced {
const root = brainDirs(vault).brain;
// Decided once per trace: every page of both passes is read the same way.
const storeOnly = isStoreOnly({ vault });
const rawMatches: RawMatch[] = [];
if (storeOnly || existsSync(root)) {
for (const absPath of walkBrainMarkdown(vault, root, storeOnly)) {
const m = directMatch(vault, absPath, identity, storeOnly);
if (m !== null) rawMatches.push(m);
}
}
// Second pass: a preference folded SOLELY from directly-derived signals
// is itself a derived entry. One evidenced by any foreign signal is a
// shared fold — reported, never auto-deleted.
const derivedSignalIds = new Set(rawMatches.filter((m) => m.kind === "signal").map((m) => m.id));
const alreadyMatched = new Set(rawMatches.map((m) => m.absPath));
if (storeOnly || existsSync(root)) {
for (const absPath of walkBrainMarkdown(vault, root, storeOnly)) {
if (alreadyMatched.has(absPath)) continue;
const kind = classifyKind(vault, absPath);
if (kind !== "preference") continue;
// Unit F: unreachable `catch { continue }` removed - see directMatch
// above. An unreadable page yields an empty map, so it has no
// `_evidenced_by` links and falls out on the next line exactly as
// the dead branch intended.
const meta = brainPageAt(vault, absPath, storeOnly)?.meta ?? {};
const evidencedBy = evidencedByLinks(meta);
const targets = evidencedBy.map((link) => wikilinkTarget(link));
if (targets.length === 0) continue;
if (!targets.some((t) => derivedSignalIds.has(t))) continue;
rawMatches.push({
absPath,
path: vaultRelative(absPath, vault),
id: readId(meta, absPath),
kind,
match: "evidenced_by",
isIndexArtifact: false,
sourceLinks: [],
evidencedBy,
});
}
}
const derived: SourceCleanupEntry[] = [];
const mentions: SourceCleanupEntry[] = [];
const structured = new Set<string>();
for (const raw of rawMatches) {
const deletable = computeDeletable(vault, raw, identity, derivedSignalIds);
const entry: SourceCleanupEntry = {
path: raw.path,
id: raw.id,
kind: raw.kind,
match: raw.match,
isIndexArtifact: raw.isIndexArtifact,
deletable,
};
if (hasStructuredProvenance(raw, identity)) structured.add(entry.path);
(deletable ? derived : mentions).push(entry);
}
return {
derived: derived.toSorted(byPath),
mentions: mentions.toSorted(byPath),
structured,
};
}
function byPath(a: SourceCleanupEntry, b: SourceCleanupEntry): number {
return a.path.localeCompare(b.path);
}
function computeDeletable(
vault: string,
raw: RawMatch,
identity: SourceIdentity,
derivedSignalIds: ReadonlySet<string>,
): boolean {
// The ingest summary page is always an index artifact to purge.
if (raw.isIndexArtifact) return true;
// Only per-item derivation pages are ever auto-deleted.
if (!inDerivationDir(vault, raw.absPath)) return false;
// A signal citing a second source is a shared observation — report it.
if (!tracesSolelyToSource(raw, identity)) return false;
// A preference folded from any foreign signal is a shared fold — report it.
// This holds however the preference was first matched: a second-pass
// `evidenced_by` fold OR a first-pass `[[source]]` wikilink match that also
// carries evidence links. Keying only on match === "evidenced_by" let a
// wikilink-matched shared preference reach `--confirm` deletion.
if (raw.kind === "preference" && raw.evidencedBy.length > 0) {
const targets = raw.evidencedBy.map((link) => wikilinkTarget(link));
if (!targets.every((t) => derivedSignalIds.has(t))) return false;
}
return true;
}
/**
* A `Brain/` page as this scan reads it, its raw text and frontmatter: the
* file on an unswitched machine; on a store-only one the page a lane entry
* or a memory record (preference, retired, signal) renders. `null` for a
* page that is not there.
*/
function brainPageAt(
vault: string,
absPath: string,
storeOnly: boolean,
): { readonly text: string; readonly meta: FrontmatterMap } | null {
if (!storeOnly) {
let text: string;
try {
text = readFileSync(absPath, "utf8");
} catch {
return null;
}
return { text, meta: parseFrontmatter(absPath)[0] };
}
const memory = storeMemoryPageAt(vault, vaultRelative(absPath, vault));
if (memory !== null) {
return { text: formatFrontmatter(memory.meta, memory.body), meta: memory.meta };
}
const text = readLaneText(vault, absPath);
return text === null ? null : { text, meta: parseLanePage(vault, absPath)[0] };
}
/**
* Every `.md` page under `root`: a walk of the tree on an unswitched
* machine; on a store-only one the lane entries and the memory records'
* pages, by the paths they had in the legacy tree. Dot folders are skipped
* in both, as the walk skips them.
*/
function walkBrainMarkdown(vault: string, root: string, storeOnly: boolean): string[] {
if (storeOnly) {
const memory = storeRecordPages(vault, { signalStates: "all" }).map((page) =>
join(vault, ...page.rel.split("/")),
);
const lanes = listLaneTree(vault, root);
return [...new Set([...memory, ...lanes])]
.filter(
(path) =>
!vaultRelative(path, vault)
.split("/")
.some((seg) => seg.startsWith(".")),
)
.toSorted();
}
const out: string[] = [];
const visit = (dir: string): void => {
let names: string[];
try {
names = readdirSync(dir);
} catch {
return;
}
for (const name of names) {
if (name.startsWith(".")) continue;
const full = join(dir, name);
let st;
try {
st = statSync(full);
} catch {
continue;
}
if (st.isDirectory()) {
visit(full);
continue;
}
if (st.isFile() && name.endsWith(".md")) out.push(full);
}
};
visit(root);
return out.toSorted();
}
/**
* Every Brain page tracing back to the EXACT source — the derived entries
* and the protected mentions, merged and sorted by path. Read-only.
*/
export function searchBySourceFile(
vault: string,
sourceFile: string,
): ReadonlyArray<SourceCleanupEntry> {
const { derived, mentions } = traceReferences(vault, exactSourceIdentity(sourceFile));
return Object.freeze([...derived, ...mentions].toSorted(byPath));
}
/**
* The vault-relative root the derived-set fold walks, and the whole of
* it: `Brain/`.
*
* Named rather than implied because a caller reporting an empty derived
* set has to be able to say whether that is a measurement or a scope
* boundary. Only `Brain/` holds per-item derivation pages, so a user note
* outside it is never a candidate for auto-deletion however tightly it
* references the subject - it is somebody's prose, and this fold does not
* reach it.
*/
export const DERIVATION_SCAN_SCOPE = "Brain/";
/**
* Every Brain page tracing back to one NOTE path, classified by the same
* blast-radius rule {@link deleteBySource} applies to an imported source:
* `deletable` marks a single-purpose derived file tracing SOLELY to the
* note, everything else is a reported mention.
*
* Read-only. The caller decides what to do with the two lists; this
* function's whole contribution is that the rule deciding which list a
* page lands in is the SAME function in both cases, so a note delete and
* a source delete cannot come to disagree about what "derived solely
* from" means.
*
* Each entry additionally carries
* {@link NoteDerivationEntry.structuredProvenance}, which the shared rule
* does not read: a note, unlike an imported source, is something ordinary
* prose links to, and the cascade needs to tell a declared derivation
* from a sentence.
*/
export function traceNoteDerivations(
vault: string,
notePath: string,
): ReadonlyArray<NoteDerivationEntry> {
const { derived, mentions, structured } = traceReferences(vault, noteIdentity(notePath));
return Object.freeze(
[...derived, ...mentions]
.toSorted(byPath)
.map((entry) =>
Object.freeze({ ...entry, structuredProvenance: structured.has(entry.path) }),
),
);
}
/**
* Original source file(s) outside `Brain/`: the imported artifact itself.
* A source that lives inside `Brain/` (or that has no on-disk file — e.g. a
* URL identity) contributes no original.
*/
function findOriginals(
vault: string,
canonical: string,
include: ((rel: string) => boolean) | undefined,
): string[] {
let abs: string;
try {
abs = ensureInsideVault(join(vault, canonical), vault);
} catch {
return [];
}
// Before the stat: an original the caller may not read is answered as
// an absent one, so neither its existence nor its removal is decided by
// a caller that cannot see it.
if (include?.(canonical) === false) return [];
if (!existsSync(abs)) return [];
try {
if (!statSync(abs).isFile()) return [];
} catch {
return [];
}
if (insideDir(abs, brainDirs(vault).brain)) return [];
return [canonical];
}
/**
* Remove one vault-relative file. `false` for an already-absent path (the
* idempotent re-run), `true` when it was removed. Any other fs error throws:
* callers that sweep many files collect per-file failures around this call
* ({@link deleteBySource} does; the retire applier consumes it the same way),
* never silently.
*/
export function removeFile(vault: string, relPath: string): boolean {
let abs: string;
try {
abs = ensureInsideVault(join(vault, relPath), vault);
} catch {
return false;
}
if (isStoreOnly({ vault }) && insideDir(abs, brainDirs(vault).brain)) {
return removeStorePage(vault, relPath, abs);
}
try {
rmSync(abs);
return true;
} catch (err) {
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") return false;
throw err;
}
}
/**
* Remove one `Brain/` page on a store-only machine, by what holds it: a
* preference or a retired page writes the preference's tombstone
* (`preference-remove`), a signal its tombstone (`signal-removed`), any other
* page removes its lane entry. `false` when there was nothing to remove.
*/
function removeStorePage(vault: string, relPath: string, abs: string): boolean {
const memory = storeMemoryPageAt(vault, relPath);
if (memory === null) {
if (!lanePageExists(vault, abs)) return false;
removeLanePage(vault, abs);
return true;
}
if (memory.kind === "signal") {
// A request to forget: the signal's tombstone, after which the fold
// drops it and its content from every read (FORMAT.md, Removal).
return removeStoreSignal(vault, memory.id);
}
// The fold keys a preference by `pref-<slug>` in both lives: a retired
// page's id (`ret-<slug>`) names no record of its own.
const preference = `pref-${stripBrainIdPrefix(memory.id)}`;
const isPrivate = memory.private;
withOperationPrivacy(isPrivate, () =>
storeWrite(
() => ({
stream: "events",
kind: PREFERENCE_REMOVE_KIND,
private: isPrivate,
body: { [AUTHORITY_FIELD]: authorityFromOrigin(), preference },
}),
{ vault },
),
);
// Removed only when the fold no longer shows the page.
return storeMemoryPageAt(vault, relPath) === null;
}
/**
* Plan and (with `confirm`) execute the removal of everything derived from
* one exact source file. Dry-run by default.
*/
export function deleteBySource(
vault: string,
sourceFile: string,
opts: DeleteBySourceOptions = {},
): SourceCleanupPlan {
const identity = exactSourceIdentity(sourceFile);
const canonical = identity.canonical;
const confirm = opts.confirm === true;
const includeOriginals = opts.includeOriginals === true;
// Lane write guard: the vault identity guard on an unswitched machine,
// the store identity check on a store-only one (design D14 item 13).
// Dry-run (the default) plans and writes nothing, so it stays ungated.
if (confirm) assertLaneWrite(vault);
// Deny-consulted evidence (write-side trust, Task 12): a confirmed
// cleanup is a mutation lane like update or append - it removes the
// derived entries and the ingest manifest entry - so the document is
// resolved against the credential subject and a deny refuses with the
// SAME typed error BEFORE anything is read for deletion, snapshotted,
// or removed, one decision-ledger row behind it. A dry run (the
// default) writes nothing, so it stays a read and stays ungated. With
// no permissions document this is a no-op and the cleanup proceeds
// exactly as before.
if (confirm) {
refuseDocumentDeny(vault, REVIEW_LANE.ingest, opts.subject, {
target: canonical,
...(opts.now !== undefined ? { now: opts.now } : {}),
});
}
// A page or a source the caller may not read is planned, counted and
// left on disk exactly as an absent one: a remote caller can neither
// learn of a reserved summary page nor delete it.
const included = (rel: string): boolean => opts.include?.(rel) !== false;
const traced = traceReferences(vault, identity);
const derived = traced.derived.filter((entry) => included(entry.path));
const mentions = traced.mentions.filter((entry) => included(entry.path));
const originals = findOriginals(vault, canonical, opts.include);
// The originals are user-vault files in both modes. On a store-only
// machine the lane guard above checks the store only, so their removal
// runs the vault adapter's guard (the bound vault, not frozen) up front.
if (confirm && includeOriginals && originals.length > 0 && isStoreOnly({ vault })) {
assertNoteVaultForWrite(vault, "delete_by_source removing originals");
}
const manifestKey =
readManifest(vault).entries[canonical] !== undefined &&
!isSourceHidden(vault, canonical, opts.include)
? canonical
: null;
const blastRadius = derived.length + mentions.length + originals.length;
/**
* The lanes this run actually destroys, in the regions the archive is
* defined over. `mentions` are reported and never deleted, so they are
* not in it; the manifest entry is a `Brain/` index artifact, so it is.
*/
const destroyed: DestructiveBlastRadius = {
brainTopLevel: derived.length > 0 || manifestKey !== null,
outsideBrainRoot: includeOriginals && originals.length > 0,
};
if (!confirm) {
return Object.freeze({
source: canonical,
confirmed: false,
includeOriginals,
derived: Object.freeze(derived),
mentions: Object.freeze(mentions),
originals: Object.freeze(originals),
manifestEntry: manifestKey,
deleted: Object.freeze([]),
failed: NO_FAILURES,
manifestEntryRemoved: false,
auditRecordId: null,
snapshotRunId: null,
snapshotPath: null,
// A dry run destroys nothing, so it promises nothing. Reporting the
// coverage the confirmed run WOULD have would be a claim about an
// archive that does not exist yet.
recoverability: classifyRecoverability({ recoveryPoint: false, blastRadius: {} }),
blastRadius,
});
}
// Confirmed path — delete the derived entries first, then (opt-in) the
// originals, then drop the manifest entry (an index artifact).
const deleted: string[] = [];
const failed: SourceCleanupFailure[] = [];
let manifestEntryRemoved = false;
// Set when the manifest entry deliberately STAYED because an original's
// removal failed (review L7) - the audit names the keep, and the result's
// `manifestEntryRemoved` stays false.
let manifestEntryKeptForFailure = false;
/**
* Attempt one removal, collecting a failure instead of throwing: one
* blocked file must not abort the entries still queued behind it. The
* collected failures ride the result and the audit, and the paths stay on
* disk for a retry.
*/
const removeCollecting = (relPath: string): void => {
try {
if (removeFile(vault, relPath)) deleted.push(relPath);
} catch (err) {
const code = (err as NodeJS.ErrnoException | null)?.code;
failed.push({
path: relPath,
message: (err as Error)?.message ?? String(err),
...(typeof code === "string" ? { code } : {}),
});
}
};
// Audit only a run that actually changed something; a no-op re-run writes
// no record (idempotent). Factored so both the success path AND a partial
// failure can persist the record of what was already removed.
const writeInvalidationAudit = (): string | null => {
if (deleted.length === 0 && !manifestEntryRemoved && !manifestEntryKeptForFailure) {
return null;
}
const agent = opts.agent?.trim() ? opts.agent.trim() : "delete_by_source";
const reason =
`${agent}: removed ${deleted.length} derived/original entr${deleted.length === 1 ? "y" : "ies"}` +
`${manifestEntryRemoved ? " and the ingest manifest entry" : ""}` +
`${manifestEntryKeptForFailure ? " (ingest manifest entry kept: an original's removal failed)" : ""}` +
`${includeOriginals ? " (originals included)" : ""}` +
(failed.length > 0
? `; failed to remove ${failed.length}: ${failed.map((f) => f.path).join(", ")}`
: "");
const record = appendContinuitySourceInvalidation(vault, {
createdAt: isoSecond(opts.now ?? new Date()),
source: { id: canonical, path: canonical, kind: "source" },
reason,
});
return record.id;
};
let auditRecordId: string | null = null;
// The destructive core: remove derived entries, opt-in originals, and
// the manifest index entry, then audit. Runs behind the D1 snapshot
// gate so every confirmed cleanup has a pre-deletion recovery point.
const runDeletion = (): void => {
try {
for (const entry of derived) removeCollecting(entry.path);
if (includeOriginals) {
for (const original of originals) removeCollecting(original);
}
// An original whose removal FAILED keeps its manifest entry (review
// L7): the source file is still on disk, so dropping the entry would
// classify it `new` at the next batch plan and the planner would
// rebuild the very derived pages this run just removed. The audit
// names the keep.
const originalRemovalFailed =
includeOriginals && originals.some((o) => failed.some((f) => f.path === o));
manifestEntryKeptForFailure = originalRemovalFailed && manifestKey !== null;
if (manifestKey !== null && existsSync(manifestPath(vault)) && !originalRemovalFailed) {
const manifest = readManifest(vault);
const entries = { ...manifest.entries };
if (entries[manifestKey] !== undefined) {
delete entries[manifestKey];
// Round-trip the contract the manifest was read with (t_586d5d8b):
// a cleanup that only removes an entry must not re-stamp the
// extraction contract the remaining entries were recorded under -
// and a v1 read (contract `null`) round-trips as v1, because its
// survivors were never reprocessed under any fingerprint. Upgrading
// them here would classify them `unchanged` forever and silently
// forfeit the owed one-time reprocess.
manifestEntryRemoved = writeManifestAtomic(vault, entries, manifest.contract);
}
}
} catch (err) {
// A manifest read/write that throws mid-cleanup would otherwise exit
// before the audit append, leaving already-deleted paths with no record
// for a retry to reconstruct. Persist what was removed (and what was
// collected as failed) so far, then rethrow; never let an audit-write
// error mask the original failure.
try {
writeInvalidationAudit();
} catch {
/* swallow: the original error below is the one that matters */
}
throw err;
}
auditRecordId = writeInvalidationAudit();
};
// Only snapshot when there is something to remove. A confirmed run over
// a source with no derived material, no included originals, and no
// manifest entry is a no-op and must stay one - snapshotting an empty
// deletion would churn `.snapshots/` for nothing.
const hasWork =
derived.length > 0 || (includeOriginals && originals.length > 0) || manifestKey !== null;
let snapshotRunId: string | null = null;
let snapshotArchivePath: string | null = null;
let recoverability: RecoverabilityVerdict;
if (hasWork) {
const gated = withDestructiveSnapshot(
vault,
BRAIN_SNAPSHOT_REASON.deleteBySource,
runDeletion,
{ now: opts.now, blastRadius: destroyed },
);
snapshotRunId = gated.snapshot.runId;
snapshotArchivePath = gated.snapshot.path;
recoverability = gated.recoverability;
} else {
runDeletion();
// No work means no snapshot AND nothing destroyed. The classifier
// says `nothing_at_risk` for that, which is neither the alarm of
// `unproven` nor the false comfort of `covered`.
recoverability = classifyRecoverability({ recoveryPoint: false, blastRadius: destroyed });
}
return Object.freeze({
source: canonical,
confirmed: true,
includeOriginals,
derived: Object.freeze(derived),
mentions: Object.freeze(mentions),
originals: Object.freeze(originals),
manifestEntry: manifestKey,
deleted: Object.freeze(deleted),
failed: Object.freeze(failed),
manifestEntryRemoved,
auditRecordId,
snapshotRunId,
snapshotPath: snapshotArchivePath,
recoverability,
blastRadius,
});
}