Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
41.5 KB
Raw Blame
/**
 * Delete and search by exact source file (C6 / t_edde2198).
 *
 * When a benchmark file, log, or accidental import pollutes a Brain, the
 * operator needs to (a) find every derived entry that traces back to one
 * EXACT source path and (b) surgically remove those entries — including
 * the ingest index artifacts (the per-source summary page and the
 * content-manifest entry) — without re-mining the vault or hand-chasing
 * summary pages.
 *
 * Three surfaces, one shared tracer:
 *   - {@link searchBySourceFile} — read-only. Every Brain page that traces
 *     to the exact source.
 *   - {@link deleteBySource} — DRY-RUN BY DEFAULT. Reports the blast radius
 *     and deletes nothing. With `confirm`, removes the derived entries and
 *     the index artifacts; original user notes are removed ONLY with an
 *     explicit `includeOriginals`. Every confirmed cleanup writes a
 *     `source_invalidation` continuity record (auditable).
 *   - {@link traceNoteDerivations} — read-only. The same classification
 *     run for a NOTE path rather than an imported source, so
 *     `noteLifecycle`'s `--delete-linked` cascade decides "derived solely
 *     from this" with this module's rule instead of a second one of its
 *     own. It removes nothing; the caller owns the ladder.
 *
 * Provenance a page can carry back to a source:
 *   - frontmatter `source_path` (the ingest summary page, kind
 *     `brain-source`),
 *   - frontmatter `session_ref` (a session-derived signal),
 *   - a `[[source]]` wikilink in the frontmatter `source` array or in the
 *     page body's `## Sources` provenance section,
 *   - a preference `evidenced_by` link to a signal that itself derives
 *     from the source (transitive fold).
 *
 * Blast-radius safety. A page is only auto-DELETED when it is a
 * single-purpose derived file that traces SOLELY to this source:
 *   - a per-item derivation page (summary / signal / preference / entity /
 *     retired), AND
 *   - it co-references no OTHER source (a signal citing a second source, or
 *     a preference folded from a second signal, is REPORTED, never
 *     deleted).
 * Everything else that merely mentions the source — daily logs, research
 * reports, MOCs, any aggregate surface — is reported as a protected
 * mention so the operator can hand-edit it. This is the "never remove more
 * than intended" guarantee the dry-run report exists to make good on.
 */

import { existsSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs";
import { join, sep } from "node:path";

import type { FrontmatterMap } from "../types.ts";
import { canonicalNotePath, ensureInsideVault, vaultRelative } from "../path-safety.ts";
import { formatFrontmatter, parseFrontmatter } from "../vault.ts";
import { AUTHORITY_FIELD, PREFERENCE_REMOVE_KIND } from "../store/constants.ts";
import {
  assertLaneWrite,
  lanePageExists,
  listLaneTree,
  parseLanePage,
  readLaneText,
  removeLanePage,
} from "../store/lane-pages.ts";
import { withOperationPrivacy } from "../store/operation-scope.ts";
import { isStoreOnly } from "../store/store-only.ts";
import { assertNoteVaultForWrite } from "../vault-adapter/index.ts";
import { stripBrainIdPrefix } from "./wikilink.ts";
import { authorityFromOrigin } from "../store/dual-write.ts";
import { storeWrite } from "../store/store-write.ts";
import { removeStoreSignal } from "./signal-state.ts";
import { storeMemoryPageAt, storeRecordPages } from "./memory-pages.ts";
import {
  BRAIN_ENTITIES_REL,
  BRAIN_INBOX_REL,
  BRAIN_PREFERENCES_REL,
  BRAIN_REPORTS_REL,
  BRAIN_RETIRED_REL,
  BRAIN_SOURCES_REL,
  brainDirs,
} from "./paths.ts";
import { BRAIN_SOURCE_KIND } from "./ingest/ingest.ts";
import { isSourceHidden } from "./intake/source-trust.ts";
import { manifestPath, readManifest, writeManifestAtomic } from "./ingest/content-manifest.ts";
import { appendContinuitySourceInvalidation } from "./continuity/store.ts";
import { isoSecond } from "./time.ts";
import {
  classifyRecoverability,
  type DestructiveBlastRadius,
  type RecoverabilityVerdict,
} from "./gates/recoverability.ts";
import { withDestructiveSnapshot } from "./snapshot-gate.ts";
import { stringArrayField, wikilinkTarget } from "./source-links.ts";
import { BRAIN_SNAPSHOT_REASON } from "./types.ts";
import { refuseDocumentDeny, type WriteSubject } from "./write-disposition.ts";
import { REVIEW_LANE } from "./write-gate.ts";

/** How a page was found to trace back to the source. */
export type SourceCleanupMatch = "source_path" | "session_ref" | "wikilink" | "evidenced_by";

/** The shared empty failure list a dry run (and every clean run) reports. */
const NO_FAILURES: ReadonlyArray<SourceCleanupFailure> = Object.freeze([]);

/** Coarse classification of the referencing page, by its home directory. */
export type SourceCleanupKind =
  | "summary"
  | "signal"
  | "preference"
  | "entity"
  | "retired"
  | "report"
  | "log"
  | "other";

export interface SourceCleanupEntry {
  /** Vault-relative POSIX path of the referencing page. */
  readonly path: string;
  /** Frontmatter `id` when present, else the file basename (no extension). */
  readonly id: string;
  readonly kind: SourceCleanupKind;
  readonly match: SourceCleanupMatch;
  /** True for the ingest summary page — an index artifact, always purged. */
  readonly isIndexArtifact: boolean;
  /**
   * True when the page is a single-purpose derived file tracing SOLELY to
   * this source and is therefore safe to auto-delete on `confirm`. False
   * for aggregate surfaces and shared-derivation pages (reported only).
   */
  readonly deletable: boolean;
}

/**
 * A traced entry plus the one thing a NOTE delete needs and a source
 * delete does not.
 *
 * The extra field is on this type rather than on
 * {@link SourceCleanupEntry} deliberately: `deleteBySource`'s payload is
 * pinned byte-for-byte by
 * `tests/core/brain/gates/recoverability.test.ts`, and its rule is
 * long-settled. Only {@link traceNoteDerivations} reports it.
 */
export interface NoteDerivationEntry extends SourceCleanupEntry {
  /**
   * True when the page names the subject in STRUCTURED provenance - a
   * `source_path`, a `session_ref`, a `source:` link, or the
   * `evidenced_by` fold - rather than only mentioning it in prose.
   *
   * {@link tracesSolelyToSource} is vacuously true for a page carrying no
   * `source:` array at all, so `deletable` alone cannot tell a derivation
   * from a paragraph that happens to write `[[Note]]`. The
   * `--delete-linked` cascade requires this as well before it will put a
   * Brain page in a deletion set.
   */
  readonly structuredProvenance: boolean;
}

/**
 * One per-entry removal failure: the vault-relative path that stayed on disk
 * and the named fs error, so a retry knows exactly what is left.
 */
export interface SourceCleanupFailure {
  readonly path: string;
  readonly message: string;
  /** The errno code the removal failed with, when the error carried one. */
  readonly code?: string;
}

export interface SourceCleanupPlan {
  /** Canonical form of the queried source. */
  readonly source: string;
  /** False on a dry run (the default); true only when `confirm` was set. */
  readonly confirmed: boolean;
  readonly includeOriginals: boolean;
  /** Single-purpose derived entries removed (or that would be) on confirm. */
  readonly derived: ReadonlyArray<SourceCleanupEntry>;
  /** Referencing pages reported but NOT auto-deleted (aggregates / shared). */
  readonly mentions: ReadonlyArray<SourceCleanupEntry>;
  /** Original source file(s) outside `Brain/`; removed only with includeOriginals. */
  readonly originals: ReadonlyArray<string>;
  /** Manifest key for the source, when the content manifest tracks it. */
  readonly manifestEntry: string | null;
  /** Paths actually removed this run (empty on a dry run). */
  readonly deleted: ReadonlyArray<string>;
  /**
   * Per-entry removal failures this run, with the named fs error (empty on a
   * dry run and on a clean run). One blocked file never aborts the cleanup:
   * every remaining entry is still attempted, the failures are reported here
   * with the failed paths left on disk, and the audit record counts both
   * sides honestly.
   */
  readonly failed: ReadonlyArray<SourceCleanupFailure>;
  readonly manifestEntryRemoved: boolean;
  /** Id of the audit continuity record, when one was written. */
  readonly auditRecordId: string | null;
  /**
   * Run id of the pre-deletion snapshot taken behind the D1 gate, or
   * null when none was taken (a dry run, or a confirmed run with nothing
   * to delete). This is the recovery point the operator can roll back to
   * FOR THE PART OF THE DELETION IT COVERS - see {@link recoverability}.
   */
  readonly snapshotRunId: string | null;
  /** Absolute path of the snapshot archive, paired with {@link snapshotRunId}. */
  readonly snapshotPath: string | null;
  /**
   * What that recovery point is actually worth for this run.
   *
   * `--include-originals` is the reason this field exists. An original is
   * only ever returned when it lies OUTSIDE `Brain/` (see
   * {@link findOriginals}), and the snapshot archive holds top-level
   * entries under `Brain/` and nothing else. So a confirmed run with that
   * flag deletes files no archive has ever held while reporting a
   * `snapshotRunId` beside them - a response that reads as fully
   * reversible and is not. The verdict names the uncovered lane instead
   * of leaving the caller to infer it.
   */
  readonly recoverability: RecoverabilityVerdict;
  /** Total referencing pages + originals (the reported blast radius). */
  readonly blastRadius: number;
}

export interface DeleteBySourceOptions {
  /** Required true to actually delete; absent/false = dry run. */
  readonly confirm?: boolean;
  /** Also remove the original source file(s) outside `Brain/`. */
  readonly includeOriginals?: boolean;
  /** Injected clock for the audit record's timestamp. */
  readonly now?: Date;
  /** Agent identity recorded in the audit reason. */
  readonly agent?: string;
  /**
   * May the caller read the page or the original at this vault-relative
   * path? Asked before anything is stat-ed or deleted, so a derived page,
   * a mention, an original or a manifest entry whose source it may not
   * read is planned, counted and left on disk exactly as an absent one
   * would be. Absent includes everything (the CLI, which reads the vault
   * directly).
   */
  readonly include?: (rel: string) => boolean;
  /**
   * The credential-minted subject of the caller (write-side-trust, Task
   * 7), when the transport resolved one. It WINS over `agent` as the
   * identity the permissions document consult answers - a caller-supplied
   * agent is content attribution, never the identity a document's rules
   * answer. Absent (stdio, the CLI) the consult falls back to the config
   * identity, exactly like every other lane.
   */
  readonly subject?: WriteSubject;
}

/**
 * The spellings that name ONE subject, and the canonical form they all
 * reduce to.
 *
 * `deleteBySource` has exactly one spelling - the canonical path of the
 * imported file, which is how the ingest pipeline stamps `source_path`
 * and how its provenance wikilinks are written. A NOTE is the same
 * subject reached from the other side and Obsidian writes it two ways,
 * `[[Projects/Note.md]]` and `[[Projects/Note]]`, so a tracer that knew
 * only the first would leave a solely-derived signal unmatched and report
 * it as a protected mention - a safe answer, but the wrong one.
 *
 * The bare `[[Note]]` spelling is deliberately NOT a member of any
 * identity built here. It resolves to one note only when exactly one note
 * in the vault carries the basename (`store/links.ts:66-69`), so
 * attributing it would let a page derived from a DIFFERENT note of the
 * same name into a deletion set. The caller still reports it: the
 * note-delete surface's own inbound probe reads all three spellings.
 */
interface SourceIdentity {
  /** Canonical form of the subject's own path. */
  readonly canonical: string;
  /** The subject as the caller spelled it, for `session_ref` equality. */
  readonly sourceFile: string;
  /** Every wikilink / `source_path` spelling that names the subject. */
  readonly canonicalSpellings: ReadonlySet<string>;
  /** The same spellings in match order, for the wikilink probes. */
  readonly spellings: ReadonlyArray<string>;
}

/** One canonical path, spelled one way: `deleteBySource`'s subject. */
function exactSourceIdentity(sourceFile: string): SourceIdentity {
  const canonical = canonicalNotePath(sourceFile);
  return Object.freeze({
    canonical,
    sourceFile,
    canonicalSpellings: new Set([canonical]),
    spellings: Object.freeze([canonical]),
  });
}

/** The `.md` suffix, matched case-insensitively as the note surfaces do. */
const MARKDOWN_SUFFIX = ".md";

/** One note path, spelled with and without its `.md` suffix. */
function noteIdentity(notePath: string): SourceIdentity {
  const canonical = canonicalNotePath(notePath);
  const bare = canonical.toLowerCase().endsWith(MARKDOWN_SUFFIX)
    ? canonical.slice(0, -MARKDOWN_SUFFIX.length)
    : canonical;
  const spellings = bare === canonical ? [canonical] : [canonical, bare];
  return Object.freeze({
    canonical,
    sourceFile: notePath,
    canonicalSpellings: new Set(spellings),
    spellings: Object.freeze(spellings),
  });
}

/** Does `target` - any spelling - name this subject? */
function namesSubject(identity: SourceIdentity, target: string): boolean {
  return identity.canonicalSpellings.has(canonicalNotePath(target));
}

/** Directories whose pages are per-item, single-source derived artifacts. */
const DERIVATION_DIRS: ReadonlyArray<string> = Object.freeze([
  BRAIN_SOURCES_REL,
  BRAIN_INBOX_REL,
  BRAIN_PREFERENCES_REL,
  BRAIN_ENTITIES_REL,
  BRAIN_RETIRED_REL,
]);

function escapeRegExp(value: string): string {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

/**
 * A wikilink to exactly `target`: `[[target]]`, `[[target|alias]]`, or
 * `[[target#heading]]`. Anchored on the `[[` boundary so `[[foo]]` never
 * matches `[[foobar]]`.
 */
function wikilinkRegExp(target: string): RegExp {
  return new RegExp(`\\[\\[${escapeRegExp(target)}(?:[|#][^\\]]*)?\\]\\]`);
}

function stringField(meta: FrontmatterMap, key: string): string | null {
  const value = meta[key];
  return typeof value === "string" && value.length > 0 ? value : null;
}

/**
 * A preference's evidence links. The writer stores the field under the
 * managed `_evidenced_by` key (with a plain `evidenced_by` on legacy
 * pages), so consult both.
 */
function evidencedByLinks(meta: FrontmatterMap): ReadonlyArray<string> {
  const managed = stringArrayField(meta, "_evidenced_by");
  return managed.length > 0 ? managed : stringArrayField(meta, "evidenced_by");
}

function classifyKind(vault: string, absPath: string): SourceCleanupKind {
  const dirs = brainDirs(vault);
  if (insideDir(absPath, join(vault, BRAIN_SOURCES_REL))) return "summary";
  if (insideDir(absPath, dirs.retired)) return "retired";
  if (insideDir(absPath, dirs.preferences)) return "preference";
  if (insideDir(absPath, dirs.inbox)) return "signal";
  if (insideDir(absPath, dirs.entities)) return "entity";
  if (insideDir(absPath, join(vault, BRAIN_REPORTS_REL))) return "report";
  if (insideDir(absPath, dirs.log)) return "log";
  return "other";
}

function insideDir(path: string, dir: string): boolean {
  return path === dir || path.startsWith(`${dir}${sep}`);
}

function inDerivationDir(vault: string, absPath: string): boolean {
  return DERIVATION_DIRS.some((rel) => insideDir(absPath, join(vault, rel)));
}

function readId(meta: FrontmatterMap, absPath: string): string {
  const id = stringField(meta, "id") ?? stringField(meta, "entity_id");
  if (id) return id;
  const base = absPath.slice(absPath.lastIndexOf(sep) + 1);
  return base.endsWith(".md") ? base.slice(0, -3) : base;
}

interface RawMatch {
  readonly absPath: string;
  readonly path: string;
  readonly id: string;
  readonly kind: SourceCleanupKind;
  readonly match: SourceCleanupMatch;
  readonly isIndexArtifact: boolean;
  /** Structured source links this page carries (signal `source` array). */
  readonly sourceLinks: ReadonlyArray<string>;
  /** Structured evidence links this page carries (preference `evidenced_by`). */
  readonly evidencedBy: ReadonlyArray<string>;
}

/**
 * Classify one Brain page against the canonical source. Returns null when
 * the page does not reference the source at all. Only DIRECT references are
 * detected here (source_path / session_ref / `[[source]]` wikilink); the
 * transitive `evidenced_by` fold is resolved in a second pass once the set
 * of directly-derived signals is known.
 */
function directMatch(
  vault: string,
  absPath: string,
  identity: SourceIdentity,
  storeOnly: boolean,
): RawMatch | null {
  const page = brainPageAt(vault, absPath, storeOnly);
  if (page === null) return null;
  const text = page.text;
  // Unit F: `parseFrontmatter` cannot throw (it reads inside its own try;
  // everything after is string work), so the `catch { meta = {} }` that
  // stood here was unreachable - and its intent is what the parser
  // already does, returning an empty map for a failed read, so a page
  // whose frontmatter is unavailable is still scanned as raw text.
  // Dropped lines are reported centrally; see the "Why most readers keep
  // the two-tuple form" section in src/core/vault.ts.
  const meta = page.meta;

  const kind = classifyKind(vault, absPath);
  const sourceLinks = stringArrayField(meta, "source");
  // Managed `_evidenced_by` links take precedence over the legacy plain key
  // (see evidencedByLinks). Reading only the plain key here let a preference
  // whose evidence lives in `_evidenced_by` slip past the foreign-evidence
  // guard in computeDeletable and be auto-deleted as a first-pass wikilink
  // match, even when it is a shared fold.
  const evidencedBy = evidencedByLinks(meta);
  const base = {
    absPath,
    path: vaultRelative(absPath, vault),
    id: readId(meta, absPath),
    kind,
    sourceLinks,
    evidencedBy,
  };

  const sourcePath = stringField(meta, "source_path");
  if (sourcePath !== null && namesSubject(identity, sourcePath)) {
    return {
      ...base,
      match: "source_path",
      isIndexArtifact: meta["kind"] === BRAIN_SOURCE_KIND,
    };
  }

  const sessionRef = stringField(meta, "session_ref");
  if (
    sessionRef !== null &&
    (sessionRef === identity.canonical || sessionRef === identity.sourceFile)
  ) {
    return { ...base, match: "session_ref", isIndexArtifact: false };
  }

  if (identity.spellings.some((spelling) => wikilinkRegExp(spelling).test(text))) {
    return { ...base, match: "wikilink", isIndexArtifact: false };
  }

  return null;
}

/**
 * A directly-derived page traces SOLELY to this source when it declares no
 * OTHER structured source link. A signal whose `source` array names a
 * second source is a shared observation — reported, never auto-deleted.
 */
function tracesSolelyToSource(raw: RawMatch, identity: SourceIdentity): boolean {
  for (const link of raw.sourceLinks) {
    if (!namesSubject(identity, wikilinkTarget(link))) return false;
  }
  return true;
}

/**
 * Does the page declare the subject as its PROVENANCE, or does it merely
 * mention it?
 *
 * Every match except `wikilink` is structured by construction: a
 * `source_path`, a `session_ref` and the `evidenced_by` fold are all
 * frontmatter this project writes. A wikilink match is structured only
 * when one of the page's own `source:` links names the subject - the same
 * spelling in a paragraph of prose is a reference, not a derivation, and
 * {@link tracesSolelyToSource} cannot tell the two apart because a page
 * with no `source:` array satisfies it vacuously.
 */
function hasStructuredProvenance(raw: RawMatch, identity: SourceIdentity): boolean {
  if (raw.match !== "wikilink") return true;
  return raw.sourceLinks.some((link) => namesSubject(identity, wikilinkTarget(link)));
}

interface Traced {
  readonly derived: ReadonlyArray<SourceCleanupEntry>;
  readonly mentions: ReadonlyArray<SourceCleanupEntry>;
  /**
   * Vault-relative paths of the pages that named the subject in
   * structured provenance. Carried beside the two lists rather than on
   * the entries so `deleteBySource`'s payload keeps the exact shape it
   * shipped with; {@link traceNoteDerivations} is the one reader.
   */
  readonly structured: ReadonlySet<string>;
}

/**
 * Walk `Brain/` and classify every page referencing the source into the
 * deletable-derived set and the reported-only mentions set. Deterministic:
 * both lists are sorted by vault-relative path.
 */
function traceReferences(vault: string, identity: SourceIdentity): Traced {
  const root = brainDirs(vault).brain;
  // Decided once per trace: every page of both passes is read the same way.
  const storeOnly = isStoreOnly({ vault });
  const rawMatches: RawMatch[] = [];
  if (storeOnly || existsSync(root)) {
    for (const absPath of walkBrainMarkdown(vault, root, storeOnly)) {
      const m = directMatch(vault, absPath, identity, storeOnly);
      if (m !== null) rawMatches.push(m);
    }
  }

  // Second pass: a preference folded SOLELY from directly-derived signals
  // is itself a derived entry. One evidenced by any foreign signal is a
  // shared fold — reported, never auto-deleted.
  const derivedSignalIds = new Set(rawMatches.filter((m) => m.kind === "signal").map((m) => m.id));
  const alreadyMatched = new Set(rawMatches.map((m) => m.absPath));
  if (storeOnly || existsSync(root)) {
    for (const absPath of walkBrainMarkdown(vault, root, storeOnly)) {
      if (alreadyMatched.has(absPath)) continue;
      const kind = classifyKind(vault, absPath);
      if (kind !== "preference") continue;
      // Unit F: unreachable `catch { continue }` removed - see directMatch
      // above. An unreadable page yields an empty map, so it has no
      // `_evidenced_by` links and falls out on the next line exactly as
      // the dead branch intended.
      const meta = brainPageAt(vault, absPath, storeOnly)?.meta ?? {};
      const evidencedBy = evidencedByLinks(meta);
      const targets = evidencedBy.map((link) => wikilinkTarget(link));
      if (targets.length === 0) continue;
      if (!targets.some((t) => derivedSignalIds.has(t))) continue;
      rawMatches.push({
        absPath,
        path: vaultRelative(absPath, vault),
        id: readId(meta, absPath),
        kind,
        match: "evidenced_by",
        isIndexArtifact: false,
        sourceLinks: [],
        evidencedBy,
      });
    }
  }

  const derived: SourceCleanupEntry[] = [];
  const mentions: SourceCleanupEntry[] = [];
  const structured = new Set<string>();
  for (const raw of rawMatches) {
    const deletable = computeDeletable(vault, raw, identity, derivedSignalIds);
    const entry: SourceCleanupEntry = {
      path: raw.path,
      id: raw.id,
      kind: raw.kind,
      match: raw.match,
      isIndexArtifact: raw.isIndexArtifact,
      deletable,
    };
    if (hasStructuredProvenance(raw, identity)) structured.add(entry.path);
    (deletable ? derived : mentions).push(entry);
  }
  return {
    derived: derived.toSorted(byPath),
    mentions: mentions.toSorted(byPath),
    structured,
  };
}

function byPath(a: SourceCleanupEntry, b: SourceCleanupEntry): number {
  return a.path.localeCompare(b.path);
}

function computeDeletable(
  vault: string,
  raw: RawMatch,
  identity: SourceIdentity,
  derivedSignalIds: ReadonlySet<string>,
): boolean {
  // The ingest summary page is always an index artifact to purge.
  if (raw.isIndexArtifact) return true;
  // Only per-item derivation pages are ever auto-deleted.
  if (!inDerivationDir(vault, raw.absPath)) return false;
  // A signal citing a second source is a shared observation — report it.
  if (!tracesSolelyToSource(raw, identity)) return false;
  // A preference folded from any foreign signal is a shared fold — report it.
  // This holds however the preference was first matched: a second-pass
  // `evidenced_by` fold OR a first-pass `[[source]]` wikilink match that also
  // carries evidence links. Keying only on match === "evidenced_by" let a
  // wikilink-matched shared preference reach `--confirm` deletion.
  if (raw.kind === "preference" && raw.evidencedBy.length > 0) {
    const targets = raw.evidencedBy.map((link) => wikilinkTarget(link));
    if (!targets.every((t) => derivedSignalIds.has(t))) return false;
  }
  return true;
}

/**
 * A `Brain/` page as this scan reads it, its raw text and frontmatter: the
 * file on an unswitched machine; on a store-only one the page a lane entry
 * or a memory record (preference, retired, signal) renders. `null` for a
 * page that is not there.
 */
function brainPageAt(
  vault: string,
  absPath: string,
  storeOnly: boolean,
): { readonly text: string; readonly meta: FrontmatterMap } | null {
  if (!storeOnly) {
    let text: string;
    try {
      text = readFileSync(absPath, "utf8");
    } catch {
      return null;
    }
    return { text, meta: parseFrontmatter(absPath)[0] };
  }
  const memory = storeMemoryPageAt(vault, vaultRelative(absPath, vault));
  if (memory !== null) {
    return { text: formatFrontmatter(memory.meta, memory.body), meta: memory.meta };
  }
  const text = readLaneText(vault, absPath);
  return text === null ? null : { text, meta: parseLanePage(vault, absPath)[0] };
}

/**
 * Every `.md` page under `root`: a walk of the tree on an unswitched
 * machine; on a store-only one the lane entries and the memory records'
 * pages, by the paths they had in the legacy tree. Dot folders are skipped
 * in both, as the walk skips them.
 */
function walkBrainMarkdown(vault: string, root: string, storeOnly: boolean): string[] {
  if (storeOnly) {
    const memory = storeRecordPages(vault, { signalStates: "all" }).map((page) =>
      join(vault, ...page.rel.split("/")),
    );
    const lanes = listLaneTree(vault, root);
    return [...new Set([...memory, ...lanes])]
      .filter(
        (path) =>
          !vaultRelative(path, vault)
            .split("/")
            .some((seg) => seg.startsWith(".")),
      )
      .toSorted();
  }
  const out: string[] = [];
  const visit = (dir: string): void => {
    let names: string[];
    try {
      names = readdirSync(dir);
    } catch {
      return;
    }
    for (const name of names) {
      if (name.startsWith(".")) continue;
      const full = join(dir, name);
      let st;
      try {
        st = statSync(full);
      } catch {
        continue;
      }
      if (st.isDirectory()) {
        visit(full);
        continue;
      }
      if (st.isFile() && name.endsWith(".md")) out.push(full);
    }
  };
  visit(root);
  return out.toSorted();
}

/**
 * Every Brain page tracing back to the EXACT source — the derived entries
 * and the protected mentions, merged and sorted by path. Read-only.
 */
export function searchBySourceFile(
  vault: string,
  sourceFile: string,
): ReadonlyArray<SourceCleanupEntry> {
  const { derived, mentions } = traceReferences(vault, exactSourceIdentity(sourceFile));
  return Object.freeze([...derived, ...mentions].toSorted(byPath));
}

/**
 * The vault-relative root the derived-set fold walks, and the whole of
 * it: `Brain/`.
 *
 * Named rather than implied because a caller reporting an empty derived
 * set has to be able to say whether that is a measurement or a scope
 * boundary. Only `Brain/` holds per-item derivation pages, so a user note
 * outside it is never a candidate for auto-deletion however tightly it
 * references the subject - it is somebody's prose, and this fold does not
 * reach it.
 */
export const DERIVATION_SCAN_SCOPE = "Brain/";

/**
 * Every Brain page tracing back to one NOTE path, classified by the same
 * blast-radius rule {@link deleteBySource} applies to an imported source:
 * `deletable` marks a single-purpose derived file tracing SOLELY to the
 * note, everything else is a reported mention.
 *
 * Read-only. The caller decides what to do with the two lists; this
 * function's whole contribution is that the rule deciding which list a
 * page lands in is the SAME function in both cases, so a note delete and
 * a source delete cannot come to disagree about what "derived solely
 * from" means.
 *
 * Each entry additionally carries
 * {@link NoteDerivationEntry.structuredProvenance}, which the shared rule
 * does not read: a note, unlike an imported source, is something ordinary
 * prose links to, and the cascade needs to tell a declared derivation
 * from a sentence.
 */
export function traceNoteDerivations(
  vault: string,
  notePath: string,
): ReadonlyArray<NoteDerivationEntry> {
  const { derived, mentions, structured } = traceReferences(vault, noteIdentity(notePath));
  return Object.freeze(
    [...derived, ...mentions]
      .toSorted(byPath)
      .map((entry) =>
        Object.freeze({ ...entry, structuredProvenance: structured.has(entry.path) }),
      ),
  );
}

/**
 * Original source file(s) outside `Brain/`: the imported artifact itself.
 * A source that lives inside `Brain/` (or that has no on-disk file — e.g. a
 * URL identity) contributes no original.
 */
function findOriginals(
  vault: string,
  canonical: string,
  include: ((rel: string) => boolean) | undefined,
): string[] {
  let abs: string;
  try {
    abs = ensureInsideVault(join(vault, canonical), vault);
  } catch {
    return [];
  }
  // Before the stat: an original the caller may not read is answered as
  // an absent one, so neither its existence nor its removal is decided by
  // a caller that cannot see it.
  if (include?.(canonical) === false) return [];
  if (!existsSync(abs)) return [];
  try {
    if (!statSync(abs).isFile()) return [];
  } catch {
    return [];
  }
  if (insideDir(abs, brainDirs(vault).brain)) return [];
  return [canonical];
}

/**
 * Remove one vault-relative file. `false` for an already-absent path (the
 * idempotent re-run), `true` when it was removed. Any other fs error throws:
 * callers that sweep many files collect per-file failures around this call
 * ({@link deleteBySource} does; the retire applier consumes it the same way),
 * never silently.
 */
export function removeFile(vault: string, relPath: string): boolean {
  let abs: string;
  try {
    abs = ensureInsideVault(join(vault, relPath), vault);
  } catch {
    return false;
  }
  if (isStoreOnly({ vault }) && insideDir(abs, brainDirs(vault).brain)) {
    return removeStorePage(vault, relPath, abs);
  }
  try {
    rmSync(abs);
    return true;
  } catch (err) {
    if ((err as NodeJS.ErrnoException)?.code === "ENOENT") return false;
    throw err;
  }
}

/**
 * Remove one `Brain/` page on a store-only machine, by what holds it: a
 * preference or a retired page writes the preference's tombstone
 * (`preference-remove`), a signal its tombstone (`signal-removed`), any other
 * page removes its lane entry. `false` when there was nothing to remove.
 */
function removeStorePage(vault: string, relPath: string, abs: string): boolean {
  const memory = storeMemoryPageAt(vault, relPath);
  if (memory === null) {
    if (!lanePageExists(vault, abs)) return false;
    removeLanePage(vault, abs);
    return true;
  }
  if (memory.kind === "signal") {
    // A request to forget: the signal's tombstone, after which the fold
    // drops it and its content from every read (FORMAT.md, Removal).
    return removeStoreSignal(vault, memory.id);
  }
  // The fold keys a preference by `pref-<slug>` in both lives: a retired
  // page's id (`ret-<slug>`) names no record of its own.
  const preference = `pref-${stripBrainIdPrefix(memory.id)}`;
  const isPrivate = memory.private;
  withOperationPrivacy(isPrivate, () =>
    storeWrite(
      () => ({
        stream: "events",
        kind: PREFERENCE_REMOVE_KIND,
        private: isPrivate,
        body: { [AUTHORITY_FIELD]: authorityFromOrigin(), preference },
      }),
      { vault },
    ),
  );
  // Removed only when the fold no longer shows the page.
  return storeMemoryPageAt(vault, relPath) === null;
}

/**
 * Plan and (with `confirm`) execute the removal of everything derived from
 * one exact source file. Dry-run by default.
 */
export function deleteBySource(
  vault: string,
  sourceFile: string,
  opts: DeleteBySourceOptions = {},
): SourceCleanupPlan {
  const identity = exactSourceIdentity(sourceFile);
  const canonical = identity.canonical;
  const confirm = opts.confirm === true;
  const includeOriginals = opts.includeOriginals === true;
  // Lane write guard: the vault identity guard on an unswitched machine,
  // the store identity check on a store-only one (design D14 item 13).
  // Dry-run (the default) plans and writes nothing, so it stays ungated.
  if (confirm) assertLaneWrite(vault);
  // Deny-consulted evidence (write-side trust, Task 12): a confirmed
  // cleanup is a mutation lane like update or append - it removes the
  // derived entries and the ingest manifest entry - so the document is
  // resolved against the credential subject and a deny refuses with the
  // SAME typed error BEFORE anything is read for deletion, snapshotted,
  // or removed, one decision-ledger row behind it. A dry run (the
  // default) writes nothing, so it stays a read and stays ungated. With
  // no permissions document this is a no-op and the cleanup proceeds
  // exactly as before.
  if (confirm) {
    refuseDocumentDeny(vault, REVIEW_LANE.ingest, opts.subject, {
      target: canonical,
      ...(opts.now !== undefined ? { now: opts.now } : {}),
    });
  }

  // A page or a source the caller may not read is planned, counted and
  // left on disk exactly as an absent one: a remote caller can neither
  // learn of a reserved summary page nor delete it.
  const included = (rel: string): boolean => opts.include?.(rel) !== false;
  const traced = traceReferences(vault, identity);
  const derived = traced.derived.filter((entry) => included(entry.path));
  const mentions = traced.mentions.filter((entry) => included(entry.path));
  const originals = findOriginals(vault, canonical, opts.include);
  // The originals are user-vault files in both modes. On a store-only
  // machine the lane guard above checks the store only, so their removal
  // runs the vault adapter's guard (the bound vault, not frozen) up front.
  if (confirm && includeOriginals && originals.length > 0 && isStoreOnly({ vault })) {
    assertNoteVaultForWrite(vault, "delete_by_source removing originals");
  }
  const manifestKey =
    readManifest(vault).entries[canonical] !== undefined &&
    !isSourceHidden(vault, canonical, opts.include)
      ? canonical
      : null;
  const blastRadius = derived.length + mentions.length + originals.length;

  /**
   * The lanes this run actually destroys, in the regions the archive is
   * defined over. `mentions` are reported and never deleted, so they are
   * not in it; the manifest entry is a `Brain/` index artifact, so it is.
   */
  const destroyed: DestructiveBlastRadius = {
    brainTopLevel: derived.length > 0 || manifestKey !== null,
    outsideBrainRoot: includeOriginals && originals.length > 0,
  };

  if (!confirm) {
    return Object.freeze({
      source: canonical,
      confirmed: false,
      includeOriginals,
      derived: Object.freeze(derived),
      mentions: Object.freeze(mentions),
      originals: Object.freeze(originals),
      manifestEntry: manifestKey,
      deleted: Object.freeze([]),
      failed: NO_FAILURES,
      manifestEntryRemoved: false,
      auditRecordId: null,
      snapshotRunId: null,
      snapshotPath: null,
      // A dry run destroys nothing, so it promises nothing. Reporting the
      // coverage the confirmed run WOULD have would be a claim about an
      // archive that does not exist yet.
      recoverability: classifyRecoverability({ recoveryPoint: false, blastRadius: {} }),
      blastRadius,
    });
  }

  // Confirmed path — delete the derived entries first, then (opt-in) the
  // originals, then drop the manifest entry (an index artifact).
  const deleted: string[] = [];
  const failed: SourceCleanupFailure[] = [];
  let manifestEntryRemoved = false;
  // Set when the manifest entry deliberately STAYED because an original's
  // removal failed (review L7) - the audit names the keep, and the result's
  // `manifestEntryRemoved` stays false.
  let manifestEntryKeptForFailure = false;

  /**
   * Attempt one removal, collecting a failure instead of throwing: one
   * blocked file must not abort the entries still queued behind it. The
   * collected failures ride the result and the audit, and the paths stay on
   * disk for a retry.
   */
  const removeCollecting = (relPath: string): void => {
    try {
      if (removeFile(vault, relPath)) deleted.push(relPath);
    } catch (err) {
      const code = (err as NodeJS.ErrnoException | null)?.code;
      failed.push({
        path: relPath,
        message: (err as Error)?.message ?? String(err),
        ...(typeof code === "string" ? { code } : {}),
      });
    }
  };

  // Audit only a run that actually changed something; a no-op re-run writes
  // no record (idempotent). Factored so both the success path AND a partial
  // failure can persist the record of what was already removed.
  const writeInvalidationAudit = (): string | null => {
    if (deleted.length === 0 && !manifestEntryRemoved && !manifestEntryKeptForFailure) {
      return null;
    }
    const agent = opts.agent?.trim() ? opts.agent.trim() : "delete_by_source";
    const reason =
      `${agent}: removed ${deleted.length} derived/original entr${deleted.length === 1 ? "y" : "ies"}` +
      `${manifestEntryRemoved ? " and the ingest manifest entry" : ""}` +
      `${manifestEntryKeptForFailure ? " (ingest manifest entry kept: an original's removal failed)" : ""}` +
      `${includeOriginals ? " (originals included)" : ""}` +
      (failed.length > 0
        ? `; failed to remove ${failed.length}: ${failed.map((f) => f.path).join(", ")}`
        : "");
    const record = appendContinuitySourceInvalidation(vault, {
      createdAt: isoSecond(opts.now ?? new Date()),
      source: { id: canonical, path: canonical, kind: "source" },
      reason,
    });
    return record.id;
  };

  let auditRecordId: string | null = null;

  // The destructive core: remove derived entries, opt-in originals, and
  // the manifest index entry, then audit. Runs behind the D1 snapshot
  // gate so every confirmed cleanup has a pre-deletion recovery point.
  const runDeletion = (): void => {
    try {
      for (const entry of derived) removeCollecting(entry.path);
      if (includeOriginals) {
        for (const original of originals) removeCollecting(original);
      }

      // An original whose removal FAILED keeps its manifest entry (review
      // L7): the source file is still on disk, so dropping the entry would
      // classify it `new` at the next batch plan and the planner would
      // rebuild the very derived pages this run just removed. The audit
      // names the keep.
      const originalRemovalFailed =
        includeOriginals && originals.some((o) => failed.some((f) => f.path === o));
      manifestEntryKeptForFailure = originalRemovalFailed && manifestKey !== null;
      if (manifestKey !== null && existsSync(manifestPath(vault)) && !originalRemovalFailed) {
        const manifest = readManifest(vault);
        const entries = { ...manifest.entries };
        if (entries[manifestKey] !== undefined) {
          delete entries[manifestKey];
          // Round-trip the contract the manifest was read with (t_586d5d8b):
          // a cleanup that only removes an entry must not re-stamp the
          // extraction contract the remaining entries were recorded under -
          // and a v1 read (contract `null`) round-trips as v1, because its
          // survivors were never reprocessed under any fingerprint. Upgrading
          // them here would classify them `unchanged` forever and silently
          // forfeit the owed one-time reprocess.
          manifestEntryRemoved = writeManifestAtomic(vault, entries, manifest.contract);
        }
      }
    } catch (err) {
      // A manifest read/write that throws mid-cleanup would otherwise exit
      // before the audit append, leaving already-deleted paths with no record
      // for a retry to reconstruct. Persist what was removed (and what was
      // collected as failed) so far, then rethrow; never let an audit-write
      // error mask the original failure.
      try {
        writeInvalidationAudit();
      } catch {
        /* swallow: the original error below is the one that matters */
      }
      throw err;
    }
    auditRecordId = writeInvalidationAudit();
  };

  // Only snapshot when there is something to remove. A confirmed run over
  // a source with no derived material, no included originals, and no
  // manifest entry is a no-op and must stay one - snapshotting an empty
  // deletion would churn `.snapshots/` for nothing.
  const hasWork =
    derived.length > 0 || (includeOriginals && originals.length > 0) || manifestKey !== null;

  let snapshotRunId: string | null = null;
  let snapshotArchivePath: string | null = null;
  let recoverability: RecoverabilityVerdict;
  if (hasWork) {
    const gated = withDestructiveSnapshot(
      vault,
      BRAIN_SNAPSHOT_REASON.deleteBySource,
      runDeletion,
      { now: opts.now, blastRadius: destroyed },
    );
    snapshotRunId = gated.snapshot.runId;
    snapshotArchivePath = gated.snapshot.path;
    recoverability = gated.recoverability;
  } else {
    runDeletion();
    // No work means no snapshot AND nothing destroyed. The classifier
    // says `nothing_at_risk` for that, which is neither the alarm of
    // `unproven` nor the false comfort of `covered`.
    recoverability = classifyRecoverability({ recoveryPoint: false, blastRadius: destroyed });
  }

  return Object.freeze({
    source: canonical,
    confirmed: true,
    includeOriginals,
    derived: Object.freeze(derived),
    mentions: Object.freeze(mentions),
    originals: Object.freeze(originals),
    manifestEntry: manifestKey,
    deleted: Object.freeze(deleted),
    failed: Object.freeze(failed),
    manifestEntryRemoved,
    auditRecordId,
    snapshotRunId,
    snapshotPath: snapshotArchivePath,
    recoverability,
    blastRadius,
  });
}