Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

174 KB
Raw Blame
import { randomUUID } from 'node:crypto';
import {
  type BigIntStats,
  type Dirent,
  existsSync,
  lstatSync,
  readdirSync,
  readFileSync,
  realpathSync,
  type Stats,
  statSync,
  watch as watchFsPath,
} from 'node:fs';
import { lstat, readdir, readFile, realpath, stat } from 'node:fs/promises';
import { basename, dirname, extname, isAbsolute, join, relative, sep } from 'node:path';
import { DOCUMENT_OPEN_BYTE_LIMIT, LINKABLE_ASSET_EXTENSIONS } from '@inkeep/open-knowledge-core';
import { filesystemIdentity, statMtime } from '@inkeep/open-knowledge-core/server';
import { isConfigDoc, isReservedForUserTree, isSystemDoc } from './cc1-broadcast.ts';
import { type ContentFilter, WATCHER_STRUCTURAL_IGNORE_DIRS } from './content-filter.ts';
import { isWithinContentDir } from './content-path.ts';
import { resolveDirectoryRoot, resolveNativePath } from './directory-root.ts';
import {
  forgetDocExtension,
  getDocExtension,
  isSupportedAssetFile,
  isSupportedDocFile,
  registerDocExtension,
  SUPPORTED_DOC_EXTENSIONS,
  stripDocExtension,
} from './doc-extensions.ts';
import { canonicalContentPathIsRefused, resolvesIntoPrivateState } from './fs-safety.ts';
import {
  classifyFsPath,
  normalizeFsPath,
  tracedMkdirSync,
  tracedRmSync,
  tracedWriteFileSync,
} from './fs-traced.ts';
import { errnoCode } from './http/handler-utils.ts';
import { getLogger } from './logger.ts';
import {
  addFreshnessCarryOver,
  addFreshnessFilesHashed,
  addFreshnessFinds,
  incrementFreshnessPasses,
  incrementFreshnessPassesIncomplete,
  incrementWatcherErrors,
  incrementWatcherUnreadableEntries,
} from './metrics.ts';
import {
  extractPageDescription,
  extractPageIcon,
  extractPageTitle,
  extractPageType,
} from './page-identity.ts';
import { classifyParcelNotification } from './parcel-recovery.ts';
import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';
import { toPosix } from './path-utils.ts';
import { containsConflictMarkers } from './reconciliation.ts';
import { isRacyStat, sampleWallClockNs, statSignature } from './stat-signature.ts';
import { getMeter, withSpan } from './telemetry.ts';
import { contentHash } from './version-hash.ts';
import {
  installWatcherClock,
  type WatcherClock,
  type WatcherTimer,
  watcherClock,
} from './watcher-clock.ts';
import {
  createResubscriber,
  FRESHNESS_HASH_BUDGET,
  FRESHNESS_MAX_INTERVAL_MS,
  nextFreshnessDelayMs,
  RESUBSCRIBE_SLOW_RETRY_MS,
  type RecoveryMode,
  resolveFreshnessInterval,
} from './watcher-freshness.ts';

export type { NativeWatcherBackend } from './parcel-subscription.ts';
export { contentHash } from './version-hash.ts';

const log = getLogger('file-watcher');

export interface AsyncSubscription {
  unsubscribe(): Promise<void>;
}

type WatcherBackend = 'parcel' | 'chokidar';

type MarkdownDiskEvent =
  | { kind: 'create'; path: string; docName: string; content: string }
  | {
      kind: 'update';
      path: string;
      docName: string;
      content: string;
      previousIndexedFields?: {
        title?: string;
        description?: string;
        type?: string;
      };
    }
  | { kind: 'delete'; path: string; docName: string }
  | {
      kind: 'rename';
      oldPath: string;
      newPath: string;
      oldDocName: string;
      newDocName: string;
      content: string;
    }
  | { kind: 'conflict'; path: string; docName: string; content: string };

type AssetDiskEvent =
  | { kind: 'asset-create'; path: string; relativePath: string }
  | { kind: 'asset-delete'; path: string; relativePath: string };

type FolderDiskEvent =
  | { kind: 'folder-create'; path: string; relativePath: string }
  | { kind: 'folder-delete'; path: string; relativePath: string };

type FileDiskEvent =
  | {
      kind: 'file-create';
      path: string;
      relativePath: string;
      size: number;
      modifiedTs: number;
      inode: string;
    }
  | {
      kind: 'file-update';
      path: string;
      relativePath: string;
      size: number;
      modifiedTs: number;
      inode: string;
    }
  | { kind: 'file-delete'; path: string; relativePath: string };

export type DiskEvent = MarkdownDiskEvent | AssetDiskEvent | FolderDiskEvent | FileDiskEvent;

export function assertNeverDiskEvent(event: never): never {
  throw new Error(`[DiskEvent] unhandled variant: ${JSON.stringify(event)}`);
}

export type AllFileEntries = Iterable<readonly [string, FileIndexEntry]>;

interface GeneralFileMembers {
  readonly regularPaths: readonly string[];
  readonly symlinks: readonly { readonly path: string; readonly targetPath: string }[];
}

export interface FileIndexEntry {
  size: number;
  modified: string;
  canonicalPath: string;
  inode: string;
  aliases: string[];
  kind: 'markdown' | 'file';
  fileMembers?: GeneralFileMembers;
  title?: string;
  icon?: string;
  description?: string;
  type?: string;
}

interface FileIndexEntryMember {
  readonly path: string;
  readonly role: 'regular' | 'symlink';
  readonly targetPath: string;
}

interface FileIndexEntryMembership {
  readonly resolved: boolean;
  readonly members: readonly FileIndexEntryMember[];
}

export function fileIndexEntryMembers(
  contentDir: string,
  name: string,
  entry: FileIndexEntry,
): FileIndexEntryMembership {
  if (entry.kind === 'file' && entry.fileMembers) {
    return {
      resolved: true,
      members: [
        ...entry.fileMembers.regularPaths.map(
          (path): FileIndexEntryMember => ({ path, role: 'regular', targetPath: path }),
        ),
        ...entry.fileMembers.symlinks.map(
          (relation): FileIndexEntryMember => ({
            path: relation.path,
            role: 'symlink',
            targetPath: relation.targetPath,
          }),
        ),
      ],
    };
  }
  const targetPath = indexedTargetPath(
    contentDir,
    entry.canonicalPath,
    entry.kind === 'markdown' ? `${name}${extname(entry.canonicalPath)}` : name,
  );
  return {
    resolved: false,
    members: [
      { path: name, role: 'regular', targetPath },
      ...entry.aliases.map((path): FileIndexEntryMember => ({ path, role: 'symlink', targetPath })),
    ],
  };
}

export function indexedTargetPath(
  contentDir: string,
  canonicalPath: string,
  recordedPath: string,
): string {
  const rel = toPosix(relative(contentDir, canonicalPath));
  if (rel === '..' || rel.startsWith('../') || isAbsolute(rel)) return recordedPath;
  return rel;
}

export interface FolderIndexEntry {
  size: 0;
  modified: string;
  canonicalPath: string;
  inode: string;
}

class WatcherFolderIndex extends Map<string, FolderIndexEntry> {
  readonly #root: string;

  constructor(root: string) {
    super();
    this.#root = root;
  }

  override set(relativePath: string, entry: FolderIndexEntry): this {
    const spelledUnderRoot =
      entry.canonicalPath === this.#root || entry.canonicalPath.startsWith(`${this.#root}${sep}`);
    return super.set(
      relativePath,
      spelledUnderRoot ? entry : { ...entry, canonicalPath: declaredPathKey(entry.canonicalPath) },
    );
  }
}

function derivePageMeta(
  content: string,
  docName: string,
): {
  title: string;
  icon: string | undefined;
  description: string | undefined;
  type: string | undefined;
} {
  const icon = extractPageIcon(content);
  const description = extractPageDescription(content);
  const type = extractPageType(content);
  return {
    title: copyDetachedFromSource(extractPageTitle(content, docName)),
    icon: icon === undefined ? undefined : copyDetachedFromSource(icon),
    description: description === undefined ? undefined : copyDetachedFromSource(description),
    type: type === undefined ? undefined : copyDetachedFromSource(type),
  };
}

function copyDetachedFromSource(value: string): string {
  return Buffer.from(value, 'utf16le').toString('utf16le');
}

function markdownIndexView(
  inner: ReadonlyMap<string, FileIndexEntry>,
): ReadonlyMap<string, FileIndexEntry> {
  const snapshot = new Map<string, FileIndexEntry>();
  for (const [k, v] of inner) {
    if (v.kind === 'markdown') snapshot.set(k, v);
  }
  return snapshot;
}

type RecheckStatus = 'completed' | 'incomplete' | 'failed' | 'closed';

export interface RecheckResult {
  status: RecheckStatus;
  candidates: number;
  dispatched: number;
  partial?: boolean;
}

interface RecoveryOutcome {
  completed: boolean;
  candidates: number;
  partial?: boolean;
}

interface DispatchCounter {
  dispatched: number;
}

export interface WatcherHandle {
  unsubscribe: () => Promise<void>;
  recheck: (mode?: RecoveryMode) => Promise<RecheckResult>;
  recoveryPassesStarted: () => number;
  recheckPaths: (paths: readonly string[]) => Promise<{ checked: number; found: number }>;
  getFileIndex: () => ReadonlyMap<string, FileIndexEntry>;
  getAllFilesIndex: () => AllFileEntries;
  getFileIndexGeneration: () => number;
  getFolderIndex: () => ReadonlyMap<string, FolderIndexEntry>;
  getAliasMap: () => ReadonlyMap<string, string>;
  getFolderAliasIndex: () => ReadonlyMap<string, string>;
  getStructuralIgnoreDirs: () => ReadonlySet<string>;
  mutateFileIndex: (event: DiskEvent) => void;
  pruneFileIndexNowExcluded: () => number;
  pruneFolderIndexNowExcluded: () => number;
  rescanFromDisk: () => Promise<void>;
  settle: () => Promise<void>;
}

export const writeTracker = new Map<string, Array<{ hash: string; timestamp: number }>>();
const WRITE_TRACKER_TTL_MS = 10_000;
const REMOVAL_TRACKER_TTL_MS = 30_000;
const deletedOwnWrites = new WeakSet<{ hash: string; timestamp: number }>();

interface OwnWriteObservation {
  key: string;
  hashes: Set<string>;
  savedDuringObservation: boolean;
}

const activeOwnWrites = new Map<string, Set<OwnWriteObservation>>();

export function registerWrite(filePath: string, hash: string): void {
  const key = declaredPathKey(filePath);
  const queue = writeTracker.get(key) ?? [];
  queue.push({ hash, timestamp: watcherClock().now() });
  writeTracker.set(key, queue);
  for (const observation of activeOwnWrites.get(key) ?? []) {
    observation.savedDuringObservation = true;
    observation.hashes.add(hash);
  }
}

function beginOwnWriteObservation(
  filePath: string,
  observations: Map<string, OwnWriteObservation>,
): void {
  if (observations.has(filePath)) return;
  const key = observedPathKey(filePath);
  const observation = {
    key,
    hashes: new Set((writeTracker.get(key) ?? []).map(({ hash }) => hash)),
    savedDuringObservation: false,
  };
  const live = activeOwnWrites.get(key) ?? new Set<OwnWriteObservation>();
  live.add(observation);
  activeOwnWrites.set(key, live);
  observations.set(filePath, observation);
}

function releaseOwnWriteObservations(observations: Map<string, OwnWriteObservation>): void {
  for (const observation of observations.values()) {
    const live = activeOwnWrites.get(observation.key);
    live?.delete(observation);
    if (live?.size === 0) activeOwnWrites.delete(observation.key);
  }
}

function discardSupersededOwnWrite(
  filePath: string,
  content: string,
  observations: ReadonlyMap<string, OwnWriteObservation>,
): boolean {
  const observation = observations.get(filePath);
  if (!observation || !isSupersededOwnWrite(filePath, content, observation)) return false;
  consumeOwnWrite(observation.key, contentHash(content));
  return true;
}

function isSupersededOwnWrite(
  filePath: string,
  content: string,
  observation: OwnWriteObservation,
): boolean {
  if (!observation.savedDuringObservation || !observation.hashes.has(contentHash(content))) {
    return false;
  }
  if (observedPathKey(filePath) !== observation.key) return false;
  try {
    return readFileSync(observation.key, 'utf-8') !== content;
  } catch (e) {
    if (errnoCode(e) !== 'ENOENT') {
      log.warn(
        { path: observation.key, eventPath: filePath, err: e },
        `Supersession recheck failed to read ${observation.key}`,
      );
    }
    return false;
  }
}

export const removalTracker = new Map<string, number>();

function declaredPathKey(filePath: string): string {
  const parent = dirname(filePath);
  if (parent === filePath) return filePath;
  try {
    return join(resolveNativePath(parent), basename(filePath));
  } catch (e) {
    const code = errnoCode(e);
    if (code === 'ENOENT') return join(declaredPathKey(parent), basename(filePath));
    log.warn(
      { path: filePath, code },
      `native path resolution failed for the declared key of ${filePath} (${code})`,
    );
    return filePath;
  }
}

function observedPathKey(filePath: string): string {
  try {
    return resolveNativePath(filePath);
  } catch (e) {
    const code = errnoCode(e);
    if (code === 'ENOENT') return declaredPathKey(filePath);
    log.warn(
      { path: filePath, code },
      `native path resolution failed for the observed key of ${filePath} (${code})`,
    );
    return filePath;
  }
}

export type RemovalDeclaration = string & { readonly __brand: 'RemovalDeclaration' };

type Assert<T extends true> = T;
type _RawStringIsNotARemovalDeclaration = Assert<string extends RemovalDeclaration ? false : true>;

/* STOP: call this before the unlink, retract it if the unlink throws, and never resolve the
   leaf; the watcher reports a removed alias at the alias, not at the target it pointed to. */
export function registerRemoval(filePath: string): RemovalDeclaration {
  const key = declaredPathKey(filePath);
  removalTracker.set(key, watcherClock().now());
  return key as RemovalDeclaration;
}

export function retractRemoval(declaration: RemovalDeclaration): void {
  removalTracker.delete(declaration);
}

function voidRemoval(filePath: string, declaredBeforeBatch: ReadonlySet<string>): void {
  if (declaredBeforeBatch.size === 0) return;
  const key = declaredPathKey(filePath);
  if (!declaredBeforeBatch.has(key)) return;
  removalTracker.delete(key);
}

/* STOP: return false on every ambiguous input; a true discards a real disk deletion. */
export function isSelfRemoval(filePath: string): boolean {
  if (removalTracker.size === 0) return false;
  const key = declaredPathKey(filePath);
  const timestamp = removalTracker.get(key);
  if (timestamp === undefined) return false;
  removalTracker.delete(key);
  return watcherClock().now() - timestamp <= REMOVAL_TRACKER_TTL_MS;
}

export function evictStaleTrackerEntries(): void {
  const now = watcherClock().now();
  for (const [path, queue] of writeTracker) {
    const fresh = queue.filter((e) => now - e.timestamp <= WRITE_TRACKER_TTL_MS);
    if (fresh.length === 0) {
      writeTracker.delete(path);
    } else if (fresh.length !== queue.length) {
      writeTracker.set(path, fresh);
    }
  }
  for (const [path, timestamp] of removalTracker) {
    if (now - timestamp > REMOVAL_TRACKER_TTL_MS) removalTracker.delete(path);
  }
}

type WatcherDropReason =
  | 'symlink-escape'
  | 'filter-excluded'
  | 'read-failed'
  | 'reserved-doc'
  | 'stat-failed';

type WatcherDecision =
  | 'dispatched'
  | 'self-write-skip'
  | 'superseded-own-write-skip'
  | 'outside-write-over-own-write'
  | 'self-removal-skip'
  | `drop-${WatcherDropReason}`;

export interface WatcherDecisionRecord {
  ts: number;
  decision: WatcherDecision;
  kind: string;
  path: string;
  pathRole: string;
}

const WATCHER_DECISION_RING_CAPACITY = 256;
const watcherDecisionRing: WatcherDecisionRecord[] = [];

const watcherDropCounts = new Map<WatcherDropReason, number>();
let watcherDispatchedCount = 0;
let watcherSelfWriteSkipCount = 0;
let watcherSupersededOwnWriteSkipCount = 0;
let watcherOutsideWriteOverOwnWriteCount = 0;
let watcherSelfRemovalSkipCount = 0;
let watcherDropsSinceLastSummary = 0;

function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath: string): void {
  watcherDecisionRing.push({
    ts: watcherClock().now(),
    decision,
    kind,
    path: normalizeFsPath(rawPath),
    pathRole: classifyFsPath(rawPath),
  });
  if (watcherDecisionRing.length > WATCHER_DECISION_RING_CAPACITY) {
    watcherDecisionRing.shift();
  }
  if (decision === 'dispatched') {
    watcherDispatchedCount++;
    return;
  }
  if (decision === 'self-write-skip') {
    watcherSelfWriteSkipCount++;
    return;
  }
  if (decision === 'superseded-own-write-skip') {
    watcherSupersededOwnWriteSkipCount++;
    return;
  }
  if (decision === 'outside-write-over-own-write') {
    watcherOutsideWriteOverOwnWriteCount++;
    return;
  }
  if (decision === 'self-removal-skip') {
    watcherSelfRemovalSkipCount++;
    return;
  }
  const reason = decision.slice('drop-'.length) as WatcherDropReason;
  watcherDropCounts.set(reason, (watcherDropCounts.get(reason) ?? 0) + 1);
  watcherDropsSinceLastSummary++;
  _fileWatcherDropsCounter().add(1, { 'disk.drop_reason': reason });
}

function recordSupersededOwnWriteSkip(kind: DiskEvent['kind'], filePath: string): void {
  log.debug(
    { kind, path: filePath, self: true, superseded: true },
    `[file-watcher] Skipped superseded own write: ${kind}`,
  );
  _fileWatcherEventsCounter().add(1, { 'disk.kind': kind, self: true, 'disk.superseded': true });
  recordWatcherDecision('superseded-own-write-skip', kind, filePath);
}

function recordOutsideWriteOverOwnWrite(filePath: string): void {
  log.debug(
    { path: filePath },
    '[file-watcher] Read a create over a pending own write as an update',
  );
  recordWatcherDecision('outside-write-over-own-write', 'create', filePath);
}

export function getWatcherDecisionRingSnapshot(): WatcherDecisionRecord[] {
  return watcherDecisionRing.map((record) => ({ ...record }));
}

const WATCHER_DROP_SUMMARY_INTERVAL_MS = 60_000;

export function logWatcherDropSummary(): void {
  if (watcherDropsSinceLastSummary === 0) return;
  const droppedSinceLastSummary = watcherDropsSinceLastSummary;
  watcherDropsSinceLastSummary = 0;
  const dropTotals: Record<string, number> = {};
  for (const [reason, count] of watcherDropCounts) {
    dropTotals[reason] = count;
  }
  log.info(
    {
      droppedSinceLastSummary,
      dropTotals,
      dispatched: watcherDispatchedCount,
      selfWriteSkips: watcherSelfWriteSkipCount,
      supersededOwnWriteSkips: watcherSupersededOwnWriteSkipCount,
      outsideWritesOverOwnWrites: watcherOutsideWriteOverOwnWriteCount,
      selfRemovalSkips: watcherSelfRemovalSkipCount,
    },
    `[file-watcher] drop summary: ${droppedSinceLastSummary} event(s) dropped since last summary`,
  );
}

export function resetWatcherDecisionDiagnostics(): void {
  watcherDecisionRing.length = 0;
  watcherDropCounts.clear();
  watcherDispatchedCount = 0;
  watcherSelfWriteSkipCount = 0;
  watcherSupersededOwnWriteSkipCount = 0;
  watcherOutsideWriteOverOwnWriteCount = 0;
  watcherSelfRemovalSkipCount = 0;
  watcherDropsSinceLastSummary = 0;
}

function eventEscapesContentDir(rawPath: string, contentDir: string): boolean {
  let lst: ReturnType<typeof lstatSync>;
  try {
    lst = lstatSync(rawPath);
  } catch (e) {
    const code = errnoCode(e);
    if (code === 'ENOENT') return false;
    log.warn(
      { path: rawPath, code },
      `lstat failed for escape check on ${rawPath} (${code}), dropping event`,
    );
    return true;
  }
  if (!lst.isSymbolicLink()) return false;
  let canonical: string;
  try {
    canonical = resolveNativePath(rawPath);
  } catch (e) {
    const code = errnoCode(e);
    if (code !== 'ENOENT' && code !== 'ELOOP') {
      log.warn(
        { path: rawPath, code },
        `realpath failed for escape check on ${rawPath} (${code}), dropping event`,
      );
    }
    return true;
  }
  return (
    !isWithinContentDir(canonical, contentDir) ||
    resolvesIntoPrivateState(rawPath, canonical, contentDir)
  );
}

export function pathToDocName(absPath: string, contentDir: string): string {
  const rel = toPosix(relative(contentDir, absPath));
  return stripDocExtension(rel);
}

function contentRelativePath(contentDir: string, absPath: string): string | null {
  const rel = relative(contentDir, absPath).replaceAll('\\', '/');
  if (!rel || rel === '.' || rel === '..' || rel.startsWith('../')) return null;
  return rel;
}

export function upsertFolderIndexEntry(
  folderIndex: Map<string, FolderIndexEntry>,
  contentDir: string,
  folderPath: string,
  stat: Pick<BigIntStats, 'dev' | 'ino' | 'mtimeNs'>,
  canonicalPath = folderPath,
): string | null {
  const relativePath = contentRelativePath(contentDir, folderPath);
  if (!relativePath) return null;
  folderIndex.set(relativePath, {
    size: 0,
    modified: statMtime(stat).toISOString(),
    canonicalPath,
    inode: filesystemIdentity(stat),
  });
  return relativePath;
}

export function removeFolderIndexEntries(
  folderIndex: Map<string, FolderIndexEntry>,
  relativePath: string,
): boolean {
  let removed = false;
  for (const path of folderIndex.keys()) {
    if (path === relativePath || path.startsWith(`${relativePath}/`)) {
      folderIndex.delete(path);
      removed = true;
    }
  }
  return removed;
}

function extractDocExtension(path: string): string | null {
  const ext = extname(path);
  if (ext === '') return null;
  const lower = ext.toLowerCase();
  if (lower === '.mdx' || lower === '.md') return ext;
  return null;
}

export const lastKnownHash = new Map<string, string>();

const lastKnownStat = new Map<string, string>();

let intakeEpoch = 0;
let openObservationWindows = 0;
const intakeEpochByPath = new Map<string, number>();

function noteIntakeMutation(path: string): void {
  intakeEpoch++;
  if (openObservationWindows > 0) intakeEpochByPath.set(path, intakeEpoch);
}

function openObservationWindow(): number {
  openObservationWindows++;
  return intakeEpoch;
}

function closeObservationWindow(): void {
  openObservationWindows--;
  if (openObservationWindows === 0) intakeEpochByPath.clear();
}

function changedSince(epoch: number, contentDir: string): ((path: string) => boolean) | undefined {
  const changed = new Set<string>();
  for (const [path, at] of intakeEpochByPath) if (at > epoch) changed.add(path);
  if (changed.size === 0) return undefined;
  return (path) => {
    let current = path;
    while (true) {
      if (changed.has(current)) return true;
      if (current === contentDir) return false;
      const parent = dirname(current);
      if (parent === current) return false;
      current = parent;
    }
  };
}

export function updateLastKnownHash(filePath: string, hash: string): void {
  noteIntakeMutation(filePath);
  lastKnownHash.set(filePath, hash);
  lastKnownStat.delete(filePath);
}

export function removeLastKnownHash(filePath: string): string | undefined {
  noteIntakeMutation(filePath);
  const hash = lastKnownHash.get(filePath);
  lastKnownHash.delete(filePath);
  lastKnownStat.delete(filePath);
  return hash;
}

function recordObservedHash(filePath: string, hash: string, signature: string | undefined): void {
  lastKnownHash.set(filePath, hash);
  if (signature === undefined) lastKnownStat.delete(filePath);
  else lastKnownStat.set(filePath, signature);
}

interface FreshnessReuse {
  readonly previousByPath: ReadonlyMap<string, FileIndexEntry>;
  readonly carriedOver: ReadonlySet<string>;
  readonly deferred: Set<string>;
  carriedSlots: number;
  freeSlots: number;
  bytesLeft: number;
  hashed: number;
}

type MarkdownReadDecision =
  | { kind: 'read' }
  | { kind: 'reuse'; entry: FileIndexEntry; hash: string }
  | { kind: 'defer'; entry: FileIndexEntry | undefined; hash: string | undefined };

function beginFreshnessReuse(
  mode: RecoveryMode,
  fileIndex: ReadonlyMap<string, FileIndexEntry>,
  carryOver: Set<string>,
): FreshnessReuse | undefined {
  const carriedOver = new Set(carryOver);
  carryOver.clear();
  if (mode === 'full') return undefined;
  const previousByPath = new Map<string, FileIndexEntry>();
  for (const entry of fileIndex.values()) {
    if (entry.kind === 'markdown') previousByPath.set(entry.canonicalPath, entry);
  }
  const carriedSlots = Math.min(carriedOver.size, FRESHNESS_HASH_BUDGET.files);
  return {
    previousByPath,
    carriedOver,
    deferred: carryOver,
    carriedSlots,
    freeSlots: FRESHNESS_HASH_BUDGET.files - carriedSlots,
    bytesLeft: FRESHNESS_HASH_BUDGET.bytes,
    hashed: 0,
  };
}

function decideMarkdownRead(
  reuse: FreshnessReuse | undefined,
  path: string,
  stats: BigIntStats,
  signature: string,
): MarkdownReadDecision {
  if (reuse === undefined) return { kind: 'read' };
  const previous = reuse.previousByPath.get(path);
  const hash = lastKnownHash.get(path);
  if (previous !== undefined && hash !== undefined && lastKnownStat.get(path) === signature) {
    return { kind: 'reuse', entry: previous, hash };
  }
  const size = Number(stats.size);
  const carried = reuse.carriedOver.has(path);
  const slotFree = carried ? reuse.carriedSlots > 0 : reuse.freeSlots > 0;
  const indexedWithoutHash = previous !== undefined && hash === undefined;
  if (indexedWithoutHash || (slotFree && (reuse.hashed === 0 || size <= reuse.bytesLeft))) {
    if (carried) reuse.carriedSlots--;
    else reuse.freeSlots--;
    reuse.bytesLeft -= size;
    reuse.hashed++;
    return { kind: 'read' };
  }
  reuse.deferred.add(path);
  return { kind: 'defer', entry: previous, hash };
}

interface RawFileEvent {
  type: 'create' | 'update' | 'delete';
  path: string;
}

type KnownDeletion = {
  type: 'delete';
  path: string;
  entryKind: 'file' | 'folder';
};

type RecoveredFileEvent = { type: 'create' | 'update'; path: string } | KnownDeletion;
type InternalRawFileEvent = RawFileEvent | KnownDeletion;

type MarkdownRemovalOrigin =
  | { kind: 'native'; docName: string; path: string }
  | {
      kind: 'recovery';
      docName: string;
      previousPath: string;
      isCurrent: () => boolean;
    };

type MarkdownRemovalObservation = {
  isCurrent: () => boolean;
  discarded: boolean;
};

const removalOrigins = new WeakMap<RawFileEvent, MarkdownRemovalOrigin>();
const removalObservations = new WeakMap<MarkdownDiskEvent, MarkdownRemovalObservation>();

export interface MarkdownEventFacts {
  readonly identity?: string;
  readonly lastHash?: string;
  readonly size?: number;
  readonly modified?: string;
}

export function usableFileIdentity(identity: string | undefined): string | undefined {
  if (identity === undefined || identity === '' || identity.endsWith(':0')) return undefined;
  return identity;
}

interface CreatedFileFacts {
  readonly identity: string | undefined;
  readonly size: number;
  readonly modified: string;
}

const markdownEventFacts = new WeakMap<MarkdownDiskEvent, MarkdownEventFacts>();

export function getMarkdownEventFacts(event: DiskEvent): MarkdownEventFacts {
  if (event.kind !== 'create' && event.kind !== 'delete' && event.kind !== 'rename') return {};
  return markdownEventFacts.get(event) ?? {};
}

function fileIdentityOf(path: string): string | undefined {
  return createdFileFactsOf(path)?.identity;
}

function createdFileFactsOf(path: string): CreatedFileFacts | undefined {
  try {
    const stats = statSync(path, { bigint: true });
    return {
      identity: usableFileIdentity(filesystemIdentity(stats)),
      size: Number(stats.size),
      modified: statMtime(stats).toISOString(),
    };
  } catch (err) {
    const code = errnoCode(err);
    if (code !== 'ENOENT' && code !== 'ENOTDIR' && code !== 'ELOOP') {
      log.debug({ err, path, code }, `could not read the file identity of ${path}`);
    }
    return undefined;
  }
}

export function getMarkdownRemovalCurrentness(
  event: Extract<DiskEvent, { kind: 'delete' | 'rename' }>,
): (() => boolean) | undefined {
  return removalObservations.get(event)?.isCurrent;
}

export function hasAdmittedMarkdownFile(
  docName: string,
  filePath: string,
  contentDir: string,
  contentFilter: Pick<ContentFilter, 'isExcluded'> | undefined,
): boolean {
  try {
    if (!statSync(filePath).isFile()) return false;
    if (!readdirSync(dirname(filePath)).includes(basename(filePath))) return false;
    if (contentFilter?.isExcluded(toPosix(relative(contentDir, filePath))) === true) return false;
    return !canonicalContentPathIsRefused(filePath, realpathSync(filePath), contentDir);
  } catch (err) {
    const code = errnoCode(err);
    if (code === undefined) {
      log.error(
        { err, docName, path: filePath },
        `[file-watcher] markdown binding for ${docName} could not be checked; treating it as absent`,
      );
    } else if (code !== 'ENOENT' && code !== 'ENOTDIR') {
      log.warn(
        { err, docName, path: filePath, code },
        `[file-watcher] markdown binding for ${docName} could not be inspected (${code}); treating it as absent`,
      );
    }
    return false;
  }
}

function observeMarkdownRemoval(
  origin: MarkdownRemovalOrigin,
  contentDir: string,
  fileIndex: ReadonlyMap<string, FileIndexEntry>,
  contentFilter: ContentFilter | undefined,
  aliasPaths: ReadonlyMap<string, string> | undefined,
): MarkdownRemovalObservation {
  const observation: MarkdownRemovalObservation = {
    discarded: false,
    isCurrent: () => {
      if (observation.discarded) return false;
      const owner = fileIndex.get(origin.docName);
      if (origin.kind === 'recovery') {
        const current =
          origin.isCurrent() &&
          (owner === undefined || owner.canonicalPath === origin.previousPath);
        observation.discarded = !current;
        return current;
      }
      const effectivePath = join(contentDir, `${origin.docName}${getDocExtension(origin.docName)}`);
      const currentPath =
        owner?.kind === 'markdown'
          ? owner.canonicalPath
          : (aliasPaths?.get(origin.docName) ??
            (existsSync(effectivePath) ? effectivePath : origin.path));
      const current = !hasAdmittedMarkdownFile(
        origin.docName,
        currentPath,
        contentDir,
        contentFilter,
      );
      observation.discarded ||= !current;
      return current;
    },
  };
  return observation;
}

export async function classifyEvents(
  rawEvents: RawFileEvent[],
  contentDir: string,
  contentFilter?: ContentFilter,
  aliasMap?: Map<string, string>,
): Promise<MarkdownDiskEvent[]> {
  const observations = new Map<string, OwnWriteObservation>();
  try {
    return await classifyEventsInternal(
      rawEvents,
      contentDir,
      observations,
      contentFilter,
      aliasMap,
    );
  } finally {
    releaseOwnWriteObservations(observations);
  }
}

async function classifyEventsInternal(
  rawEvents: RawFileEvent[],
  contentDir: string,
  observations: Map<string, OwnWriteObservation>,
  contentFilter?: ContentFilter,
  aliasMap?: Map<string, string>,
  aliasContext?: {
    fileIndex: ReadonlyMap<string, FileIndexEntry>;
    aliasPaths: ReadonlyMap<string, string>;
    canonicalRecords: Set<RawFileEvent>;
    removals: ReadonlyMap<RawFileEvent, MarkdownRemovalObservation>;
  },
): Promise<MarkdownDiskEvent[]> {
  const deletes: RawFileEvent[] = [];
  const creates: RawFileEvent[] = [];
  const updates: RawFileEvent[] = [];

  for (const event of rawEvents) {
    if (!isSupportedDocFile(event.path)) continue;

    if (contentFilter) {
      const relPath = toPosix(relative(contentDir, event.path));
      if (contentFilter.isExcluded(relPath)) {
        recordWatcherDecision('drop-filter-excluded', event.type, event.path);
        continue;
      }
    }

    switch (event.type) {
      case 'delete': {
        if (aliasContext) {
          const name = pathToDocName(event.path, contentDir);
          const owner = aliasContext.fileIndex.get(name);
          const canonicalPath = owner?.kind === 'markdown' ? owner.canonicalPath : undefined;
          const aliasPath = aliasMap?.has(name) ? aliasContext.aliasPaths.get(name) : undefined;
          if (
            (canonicalPath !== undefined || aliasPath !== undefined) &&
            canonicalPath !== event.path &&
            aliasPath !== event.path
          ) {
            removeLastKnownHash(event.path);
            continue;
          }
        }
        if (writeTracker.size > 0)
          for (const write of writeTracker.get(observedPathKey(event.path)) ?? [])
            deletedOwnWrites.add(write);
        deletes.push(event);
        break;
      }
      case 'create':
        if (lastKnownHash.has(event.path)) {
          updates.push(event);
        } else {
          creates.push(event);
        }
        break;
      case 'update':
        updates.push(event);
        break;
    }
  }

  const createContents = new Map<string, string>();
  const createFacts = new Map<string, CreatedFileFacts>();
  const updateContents = new Map<string, string>();
  for (const event of creates) {
    beginOwnWriteObservation(event.path, observations);
    try {
      createContents.set(event.path, await readFile(event.path, 'utf-8'));
      const facts = createdFileFactsOf(event.path);
      if (facts !== undefined) createFacts.set(event.path, facts);
    } catch (e) {
      recordWatcherDecision('drop-read-failed', event.type, event.path);
      if (errnoCode(e) !== 'ENOENT') {
        log.warn({ path: event.path, err: e }, `Failed to read ${event.path}`);
      }
    }
  }
  for (const event of updates) {
    beginOwnWriteObservation(event.path, observations);
    try {
      updateContents.set(event.path, await readFile(event.path, 'utf-8'));
    } catch (e) {
      recordWatcherDecision('drop-read-failed', event.type, event.path);
      if (errnoCode(e) !== 'ENOENT') {
        log.warn({ path: event.path, err: e }, `Failed to read ${event.path}`);
      }
    }
  }

  for (const [contents, type] of [
    [createContents, 'create'],
    [updateContents, 'update'],
  ] as const) {
    for (const [filePath, content] of contents) {
      if (!discardSupersededOwnWrite(filePath, content, observations)) continue;
      contents.delete(filePath);
      recordSupersededOwnWriteSkip(type, filePath);
    }
  }

  for (const event of creates) {
    const content = createContents.get(event.path);
    if (content === undefined || !overwritesOwnWrite(event.path, content)) continue;
    createContents.delete(event.path);
    createFacts.delete(event.path);
    updateContents.set(event.path, content);
    updates.push(event);
    recordOutsideWriteOverOwnWrite(event.path);
  }

  function resolveDocName(event: RawFileEvent): string {
    const rawPath = event.path;
    const raw = pathToDocName(rawPath, contentDir);
    if (!aliasMap) return raw;

    let lst: ReturnType<typeof lstatSync> | null = null;
    try {
      lst = lstatSync(rawPath);
    } catch (e) {
      const code = errnoCode(e);
      if (code !== 'ENOENT') {
        log.warn({ path: rawPath, err: e }, `resolveDocName lstat failed for ${rawPath}`);
      }
      if (aliasMap.has(raw)) {
        aliasMap.delete(raw);
        return raw;
      }
      return raw;
    }

    if (!lst.isSymbolicLink()) {
      const owner = aliasContext?.fileIndex.get(raw);
      const aliasPath = aliasContext?.aliasPaths.get(raw);
      if (
        event.type !== 'delete' &&
        owner?.kind === 'markdown' &&
        owner.canonicalPath === rawPath &&
        aliasMap.has(raw) &&
        aliasPath !== undefined &&
        aliasPath !== rawPath
      ) {
        aliasContext?.canonicalRecords.add(event);
        return raw;
      }
      if (aliasMap.has(raw)) aliasMap.delete(raw);
      return raw;
    }

    let canonical: string;
    try {
      canonical = resolveNativePath(rawPath);
    } catch (e) {
      const code = errnoCode(e);
      if (code !== 'ENOENT' && code !== 'ELOOP') {
        log.warn({ path: rawPath, err: e }, `resolveDocName realpath failed for ${rawPath}`);
      }
      aliasMap.delete(raw);
      return raw;
    }

    if (!isWithinContentDir(canonical, contentDir)) {
      aliasMap.delete(raw);
      return raw;
    }

    const canonicalDocName = pathToDocName(canonical, contentDir);
    aliasMap.set(raw, canonicalDocName);
    return canonicalDocName;
  }

  const results: MarkdownDiskEvent[] = [];
  const pairedCreates = new Set<string>();
  const pairedDeletes = new Set<string>();
  const deletedFacts = new Map<string, MarkdownEventFacts>();

  for (const del of deletes) {
    if (aliasContext?.removals.get(del)?.isCurrent() === false) continue;
    const indexed = aliasContext?.fileIndex.get(pathToDocName(del.path, contentDir));
    const indexedHere =
      indexed?.kind === 'markdown' && indexed.canonicalPath === del.path ? indexed : undefined;
    const deletedIdentity = usableFileIdentity(indexedHere?.inode);
    const deletedHash = removeLastKnownHash(del.path);
    deletedFacts.set(del.path, {
      identity: deletedIdentity,
      lastHash: deletedHash,
      size: indexedHere?.size,
      modified: indexedHere?.modified,
    });
    if (!deletedHash && deletedIdentity === undefined) continue;

    for (const create of creates) {
      if (pairedCreates.has(create.path)) continue;
      const content = createContents.get(create.path);
      if (content === undefined) continue;
      const hash = contentHash(content);
      const created = createFacts.get(create.path);
      const sameUnchangedFile =
        deletedIdentity !== undefined &&
        created?.identity === deletedIdentity &&
        created.size === indexedHere?.size &&
        created.modified === indexedHere?.modified;
      if (hash === deletedHash || sameUnchangedFile) {
        pairedCreates.add(create.path);
        pairedDeletes.add(del.path);
        updateLastKnownHash(create.path, hash);
        const renamed: MarkdownDiskEvent = {
          kind: 'rename',
          oldPath: del.path,
          newPath: create.path,
          oldDocName: resolveDocName(del),
          newDocName: resolveDocName(create),
          content,
        };
        const observation = aliasContext?.removals.get(del);
        if (observation) removalObservations.set(renamed, observation);
        markdownEventFacts.set(renamed, {
          ...createFacts.get(create.path),
          lastHash: deletedHash,
        });
        results.push(renamed);
        break;
      }
    }
  }

  for (const del of deletes) {
    if (pairedDeletes.has(del.path)) continue;
    const observation = aliasContext?.removals.get(del);
    if (observation?.isCurrent() !== false) removeLastKnownHash(del.path);
    const origin = removalOrigins.get(del);
    const event: MarkdownDiskEvent = {
      kind: 'delete',
      path: del.path,
      docName: origin?.kind === 'recovery' ? origin.docName : resolveDocName(del),
    };
    if (observation) removalObservations.set(event, observation);
    const facts = deletedFacts.get(del.path);
    if (facts) markdownEventFacts.set(event, facts);
    results.push(event);
  }

  for (const create of creates) {
    if (pairedCreates.has(create.path)) continue;
    const content = createContents.get(create.path);
    if (content === undefined) continue;
    const hash = contentHash(content);
    updateLastKnownHash(create.path, hash);

    if (containsConflictMarkers(content)) {
      results.push({
        kind: 'conflict',
        path: create.path,
        docName: resolveDocName(create),
        content,
      });
    } else {
      const created: MarkdownDiskEvent = {
        kind: 'create',
        path: create.path,
        docName: resolveDocName(create),
        content,
      };
      markdownEventFacts.set(created, { ...createFacts.get(create.path) });
      results.push(created);
    }
  }

  for (const update of updates) {
    const content = updateContents.get(update.path);
    if (content === undefined) continue;
    const hash = contentHash(content);
    updateLastKnownHash(update.path, hash);

    if (containsConflictMarkers(content)) {
      results.push({
        kind: 'conflict',
        path: update.path,
        docName: resolveDocName(update),
        content,
      });
    } else {
      results.push({
        kind: 'update',
        path: update.path,
        docName: resolveDocName(update),
        content,
      });
    }
  }

  return results;
}

function declaredDiskEvent(event: DiskEvent): DiskEvent {
  if (event.kind === 'rename') {
    return {
      ...event,
      oldPath: declaredPathKey(event.oldPath),
      newPath: declaredPathKey(event.newPath),
    };
  }
  return { ...event, path: declaredPathKey(event.path) };
}

export function isSelfWrite(filePath: string, hash: string): boolean {
  if (consumeOwnWrite(filePath, hash)) return true;
  writeTracker.delete(filePath);
  return false;
}

function overwritesOwnWrite(filePath: string, content: string): boolean {
  if (writeTracker.size === 0) return false;
  const queue = writeTracker.get(observedPathKey(filePath));
  const newest = queue?.at(-1);
  if (queue === undefined || newest === undefined || deletedOwnWrites.has(newest)) return false;
  const hash = contentHash(content);
  return queue.every((write) => write.hash !== hash);
}

function consumeOwnWrite(key: string, hash: string): boolean {
  const queue = writeTracker.get(key);
  if (!queue) return false;
  const idx = queue.findIndex((e) => e.hash === hash);
  if (idx < 0) return false;
  queue.splice(0, idx + 1);
  if (queue.length === 0) writeTracker.delete(key);
  return true;
}

interface DiskObservation {
  fileIndex: Map<string, FileIndexEntry>;
  generalFileIndex: GeneralFileIndex;
  folderIndex: Map<string, FolderIndexEntry>;
  aliasMap: Map<string, string>;
  aliasPaths: Map<string, string>;
  folderAliasIndex: Map<string, string>;
  hashes: Map<string, string>;
  unreadable: Map<string, string>;
  signatures: Map<string, string>;
  sampledAtNs: bigint;
  reuse: FreshnessReuse | undefined;
  extensions: Array<{ docName: string; ext: string }>;
  preferredExtensions: Map<string, string>;
  structuralIgnoreDirs: Set<string>;
  paths: Set<string>;
  complete: boolean;
  stopped?: () => boolean;
}

const reportedUnreadable = new Map<string, string>();

function markUnreadable(
  observation: DiskObservation,
  path: string,
  code: string | undefined,
): void {
  observation.unreadable.set(path, code ?? 'unknown');
}

function reportUnreadableEntries(observation: DiskObservation): void {
  for (const [path, code] of observation.unreadable) {
    if (reportedUnreadable.has(path)) continue;
    reportedUnreadable.set(path, code);
    incrementWatcherUnreadableEntries();
    log.warn(
      { path, code },
      `Cannot read ${path} (${code}); it keeps its last known state until it can be read`,
    );
  }
  for (const path of reportedUnreadable.keys()) {
    if (!observation.unreadable.has(path)) reportedUnreadable.delete(path);
  }
}

function isUnderUnreadable(observation: DiskObservation, path: string): boolean {
  for (const unreadable of observation.unreadable.keys()) {
    if (path === unreadable || path.startsWith(`${unreadable}${sep}`)) return true;
  }
  return false;
}

function observeMarkdownSignature(
  observation: DiskObservation,
  path: string,
  stats: BigIntStats,
  signature: string,
): void {
  if (!isRacyStat(stats, observation.sampledAtNs)) observation.signatures.set(path, signature);
}

function reusedMarkdownEntry(
  decision: Exclude<MarkdownReadDecision, { kind: 'read' }>,
  observation: DiskObservation,
  path: string,
  identity: string,
  aliases: string[],
  signature: string,
): FileIndexEntry | undefined {
  if (decision.entry === undefined || decision.hash === undefined) return undefined;
  observation.hashes.set(path, decision.hash);
  observation.paths.add(path);
  if (decision.kind === 'reuse') observation.signatures.set(path, signature);
  return { ...decision.entry, canonicalPath: path, inode: identity, aliases };
}

type GeneralIdentity = ReturnType<typeof filesystemIdentity>;
type KnownGeneralEntry = FileIndexEntry & { kind: 'file'; fileMembers: GeneralFileMembers };
type GeneralFileState =
  | { kind: 'known'; entry: KnownGeneralEntry; identity: GeneralIdentity; sequence: number }
  | { kind: 'unresolved'; reportedPath: string; sequence: number };
type GeneralMetadata = { size: number; modified: string };

const generalFileStates = new WeakMap<FileIndexEntry, GeneralFileState>();
let generalFileSequence = 0;

function knownGeneralState(
  entry: FileIndexEntry,
): Extract<GeneralFileState, { kind: 'known' }> | null {
  const state = generalFileStates.get(entry);
  return state?.kind === 'known' ? state : null;
}

function sameGeneralIdentity(left: GeneralIdentity, right: GeneralIdentity): boolean {
  return left === right;
}

function addGeneralOwner(owners: Map<string, Set<string>>, key: string, name: string): void {
  const names = owners.get(key);
  if (names) names.add(name);
  else owners.set(key, new Set([name]));
}

function removeGeneralOwner(owners: Map<string, Set<string>>, key: string, name: string): void {
  const names = owners.get(key);
  if (!names) return;
  names.delete(name);
  if (names.size === 0) owners.delete(key);
}

class GeneralFileIndex implements ReadonlyMap<string, FileIndexEntry> {
  readonly #contentDir: string;
  readonly #entries = new Map<string, FileIndexEntry>();
  readonly #linked = new Map<
    string,
    { readonly members: readonly string[]; readonly identity: string | null }
  >();
  readonly #members = new Map<string, Set<string>>();
  readonly #identities = new Map<string, Set<string>>();

  constructor(contentDir: string) {
    this.#contentDir = contentDir;
  }

  get size(): number {
    return this.#entries.size;
  }

  get(name: string): FileIndexEntry | undefined {
    return this.#entries.get(name);
  }

  has(name: string): boolean {
    return this.#entries.has(name);
  }

  keys(): MapIterator<string> {
    return this.#entries.keys();
  }

  values(): MapIterator<FileIndexEntry> {
    return this.#entries.values();
  }

  entries(): MapIterator<[string, FileIndexEntry]> {
    return this.#entries.entries();
  }

  [Symbol.iterator](): MapIterator<[string, FileIndexEntry]> {
    return this.#entries[Symbol.iterator]();
  }

  forEach(
    callback: (
      entry: FileIndexEntry,
      name: string,
      index: ReadonlyMap<string, FileIndexEntry>,
    ) => void,
    thisArg?: unknown,
  ): void {
    for (const [name, entry] of this.#entries) callback.call(thisArg, entry, name, this);
  }

  ownersOfPath(path: string): ReadonlySet<string> | undefined {
    return this.#members.get(path);
  }

  ownersOfIdentity(identity: GeneralIdentity): ReadonlySet<string> | undefined {
    return this.#identities.get(identity);
  }

  set(name: string, entry: FileIndexEntry): void {
    Object.freeze(entry.aliases);
    Object.freeze(entry);
    this.#unlink(name);
    this.#entries.set(name, entry);
    this.#link(name, entry);
  }

  delete(name: string): void {
    this.#unlink(name);
    this.#entries.delete(name);
  }

  clear(): void {
    this.#entries.clear();
    this.#linked.clear();
    this.#members.clear();
    this.#identities.clear();
  }

  #link(name: string, entry: FileIndexEntry): void {
    const members = fileIndexEntryMembers(this.#contentDir, name, entry).members.map(
      (member) => member.path,
    );
    for (const member of members) addGeneralOwner(this.#members, member, name);
    const state = knownGeneralState(entry);
    const identity = state ? state.identity : null;
    if (identity !== null) addGeneralOwner(this.#identities, identity, name);
    this.#linked.set(name, { members, identity });
  }

  #unlink(name: string): void {
    const linked = this.#linked.get(name);
    if (!linked) return;
    for (const member of linked.members) removeGeneralOwner(this.#members, member, name);
    if (linked.identity !== null) removeGeneralOwner(this.#identities, linked.identity, name);
    this.#linked.delete(name);
  }
}

function generalOwnerNames(index: GeneralFileIndex, path: string): string[] {
  return [...(index.ownersOfPath(path) ?? [])];
}

function generalFactsFor(
  index: GeneralFileIndex,
  paths: Iterable<string>,
): Map<string, GeneralPathFact> {
  const names = new Set<string>();
  for (const path of paths) {
    for (const name of index.ownersOfPath(path) ?? []) names.add(name);
  }
  const facts = new Map<string, GeneralPathFact>();
  for (const name of names) {
    const entry = index.get(name);
    if (entry) addGeneralEntryFacts(facts, name, entry);
  }
  return facts;
}

function findKnownGeneralGroup(
  index: GeneralFileIndex,
  identity: GeneralIdentity,
): [string, KnownGeneralEntry] | null {
  for (const name of index.ownersOfIdentity(identity) ?? []) {
    const entry = index.get(name);
    const state = entry && knownGeneralState(entry);
    if (state && sameGeneralIdentity(state.identity, identity)) return [name, state.entry];
  }
  return null;
}

function storeKnownGeneralGroup(
  index: GeneralFileIndex,
  contentDir: string,
  previousName: string | null,
  identity: GeneralIdentity,
  regularPaths: readonly string[],
  symlinks: GeneralFileMembers['symlinks'],
  metadata: GeneralMetadata,
  sequence = ++generalFileSequence,
): void {
  const regular = [...new Set(regularPaths)].toSorted();
  if (previousName !== null) {
    const current = index.get(previousName);
    const currentState = current && knownGeneralState(current);
    if (currentState && sameGeneralIdentity(currentState.identity, identity)) {
      index.delete(previousName);
    }
  }
  if (regular.length === 0) return;
  const representative = previousName && regular.includes(previousName) ? previousName : regular[0];
  const relations = [...new Map(symlinks.map((relation) => [relation.path, relation])).values()]
    .filter((relation) => !regular.includes(relation.path))
    .toSorted((left, right) => left.path.localeCompare(right.path));
  const entry: KnownGeneralEntry = {
    kind: 'file',
    canonicalPath: join(contentDir, representative),
    inode: identity,
    size: metadata.size,
    modified: metadata.modified,
    aliases: relations.map((relation) => relation.path),
    fileMembers: { regularPaths: regular, symlinks: relations },
  };
  generalFileStates.set(entry, { kind: 'known', entry, identity, sequence });
  index.set(representative, entry);
}

function storeUnresolvedGeneralEntry(
  index: GeneralFileIndex,
  contentDir: string,
  event: Extract<FileDiskEvent, { kind: 'file-create' | 'file-update' }>,
): void {
  const name = toPosix(relative(contentDir, event.path));
  const prior = index.get(name);
  const entry: FileIndexEntry = {
    kind: 'file',
    canonicalPath: event.path,
    inode: event.inode || prior?.inode || '',
    size: event.size,
    modified: new Date(event.modifiedTs).toISOString(),
    aliases:
      prior?.kind === 'file'
        ? fileIndexEntryMembers(contentDir, name, prior)
            .members.filter((member) => member.role === 'symlink')
            .map((member) => member.path)
        : [],
  };
  generalFileStates.set(entry, {
    kind: 'unresolved',
    reportedPath: event.relativePath,
    sequence: ++generalFileSequence,
  });
  index.set(name, entry);
}

function removeKnownGeneralPath(index: GeneralFileIndex, contentDir: string, path: string): void {
  for (const name of generalOwnerNames(index, path)) {
    const entry = index.get(name);
    if (!entry) continue;
    const state = knownGeneralState(entry);
    if (!state) {
      if (name === path) index.delete(name);
      else if (entry.kind === 'file' && entry.aliases.includes(path)) {
        index.set(name, {
          ...entry,
          aliases: entry.aliases.filter((alias) => alias !== path),
        });
      }
      continue;
    }
    const members = state.entry.fileMembers;
    const regular = members.regularPaths.filter((member) => member !== path);
    const symlinks = members.symlinks.filter(
      (relation) =>
        relation.path !== path &&
        (regular.length === members.regularPaths.length || relation.targetPath !== path),
    );
    if (
      regular.length === members.regularPaths.length &&
      symlinks.length === members.symlinks.length
    )
      continue;
    storeKnownGeneralGroup(
      index,
      contentDir,
      name,
      state.identity,
      regular,
      symlinks,
      entry,
      state.sequence,
    );
  }
}

type GeneralRegistrationResult =
  | { kind: 'applied'; changedPaths: ReadonlySet<string> }
  | { kind: 'unobserved' };

function registerKnownGeneralPath(
  index: GeneralFileIndex,
  contentDir: string,
  canonicalName: string,
  lexicalName: string,
  leafSymlink: boolean,
  identity: GeneralIdentity,
  metadata: GeneralMetadata,
  contentFilter?: ContentFilter,
  confirmedAbsentFormerPath?: string,
): GeneralRegistrationResult {
  const changes = new Map<string, GeneralPathFact | null>();
  const selectedFact = (path: string): GeneralPathFact | undefined =>
    changes.has(path) ? (changes.get(path) ?? undefined) : generalFactsFor(index, [path]).get(path);
  const prior = selectedFact(lexicalName);
  const priorIdentity =
    prior?.kind === 'regular' || prior?.kind === 'symlink' ? prior.identity : null;
  const priorOwner = priorIdentity && findKnownGeneralGroup(index, priorIdentity)?.[1];
  const roleChanged =
    prior !== undefined &&
    prior.kind !== 'unresolved' &&
    (prior.kind === 'symlink') !== leafSymlink;
  const targetChanged =
    prior?.kind === 'symlink' && leafSymlink && prior.targetPath !== canonicalName;
  const sequence = ++generalFileSequence;
  const primaryPath = leafSymlink ? lexicalName : canonicalName;
  const primary: GeneralPathFact = leafSymlink
    ? {
        kind: 'symlink',
        path: lexicalName,
        targetPath: canonicalName,
        identity,
        metadata,
        sequence,
      }
    : { kind: 'regular', path: primaryPath, identity, metadata, sequence };

  if (roleChanged || targetChanged) {
    for (const relation of priorOwner?.fileMembers.symlinks ?? []) {
      if (relation.path === lexicalName) continue;
      const path = join(contentDir, relation.path);
      try {
        const actualTarget = resolveNativePath(path);
        if (!isWithinContentDir(actualTarget, contentDir)) {
          changes.set(relation.path, null);
          continue;
        }
        const targetPath = toPosix(relative(contentDir, actualTarget));
        if (
          isSystemDoc(targetPath) ||
          isConfigDoc(targetPath) ||
          contentFilter?.isPathIgnored(relation.path) ||
          contentFilter?.isPathIgnored(targetPath)
        ) {
          changes.set(relation.path, null);
          continue;
        }
        const targetStat =
          targetPath === canonicalName ? null : statSync(actualTarget, { bigint: true });
        if (targetStat && !targetStat.isFile()) {
          changes.set(relation.path, null);
          continue;
        }
        const targetIdentity = targetStat ? filesystemIdentity(targetStat) : identity;
        const targetMetadata = targetStat
          ? { size: Number(targetStat.size), modified: statMtime(targetStat).toISOString() }
          : metadata;
        const existingTarget = selectedFact(targetPath);
        if (
          existingTarget?.kind !== 'regular' ||
          !sameGeneralIdentity(existingTarget.identity, targetIdentity)
        ) {
          changes.set(targetPath, {
            kind: 'regular',
            path: targetPath,
            identity: targetIdentity,
            metadata: targetMetadata,
            sequence,
          });
        }
        changes.set(relation.path, {
          kind: 'symlink',
          path: relation.path,
          targetPath,
          identity: targetIdentity,
          metadata: targetMetadata,
          sequence,
        });
      } catch (err) {
        const code = errnoCode(err);
        if (code === 'ENOENT' || code === 'ENOTDIR' || code === 'ELOOP') {
          changes.set(relation.path, null);
          continue;
        }
        log.warn({ path, code, err }, 'dependent symlink target read failed');
        return { kind: 'unobserved' };
      }
    }
  }

  if (confirmedAbsentFormerPath) changes.set(confirmedAbsentFormerPath, null);
  if (leafSymlink) {
    const current = selectedFact(canonicalName);
    if (current?.kind !== 'regular' || !sameGeneralIdentity(current.identity, identity)) {
      changes.set(canonicalName, {
        kind: 'regular',
        path: canonicalName,
        identity,
        metadata,
        sequence,
      });
    }
  }
  changes.set(primaryPath, primary);

  const affected = new Set<string>();
  const include = (names: ReadonlySet<string> | undefined): void => {
    for (const name of names ?? []) affected.add(name);
  };
  include(index.ownersOfIdentity(identity));
  for (const [path, fact] of changes) {
    include(index.ownersOfPath(path));
    if (fact?.kind === 'regular') include(index.ownersOfIdentity(fact.identity));
    else if (fact?.kind === 'symlink') include(index.ownersOfPath(fact.targetPath));
  }
  const previousFacts = new Map<string, GeneralPathFact>();
  const representatives = new Map<string, string>();
  for (const name of affected) {
    const entry = index.get(name);
    if (!entry) continue;
    addGeneralEntryFacts(previousFacts, name, entry);
    const state = knownGeneralState(entry);
    const key = state?.identity;
    if (key && !representatives.has(key)) representatives.set(key, name);
  }
  const selected = new Map(previousFacts);
  for (const [path, fact] of changes) {
    if (fact) selected.set(path, fact);
    else selected.delete(path);
  }
  const metadataEffects = new Map<string, GeneralMetadataEffect>([
    [identity, { ...metadata, sequence }],
  ]);
  for (const name of affected) index.delete(name);
  storeGeneralFacts(index, contentDir, selected, representatives, metadataEffects);
  const comparedPaths = new Set([...previousFacts.keys(), ...selected.keys()]);
  const nextFacts = generalFactsFor(index, comparedPaths);
  const changedPaths = new Set<string>();
  for (const path of comparedPaths) {
    const before = previousFacts.get(path);
    const after = nextFacts.get(path);
    if (before?.kind !== after?.kind) {
      changedPaths.add(path);
    } else if (before?.kind === 'symlink' && after?.kind === 'symlink') {
      if (
        before.targetPath !== after.targetPath ||
        !sameGeneralIdentity(before.identity, after.identity)
      ) {
        changedPaths.add(path);
      }
    } else if (before?.kind === 'regular' && after?.kind === 'regular') {
      if (!sameGeneralIdentity(before.identity, after.identity)) changedPaths.add(path);
    }
  }
  return { kind: 'applied', changedPaths };
}

function generalMemberPaths(
  contentDir: string,
  index: ReadonlyMap<string, FileIndexEntry>,
): Set<string> {
  const paths = new Set<string>();
  for (const [name, entry] of index) {
    if (entry.kind !== 'file') continue;
    for (const member of fileIndexEntryMembers(contentDir, name, entry).members) {
      paths.add(member.path);
    }
  }
  return paths;
}

type GeneralPathFact =
  | {
      kind: 'regular';
      path: string;
      identity: GeneralIdentity;
      metadata: GeneralMetadata;
      sequence: number;
    }
  | {
      kind: 'symlink';
      path: string;
      targetPath: string;
      identity: GeneralIdentity;
      metadata: GeneralMetadata;
      sequence: number;
    }
  | { kind: 'unresolved'; path: string; entry: FileIndexEntry; sequence: number };

type GeneralMetadataEffect = GeneralMetadata & { sequence: number };
type GeneralMutationRecord = {
  generalNames: Set<string>;
  generalEffects: Map<string, GeneralPathFact | null>;
  generalMetadata: Map<string, GeneralMetadataEffect>;
};

function addGeneralEntryFacts(
  facts: Map<string, GeneralPathFact>,
  name: string,
  entry: FileIndexEntry,
): void {
  if (entry.kind !== 'file') return;
  const state = knownGeneralState(entry);
  if (!state) {
    facts.set(name, {
      kind: 'unresolved',
      path: name,
      entry,
      sequence: generalFileStates.get(entry)?.sequence ?? 0,
    });
    return;
  }
  const knownEntry = state.entry;
  const metadata = { size: knownEntry.size, modified: knownEntry.modified };
  for (const path of knownEntry.fileMembers.regularPaths) {
    facts.set(path, {
      kind: 'regular',
      path,
      identity: state.identity,
      metadata,
      sequence: state.sequence,
    });
  }
  for (const relation of knownEntry.fileMembers.symlinks) {
    facts.set(relation.path, {
      kind: 'symlink',
      path: relation.path,
      targetPath: relation.targetPath,
      identity: state.identity,
      metadata,
      sequence: state.sequence,
    });
  }
}

function generalPathFacts(
  index: ReadonlyMap<string, FileIndexEntry>,
): Map<string, GeneralPathFact> {
  const facts = new Map<string, GeneralPathFact>();
  for (const [name, entry] of index) addGeneralEntryFacts(facts, name, entry);
  return facts;
}

function generalRepresentativeNames(
  index: ReadonlyMap<string, FileIndexEntry>,
): Map<string, string> {
  const names = new Map<string, string>();
  for (const [name, entry] of index) {
    const state = knownGeneralState(entry);
    if (!state) continue;
    const key = state.identity;
    if (!names.has(key)) names.set(key, name);
  }
  return names;
}

function rebuildGeneralFileIndex(
  index: GeneralFileIndex,
  contentDir: string,
  facts: ReadonlyMap<string, GeneralPathFact>,
  previousRepresentatives: ReadonlyMap<string, string>,
  metadataEffects: ReadonlyMap<string, GeneralMetadataEffect> = new Map(),
): void {
  index.clear();
  storeGeneralFacts(index, contentDir, facts, previousRepresentatives, metadataEffects);
}

function storeGeneralFacts(
  index: GeneralFileIndex,
  contentDir: string,
  facts: ReadonlyMap<string, GeneralPathFact>,
  previousRepresentatives: ReadonlyMap<string, string>,
  metadataEffects: ReadonlyMap<string, GeneralMetadataEffect>,
): void {
  const grouped = new Map<
    string,
    {
      identity: GeneralIdentity;
      regularPaths: string[];
      symlinks: GeneralFileMembers['symlinks'][number][];
      metadata: GeneralMetadata;
      sequence: number;
    }
  >();
  for (const fact of facts.values()) {
    if (fact.kind !== 'regular') continue;
    const key = fact.identity;
    const group = grouped.get(key);
    if (group) {
      group.regularPaths.push(fact.path);
      if (fact.sequence >= group.sequence) {
        group.metadata = fact.metadata;
        group.sequence = fact.sequence;
      }
    } else {
      grouped.set(key, {
        identity: fact.identity,
        regularPaths: [fact.path],
        symlinks: [],
        metadata: fact.metadata,
        sequence: fact.sequence,
      });
    }
  }
  for (const fact of facts.values()) {
    if (fact.kind !== 'symlink') continue;
    const target = facts.get(fact.targetPath);
    if (target?.kind !== 'regular') continue;
    grouped.get(target.identity)?.symlinks.push({
      path: fact.path,
      targetPath: fact.targetPath,
    });
  }
  for (const [key, group] of grouped) {
    const priorName = previousRepresentatives.get(key) ?? null;
    const effect = metadataEffects.get(key);
    storeKnownGeneralGroup(
      index,
      contentDir,
      priorName,
      group.identity,
      group.regularPaths,
      group.symlinks,
      effect ?? group.metadata,
      effect?.sequence ?? group.sequence,
    );
  }
  for (const fact of facts.values()) {
    if (fact.kind !== 'unresolved' || index.has(fact.path)) continue;
    index.set(fact.path, fact.entry);
  }
}

type GeneralScanAccumulator = {
  facts: Map<string, GeneralPathFact>;
  metadataEffects: Map<string, GeneralMetadataEffect>;
};

function materializeScanGeneralIndex(
  observation: DiskObservation,
  contentDir: string,
  scan: GeneralScanAccumulator,
): void {
  const representatives = generalRepresentativeNames(observation.generalFileIndex);
  for (const fact of scan.facts.values()) {
    if (fact.kind !== 'regular') continue;
    const key = fact.identity;
    if (!representatives.has(key)) representatives.set(key, fact.path);
  }
  const next = new GeneralFileIndex(contentDir);
  rebuildGeneralFileIndex(next, contentDir, scan.facts, representatives, scan.metadataEffects);
  observation.generalFileIndex = next;
}

function recordScannedGeneralPath(
  observation: DiskObservation,
  scan: GeneralScanAccumulator,
  contentDir: string,
  canonicalName: string,
  lexicalName: string,
  leafSymlink: boolean,
  identity: GeneralIdentity,
  metadata: GeneralMetadata,
  contentFilter: ContentFilter | undefined,
): GeneralRegistrationResult['kind'] {
  const prior = scan.facts.get(lexicalName);
  const roleChanged =
    prior !== undefined &&
    prior.kind !== 'unresolved' &&
    (prior.kind === 'symlink') !== leafSymlink;
  const targetChanged =
    prior?.kind === 'symlink' && leafSymlink && prior.targetPath !== canonicalName;
  if (roleChanged || targetChanged) {
    materializeScanGeneralIndex(observation, contentDir, scan);
    const registered = registerKnownGeneralPath(
      observation.generalFileIndex,
      contentDir,
      canonicalName,
      lexicalName,
      leafSymlink,
      identity,
      metadata,
      contentFilter,
    );
    scan.facts = generalPathFacts(observation.generalFileIndex);
    scan.metadataEffects.clear();
    return registered.kind;
  }

  const sequence = ++generalFileSequence;
  if (leafSymlink) {
    const target = scan.facts.get(canonicalName);
    if (target?.kind !== 'regular' || !sameGeneralIdentity(target.identity, identity)) {
      scan.facts.set(canonicalName, {
        kind: 'regular',
        path: canonicalName,
        identity,
        metadata,
        sequence,
      });
    }
    scan.facts.set(lexicalName, {
      kind: 'symlink',
      path: lexicalName,
      targetPath: canonicalName,
      identity,
      metadata,
      sequence,
    });
  } else {
    scan.facts.set(canonicalName, {
      kind: 'regular',
      path: canonicalName,
      identity,
      metadata,
      sequence,
    });
  }
  scan.metadataEffects.set(identity, { ...metadata, sequence });
  return 'applied';
}

function recordGeneralPublicEffect(
  record: GeneralMutationRecord,
  index: GeneralFileIndex,
  contentDir: string,
  event: FileDiskEvent,
  changedPaths: ReadonlySet<string> = new Set(),
): void {
  const canonicalName = toPosix(relative(contentDir, event.path));
  const names =
    event.kind === 'file-delete'
      ? new Set([event.relativePath])
      : new Set([canonicalName, event.relativePath]);
  const recorded = new Set([...names, ...changedPaths]);
  const facts = generalFactsFor(index, recorded);
  for (const name of recorded) {
    record.generalNames.add(name);
    record.generalEffects.set(name, facts.get(name) ?? null);
  }
  if (event.kind === 'file-delete') return;
  const fact = facts.get(canonicalName) ?? facts.get(event.relativePath);
  if (fact?.kind !== 'regular' && fact?.kind !== 'symlink') return;
  record.generalMetadata.set(fact.identity, {
    size: event.size,
    modified: new Date(event.modifiedTs).toISOString(),
    sequence: fact.sequence,
  });
}

function recordObservedExtension(
  observation: DiskObservation,
  docName: string,
  ext: string,
): boolean {
  observation.extensions.push({ docName, ext });
  const existing = observation.preferredExtensions.get(docName);
  const orderedExtensions: readonly string[] = SUPPORTED_DOC_EXTENSIONS;
  if (
    !existing ||
    orderedExtensions.indexOf(ext.toLowerCase()) < orderedExtensions.indexOf(existing.toLowerCase())
  ) {
    observation.preferredExtensions.set(docName, ext);
    return true;
  }
  return existing.toLowerCase() === ext.toLowerCase();
}

async function scanDisk(
  dir: string,
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  observation: DiskObservation,
  generalScan: GeneralScanAccumulator,
  visitedInodes?: Set<string>,
  generalVisitedInodes?: Set<string>,
): Promise<void> {
  const visited = visitedInodes ?? new Set<string>();
  const generalVisited = generalVisitedInodes ?? new Set<string>();
  if (observation.stopped?.()) {
    observation.complete = false;
    return;
  }
  try {
    const entries = await readdir(dir, { withFileTypes: true });
    for (const entry of entries) {
      const fullPath = join(dir, entry.name);
      let lst: BigIntStats;
      try {
        lst = await lstat(fullPath, { bigint: true });
      } catch (e) {
        const code = errnoCode(e);
        if (code !== 'ENOENT') markUnreadable(observation, fullPath, code);
        continue;
      }
      const lstIdentity = filesystemIdentity(lst);

      if (lst.isSymbolicLink()) {
        let canonical: string;
        try {
          canonical = await realpath(fullPath);
        } catch (e) {
          const code = errnoCode(e);
          if (code === 'ENOENT' || code === 'ELOOP') {
            log.warn({ path: fullPath, code }, `Broken/cyclic symlink at ${fullPath}, skipping`);
          } else {
            markUnreadable(observation, fullPath, code);
          }
          continue;
        }

        if (
          !isWithinContentDir(canonical, contentDir) ||
          resolvesIntoPrivateState(fullPath, canonical, contentDir)
        ) {
          log.warn(
            { path: fullPath, canonical },
            `Symlink escape: ${fullPath} → ${canonical}, skipping`,
          );
          continue;
        }

        try {
          const canonStat = await stat(canonical, { bigint: true });
          const canonIdentity = filesystemIdentity(canonStat);
          if (canonStat.isFile() && !isSupportedDocFile(entry.name)) {
            const canonicalName = toPosix(relative(contentDir, canonical));
            const lexicalName = toPosix(relative(contentDir, fullPath));
            if (
              contentFilter?.isPathIgnored(canonicalName) ||
              contentFilter?.isPathIgnored(lexicalName) ||
              isSystemDoc(canonicalName) ||
              isConfigDoc(canonicalName) ||
              isSystemDoc(lexicalName) ||
              isConfigDoc(lexicalName)
            )
              continue;
            if (visited.has(canonIdentity) && !generalVisited.has(canonIdentity)) continue;
            visited.add(canonIdentity);
            generalVisited.add(canonIdentity);
            const registered = recordScannedGeneralPath(
              observation,
              generalScan,
              contentDir,
              canonicalName,
              lexicalName,
              true,
              canonIdentity,
              { size: Number(canonStat.size), modified: statMtime(canonStat).toISOString() },
              contentFilter,
            );
            if (registered === 'unobserved') observation.complete = false;
            observation.paths.add(fullPath);
            continue;
          }
          if (visited.has(canonIdentity)) {
            if (canonStat.isFile() && isSupportedDocFile(entry.name)) {
              const aliasDocName = pathToDocName(fullPath, contentDir);
              const canonicalDocName = pathToDocName(canonical, contentDir);
              observation.aliasMap.set(aliasDocName, canonicalDocName);
              observation.aliasPaths.set(aliasDocName, fullPath);
              observation.paths.add(fullPath);
              const existing = observation.fileIndex.get(canonicalDocName);
              if (existing && !existing.aliases.includes(aliasDocName)) {
                existing.aliases.push(aliasDocName);
              }
            } else if (canonStat.isDirectory()) {
              const relPath = contentRelativePath(contentDir, fullPath);
              if (!contentFilter || (relPath && !contentFilter.isDirExcluded(relPath))) {
                observation.folderAliasIndex.set(
                  toPosix(relative(contentDir, fullPath)),
                  toPosix(relative(contentDir, canonical)),
                );
                observation.paths.add(fullPath);
              }
            }
            continue;
          }
          visited.add(canonIdentity);

          if (canonStat.isDirectory()) {
            const relPath = contentRelativePath(contentDir, fullPath);
            if (contentFilter) {
              if (!relPath || contentFilter.isDirExcluded(relPath)) continue;
            }
            observation.folderAliasIndex.set(
              toPosix(relative(contentDir, fullPath)),
              toPosix(relative(contentDir, canonical)),
            );
            observation.paths.add(fullPath);
            await scanDisk(
              canonical,
              contentDir,
              contentFilter,
              observation,
              generalScan,
              visited,
              generalVisited,
            );
          } else if (canonStat.isFile() && isSupportedDocFile(entry.name)) {
            if (contentFilter) {
              const relPath = toPosix(relative(contentDir, canonical));
              if (contentFilter.isExcluded(relPath)) continue;
            }
            const aliasDocName = pathToDocName(fullPath, contentDir);
            const canonicalDocName = pathToDocName(canonical, contentDir);
            observation.aliasMap.set(aliasDocName, canonicalDocName);
            observation.aliasPaths.set(aliasDocName, fullPath);
            observation.paths.add(fullPath);

            const signature = statSignature(canonStat);
            const decision = decideMarkdownRead(observation.reuse, canonical, canonStat, signature);
            if (decision.kind !== 'read') {
              const reused = reusedMarkdownEntry(
                decision,
                observation,
                canonical,
                canonIdentity,
                [aliasDocName],
                signature,
              );
              if (reused === undefined) continue;
              const ext = extractDocExtension(canonical);
              if (ext && !recordObservedExtension(observation, canonicalDocName, ext)) continue;
              observation.fileIndex.set(canonicalDocName, reused);
              continue;
            }
            try {
              const content = await readFile(canonical, 'utf-8');
              const hash = contentHash(content);
              observation.hashes.set(canonical, hash);
              observeMarkdownSignature(observation, canonical, canonStat, signature);
              observation.paths.add(canonical);
              const ext = extractDocExtension(canonical);
              if (ext && !recordObservedExtension(observation, canonicalDocName, ext)) continue;
              observation.fileIndex.set(canonicalDocName, {
                size: Number(canonStat.size),
                modified: statMtime(canonStat).toISOString(),
                canonicalPath: canonical,
                inode: canonIdentity,
                aliases: [aliasDocName],
                kind: 'markdown',
                ...derivePageMeta(content, canonicalDocName),
              });
            } catch (err) {
              const code = errnoCode(err);
              if (code !== 'ENOENT') markUnreadable(observation, canonical, code);
            }
          }
        } catch (e) {
          const code = errnoCode(e);
          if (code !== 'ENOENT' && code !== 'ENOTDIR') {
            markUnreadable(observation, fullPath, code);
          } else {
            log.warn({ path: canonical, err: e }, `Failed to stat symlink target ${canonical}`);
          }
        }
      } else if (lst.isDirectory()) {
        const relPath = contentRelativePath(contentDir, fullPath);
        if (contentFilter) {
          if (!relPath || contentFilter.isDirExcluded(relPath)) {
            const structural = relPath && structuralIgnoreOccurrence(relPath);
            if (structural) observation.structuralIgnoreDirs.add(structural);
            continue;
          }
        }
        upsertFolderIndexEntry(observation.folderIndex, contentDir, fullPath, lst);
        observation.paths.add(fullPath);
        await scanDisk(
          fullPath,
          contentDir,
          contentFilter,
          observation,
          generalScan,
          visited,
          generalVisited,
        );
      } else if (lst.isFile() && isSupportedDocFile(entry.name)) {
        if (visited.has(lstIdentity)) continue;
        visited.add(lstIdentity);

        if (contentFilter) {
          const relPath = toPosix(relative(contentDir, fullPath));
          if (contentFilter.isExcluded(relPath)) continue;
        }
        const signature = statSignature(lst);
        const decision = decideMarkdownRead(observation.reuse, fullPath, lst, signature);
        if (decision.kind !== 'read') {
          const reused = reusedMarkdownEntry(
            decision,
            observation,
            fullPath,
            lstIdentity,
            [],
            signature,
          );
          if (reused === undefined) continue;
          const docName = pathToDocName(fullPath, contentDir);
          const ext = extractDocExtension(fullPath);
          if (ext && !recordObservedExtension(observation, docName, ext)) continue;
          observation.fileIndex.set(docName, reused);
          continue;
        }
        try {
          const content = await readFile(fullPath, 'utf-8');
          observation.hashes.set(fullPath, contentHash(content));
          observeMarkdownSignature(observation, fullPath, lst, signature);
          observation.paths.add(fullPath);

          const docName = pathToDocName(fullPath, contentDir);
          const ext = extractDocExtension(fullPath);
          if (ext && !recordObservedExtension(observation, docName, ext)) continue;
          observation.fileIndex.set(docName, {
            size: Number(lst.size),
            modified: statMtime(lst).toISOString(),
            canonicalPath: fullPath,
            inode: lstIdentity,
            aliases: [],
            kind: 'markdown',
            ...derivePageMeta(content, docName),
          });
        } catch (err) {
          const code = errnoCode(err);
          if (code !== 'ENOENT') markUnreadable(observation, fullPath, code);
        }
      } else if (lst.isFile()) {
        if (contentFilter) {
          const relPath = toPosix(relative(contentDir, fullPath));
          if (contentFilter.isPathIgnored(relPath)) continue;
        }
        const relativePath = toPosix(relative(contentDir, fullPath));
        if (isSystemDoc(relativePath) || isConfigDoc(relativePath)) continue;
        if (visited.has(lstIdentity) && !generalVisited.has(lstIdentity)) continue;
        visited.add(lstIdentity);
        generalVisited.add(lstIdentity);
        const registered = recordScannedGeneralPath(
          observation,
          generalScan,
          contentDir,
          relativePath,
          relativePath,
          false,
          lstIdentity,
          { size: Number(lst.size), modified: statMtime(lst).toISOString() },
          contentFilter,
        );
        if (registered === 'unobserved') observation.complete = false;
        observation.paths.add(fullPath);
      }
    }
  } catch (err) {
    const code = errnoCode(err);
    if (dir === contentDir) {
      observation.complete = false;
      log.warn({ dir, err }, `Failed to read the content directory ${dir}`);
    } else if (code !== 'ENOENT' && code !== 'ENOTDIR') {
      markUnreadable(observation, dir, code);
    }
  }
}

async function observeDisk(
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  reuse?: FreshnessReuse,
  stopped?: () => boolean,
): Promise<DiskObservation> {
  const observation: DiskObservation = {
    fileIndex: new Map(),
    generalFileIndex: new GeneralFileIndex(contentDir),
    folderIndex: new Map(),
    aliasMap: new Map(),
    aliasPaths: new Map(),
    folderAliasIndex: new Map(),
    hashes: new Map(),
    unreadable: new Map(),
    signatures: new Map(),
    sampledAtNs: sampleWallClockNs(),
    reuse,
    extensions: [],
    preferredExtensions: new Map(),
    structuralIgnoreDirs: new Set(),
    paths: new Set(),
    complete: true,
    stopped,
  };
  const generalScan: GeneralScanAccumulator = {
    facts: new Map(),
    metadataEffects: new Map(),
  };
  await scanDisk(contentDir, contentDir, contentFilter, observation, generalScan);
  if (observation.stopped?.()) {
    observation.complete = false;
    return observation;
  }
  reportUnreadableEntries(observation);
  materializeScanGeneralIndex(observation, contentDir, generalScan);
  for (const [name, entry] of new Map(observation.generalFileIndex)) {
    const state = knownGeneralState(entry);
    if (!state) continue;
    const members = state.entry.fileMembers;
    const first = members.regularPaths.toSorted()[0];
    if (name === first) continue;
    observation.generalFileIndex.delete(name);
    storeKnownGeneralGroup(
      observation.generalFileIndex,
      contentDir,
      null,
      state.identity,
      members.regularPaths,
      members.symlinks,
      entry,
      state.sequence,
    );
  }
  for (const [alias, canonical] of observation.aliasMap) {
    const entry = observation.fileIndex.get(canonical);
    if (entry && !entry.aliases.includes(alias)) entry.aliases.push(alias);
  }
  return observation;
}

type ObservedInventory = Pick<DiskObservation, 'fileIndex' | 'folderIndex' | 'folderAliasIndex'>;

function generalAliasIsAdmitted(
  alias: string,
  targetPath: string,
  contentFilter: ContentFilter | undefined,
): boolean {
  return !contentFilter?.isPathIgnored(alias) && !contentFilter?.isPathIgnored(targetPath);
}

function selectObservedInventory(
  observation: DiskObservation,
  contentDir: string,
  contentFilter: ContentFilter | undefined,
): ObservedInventory & { generalFacts: Map<string, GeneralPathFact> } {
  return {
    fileIndex: new Map(
      [...observation.fileIndex].filter(
        ([, entry]) =>
          !contentFilter?.isExcluded(toPosix(relative(contentDir, entry.canonicalPath))),
      ),
    ),
    generalFacts: new Map(
      [...generalPathFacts(observation.generalFileIndex)].filter(([, fact]) =>
        fact.kind === 'symlink'
          ? generalAliasIsAdmitted(fact.path, fact.targetPath, contentFilter)
          : !contentFilter?.isPathIgnored(fact.path),
      ),
    ),
    folderIndex: new Map(
      [...observation.folderIndex].filter(([path]) => !contentFilter?.isDirExcluded(path)),
    ),
    folderAliasIndex: new Map(
      [...observation.folderAliasIndex].filter(([alias]) => !contentFilter?.isDirExcluded(alias)),
    ),
  };
}

function publishDiskObservation(
  observation: DiskObservation,
  inventory: ObservedInventory,
  fileIndex: Map<string, FileIndexEntry>,
  generalFileIndex: GeneralFileIndex,
  folderIndex: Map<string, FolderIndexEntry>,
  aliasMap: Map<string, string>,
  folderAliasIndex: Map<string, string>,
  structuralIgnoreDirs?: Set<string>,
  publishGeneral = true,
): void {
  for (const { docName, ext } of observation.extensions) {
    const reg = registerDocExtension(docName, ext);
    if (reg.shadowed) {
      log.warn(
        { docName, effective: reg.effective, shadowed: reg.shadowed },
        `docName "${docName}" has both "${reg.effective}" and "${reg.shadowed}" on disk; "${reg.effective}" wins (industry convention). Rename or delete one to disambiguate.`,
      );
    }
  }
  for (const [path, hash] of observation.hashes) {
    recordObservedHash(path, hash, observation.signatures.get(path));
  }
  for (const [docName, entry] of inventory.fileIndex) {
    if (getDocExtension(docName).toLowerCase() !== extname(entry.canonicalPath).toLowerCase()) {
      continue;
    }
    fileIndex.set(docName, entry);
  }
  if (publishGeneral) {
    for (const [name, entry] of observation.generalFileIndex) {
      generalFileIndex.set(name, entry);
    }
  }
  for (const [path, entry] of inventory.folderIndex) folderIndex.set(path, entry);
  for (const [alias, canonical] of observation.aliasMap) aliasMap.set(alias, canonical);
  for (const [alias, canonical] of inventory.folderAliasIndex) {
    folderAliasIndex.set(alias, canonical);
  }
  for (const path of observation.structuralIgnoreDirs) structuralIgnoreDirs?.add(path);
}

function updateGeneralFileIndex(
  event: FileDiskEvent,
  generalFileIndex: Map<string, FileIndexEntry>,
  contentDir: string,
  canonicalPath = event.path,
): void {
  if (isSystemDoc(event.relativePath) || isConfigDoc(event.relativePath)) return;
  if (event.kind === 'file-delete') {
    const direct = generalFileIndex.get(event.relativePath);
    if (direct?.kind === 'file') {
      generalFileIndex.delete(event.relativePath);
      return;
    }
    for (const [name, entry] of generalFileIndex) {
      if (entry.kind !== 'file' || !entry.aliases.includes(event.relativePath)) continue;
      generalFileIndex.set(name, {
        ...entry,
        aliases: entry.aliases.filter((alias) => alias !== event.relativePath),
      });
      return;
    }
    return;
  }

  const canonicalName = toPosix(relative(contentDir, canonicalPath));
  for (const [name, entry] of generalFileIndex) {
    if (name === canonicalName || entry.kind !== 'file') continue;
    if (!entry.aliases.includes(event.relativePath)) continue;
    generalFileIndex.set(name, {
      ...entry,
      aliases: entry.aliases.filter((alias) => alias !== event.relativePath),
    });
  }
  const prior = generalFileIndex.get(canonicalName);
  const aliases = new Set(prior?.kind === 'file' ? prior.aliases : []);
  if (event.relativePath !== canonicalName) aliases.add(event.relativePath);
  aliases.delete(canonicalName);
  generalFileIndex.set(canonicalName, {
    size: event.size,
    modified: new Date(event.modifiedTs).toISOString(),
    canonicalPath,
    inode: event.inode || (prior?.kind === 'file' ? prior.inode : ''),
    aliases: [...aliases],
    kind: 'file',
  });
}

export function updateFileIndex(event: DiskEvent, fileIndex: Map<string, FileIndexEntry>): void {
  if (
    event.kind === 'asset-create' ||
    event.kind === 'asset-delete' ||
    event.kind === 'folder-create' ||
    event.kind === 'folder-delete'
  ) {
    return;
  }
  if (
    event.kind === 'file-create' ||
    event.kind === 'file-update' ||
    event.kind === 'file-delete'
  ) {
    const docName = event.relativePath;
    if (isSystemDoc(docName) || isConfigDoc(docName)) return;
    if (event.kind === 'file-delete') {
      const existing = fileIndex.get(docName);
      if (existing && existing.kind === 'file') {
        fileIndex.delete(docName);
      }
      return;
    }
    const existing = fileIndex.get(docName);
    if (existing && existing.kind === 'markdown') return;
    fileIndex.set(docName, {
      size: event.size,
      modified: new Date(event.modifiedTs).toISOString(),
      canonicalPath: existing?.canonicalPath ?? event.path,
      inode: event.inode || existing?.inode || '',
      aliases: existing?.aliases ?? [],
      kind: 'file',
    });
    return;
  }
  const docName = event.kind === 'rename' ? event.newDocName : event.docName;
  if (isReservedForUserTree(docName)) return;
  switch (event.kind) {
    case 'create':
    case 'update':
    case 'conflict': {
      const docName = event.docName;
      const existing = fileIndex.get(docName);
      const ext = extractDocExtension(event.path);
      if (ext) registerDocExtension(docName, ext);
      const canonicalPath = existing?.canonicalPath ?? event.path;
      fileIndex.set(docName, {
        size: Buffer.byteLength(event.content, 'utf-8'),
        modified: new Date().toISOString(),
        canonicalPath,
        inode: fileIdentityOf(canonicalPath) ?? existing?.inode ?? '',
        aliases: existing?.aliases ?? [],
        kind: 'markdown',
        ...derivePageMeta(event.content, docName),
      });
      break;
    }
    case 'delete': {
      if (fileIndex.has(event.docName)) {
        fileIndex.delete(event.docName);
        forgetDocExtension(event.docName);
      } else {
        for (const [, entry] of fileIndex) {
          const idx = entry.aliases.indexOf(event.docName);
          if (idx !== -1) {
            entry.aliases.splice(idx, 1);
            break;
          }
        }
      }
      break;
    }
    case 'rename': {
      const existing = fileIndex.get(event.oldDocName);
      fileIndex.delete(event.oldDocName);
      forgetDocExtension(event.oldDocName);
      const ext = extractDocExtension(event.newPath);
      if (ext) registerDocExtension(event.newDocName, ext);
      fileIndex.set(event.newDocName, {
        size: Buffer.byteLength(event.content, 'utf-8'),
        modified: new Date().toISOString(),
        canonicalPath: event.newPath,
        inode: fileIdentityOf(event.newPath) ?? existing?.inode ?? '',
        aliases: existing?.aliases ?? [],
        kind: 'markdown',
        ...derivePageMeta(event.content, event.newDocName),
      });
      break;
    }
    default:
      assertNeverDiskEvent(event);
  }
}

function updateFolderIndexFromRawEvents(
  rawEvents: Array<{ type: 'create' | 'update' | 'delete'; path: string }>,
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  folderIndex: Map<string, FolderIndexEntry>,
): { events: FolderDiskEvent[]; untrackedFiles: string[] } {
  const events: FolderDiskEvent[] = [];
  const untrackedFiles: string[] = [];

  for (const raw of rawEvents) {
    const relativePath = contentRelativePath(contentDir, raw.path);
    if (!relativePath) continue;

    if (raw.type === 'delete') {
      if (removeFolderIndexEntries(folderIndex, relativePath)) {
        events.push({ kind: 'folder-delete', path: raw.path, relativePath });
      }
      continue;
    }

    let lst: BigIntStats;
    try {
      lst = lstatSync(raw.path, { bigint: true });
    } catch (err) {
      const code = errnoCode(err);
      if (code !== 'ENOENT') {
        log.warn({ path: raw.path, code }, `folder lstat failed for ${raw.path} (${code})`);
      }
      continue;
    }

    let folderStat: BigIntStats | null = null;
    let canonicalPath = raw.path;
    if (lst.isDirectory()) {
      folderStat = lst;
    } else if (lst.isSymbolicLink()) {
      try {
        canonicalPath = resolveNativePath(raw.path);
        if (!isWithinContentDir(canonicalPath, contentDir)) continue;
        const stat = statSync(canonicalPath, { bigint: true });
        if (stat.isDirectory()) folderStat = stat;
      } catch (err) {
        const code = errnoCode(err);
        if (code !== 'ENOENT') {
          log.warn(
            { path: raw.path, code },
            `folder symlink resolve failed for ${raw.path} (${code})`,
          );
        }
        folderStat = null;
      }
    }
    if (!folderStat) continue;
    if (contentFilter?.isDirExcluded(relativePath)) continue;

    const hadFolder = folderIndex.has(relativePath);
    upsertFolderIndexEntry(folderIndex, contentDir, raw.path, folderStat, canonicalPath);
    if (!hadFolder) {
      events.push({ kind: 'folder-create', path: raw.path, relativePath });
      scanForUntrackedSubfolders(
        canonicalPath,
        contentDir,
        contentFilter,
        folderIndex,
        events,
        untrackedFiles,
      );
    }
  }

  return { events, untrackedFiles };
}

function scanForUntrackedSubfolders(
  startPath: string,
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  folderIndex: Map<string, FolderIndexEntry>,
  events: FolderDiskEvent[],
  untrackedFiles: string[],
): void {
  const queue: string[] = [startPath];
  while (queue.length > 0) {
    const dir = queue.shift();
    if (dir === undefined) continue;

    let entries: Dirent[];
    try {
      entries = readdirSync(dir, { withFileTypes: true });
    } catch (err) {
      const code = errnoCode(err);
      if (code !== 'ENOENT') {
        log.warn({ dir, code }, `folder rescan readdir failed for ${dir} (${code})`);
      }
      continue;
    }

    for (const entry of entries) {
      if (entry.isFile()) {
        untrackedFiles.push(join(dir, entry.name));
        continue;
      }
      if (!entry.isDirectory()) continue;

      const fullPath = join(dir, entry.name);
      const relPath = contentRelativePath(contentDir, fullPath);
      if (!relPath) continue;
      if (contentFilter?.isDirExcluded(relPath)) continue;

      let stat: BigIntStats;
      try {
        stat = lstatSync(fullPath, { bigint: true });
      } catch (err) {
        const code = errnoCode(err);
        if (code !== 'ENOENT') {
          log.warn(
            { path: fullPath, code },
            `folder rescan lstat failed for ${fullPath} (${code})`,
          );
        }
        continue;
      }
      if (!stat.isDirectory()) continue;

      if (!folderIndex.has(relPath)) {
        upsertFolderIndexEntry(folderIndex, contentDir, fullPath, stat);
        events.push({ kind: 'folder-create', path: fullPath, relativePath: relPath });
      }
      queue.push(fullPath);
    }
  }
}

async function handleRawEventsInternal(
  rawEvents: InternalRawFileEvent[],
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  fileIndex: Map<string, FileIndexEntry>,
  folderIndex: Map<string, FolderIndexEntry>,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  aliasMap?: Map<string, string>,
  admitEvent?: (event: DiskEvent) => boolean,
  generalFileIndex?: GeneralFileIndex,
  aliasPaths?: Map<string, string>,
): Promise<void> {
  for (const event of rawEvents) noteIntakeMutation(event.path);
  const declaredBeforeBatch: ReadonlySet<string> = new Set(removalTracker.keys());
  const safeEvents = rawEvents.filter((e) => {
    if (!eventEscapesContentDir(e.path, contentDir)) return true;
    recordWatcherDecision('drop-symlink-escape', e.type, e.path);
    log.warn({ path: e.path, type: e.type }, `Symlink escape: ${e.path}, dropping ${e.type} event`);
    return false;
  });

  const folderInputs = safeEvents.filter(
    (event) => !('entryKind' in event) || event.entryKind !== 'file',
  );
  let knownFilePaths: Set<string> | undefined;
  const fileInputs = safeEvents.filter((event) => {
    if (event.type !== 'delete') return true;
    if ('entryKind' in event) return event.entryKind === 'file';
    const relativePath = contentRelativePath(contentDir, event.path);
    if (relativePath === null || !folderIndex.has(relativePath)) return true;
    if (!knownFilePaths) {
      knownFilePaths = new Set([...fileIndex.values()].map((entry) => entry.canonicalPath));
      for (const name of generalMemberPaths(contentDir, generalFileIndex ?? fileIndex)) {
        knownFilePaths.add(join(contentDir, name));
      }
      for (const path of aliasPaths?.values() ?? []) knownFilePaths.add(path);
    }
    return (
      knownFilePaths.has(event.path) ||
      (aliasPaths === undefined &&
        isSupportedDocFile(event.path) &&
        aliasMap?.has(pathToDocName(event.path, contentDir)) === true)
    );
  });

  const { events: folderEvents, untrackedFiles } = updateFolderIndexFromRawEvents(
    folderInputs,
    contentDir,
    contentFilter,
    folderIndex,
  );

  let rescuedCreates: RawFileEvent[] = [];
  if (untrackedFiles.length > 0) {
    const batchPaths = new Set(safeEvents.map((e) => e.path));
    const knownCanonicalPaths = new Set<string>();
    for (const entry of fileIndex.values()) knownCanonicalPaths.add(entry.canonicalPath);
    for (const name of generalMemberPaths(contentDir, generalFileIndex ?? fileIndex)) {
      knownCanonicalPaths.add(join(contentDir, name));
    }
    rescuedCreates = untrackedFiles
      .filter(
        (path) =>
          !batchPaths.has(path) && !lastKnownHash.has(path) && !knownCanonicalPaths.has(path),
      )
      .filter((path) => {
        if (!eventEscapesContentDir(path, contentDir)) return true;
        recordWatcherDecision('drop-symlink-escape', 'create', path);
        return false;
      })
      .map((path) => ({ type: 'create' as const, path }));
    if (rescuedCreates.length > 0) {
      log.debug(
        {
          count: rescuedCreates.length,
          paths: rescuedCreates.map((e) => toPosix(relative(contentDir, e.path))),
        },
        '[file-watcher] re-injecting rescued file creates from new-folder rescan',
      );
    }
  }
  const batchPaths = new Set(safeEvents.map((event) => event.path));
  const collapsedDeletes: RawFileEvent[] = [];
  for (const folderEvent of folderEvents) {
    if (folderEvent.kind !== 'folder-delete') continue;
    const prefix = `${folderEvent.relativePath}/`;
    for (const [indexedPath, entry] of fileIndex) {
      if (!indexedPath.startsWith(prefix) || batchPaths.has(entry.canonicalPath)) continue;
      batchPaths.add(entry.canonicalPath);
      collapsedDeletes.push({ type: 'delete', path: entry.canonicalPath });
    }
    for (const name of generalMemberPaths(contentDir, generalFileIndex ?? fileIndex)) {
      if (!name.startsWith(prefix)) continue;
      const path = join(contentDir, name);
      if (batchPaths.has(path)) continue;
      batchPaths.add(path);
      collapsedDeletes.push({ type: 'delete', path });
    }
  }

  const fileEvents = fileInputs.concat(rescuedCreates, collapsedDeletes);
  const removedSameNameAliases = new Map<string, { name: string; targetName: string }>();
  for (const event of fileEvents) {
    if (event.type !== 'delete' || !isSupportedDocFile(event.path)) continue;
    const name = pathToDocName(event.path, contentDir);
    const owner = fileIndex.get(name);
    const targetName = aliasMap?.get(name);
    if (
      aliasPaths?.get(name) === event.path &&
      targetName !== undefined &&
      owner?.kind === 'markdown' &&
      owner.canonicalPath !== event.path &&
      !contentFilter?.isExcluded(toPosix(relative(contentDir, event.path)))
    ) {
      removedSameNameAliases.set(event.path, { name, targetName });
    }
  }
  for (const [path, { name, targetName }] of removedSameNameAliases) {
    if (admitEvent && !admitEvent({ kind: 'delete', path, docName: name })) continue;
    aliasMap?.delete(name);
    aliasPaths?.delete(name);
    const target = fileIndex.get(targetName);
    if (target) {
      fileIndex.set(targetName, {
        ...target,
        aliases: target.aliases.filter((alias) => alias !== name),
      });
    }
    removeLastKnownHash(path);
    isSelfRemoval(path);
  }

  const mdEvents = fileEvents.filter(
    (event) =>
      isSupportedDocFile(event.path) &&
      !(event.type === 'delete' && removedSameNameAliases.has(event.path)),
  );
  const assetEvents = fileEvents.filter((e) =>
    isSupportedAssetFile(e.path, LINKABLE_ASSET_EXTENSIONS),
  );
  const nonMdRawEvents = fileEvents.filter((e) => !isSupportedDocFile(e.path));
  if (
    mdEvents.length === 0 &&
    assetEvents.length === 0 &&
    folderEvents.length === 0 &&
    nonMdRawEvents.length === 0
  ) {
    return;
  }

  const canonicalAliasRecords = new Set<RawFileEvent>();
  const removals = new Map<RawFileEvent, MarkdownRemovalObservation>();
  for (const event of mdEvents) {
    if (event.type !== 'delete') continue;
    const origin = removalOrigins.get(event) ?? {
      kind: 'native',
      docName: pathToDocName(event.path, contentDir),
      path: event.path,
    };
    removals.set(
      event,
      observeMarkdownRemoval(origin, contentDir, fileIndex, contentFilter, aliasPaths),
    );
  }
  const observations = new Map<string, OwnWriteObservation>();
  const admittedMarkdownPaths = new Set<string>();
  try {
    const diskEvents =
      mdEvents.length > 0
        ? await classifyEventsInternal(
            mdEvents,
            contentDir,
            observations,
            contentFilter,
            aliasMap,
            {
              fileIndex,
              aliasPaths: aliasPaths ?? new Map(),
              canonicalRecords: canonicalAliasRecords,
              removals,
            },
          )
        : [];

    for (const event of diskEvents) {
      if (
        event.kind !== 'delete' &&
        event.kind !== 'rename' &&
        discardSupersededOwnWrite(event.path, event.content, observations)
      ) {
        recordSupersededOwnWriteSkip(event.kind, event.path);
        continue;
      }
      if (admitEvent && !admitEvent(event)) continue;
      if (event.kind === 'rename') {
        admittedMarkdownPaths.add(event.oldPath);
        admittedMarkdownPaths.add(event.newPath);
      } else {
        admittedMarkdownPaths.add(event.path);
      }
      let isSelf = false;
      let indexEvent = event;

      const previousIndexedFields =
        event.kind === 'update'
          ? (() => {
              const previous = fileIndex.get(event.docName);
              if (previous?.kind !== 'markdown') return undefined;
              return {
                title: previous.title,
                description: previous.description,
                type: previous.type,
              };
            })()
          : undefined;

      if (event.kind !== 'delete' && event.kind !== 'rename') {
        const hash = contentHash(event.content);
        const checkPath = observedPathKey(event.path);
        isSelf = isSelfWrite(checkPath, hash);
        voidRemoval(event.path, declaredBeforeBatch);
      } else if (event.kind === 'rename') {
        const hash = contentHash(event.content);
        const checkPath = observedPathKey(event.newPath);
        isSelf = isSelfWrite(checkPath, hash);
        indexEvent = { ...event, newPath: checkPath };
        voidRemoval(event.oldPath, declaredBeforeBatch);
        voidRemoval(event.newPath, declaredBeforeBatch);
      } else {
        isSelf = isSelfRemoval(event.path);
      }

      const dispatchedEvent =
        event.kind === 'update' && previousIndexedFields !== undefined
          ? { ...event, previousIndexedFields }
          : event;
      const staleNamespace =
        (event.kind === 'delete' || event.kind === 'rename') &&
        getMarkdownRemovalCurrentness(event)?.() === false;

      if (!staleNamespace) updateFileIndex(indexEvent, fileIndex);
      else if (indexEvent.kind === 'rename') {
        updateFileIndex(
          {
            kind: 'create',
            path: indexEvent.newPath,
            docName: indexEvent.newDocName,
            content: indexEvent.content,
          },
          fileIndex,
        );
      }

      if (contentFilter && !isSelf && !staleNamespace) {
        switch (event.kind) {
          case 'create':
            contentFilter.incrementMdDir(dirname(event.docName));
            break;
          case 'delete':
            contentFilter.decrementMdDir(dirname(event.docName));
            break;
          case 'rename':
            contentFilter.decrementMdDir(dirname(event.oldDocName));
            contentFilter.incrementMdDir(dirname(event.newDocName));
            break;
          case 'update':
          case 'conflict':
            break;
          default:
            assertNeverDiskEvent(event);
        }
      }

      if (isSelf) {
        const selfKind = event.kind === 'delete' ? 'self-removal' : 'self-write';
        log.debug(
          {
            kind: event.kind,
            path: event.kind === 'rename' ? event.newPath : event.path,
            self: true,
          },
          `[file-watcher] Skipped ${selfKind}: ${event.kind}`,
        );
        _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: true });
        recordWatcherDecision(
          `${selfKind}-skip`,
          event.kind,
          event.kind === 'rename' ? event.newPath : event.path,
        );
        continue;
      }

      log.debug(
        {
          kind: event.kind,
          path: event.kind === 'rename' ? event.newPath : event.path,
          staleNamespace,
        },
        `[file-watcher] Dispatching: ${event.kind}`,
      );
      _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: false });
      const rawPath = event.kind === 'rename' ? event.newPath : event.path;
      recordWatcherDecision('dispatched', event.kind, rawPath);
      await withSpan(
        'file_watcher.process_event',
        {
          attributes: {
            'disk.kind': event.kind,
            'disk.path': normalizeFsPath(rawPath),
            'disk.path.role': classifyFsPath(rawPath),
          },
        },
        async () => onDiskEvent(dispatchedEvent),
      );
    }
  } finally {
    releaseOwnWriteObservations(observations);
    if ([...removals.values()].some((observation) => observation.discarded)) {
      contentFilter?.rebuildDirCount();
    }
  }

  for (const event of folderEvents) {
    if (admitEvent && !admitEvent(event)) continue;
    log.debug({ kind: event.kind, path: event.path }, `[file-watcher] Dispatching: ${event.kind}`);
    _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: false });
    recordWatcherDecision('dispatched', event.kind, event.path);
    await withSpan(
      'file_watcher.process_event',
      {
        attributes: {
          'disk.kind': event.kind,
          'disk.path': normalizeFsPath(event.path),
          'disk.path.role': classifyFsPath(event.path),
        },
      },
      async () => onDiskEvent(event),
    );
  }

  for (const raw of assetEvents) {
    if (contentFilter) {
      const relPath = toPosix(relative(contentDir, raw.path));
      if (contentFilter.isPathIgnored(relPath)) {
        recordWatcherDecision('drop-filter-excluded', raw.type, raw.path);
        continue;
      }
    }
    const relativePath = toPosix(relative(contentDir, raw.path));
    const event: DiskEvent =
      raw.type === 'delete'
        ? { kind: 'asset-delete', path: raw.path, relativePath }
        : { kind: 'asset-create', path: raw.path, relativePath };
    if (admitEvent && !admitEvent(event)) continue;
    recordWatcherDecision('dispatched', event.kind, raw.path);
    await onDiskEvent(event);
  }

  for (const raw of nonMdRawEvents) {
    const relativePath = toPosix(relative(contentDir, raw.path));
    if (contentFilter?.isPathIgnored(relativePath)) {
      recordWatcherDecision('drop-filter-excluded', raw.type, raw.path);
      continue;
    }
    if (isSystemDoc(relativePath) || isConfigDoc(relativePath)) {
      recordWatcherDecision('drop-reserved-doc', raw.type, raw.path);
      continue;
    }

    if (raw.type === 'delete') {
      const event: DiskEvent = { kind: 'file-delete', path: raw.path, relativePath };
      if (admitEvent && !admitEvent(event)) continue;
      if (generalFileIndex) removeKnownGeneralPath(generalFileIndex, contentDir, relativePath);
      else updateGeneralFileIndex(event, fileIndex, contentDir);
      recordWatcherDecision('dispatched', event.kind, raw.path);
      await onDiskEvent(event);
      continue;
    }

    let st: BigIntStats;
    let leafSymlink = false;
    let canonicalPath: string;
    let canonicalContentDir: string;
    try {
      canonicalContentDir = resolveNativePath(contentDir);
      st = lstatSync(raw.path, { bigint: true });
      leafSymlink = st.isSymbolicLink();
      canonicalPath = resolveNativePath(raw.path);
      if (leafSymlink) st = statSync(canonicalPath, { bigint: true });
    } catch (e) {
      const code = errnoCode(e);
      recordWatcherDecision('drop-stat-failed', raw.type, raw.path);
      if (code !== 'ENOENT') {
        log.warn({ path: raw.path, code }, `file-event lstat failed for ${raw.path} (${code})`);
      }
      continue;
    }
    if (!st.isFile()) continue;
    if (!isWithinContentDir(canonicalPath, canonicalContentDir)) {
      recordWatcherDecision('drop-symlink-escape', raw.type, raw.path);
      log.warn({ path: raw.path, canonicalPath }, `Symlink escape: ${raw.path}, dropping event`);
      continue;
    }
    const canonicalName = toPosix(relative(canonicalContentDir, canonicalPath));
    if (contentFilter?.isPathIgnored(canonicalName)) {
      recordWatcherDecision('drop-filter-excluded', raw.type, raw.path);
      continue;
    }
    if (isSystemDoc(canonicalName) || isConfigDoc(canonicalName)) {
      recordWatcherDecision('drop-reserved-doc', raw.type, raw.path);
      continue;
    }

    const event: FileDiskEvent =
      raw.type === 'create'
        ? {
            kind: 'file-create',
            path: raw.path,
            relativePath,
            size: Number(st.size),
            modifiedTs: statMtime(st).getTime(),
            inode: filesystemIdentity(st),
          }
        : {
            kind: 'file-update',
            path: raw.path,
            relativePath,
            size: Number(st.size),
            modifiedTs: statMtime(st).getTime(),
            inode: filesystemIdentity(st),
          };
    if (admitEvent && !admitEvent(event)) continue;
    let formerOwner: [string, FileIndexEntry] | undefined;
    let formerTarget: string | null = null;
    if (generalFileIndex) {
      for (const name of generalOwnerNames(generalFileIndex, relativePath)) {
        const entry = generalFileIndex.get(name);
        if (entry?.kind !== 'file') continue;
        const relation = knownGeneralState(entry)?.entry.fileMembers.symlinks.find(
          (candidate) => candidate.path === relativePath,
        );
        if (!relation || relation.targetPath === canonicalName) continue;
        formerOwner = [name, entry];
        formerTarget = join(canonicalContentDir, relation.targetPath);
        break;
      }
    } else {
      for (const owner of fileIndex) {
        if (owner[1].kind !== 'file') continue;
        if (owner[0] !== canonicalName && owner[1].aliases.includes(relativePath)) {
          formerOwner = owner;
          formerTarget = owner[1].canonicalPath;
          break;
        }
      }
    }
    let confirmedAbsentFormerPath: string | undefined;
    if (formerOwner && formerTarget) {
      try {
        const formerPath = realpathSync(formerTarget);
        statSync(formerPath);
      } catch (err) {
        const code = errnoCode(err);
        if (code !== 'ENOENT' && code !== 'ENOTDIR' && code !== 'ELOOP') {
          recordWatcherDecision('drop-stat-failed', raw.type, raw.path);
          log.warn({ path: formerTarget, code, err }, 'file watcher former target read failed');
          continue;
        }
        if (generalFileIndex) {
          confirmedAbsentFormerPath = toPosix(relative(canonicalContentDir, formerTarget));
        } else {
          fileIndex.delete(formerOwner[0]);
        }
      }
    }
    if (generalFileIndex) {
      const registered = registerKnownGeneralPath(
        generalFileIndex,
        canonicalContentDir,
        canonicalName,
        relativePath,
        leafSymlink,
        filesystemIdentity(st),
        { size: event.size, modified: new Date(event.modifiedTs).toISOString() },
        contentFilter,
        confirmedAbsentFormerPath,
      );
      if (registered.kind === 'unobserved') {
        recordWatcherDecision('drop-stat-failed', raw.type, raw.path);
        continue;
      }
    } else {
      updateGeneralFileIndex(event, fileIndex, canonicalContentDir, canonicalPath);
    }
    recordWatcherDecision('dispatched', event.kind, raw.path);
    await onDiskEvent(event);
  }
  if (aliasPaths) {
    for (const event of mdEvents) {
      if (!admittedMarkdownPaths.has(event.path) || canonicalAliasRecords.has(event)) continue;
      const name = pathToDocName(event.path, contentDir);
      if (event.type !== 'delete' && aliasMap?.has(name)) {
        aliasPaths.set(name, event.path);
      } else if (aliasPaths.get(name) === event.path) {
        aliasPaths.delete(name);
      }
    }
  }
}

export async function handleRawEvents(
  rawEvents: Array<{ type: 'create' | 'update' | 'delete'; path: string }>,
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  fileIndex: Map<string, FileIndexEntry>,
  folderIndex: Map<string, FolderIndexEntry>,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  aliasMap?: Map<string, string>,
): Promise<void> {
  return handleRawEventsInternal(
    rawEvents,
    contentDir,
    contentFilter,
    fileIndex,
    folderIndex,
    onDiskEvent,
    aliasMap,
  );
}

let _fwEventsCounterCache: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null = null;
function _fileWatcherEventsCounter() {
  _fwEventsCounterCache ||= getMeter().createCounter('ok.file_watcher.events', {
    description: 'Number of file-watcher events classified by kind',
  });
  return _fwEventsCounterCache;
}

let _fwDropsCounterCache: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null = null;
function _fileWatcherDropsCounter() {
  _fwDropsCounterCache ||= getMeter().createCounter('ok.file_watcher.drops', {
    description: 'File-watcher events dropped before dispatch, by bounded reason',
  });
  return _fwDropsCounterCache;
}

const GLOB_METACHARACTER_RE = /[*?[\]{}()!+@|\\]/;

function structuralIgnoreOccurrence(relativePath: string): string | null {
  for (const dir of WATCHER_STRUCTURAL_IGNORE_DIRS) {
    if (relativePath === dir || relativePath.endsWith(`/${dir}`)) return relativePath;
  }
  return null;
}

type RecoveredAliasCandidate =
  | { kind: 'delete'; alias: string; targetPath: string }
  | {
      kind: 'target-update';
      alias: string;
      targetPath: string;
      targetIdentity: GeneralIdentity;
    };

function protectedGeneralNames(
  observed: DiskObservation,
  contentDir: string,
  previousGeneralFacts: ReadonlyMap<string, GeneralPathFact>,
  observedGeneralFacts: ReadonlyMap<string, GeneralPathFact>,
  changedDuringWalk: ((path: string) => boolean) | undefined,
): Set<string> {
  const names = new Set<string>();
  if (observed.unreadable.size === 0 && changedDuringWalk === undefined) return names;
  const keepKnownAt = (name: string): boolean => {
    const path = join(contentDir, name);
    return isUnderUnreadable(observed, path) || changedDuringWalk?.(path) === true;
  };
  const identities = new Set<GeneralIdentity>();
  for (const [name, fact] of previousGeneralFacts) {
    if (fact.kind !== 'unresolved' && keepKnownAt(name)) identities.add(fact.identity);
  }
  for (const [name, fact] of previousGeneralFacts) {
    if (
      keepKnownAt(name) ||
      (fact.kind !== 'unresolved' && identities.has(fact.identity)) ||
      (fact.kind === 'symlink' && keepKnownAt(fact.targetPath))
    ) {
      names.add(name);
    }
  }
  for (const name of observedGeneralFacts.keys()) {
    if (keepKnownAt(name)) names.add(name);
  }
  return names;
}

interface HandedObservation {
  readonly observed: DiskObservation;
  readonly sinceEpoch: number;
  readonly pass: number;
}

async function recoverDiskChanges(
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  fileIndex: Map<string, FileIndexEntry>,
  generalFileIndex: GeneralFileIndex,
  folderIndex: Map<string, FolderIndexEntry>,
  aliasMap: Map<string, string>,
  aliasPaths: Map<string, string>,
  folderAliasIndex: Map<string, string>,
  publicMutations: RecoveryPublicMutations,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  onAfterMutation: () => void,
  getFileIndexGeneration: () => number,
  counter: DispatchCounter,
  onRawBatch?: (absPaths: readonly string[]) => void,
  reuse?: FreshnessReuse,
  handed?: HandedObservation,
): Promise<RecoveryOutcome> {
  const countingOnDiskEvent = async (event: DiskEvent): Promise<void> => {
    counter.dispatched++;
    await onDiskEvent(event);
  };
  const knownPaths = new Set<string>();
  for (const entry of fileIndex.values()) knownPaths.add(entry.canonicalPath);
  for (const name of generalMemberPaths(contentDir, generalFileIndex)) {
    knownPaths.add(join(contentDir, name));
  }
  for (const relativePath of folderIndex.keys()) knownPaths.add(join(contentDir, relativePath));
  for (const path of aliasPaths.values()) knownPaths.add(path);
  for (const alias of folderAliasIndex.keys()) knownPaths.add(join(contentDir, alias));

  const previousFiles = new Map(fileIndex);
  const previousGeneralFiles = new Map(generalFileIndex);
  const previousFolders = new Map(folderIndex);
  const previousHashes = new Map(lastKnownHash);
  const previousAliases = new Map(aliasMap);
  const previousAliasPaths = new Map(aliasPaths);
  const previousFolderAliases = new Map(folderAliasIndex);

  let observed: DiskObservation;
  if (handed) {
    observed = handed.observed;
  } else {
    contentFilter?.refreshInPlaceSkillDirs();
    observed = await observeDisk(contentDir, contentFilter, reuse);
  }
  if (!observed.complete) return { completed: false, candidates: 0 };
  const changedDuringWalk = handed ? changedSince(handed.sinceEpoch, contentDir) : undefined;
  const intakeOwns = (path: string): boolean => changedDuringWalk?.(path) === true;

  const markdownPublicMutationFor = (name: string, entry: FileIndexEntry): boolean => {
    const names = publicMutations.markdownNames;
    if (names.size === 0) return false;
    if (names.has(name)) return true;
    const canonicalName = pathToDocName(entry.canonicalPath, contentDir);
    return names.has(canonicalName) || entry.aliases.some((alias) => names.has(alias));
  };
  const previousGeneralFacts = generalPathFacts(previousGeneralFiles);
  const observedGeneralFacts = generalPathFacts(observed.generalFileIndex);
  const unreadableAt = (path: string): boolean =>
    (observed.unreadable.size > 0 && isUnderUnreadable(observed, path)) || intakeOwns(path);
  const protectedGeneral = protectedGeneralNames(
    observed,
    contentDir,
    previousGeneralFacts,
    observedGeneralFacts,
    changedDuringWalk,
  );

  const rawEvents: RecoveredFileEvent[] = [];
  for (const [docName, previous] of previousFiles) {
    const current = observed.fileIndex.get(docName);
    if (
      markdownPublicMutationFor(docName, previous) ||
      (current && markdownPublicMutationFor(docName, current))
    )
      continue;
    if (current === undefined && unreadableAt(previous.canonicalPath)) continue;
    if (intakeOwns(previous.canonicalPath) || (current && intakeOwns(current.canonicalPath))) {
      continue;
    }
    if (current?.canonicalPath !== previous.canonicalPath) {
      const removal: KnownDeletion = {
        type: 'delete',
        path: previous.canonicalPath,
        entryKind: 'file',
      };
      removalOrigins.set(removal, {
        kind: 'recovery',
        docName,
        previousPath: previous.canonicalPath,
        isCurrent: () =>
          !markdownPublicMutationFor(docName, previous) &&
          !(current && markdownPublicMutationFor(docName, current)),
      });
      rawEvents.push(removal);
    } else if (
      previousHashes.get(previous.canonicalPath) !== observed.hashes.get(current.canonicalPath)
    ) {
      rawEvents.push({ type: 'update', path: current.canonicalPath });
    }
  }
  for (const [name, previous] of previousGeneralFacts) {
    if (
      previous.kind === 'symlink' ||
      publicMutations.generalEffects.has(name) ||
      protectedGeneral.has(name)
    )
      continue;
    const current = observedGeneralFacts.get(name);
    if (current === undefined) {
      rawEvents.push({ type: 'delete', path: join(contentDir, name), entryKind: 'file' });
    } else if (
      current.kind === 'symlink' ||
      (current.kind === 'regular' &&
        (previous.kind === 'unresolved' ||
          !sameGeneralIdentity(previous.identity, current.identity) ||
          previous.metadata.size !== current.metadata.size ||
          previous.metadata.modified !== current.metadata.modified))
    ) {
      rawEvents.push({ type: 'update', path: join(contentDir, name) });
    }
  }
  for (const [docName, current] of observed.fileIndex) {
    const previous = previousFiles.get(docName);
    if (
      markdownPublicMutationFor(docName, current) ||
      (previous && markdownPublicMutationFor(docName, previous)) ||
      intakeOwns(current.canonicalPath) ||
      (previous && intakeOwns(previous.canonicalPath))
    )
      continue;
    if (current.canonicalPath !== previous?.canonicalPath) {
      rawEvents.push({ type: 'create', path: current.canonicalPath });
    }
  }
  for (const [name, current] of observedGeneralFacts) {
    if (
      current.kind !== 'regular' ||
      publicMutations.generalEffects.has(name) ||
      protectedGeneral.has(name)
    )
      continue;
    const previous = previousGeneralFacts.get(name);
    if (previous === undefined) {
      rawEvents.push({ type: 'create', path: join(contentDir, name) });
    } else if (previous.kind === 'symlink') {
      rawEvents.push({ type: 'update', path: join(contentDir, name) });
    }
  }
  for (const [relativePath, previous] of folderIndex) {
    if (!observed.folderIndex.has(relativePath)) {
      if (unreadableAt(join(contentDir, relativePath))) continue;
      rawEvents.push({ type: 'delete', path: join(contentDir, relativePath), entryKind: 'folder' });
    } else if (
      observed.folderIndex.get(relativePath)?.canonicalPath !== previous.canonicalPath &&
      !intakeOwns(join(contentDir, relativePath))
    ) {
      rawEvents.push({ type: 'update', path: join(contentDir, relativePath) });
    }
  }
  for (const [relativePath, current] of observed.folderIndex) {
    if (
      !folderIndex.has(relativePath) &&
      !intakeOwns(join(contentDir, relativePath)) &&
      !intakeOwns(current.canonicalPath)
    )
      rawEvents.push({ type: 'create', path: current.canonicalPath });
  }
  for (const [alias, path] of aliasPaths) {
    if (!observed.aliasPaths.has(alias) && !unreadableAt(path))
      rawEvents.push({ type: 'delete', path, entryKind: 'file' });
  }

  const deduplicated = [
    ...new Map(
      rawEvents.map((event) => [
        event.type === 'delete'
          ? `${event.type}:${event.entryKind}:${event.path}`
          : `${event.type}:${event.path}`,
        event,
      ]),
    ).values(),
  ];
  if (deduplicated.length > 0) {
    const admitRecoveredEvent = (event: DiskEvent): boolean => {
      const markdownNames = publicMutations.markdownNames;
      const generalNames = publicMutations.generalNames;
      if (markdownNames.size === 0 && generalNames.size === 0) return true;
      if (event.kind === 'rename') {
        return !markdownNames.has(event.oldDocName) && !markdownNames.has(event.newDocName);
      }
      if (event.kind === 'folder-create' || event.kind === 'folder-delete') {
        return ![...markdownNames, ...generalNames].some(
          (name) => name === event.relativePath || name.startsWith(`${event.relativePath}/`),
        );
      }
      if (!('docName' in event)) {
        const pathName = toPosix(relative(contentDir, event.path));
        return (
          !publicMutations.generalEffects.has(event.relativePath) &&
          !publicMutations.generalEffects.has(pathName)
        );
      }
      if (markdownNames.has(event.docName)) return false;
      for (const [indexedName, entry] of fileIndex) {
        if (entry.canonicalPath === event.path && markdownPublicMutationFor(indexedName, entry))
          return false;
      }
      return true;
    };
    await handleRawEventsInternal(
      deduplicated,
      contentDir,
      contentFilter,
      fileIndex,
      folderIndex,
      countingOnDiskEvent,
      aliasMap,
      admitRecoveredEvent,
      generalFileIndex,
      aliasPaths,
    );
  }

  const publishedDuringPass = (path: string): boolean =>
    publicMutations.markdownNames.size > 0 &&
    publicMutations.markdownNames.has(pathToDocName(path, contentDir));
  for (const [path, prior] of previousHashes) {
    if (
      !observed.hashes.has(path) &&
      lastKnownHash.get(path) === prior &&
      !unreadableAt(path) &&
      isWithinContentDir(path, contentDir) &&
      !publishedDuringPass(path)
    ) {
      removeLastKnownHash(path);
    }
  }
  for (const [path, hash] of observed.hashes) {
    if (publishedDuringPass(path) || intakeOwns(path)) continue;
    const current = lastKnownHash.get(path);
    if (current === undefined || current === previousHashes.get(path) || current === hash) {
      recordObservedHash(path, hash, observed.signatures.get(path));
    }
  }

  for (const [name, prior] of previousFiles) {
    const current = fileIndex.get(name);
    if (
      !markdownPublicMutationFor(name, prior) &&
      !observed.fileIndex.has(name) &&
      current === prior &&
      !unreadableAt(prior.canonicalPath)
    ) {
      fileIndex.delete(name);
      forgetDocExtension(name);
    }
  }
  const inventory = selectObservedInventory(observed, contentDir, contentFilter);
  for (const [name, entry] of observed.fileIndex) {
    if (markdownPublicMutationFor(name, entry)) continue;
    const live = fileIndex.get(name);
    if (intakeOwns(entry.canonicalPath) || (live && intakeOwns(live.canonicalPath))) continue;
    if (inventory.fileIndex.has(name)) fileIndex.set(name, entry);
    else if (fileIndex.get(name)?.canonicalPath === entry.canonicalPath) fileIndex.delete(name);
  }
  const selectedGeneralFacts = inventory.generalFacts;
  for (const name of protectedGeneral) {
    const fact = previousGeneralFacts.get(name);
    if (fact) selectedGeneralFacts.set(name, fact);
    else selectedGeneralFacts.delete(name);
  }
  for (const [path, effect] of publicMutations.generalEffects) {
    if (effect) selectedGeneralFacts.set(path, effect);
    else selectedGeneralFacts.delete(path);
  }
  rebuildGeneralFileIndex(
    generalFileIndex,
    contentDir,
    selectedGeneralFacts,
    generalRepresentativeNames(previousGeneralFiles),
    publicMutations.generalMetadata,
  );

  for (const [path, prior] of previousFolders) {
    if (
      !observed.folderIndex.has(path) &&
      folderIndex.get(path) === prior &&
      !unreadableAt(join(contentDir, path))
    )
      folderIndex.delete(path);
  }
  for (const [path, entry] of inventory.folderIndex) {
    if (intakeOwns(join(contentDir, path))) continue;
    const current = folderIndex.get(path);
    if (
      current === undefined ||
      current === previousFolders.get(path) ||
      (current.canonicalPath === entry.canonicalPath &&
        current.modified === entry.modified &&
        current.inode === entry.inode)
    ) {
      folderIndex.set(path, entry);
    }
  }
  for (const [alias, prior] of previousAliases) {
    const aliasPath = previousAliasPaths.get(alias);
    if (aliasPath !== undefined && unreadableAt(aliasPath)) continue;
    if (!observed.aliasMap.has(alias) && aliasMap.get(alias) === prior) aliasMap.delete(alias);
  }
  for (const [alias, canonical] of observed.aliasMap) {
    const aliasPath = observed.aliasPaths.get(alias) ?? previousAliasPaths.get(alias);
    if (aliasPath !== undefined && intakeOwns(aliasPath)) continue;
    const current = aliasMap.get(alias);
    if (current === undefined || current === previousAliases.get(alias))
      aliasMap.set(alias, canonical);
  }
  for (const [alias, prior] of previousAliasPaths) {
    if (!observed.aliasPaths.has(alias) && aliasPaths.get(alias) === prior && !unreadableAt(prior))
      aliasPaths.delete(alias);
  }
  for (const [alias, path] of observed.aliasPaths) {
    if (intakeOwns(path)) continue;
    const current = aliasPaths.get(alias);
    if (current === undefined || current === previousAliasPaths.get(alias))
      aliasPaths.set(alias, path);
  }
  for (const [alias, prior] of previousFolderAliases) {
    if (
      !observed.folderAliasIndex.has(alias) &&
      folderAliasIndex.get(alias) === prior &&
      !unreadableAt(join(contentDir, alias))
    ) {
      folderAliasIndex.delete(alias);
    }
  }
  for (const [alias, canonical] of inventory.folderAliasIndex) {
    if (intakeOwns(join(contentDir, alias))) continue;
    const current = folderAliasIndex.get(alias);
    if (current === undefined || current === previousFolderAliases.get(alias)) {
      folderAliasIndex.set(alias, canonical);
    }
  }
  const publicParentDirs = new Set<string>();
  for (const name of [...publicMutations.markdownNames, ...publicMutations.generalNames]) {
    let parent = dirname(name);
    while (parent !== '.' && parent !== '') {
      publicParentDirs.add(parent);
      const next = dirname(parent);
      if (next === parent) break;
      parent = next;
    }
  }
  for (const relativePath of publicParentDirs) {
    const path = join(contentDir, relativePath);
    if (contentRelativePath(contentDir, path) !== relativePath) continue;
    if (contentFilter?.isDirExcluded(relativePath)) {
      removeFolderIndexEntries(folderIndex, relativePath);
      folderAliasIndex.delete(relativePath);
      continue;
    }
    let folderStat: BigIntStats;
    let canonicalPath = path;
    try {
      const lst = lstatSync(path, { bigint: true });
      if (lst.isSymbolicLink()) {
        canonicalPath = resolveNativePath(path);
        if (!isWithinContentDir(canonicalPath, contentDir)) {
          removeFolderIndexEntries(folderIndex, relativePath);
          folderAliasIndex.delete(relativePath);
          continue;
        }
        folderStat = statSync(canonicalPath, { bigint: true });
      } else {
        folderStat = lst;
      }
    } catch (err) {
      const code = errnoCode(err);
      if (code === 'ENOENT' || code === 'ENOTDIR' || code === 'ELOOP') {
        removeFolderIndexEntries(folderIndex, relativePath);
        folderAliasIndex.delete(relativePath);
      } else {
        log.warn({ path, err }, `Failed to refresh public mutation folder ${path}`);
      }
      continue;
    }
    if (!folderStat.isDirectory()) {
      removeFolderIndexEntries(folderIndex, relativePath);
      folderAliasIndex.delete(relativePath);
      continue;
    }
    upsertFolderIndexEntry(folderIndex, contentDir, path, folderStat, canonicalPath);
    if (canonicalPath === path) folderAliasIndex.delete(relativePath);
    else folderAliasIndex.set(relativePath, toPosix(relative(contentDir, canonicalPath)));
  }
  const aliasChanges: RecoveredAliasCandidate[] = [];
  const finalGeneralFacts = generalPathFacts(generalFileIndex);
  for (const [alias, previous] of previousGeneralFacts) {
    if (previous.kind !== 'symlink' || publicMutations.generalEffects.has(alias)) continue;
    if (finalGeneralFacts.has(alias)) continue;
    if (!generalAliasIsAdmitted(alias, previous.targetPath, contentFilter)) continue;
    aliasChanges.push({ kind: 'delete', alias, targetPath: previous.targetPath });
  }
  for (const [alias, current] of finalGeneralFacts) {
    if (current.kind !== 'symlink' || publicMutations.generalEffects.has(alias)) continue;
    if (!generalAliasIsAdmitted(alias, current.targetPath, contentFilter)) continue;
    const previous = previousGeneralFacts.get(alias);
    if (previous?.kind === 'symlink' && previous.targetPath === current.targetPath) continue;
    const target = finalGeneralFacts.get(current.targetPath);
    if (target?.kind !== 'regular') continue;
    aliasChanges.push({
      kind: 'target-update',
      alias,
      targetPath: current.targetPath,
      targetIdentity: target.identity,
    });
  }
  if (aliasChanges.length > 0) {
    onAfterMutation();
    let currentGeneration = getFileIndexGeneration();
    let currentGeneralFacts = generalPathFacts(generalFileIndex);
    for (const candidate of aliasChanges) {
      if (!generalAliasIsAdmitted(candidate.alias, candidate.targetPath, contentFilter)) continue;
      const generation = getFileIndexGeneration();
      if (generation !== currentGeneration) {
        currentGeneralFacts = generalPathFacts(generalFileIndex);
        currentGeneration = generation;
      }
      let event: FileDiskEvent;
      if (candidate.kind === 'delete') {
        if (
          publicMutations.generalEffects.has(candidate.alias) ||
          currentGeneralFacts.has(candidate.alias)
        )
          continue;
        event = {
          kind: 'file-delete',
          path: join(contentDir, candidate.alias),
          relativePath: candidate.alias,
        };
      } else {
        if (
          publicMutations.generalEffects.has(candidate.alias) ||
          publicMutations.generalEffects.has(candidate.targetPath)
        )
          continue;
        const alias = currentGeneralFacts.get(candidate.alias);
        const target = currentGeneralFacts.get(candidate.targetPath);
        if (
          alias?.kind !== 'symlink' ||
          alias.targetPath !== candidate.targetPath ||
          target?.kind !== 'regular' ||
          !sameGeneralIdentity(target.identity, candidate.targetIdentity)
        )
          continue;
        event = {
          kind: 'file-update',
          path: join(contentDir, candidate.targetPath),
          relativePath: candidate.targetPath,
          size: target.metadata.size,
          modifiedTs: Date.parse(target.metadata.modified),
          inode: target.identity,
        };
      }
      recordWatcherDecision('dispatched', event.kind, event.path);
      await countingOnDiskEvent(event);
    }
  }
  if (handed) {
    if (deduplicated.length > 0) onRawBatch?.(deduplicated.map((event) => event.path));
  } else {
    onRawBatch?.([...new Set([...observed.paths, ...knownPaths])]);
  }
  return {
    completed: true,
    candidates: deduplicated.length,
    partial: observed.unreadable.size > 0,
  };
}

export function toParcelIgnorePaths(
  watcherIgnoreGlobs: readonly string[],
  discoveredStructuralDirs: Iterable<string> = [],
): string[] {
  const prefixIgnorable = new Set<string>(WATCHER_STRUCTURAL_IGNORE_DIRS);
  const entries = new Set(watcherIgnoreGlobs.filter((entry) => prefixIgnorable.has(entry)));
  for (const dir of discoveredStructuralDirs) entries.add(dir);
  return [...entries].filter((entry) => !GLOB_METACHARACTER_RE.test(entry));
}

type RecoveryPublicMutations = GeneralMutationRecord & {
  markdownNames: Set<string>;
  hashEffects: Map<string, string | null>;
};

/* WARN: LEASE_RESET_MS in packages/app/tests/stress/_helpers/reuse/lease-fixture.ts must exceed this bound plus the rest of a slot reset. */
export const INTERIM_SETTLE_NO_PROGRESS_MS = 20_000;

interface RecoveryDriverDeps {
  runOrdered: (work: () => Promise<void>) => Promise<void>;
  isClosed: () => boolean;
  recover: (
    mutations: RecoveryPublicMutations,
    counter: DispatchCounter,
    mode: RecoveryMode,
    handed: HandedObservation | undefined,
  ) => Promise<RecoveryOutcome>;
  setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void;
  onAfterMutation: () => void;
  recoveryHooks: RecoveryHooks;
  backendName: WatcherBackend;
}

interface RecoveryHooks {
  readonly onRecoveryComplete?: () => Promise<void>;
  readonly onRecoveryPassCompleted?: (pass: number, whole: boolean) => Promise<void>;
}

interface RecoveryDriver {
  requestRecovery: (mode: RecoveryMode, handed?: HandedObservation) => Promise<RecheckResult>;
  isIdle: () => boolean;
  passesStarted: () => number;
  beginPass: () => number;
}

function createRecoveryDriver(deps: RecoveryDriverDeps): RecoveryDriver {
  let recoveryRunning = false;
  let nextRecovery: Promise<RecheckResult> | undefined;
  let nextMode: RecoveryMode = 'metadata';
  let nextHanded: HandedObservation | undefined;
  let followUpRecovery: PromiseWithResolvers<RecheckResult> | undefined;
  let passesStarted = 0;
  const closed = (): RecheckResult => ({ status: 'closed', candidates: 0, dispatched: 0 });
  const recoverOnce = async (): Promise<RecheckResult> => {
    const mode = nextMode;
    const handed = nextHanded;
    nextRecovery = undefined;
    nextMode = 'metadata';
    nextHanded = undefined;
    if (deps.isClosed()) return closed();
    const pass = handed?.pass ?? ++passesStarted;
    recoveryRunning = true;
    const recoveryMutations: RecoveryPublicMutations = {
      markdownNames: new Set(),
      generalNames: new Set(),
      generalEffects: new Map(),
      generalMetadata: new Map(),
      hashEffects: new Map(),
    };
    const counter: DispatchCounter = { dispatched: 0 };
    let result: RecheckResult = { status: 'failed', candidates: 0, dispatched: 0 };
    try {
      deps.setActivePublicMutations(recoveryMutations);
      const outcome = await deps.recover(recoveryMutations, counter, mode, handed);
      deps.onAfterMutation();
      if (outcome.completed) {
        if (!outcome.partial) await deps.recoveryHooks.onRecoveryComplete?.();
        await deps.recoveryHooks.onRecoveryPassCompleted?.(pass, !outcome.partial);
        result = {
          status: 'completed',
          candidates: outcome.candidates,
          dispatched: counter.dispatched,
          ...(outcome.partial ? { partial: true } : {}),
        };
        const finished = {
          mode: handed ? 'timed' : mode,
          candidates: outcome.candidates,
          dispatched: counter.dispatched,
          partial: outcome.partial === true,
        };
        if (mode === 'full' || counter.dispatched > 0) {
          log.info(finished, `${deps.backendName} watcher recovery finished`);
        } else {
          log.debug(finished, `${deps.backendName} watcher recovery finished`);
        }
      } else {
        result = { status: 'incomplete', candidates: 0, dispatched: counter.dispatched };
        log.warn(
          { dispatched: counter.dispatched },
          `${deps.backendName} watcher recovery incomplete; known state retained`,
        );
      }
    } catch (handleErr) {
      log.error({ err: handleErr }, `${deps.backendName} batch error`);
      result = { status: 'failed', candidates: 0, dispatched: counter.dispatched };
    } finally {
      try {
        for (const [path, hash] of recoveryMutations.hashEffects) {
          if (hash === null) removeLastKnownHash(path);
          else updateLastKnownHash(path, hash);
        }
      } finally {
        deps.setActivePublicMutations(null);
        recoveryRunning = false;
        const followUp = followUpRecovery;
        followUpRecovery = undefined;
        if (followUp) followUp.resolve(runRecovery());
      }
    }
    return result;
  };
  const runRecovery = (): Promise<RecheckResult> => {
    let settled = closed();
    return deps
      .runOrdered(async () => {
        settled = await recoverOnce();
      })
      .then(() => settled);
  };
  const requestRecovery = (
    mode: RecoveryMode,
    handed?: HandedObservation,
  ): Promise<RecheckResult> => {
    if (mode === 'full') nextMode = 'full';
    if (nextRecovery) {
      nextHanded = undefined;
      return nextRecovery;
    }
    nextHanded = mode === 'full' ? undefined : handed;
    if (recoveryRunning) {
      followUpRecovery = Promise.withResolvers<RecheckResult>();
      nextRecovery = followUpRecovery.promise;
    } else {
      nextRecovery = runRecovery();
    }
    return nextRecovery;
  };
  return {
    requestRecovery,
    isIdle: () => !recoveryRunning && nextRecovery === undefined,
    passesStarted: () => passesStarted,
    beginPass: () => ++passesStarted,
  };
}

interface WatcherBackendSubscription extends AsyncSubscription {
  runOrdered(work: () => Promise<void>): Promise<void>;
  requestRecovery(mode: RecoveryMode, handed?: HandedObservation): Promise<RecheckResult>;
  isRecoveryIdle(): boolean;
  recoveryPassesStarted(): number;
  beginRecoveryPass(): number;
  noticeRootMissing(): void;
}

async function startParcelWatcher(
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  fileIndex: Map<string, FileIndexEntry>,
  generalFileIndex: GeneralFileIndex,
  folderIndex: Map<string, FolderIndexEntry>,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  aliasMap: Map<string, string>,
  aliasPaths: Map<string, string>,
  folderAliasIndex: Map<string, string>,
  setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void,
  onAfterMutation: () => void,
  getFileIndexGeneration: () => number,
  platform: NodeJS.Platform,
  onRawBatch?: (absPaths: readonly string[]) => void,
  discoveredStructuralDirs?: ReadonlySet<string>,
  recoveryHooks: RecoveryHooks = {},
  clock: WatcherClock = watcherClock(),
  nativeBackend?: NativeWatcherBackend,
  carryOver: Set<string> = new Set(),
  coverageWhileLost: string = coverageWithoutWatcher(null),
): Promise<(WatcherBackendSubscription & { settle(): Promise<void> }) | null> {
  let parcel: NativeWatcherBackend;
  if (nativeBackend) {
    parcel = nativeBackend;
  } else {
    try {
      parcel = await import('@parcel/watcher');
    } catch (err) {
      log.debug(
        { err: err instanceof Error ? err.message : String(err) },
        '[file-watcher] @parcel/watcher import failed; falling back to chokidar',
      );
      return null;
    }
  }

  try {
    const subscribeOpts = () =>
      contentFilter
        ? {
            ignore: toParcelIgnorePaths(
              contentFilter.getWatcherIgnoreGlobs(),
              discoveredStructuralDirs ?? [],
            ),
          }
        : undefined;

    let closed = false;
    let failure: Error | undefined;
    const settleFences = new Map<string, PromiseWithResolvers<void>>();
    const settleProgress = new Set<() => void>();
    let workTail = Promise.resolve();
    const runOrdered = (work: () => Promise<void>): Promise<void> => {
      if (closed) return Promise.resolve();
      const next = workTail.then(work);
      workTail = next
        .catch((err) => {
          log.error({ err }, 'parcel batch error');
        })
        .then(() => {
          for (const progressed of settleProgress) progressed();
        });
      return next;
    };
    const { requestRecovery, isIdle, passesStarted, beginPass } = createRecoveryDriver({
      runOrdered,
      isClosed: () => closed,
      recover: (recoveryMutations, counter, mode, handed) =>
        recoverDiskChanges(
          contentDir,
          contentFilter,
          fileIndex,
          generalFileIndex,
          folderIndex,
          aliasMap,
          aliasPaths,
          folderAliasIndex,
          recoveryMutations,
          onDiskEvent,
          onAfterMutation,
          getFileIndexGeneration,
          counter,
          onRawBatch,
          handed ? undefined : beginFreshnessReuse(mode, fileIndex, carryOver),
          handed,
        ),
      setActivePublicMutations,
      onAfterMutation,
      recoveryHooks,
      backendName: 'parcel',
    });
    const notificationsFor =
      (generation: number): Parameters<typeof subscribeParcel>[2] =>
      (err, events) => {
        if (closed || !resubscriber.isLive(generation)) return;
        const notification = classifyParcelNotification(err, events);
        if (notification.kind === 'error') {
          log.error({ err: notification.error }, 'parcel watcher callback error');
          failure = new Error(
            `the file watcher is subscribing again after an error and cannot settle until it is back: ${notification.error.message}`,
          );
          for (const fence of settleFences.values()) fence.reject(failure);
          incrementWatcherErrors();
          resubscriber.lost('error', generation, notification.error);
          return;
        }
        if (notification.kind === 'rescan') {
          log.warn(
            { err: notification.error, eventCount: notification.events.length },
            'parcel watcher recovery requested',
          );
        }
        const records = runOrdered(async () => {
          if (closed) return;
          try {
            onRawBatch?.(notification.events.map((event) => event.path));
            await handleRawEventsInternal(
              notification.events.map((event) => ({ type: event.type, path: event.path })),
              contentDir,
              contentFilter,
              fileIndex,
              folderIndex,
              onDiskEvent,
              aliasMap,
              undefined,
              generalFileIndex,
              aliasPaths,
            );
            onAfterMutation();
          } catch (handleErr) {
            log.error({ err: handleErr }, 'parcel batch error');
          }
          for (const event of notification.events)
            settleFences.get(basename(event.path))?.resolve();
        });
        if (
          notification.events.some((event) => event.type === 'delete' && event.path === contentDir)
        ) {
          resubscriber.lost('root-delete', generation);
        }
        if (notification.kind !== 'rescan') return records;
        const fencesBeforeRescan = [...settleFences.values()];
        return Promise.all([
          records,
          requestRecovery('full').then(() => {
            for (const fence of fencesBeforeRescan) fence.resolve();
          }),
        ]);
      };
    const resubscriber = createResubscriber({
      clock,
      root: contentDir,
      backendName: 'parcel',
      coverageWhileLost,
      onGaveUp: (err) => {
        const message = err instanceof Error ? err.message : String(err);
        failure = new Error(
          `the file watcher gave up subscribing after repeated failures (${message}); it tries again every ${RESUBSCRIBE_SLOW_RETRY_MS / 60_000} minutes and cannot settle until it is back`,
        );
        for (const fence of settleFences.values()) fence.reject(failure);
      },
      subscribe: (generation) =>
        subscribeParcel(
          parcel,
          contentDir,
          notificationsFor(generation),
          subscribeOpts(),
          platform,
        ),
      onResubscribed: () => {
        failure = undefined;
        void requestRecovery('full');
      },
    });
    await resubscriber.start();

    return {
      runOrdered,
      requestRecovery,
      isRecoveryIdle: isIdle,
      recoveryPassesStarted: passesStarted,
      beginRecoveryPass: beginPass,
      noticeRootMissing: () => resubscriber.noticeRootMissing(),
      async settle() {
        if (closed) return;
        if (failure) throw failure;
        const name = `watcher-settle-fence-${randomUUID()}`;
        const fencePath = join(contentDir, '.ok', name);
        const reached = Promise.withResolvers<void>();
        let quiet: WatcherTimer | undefined;
        const progressed = (): void => {
          clock.clearTimeout(quiet);
          quiet = clock.setTimeout(() => {
            reached.reject(
              new Error(
                `the file watcher handled no batch for ${INTERIM_SETTLE_NO_PROGRESS_MS} ms before it saw its settle fence`,
              ),
            );
          }, INTERIM_SETTLE_NO_PROGRESS_MS);
        };
        settleFences.set(name, reached);
        let fenceWritten = false;
        try {
          tracedMkdirSync(dirname(fencePath), { recursive: true });
          tracedWriteFileSync(fencePath, '');
          fenceWritten = true;
          settleProgress.add(progressed);
          progressed();
          await reached.promise;
        } finally {
          clock.clearTimeout(quiet);
          settleProgress.delete(progressed);
          settleFences.delete(name);
          if (fenceWritten) tracedRmSync(fencePath, { force: true });
        }
      },
      async unsubscribe() {
        closed = true;
        for (const fence of settleFences.values()) {
          fence.reject(new Error('the file watcher stopped before it saw its settle fence'));
        }
        await resubscriber.close();
        await workTail;
      },
    };
  } catch (err) {
    log.warn({ err }, '@parcel/watcher subscribe failed, falling back to chokidar');
    return null;
  }
}

export function isChokidarPathIgnored(
  contentDir: string,
  contentFilter: ContentFilter,
  filePath: string,
  stats?: Stats,
): boolean {
  const rel = toPosix(relative(contentDir, filePath));
  if (rel === '' || rel === '.') return false;
  let isDirectory: boolean;
  if (stats) {
    isDirectory = stats.isDirectory();
  } else {
    try {
      isDirectory = lstatSync(filePath).isDirectory();
    } catch {
      return false;
    }
  }
  if (isDirectory) {
    const m = /^(\.[A-Za-z0-9_-]+)(\/skills(\/[^/]+)?)?$/.exec(rel);
    if (m && m[1] !== '.ok') {
      if (m[2] !== undefined) return false;
      if (existsSync(join(contentDir, rel, 'skills'))) return false;
    }
    return contentFilter.isDirExcluded(rel);
  }
  return contentFilter.isExcluded(rel);
}

const CHOKIDAR_READY_TIMEOUT_MS = 10_000;
export const CHOKIDAR_BATCH_WINDOW_MS = 50;

/* UPSTREAM(nodejs/node#52601, libuv/libuv#3866, libuv/libuv#637): a macOS directory fs.watch
   returns before FSEvents covers its path; closing one waits for the stream rebuild that covers
   every earlier registration. */
function waitForPendingFsEventsRegistrations(dir: string, platform: NodeJS.Platform): void {
  if (platform !== 'darwin') return;
  try {
    watchFsPath(dir).close();
  } catch (err) {
    log.warn(
      { err },
      'could not wait for FSEvents to cover the chokidar-watched directories; a change made right after start may be missed',
    );
  }
}

const CHOKIDAR_NON_FATAL_ERROR_CODES: ReadonlySet<string> = new Set([
  'EACCES',
  'EPERM',
  'ENOSPC',
  'EMFILE',
  'ENFILE',
]);

async function startChokidarWatcher(
  contentDir: string,
  contentFilter: ContentFilter | undefined,
  fileIndex: Map<string, FileIndexEntry>,
  generalFileIndex: GeneralFileIndex,
  folderIndex: Map<string, FolderIndexEntry>,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  aliasMap: Map<string, string>,
  aliasPaths: Map<string, string>,
  folderAliasIndex: Map<string, string>,
  setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void,
  onAfterMutation: () => void,
  getFileIndexGeneration: () => number,
  platform: NodeJS.Platform,
  clock: WatcherClock,
  onRawBatch?: (absPaths: readonly string[]) => void,
  recoveryHooks: RecoveryHooks = {},
  carryOver: Set<string> = new Set(),
  coverageWhileLost: string = coverageWithoutWatcher(null),
): Promise<WatcherBackendSubscription> {
  const { watch } = await import('chokidar');

  let closed = false;
  let workTail = Promise.resolve();
  const runOrdered = (work: () => Promise<void>): Promise<void> => {
    if (closed) return Promise.resolve();
    const next = workTail.then(work);
    workTail = next.catch((err) => {
      log.error({ err }, 'chokidar batch error');
    });
    return next;
  };
  const { requestRecovery, isIdle, passesStarted, beginPass } = createRecoveryDriver({
    runOrdered,
    isClosed: () => closed,
    recover: (recoveryMutations, counter, mode, handed) =>
      recoverDiskChanges(
        contentDir,
        contentFilter,
        fileIndex,
        generalFileIndex,
        folderIndex,
        aliasMap,
        aliasPaths,
        folderAliasIndex,
        recoveryMutations,
        onDiskEvent,
        onAfterMutation,
        getFileIndexGeneration,
        counter,
        onRawBatch,
        handed ? undefined : beginFreshnessReuse(mode, fileIndex, carryOver),
        handed,
      ),
    setActivePublicMutations,
    onAfterMutation,
    recoveryHooks,
    backendName: 'chokidar',
  });

  let pendingEvents: InternalRawFileEvent[] = [];
  let batchTimer: WatcherTimer | null = null;

  function queueEvent(event: InternalRawFileEvent) {
    pendingEvents.push(event);
    batchTimer ||= clock.setTimeout(() => {
      const batch = pendingEvents;
      pendingEvents = [];
      batchTimer = null;
      void runOrdered(async () => {
        try {
          onRawBatch?.(batch.map((e) => e.path));
          await handleRawEventsInternal(
            batch,
            contentDir,
            contentFilter,
            fileIndex,
            folderIndex,
            onDiskEvent,
            aliasMap,
            undefined,
            generalFileIndex,
            aliasPaths,
          );
          onAfterMutation();
        } catch (err) {
          log.error({ err }, 'chokidar batch error');
        }
      });
    }, CHOKIDAR_BATCH_WINDOW_MS);
  }

  const reportedNonFatalCodes = new Set<string>();
  const watchAs = async (generation: number): Promise<AsyncSubscription> => {
    const watcher = watch(contentDir, {
      ignoreInitial: true,
      followSymlinks: false,
      ignored: contentFilter
        ? (filePath: string, stats?: Stats) =>
            isChokidarPathIgnored(contentDir, contentFilter, filePath, stats)
        : undefined,
    });
    const live = (): boolean => !closed && resubscriber.isLive(generation);
    const queueLive = (event: InternalRawFileEvent): void => {
      if (live()) queueEvent(event);
    };

    watcher.on('error', (err) => {
      if (!live()) return;
      incrementWatcherErrors();
      const code = errnoCode(err);
      if (code !== undefined && CHOKIDAR_NON_FATAL_ERROR_CODES.has(code)) {
        if (reportedNonFatalCodes.has(code)) return;
        reportedNonFatalCodes.add(code);
        log.warn(
          { err, code },
          `chokidar reported ${code}; the watcher stays up, and for what it cannot watch ${coverageWhileLost}`,
        );
        return;
      }
      log.error({ err }, 'chokidar error');
      resubscriber.lost('error', generation, err);
    });
    watcher.on('add', (path) => queueLive({ type: 'create', path }));
    watcher.on('change', (path) => queueLive({ type: 'update', path }));
    watcher.on('unlink', (path) => queueLive({ type: 'delete', path, entryKind: 'file' }));
    watcher.on('addDir', (path) => queueLive({ type: 'create', path }));
    watcher.on('unlinkDir', (path) => {
      queueLive({ type: 'delete', path, entryKind: 'folder' });
      if (path === contentDir && live()) resubscriber.lost('root-delete', generation);
    });

    await new Promise<void>((resolveReady) => {
      let settled = false;
      const settle = () => {
        if (settled) return;
        settled = true;
        resolveReady();
      };
      watcher.once('ready', settle);
      clock.setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();
    });

    waitForPendingFsEventsRegistrations(contentDir, platform);
    return { unsubscribe: () => watcher.close() };
  };

  const resubscriber = createResubscriber({
    clock,
    root: contentDir,
    backendName: 'chokidar',
    coverageWhileLost,
    subscribe: watchAs,
    onResubscribed: () => {
      void requestRecovery('full');
    },
  });
  await resubscriber.start();

  return {
    runOrdered,
    requestRecovery,
    isRecoveryIdle: isIdle,
    recoveryPassesStarted: passesStarted,
    beginRecoveryPass: beginPass,
    noticeRootMissing: () => resubscriber.noticeRootMissing(),
    unsubscribe: async () => {
      closed = true;
      if (batchTimer) {
        clock.clearTimeout(batchTimer);
        batchTimer = null;
        pendingEvents = [];
      }
      await resubscriber.close();
      await workTail;
    },
  };
}

function coverageWithoutWatcher(freshnessIntervalMs: number | null): string {
  if (freshnessIntervalMs === null) {
    return 'changes made outside the app are found only by an explicit re-check';
  }
  const fastest = freshnessIntervalMs / 1000;
  const slowest = Math.max(freshnessIntervalMs, FRESHNESS_MAX_INTERVAL_MS) / 1000;
  return `the timed re-check still finds changes made outside the app, every ${fastest} to ${slowest} seconds`;
}

export interface StartWatcherOptions {
  forceBackend?: 'parcel' | 'chokidar';
  onRawBatch?: (absPaths: readonly string[]) => void;
  onRecoveryComplete?: () => Promise<void>;
  onRecoveryPassCompleted?: (pass: number, whole: boolean) => Promise<void>;
  platform?: NodeJS.Platform;
  nativeBackend?: NativeWatcherBackend;
  clock?: WatcherClock;
  freshnessIntervalMs?: number | null;
}

export async function startWatcher(
  contentDirRaw: string,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  contentFilter?: ContentFilter,
  opts: StartWatcherOptions = {},
): Promise<WatcherHandle> {
  const restoreClock = opts.clock ? installWatcherClock(opts.clock) : (): void => {};
  const clock = opts.clock ?? watcherClock();
  try {
    return await startWatcherWithClock(
      contentDirRaw,
      onDiskEvent,
      contentFilter,
      opts,
      clock,
      restoreClock,
    );
  } catch (e) {
    restoreClock();
    throw e;
  }
}

async function startWatcherWithClock(
  contentDirRaw: string,
  onDiskEvent: (event: DiskEvent) => Promise<void>,
  contentFilter: ContentFilter | undefined,
  opts: StartWatcherOptions,
  clock: WatcherClock,
  restoreClock: () => void,
): Promise<WatcherHandle> {
  const { onRawBatch, onRecoveryComplete, onRecoveryPassCompleted } = opts;
  const recoveryHooks: RecoveryHooks = { onRecoveryComplete, onRecoveryPassCompleted };
  const contentDir = resolveDirectoryRoot(contentDirRaw, {
    root: 'content',
    component: 'file-watcher',
  });

  const fileIndex = new Map<string, FileIndexEntry>();
  const generalFileIndex = new GeneralFileIndex(contentDir);
  const allFileEntries: AllFileEntries = {
    *[Symbol.iterator]() {
      yield* fileIndex;
      yield* generalFileIndex;
    },
  };
  const folderIndex = new WatcherFolderIndex(contentDir);
  const aliasMap = new Map<string, string>();
  const aliasPaths = new Map<string, string>();
  const folderAliasIndex = new Map<string, string>();
  let activePublicMutations: RecoveryPublicMutations | null = null;
  let activeReseedMutations: GeneralMutationRecord | null = null;

  let fileIndexGeneration = 0;
  let cachedMarkdownView: ReadonlyMap<string, FileIndexEntry> | null = null;
  let cachedMarkdownViewGeneration = -1;
  const bumpFileIndexGeneration = (): void => {
    fileIndexGeneration++;
  };

  const structuralIgnoreDirs = new Set<string>();

  const initialObservation = await observeDisk(contentDir, contentFilter);
  publishDiskObservation(
    initialObservation,
    initialObservation,
    fileIndex,
    generalFileIndex,
    folderIndex,
    aliasMap,
    folderAliasIndex,
    structuralIgnoreDirs,
  );
  for (const [alias, path] of initialObservation.aliasPaths) aliasPaths.set(alias, path);
  bumpFileIndexGeneration();

  const evictionInterval = clock.setInterval(evictStaleTrackerEntries, WRITE_TRACKER_TTL_MS);
  const dropSummaryInterval = clock.setInterval(
    logWatcherDropSummary,
    WATCHER_DROP_SUMMARY_INTERVAL_MS,
  );

  const freshnessIntervalMs = resolveFreshnessInterval(opts.freshnessIntervalMs);
  const coverageWhileLost = coverageWithoutWatcher(freshnessIntervalMs);
  const carryOver = new Set<string>();
  let subscription: WatcherBackendSubscription;
  let runOrderedRescan: ((work: () => Promise<void>) => Promise<void>) | undefined;
  let settleSubscription: (() => Promise<void>) | undefined;
  let backend: WatcherBackend;
  const forceChokidar = process.env.OK_FILE_WATCHER_BACKEND === 'chokidar';
  const platform = opts.platform ?? process.platform;
  try {
    const parcelSub =
      forceChokidar || opts.forceBackend === 'chokidar'
        ? null
        : await startParcelWatcher(
            contentDir,
            contentFilter,
            fileIndex,
            generalFileIndex,
            folderIndex,
            onDiskEvent,
            aliasMap,
            aliasPaths,
            folderAliasIndex,
            (mutations) => {
              activePublicMutations = mutations;
            },
            bumpFileIndexGeneration,
            () => fileIndexGeneration,
            platform,
            onRawBatch,
            structuralIgnoreDirs,
            recoveryHooks,
            clock,
            opts.nativeBackend,
            carryOver,
            coverageWhileLost,
          );
    if (parcelSub) {
      subscription = parcelSub;
      runOrderedRescan = parcelSub.runOrdered;
      settleSubscription = () => parcelSub.settle();
      backend = 'parcel';
    } else {
      if (opts.forceBackend === 'parcel') {
        throw new Error('@parcel/watcher unavailable (forced backend)');
      }
      const chokidarSub = await startChokidarWatcher(
        contentDir,
        contentFilter,
        fileIndex,
        generalFileIndex,
        folderIndex,
        onDiskEvent,
        aliasMap,
        aliasPaths,
        folderAliasIndex,
        (mutations) => {
          activePublicMutations = mutations;
        },
        bumpFileIndexGeneration,
        () => fileIndexGeneration,
        platform,
        clock,
        onRawBatch,
        recoveryHooks,
        carryOver,
        coverageWhileLost,
      );
      subscription = chokidarSub;
      backend = 'chokidar';
    }
  } catch (e) {
    clock.clearInterval(evictionInterval);
    clock.clearInterval(dropSummaryInterval);
    throw e;
  }

  const backendSubscription = subscription;
  let handleClosed = false;
  let freshnessTimer: WatcherTimer | null = null;
  let freshnessPassInFlight: Promise<void> | null = null;
  const scheduleFreshnessPass = (baseMs: number, delayMs: number): void => {
    if (handleClosed) return;
    freshnessTimer = clock.setTimeout(() => {
      freshnessTimer = null;
      const pass = runFreshnessPass(baseMs);
      freshnessPassInFlight = pass;
      void pass.finally(() => {
        if (freshnessPassInFlight === pass) freshnessPassInFlight = null;
      });
    }, delayMs);
    freshnessTimer.unref();
  };
  const walkForFreshness = async (): Promise<DiskObservation | null> => {
    contentFilter?.refreshInPlaceSkillDirs();
    const reuse = beginFreshnessReuse('metadata', fileIndex, carryOver);
    const observed = await observeDisk(contentDir, contentFilter, reuse, () => handleClosed);
    if (handleClosed) return null;
    if (reuse) {
      addFreshnessFilesHashed(reuse.hashed);
      addFreshnessCarryOver(reuse.deferred.size);
    }
    return observed;
  };
  const runFreshnessPass = async (baseMs: number): Promise<void> => {
    if (handleClosed) return;
    if (!existsSync(contentDir)) {
      backendSubscription.noticeRootMissing();
      scheduleFreshnessPass(baseMs, baseMs);
      return;
    }
    if (!backendSubscription.isRecoveryIdle()) {
      scheduleFreshnessPass(baseMs, baseMs);
      return;
    }
    const startedAt = clock.now();
    let elapsedMs = 0;
    const sinceEpoch = openObservationWindow();
    const pass = backendSubscription.beginRecoveryPass();
    try {
      const observed = await walkForFreshness();
      if (observed === null) return;
      const result = observed.complete
        ? await backendSubscription.requestRecovery('metadata', { observed, sinceEpoch, pass })
        : { status: 'incomplete' as const, candidates: 0, dispatched: 0 };
      if (handleClosed) return;
      elapsedMs = clock.now() - startedAt;
      addFreshnessFinds(result.dispatched);
      if (result.status === 'incomplete' || result.status === 'failed') {
        incrementFreshnessPassesIncomplete();
      }
      log.debug({ ...result, elapsedMs }, 'freshness re-check pass finished');
    } catch (err) {
      if (handleClosed) return;
      incrementFreshnessPassesIncomplete();
      log.warn({ err }, 'freshness re-check pass failed');
    } finally {
      closeObservationWindow();
      if (!handleClosed) {
        incrementFreshnessPasses();
        scheduleFreshnessPass(baseMs, nextFreshnessDelayMs(elapsedMs, baseMs));
      }
    }
  };
  if (freshnessIntervalMs !== null) scheduleFreshnessPass(freshnessIntervalMs, freshnessIntervalMs);

  log.info({ contentDir, backend, freshnessIntervalMs }, 'watching for external .md changes');

  return {
    async unsubscribe() {
      handleClosed = true;
      clock.clearTimeout(freshnessTimer);
      freshnessTimer = null;
      clock.clearInterval(evictionInterval);
      clock.clearInterval(dropSummaryInterval);
      try {
        await backendSubscription.unsubscribe();
      } finally {
        await freshnessPassInFlight;
        writeTracker.clear();
        removalTracker.clear();
        lastKnownHash.clear();
        lastKnownStat.clear();
        reportedUnreadable.clear();
        resetWatcherDecisionDiagnostics();
        restoreClock();
      }
    },
    recheck(mode = 'full') {
      return backendSubscription.requestRecovery(mode);
    },
    recoveryPassesStarted() {
      return backendSubscription.recoveryPassesStarted();
    },
    async recheckPaths(paths) {
      const unique = [...new Set(paths)];
      if (unique.length === 0) return { checked: 0, found: 0 };
      let found = 0;
      let checked = unique.length;
      const rootMissing = (): boolean => {
        if (existsSync(contentDir)) return false;
        checked = 0;
        found = 0;
        log.warn(
          { contentDir, paths: unique.length },
          `skipped re-reading ${unique.length} documents: the content folder ${contentDir} is missing`,
        );
        backendSubscription.noticeRootMissing();
        return true;
      };
      await backendSubscription.runOrdered(async () => {
        if (rootMissing()) return;
        const indexedPaths = new Set<string>();
        for (const entry of fileIndex.values()) indexedPaths.add(entry.canonicalPath);
        const changed: InternalRawFileEvent[] = [];
        for (const path of unique) {
          const known = lastKnownHash.get(path);
          const tracked = known !== undefined || indexedPaths.has(path);
          try {
            lstatSync(path);
          } catch (err) {
            const code = errnoCode(err);
            if (code === 'ENOENT' || code === 'ENOTDIR') {
              if (rootMissing()) return;
              if (tracked) changed.push({ type: 'delete', path, entryKind: 'file' });
            } else {
              log.warn({ err, path, code }, `could not re-read ${path}`);
            }
            continue;
          }
          if (
            !hasAdmittedMarkdownFile(
              pathToDocName(path, contentDir),
              path,
              contentDir,
              contentFilter,
            )
          ) {
            log.debug({ path }, `skipped a re-read of ${path}: not an admitted Markdown file`);
            continue;
          }
          let content: string;
          try {
            if (statSync(path).size > DOCUMENT_OPEN_BYTE_LIMIT) {
              log.debug({ path }, `skipped a re-read of ${path}: larger than an open document`);
              continue;
            }
            content = await readFile(path, 'utf-8');
          } catch (err) {
            log.warn({ err, path }, `could not re-read ${path}`);
            continue;
          }
          if (contentHash(content) === known) continue;
          changed.push({ type: tracked ? 'update' : 'create', path });
        }
        found = changed.length;
        if (changed.length === 0) return;
        onRawBatch?.(changed.map((event) => event.path));
        await handleRawEventsInternal(
          changed,
          contentDir,
          contentFilter,
          fileIndex,
          folderIndex,
          onDiskEvent,
          aliasMap,
          undefined,
          generalFileIndex,
          aliasPaths,
        );
        bumpFileIndexGeneration();
      });
      return { checked, found };
    },
    getFileIndex() {
      if (cachedMarkdownView && cachedMarkdownViewGeneration === fileIndexGeneration) {
        return cachedMarkdownView;
      }
      cachedMarkdownView = markdownIndexView(fileIndex);
      cachedMarkdownViewGeneration = fileIndexGeneration;
      return cachedMarkdownView;
    },
    getAllFilesIndex() {
      return allFileEntries;
    },
    getFileIndexGeneration() {
      return fileIndexGeneration;
    },
    getFolderIndex() {
      return folderIndex;
    },
    getAliasMap() {
      return aliasMap;
    },
    getFolderAliasIndex() {
      return folderAliasIndex;
    },
    getStructuralIgnoreDirs() {
      return structuralIgnoreDirs;
    },
    mutateFileIndex(reportedEvent) {
      const event = declaredDiskEvent(reportedEvent);
      if (event.kind === 'rename') {
        noteIntakeMutation(event.oldPath);
        noteIntakeMutation(event.newPath);
      } else {
        noteIntakeMutation(event.path);
      }
      if (event.kind === 'rename') {
        activePublicMutations?.markdownNames.add(event.oldDocName);
        activePublicMutations?.markdownNames.add(event.newDocName);
      } else if ('docName' in event) {
        activePublicMutations?.markdownNames.add(event.docName);
      }
      if (
        event.kind === 'file-create' ||
        event.kind === 'file-update' ||
        event.kind === 'file-delete'
      ) {
        let changedPaths: ReadonlySet<string> = new Set();
        if (event.kind === 'file-delete') {
          const before = generalFactsFor(generalFileIndex, [event.relativePath]);
          removeKnownGeneralPath(generalFileIndex, contentDir, event.relativePath);
          const after = generalFactsFor(generalFileIndex, before.keys());
          changedPaths = new Set([...before.keys()].filter((path) => !after.has(path)));
        } else {
          try {
            const lexicalPath = join(contentDir, event.relativePath);
            const lexicalStat = lstatSync(lexicalPath, { bigint: true });
            const leafSymlink = lexicalStat.isSymbolicLink();
            const canonicalPath = resolveNativePath(lexicalPath);
            if (!isWithinContentDir(canonicalPath, contentDir)) {
              throw new Error('General file path resolves outside content directory');
            }
            const physicalStat = leafSymlink
              ? statSync(canonicalPath, { bigint: true })
              : lexicalStat;
            const registered = registerKnownGeneralPath(
              generalFileIndex,
              contentDir,
              toPosix(relative(contentDir, canonicalPath)),
              event.relativePath,
              leafSymlink,
              filesystemIdentity(physicalStat),
              { size: event.size, modified: new Date(event.modifiedTs).toISOString() },
              contentFilter,
            );
            if (registered.kind === 'unobserved') return;
            changedPaths = registered.changedPaths;
          } catch (err) {
            log.warn({ path: event.path, err }, 'general file identity observation failed');
            const canonicalName = toPosix(relative(contentDir, event.path));
            let retained = false;
            const candidates = new Set([
              ...generalOwnerNames(generalFileIndex, canonicalName),
              ...generalOwnerNames(generalFileIndex, event.relativePath),
            ]);
            for (const name of candidates) {
              const entry = generalFileIndex.get(name);
              const state = entry && knownGeneralState(entry);
              if (!state) continue;
              const members = state.entry.fileMembers;
              if (
                !members.regularPaths.includes(canonicalName) &&
                !members.symlinks.some((relation) => relation.path === event.relativePath)
              )
                continue;
              storeKnownGeneralGroup(
                generalFileIndex,
                contentDir,
                name,
                state.identity,
                members.regularPaths,
                members.symlinks,
                { size: event.size, modified: new Date(event.modifiedTs).toISOString() },
              );
              retained = true;
              break;
            }
            if (!retained) storeUnresolvedGeneralEntry(generalFileIndex, contentDir, event);
          }
        }
        if (activePublicMutations) {
          recordGeneralPublicEffect(
            activePublicMutations,
            generalFileIndex,
            contentDir,
            event,
            changedPaths,
          );
        }
        if (activeReseedMutations) {
          recordGeneralPublicEffect(
            activeReseedMutations,
            generalFileIndex,
            contentDir,
            event,
            changedPaths,
          );
        }
      } else {
        updateFileIndex(event, fileIndex);
      }
      const hashEffects = activePublicMutations?.hashEffects;
      if (hashEffects) {
        switch (event.kind) {
          case 'create':
          case 'update':
          case 'conflict':
            if (!isReservedForUserTree(event.docName)) {
              hashEffects.set(event.path, contentHash(event.content));
            }
            break;
          case 'delete':
            if (!isReservedForUserTree(event.docName)) hashEffects.set(event.path, null);
            break;
          case 'rename':
            if (!isReservedForUserTree(event.newDocName)) {
              hashEffects.set(event.oldPath, null);
              hashEffects.set(event.newPath, contentHash(event.content));
            }
            break;
          case 'asset-create':
          case 'asset-delete':
          case 'folder-create':
          case 'folder-delete':
          case 'file-create':
          case 'file-update':
          case 'file-delete':
            break;
          default:
            assertNeverDiskEvent(event);
        }
      }
      bumpFileIndexGeneration();
    },
    pruneFileIndexNowExcluded() {
      if (!contentFilter) return 0;
      noteIntakeMutation(contentDir);
      let pruned = 0;
      let changed = false;
      const removedGeneralPaths = new Set<string>();
      for (const [docName, entry] of fileIndex) {
        const relPath = toPosix(relative(contentDir, entry.canonicalPath));
        if (contentFilter.isExcluded(relPath)) {
          fileIndex.delete(docName);
          pruned++;
        }
      }
      for (const [name, entry] of generalFileIndex) {
        const state = knownGeneralState(entry);
        if (!state) {
          const relPath = toPosix(relative(contentDir, entry.canonicalPath));
          if (contentFilter.isPathIgnored(relPath)) {
            generalFileIndex.delete(name);
            removedGeneralPaths.add(name);
            pruned++;
          }
          continue;
        }
        const members = state.entry.fileMembers;
        const regular = members.regularPaths.filter((path) => !contentFilter.isPathIgnored(path));
        const symlinks = members.symlinks.filter(
          (relation) =>
            !contentFilter.isPathIgnored(relation.path) &&
            !contentFilter.isPathIgnored(relation.targetPath) &&
            regular.includes(relation.targetPath),
        );
        if (
          regular.length === members.regularPaths.length &&
          symlinks.length === members.symlinks.length
        )
          continue;
        const retained = new Set([...regular, ...symlinks.map((relation) => relation.path)]);
        for (const path of members.regularPaths) {
          if (!retained.has(path)) removedGeneralPaths.add(path);
        }
        for (const relation of members.symlinks) {
          if (!retained.has(relation.path)) removedGeneralPaths.add(relation.path);
        }
        if (regular.length === 0) pruned++;
        storeKnownGeneralGroup(
          generalFileIndex,
          contentDir,
          name,
          state.identity,
          regular,
          symlinks,
          entry,
          state.sequence,
        );
        changed = true;
      }
      for (const record of [activePublicMutations, activeReseedMutations]) {
        if (!record) continue;
        for (const path of removedGeneralPaths) {
          if (record.generalEffects.get(path)) record.generalEffects.set(path, null);
        }
      }
      if (changed || pruned > 0) bumpFileIndexGeneration();
      return pruned;
    },
    pruneFolderIndexNowExcluded() {
      if (!contentFilter) return 0;
      noteIntakeMutation(contentDir);
      let pruned = 0;
      let aliasesChanged = false;
      for (const folderPath of folderIndex.keys()) {
        if (contentFilter.isDirExcluded(folderPath)) {
          folderIndex.delete(folderPath);
          pruned++;
        }
      }
      for (const alias of folderAliasIndex.keys()) {
        if (contentFilter.isDirExcluded(alias)) {
          folderAliasIndex.delete(alias);
          aliasesChanged = true;
        }
      }
      if (pruned > 0 || aliasesChanged) bumpFileIndexGeneration();
      return pruned;
    },
    async settle() {
      if (settleSubscription === undefined) {
        throw new Error(
          'settling disk events needs the parcel watcher backend, and this watcher runs on chokidar',
        );
      }
      await settleSubscription();
    },
    async rescanFromDisk() {
      const reseed = async () => {
        noteIntakeMutation(contentDir);
        const mutations: GeneralMutationRecord = {
          generalNames: new Set(),
          generalEffects: new Map(),
          generalMetadata: new Map(),
        };
        activeReseedMutations = mutations;
        try {
          const observation = await observeDisk(contentDir, contentFilter);
          const inventory = selectObservedInventory(observation, contentDir, contentFilter);
          publishDiskObservation(
            observation,
            inventory,
            fileIndex,
            generalFileIndex,
            folderIndex,
            aliasMap,
            folderAliasIndex,
            undefined,
            false,
          );
          const previous = new Map(generalFileIndex);
          const facts = generalPathFacts(generalFileIndex);
          for (const [path, fact] of inventory.generalFacts) {
            if (!mutations.generalEffects.has(path)) facts.set(path, fact);
          }
          for (const [path, effect] of mutations.generalEffects) {
            if (effect) facts.set(path, effect);
            else facts.delete(path);
          }
          rebuildGeneralFileIndex(
            generalFileIndex,
            contentDir,
            facts,
            generalRepresentativeNames(previous),
            mutations.generalMetadata,
          );
          for (const [alias, path] of observation.aliasPaths) aliasPaths.set(alias, path);
          bumpFileIndexGeneration();
          noteIntakeMutation(contentDir);
        } finally {
          activeReseedMutations = null;
        }
      };
      if (runOrderedRescan) return runOrderedRescan(reseed);
      return reseed();
    },
  };
}

export async function reconcileFileIndexAfterFilterRebuild(
  watcher: WatcherHandle | null | undefined,
): Promise<{
  prunedFiles: number;
  prunedFolders: number;
}> {
  if (!watcher) return { prunedFiles: 0, prunedFolders: 0 };
  const prunedFiles = watcher.pruneFileIndexNowExcluded();
  const prunedFolders = watcher.pruneFolderIndexNowExcluded();
  await watcher.rescanFromDisk();
  return { prunedFiles, prunedFolders };
}