Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

38.0 KB
Raw Blame
import { readFile } from 'node:fs/promises';
import {
  createTargetNamespace,
  type DerivedViewChannel,
  type TargetNamespace,
} from '@inkeep/open-knowledge-core';
import {
  type BacklinkEntry,
  BacklinkIndex,
  type GraphNode,
  type HubEntry,
  isOrphanMode,
  type OrphanMode,
} from './backlink-index.ts';
import { isLinkIndexExcludedDoc, isManagedArtifactDoc } from './cc1-broadcast.ts';
import type { ContentFilter } from './content-filter.ts';
import { docNameToRelativePath } from './doc-extensions.ts';
import { recordIndexGenerationLag } from './index-telemetry.ts';
import type { LocalTargetAssessment } from './local-target-assessment.ts';
import {
  LocalTargetIndex,
  type LocalTargetIndexStats,
  type LocalTargetRebuildInventory,
  type LocalTargetSourceAssessments,
} from './local-target-index.ts';
import type { WatcherLocalTargetInventory } from './local-target-inventory.ts';
import { getLogger } from './logger.ts';
import { TagIndex, type TagSummaryEntry } from './tag-index.ts';

const log = getLogger('derived-document-index');
const SLOW_STARTUP_LOG_THRESHOLD_MS = 10_000;

const DERIVED_INDEX_SAVE_COALESCE_MS = 2000;
const DERIVED_INDEX_SAVE_MIN_INTERVAL_MS = 30_000;
const LOCAL_TARGET_REBUILD_RETRY_BASE_MS = 1000;
const LOCAL_TARGET_REBUILD_RETRY_MAX_MS = 30_000;
const LOCAL_TARGET_FILE_EXISTENCE_SWEEP_MS = 5000;

export type DerivedDocumentRelationChannel = Extract<
  DerivedViewChannel,
  'files' | 'backlinks' | 'graph' | 'tags' | 'local-targets'
>;

type ForwardLinkEntry = ReturnType<BacklinkIndex['getForwardLinkEntries']>[number];
type LinkGraph = ReturnType<BacklinkIndex['getLinkGraph']>;
type DeadLinkEntry = ReturnType<BacklinkIndex['getDeadLinks']>[number];
type TagDocMatch = ReturnType<TagIndex['getDocsForTagWithMatches']>[number];

export type DerivedGraphNode = GraphNode;
export type DerivedOrphanMode = OrphanMode;

export function isDerivedOrphanMode(value: unknown): value is DerivedOrphanMode {
  return typeof value === 'string' && isOrphanMode(value);
}

export interface DerivedIndexStartupBacklinksResult {
  backlinkIndexDegraded: boolean;
}

export interface DerivedIndexStartupSettlement {
  tagIndexDegraded: boolean;
}

export interface DerivedDocumentIndexBranchTransition {
  readonly branch: string;
}

export interface DerivedDocumentIndexLivePort {
  captureLiveUpdateToken(): number | null;
  recordLiveDocument(documentName: string, markdown: string, token: number): Promise<void>;
}

export interface DerivedDocumentIndexPersistencePort {
  recordDurableStore(documentName: string, markdown: string): Promise<void>;
}

export interface DerivedDocumentIndexTestPort {
  resetDocumentForTest(documentName: string): Promise<void>;
  rescanBacklinksForTest(): Promise<void>;
}

export type DerivedDocumentIndexMutation =
  | { kind: 'upsert'; documentName: string; markdown: string }
  | { kind: 'delete'; documentName: string }
  | {
      kind: 'rename';
      oldDocumentName: string;
      newDocumentName: string;
      markdown: string;
    }
  | { kind: 'link-rewrite'; documentName: string; markdown: string };

export interface DerivedDocumentIndexApiPort {
  readonly testOnly?: DerivedDocumentIndexTestPort;
  recordDirectMutations(mutations: readonly DerivedDocumentIndexMutation[]): Promise<void>;
  recordDirectDocument(documentName: string, markdown: string): Promise<void>;
  recordDirectDelete(documentName: string): Promise<void>;
  recordDirectRename(
    oldDocumentName: string,
    newDocumentName: string,
    markdown: string,
  ): Promise<void>;
  recordLinkRewrite(documentName: string, markdown: string): Promise<void>;
  isReady(): boolean;
  getBacklinks(documentName: string): Promise<BacklinkEntry[]>;
  getBacklinkCount(documentName: string): Promise<number>;
  getBacklinkCounts(documentNames: readonly string[]): Promise<Record<string, number>>;
  getForwardLinkEntries(documentName: string): Promise<ForwardLinkEntry[]>;
  getLinkGraphNeighborhood(documentName: string, degrees: number): Promise<LinkGraph>;
  getLinkGraph(): Promise<LinkGraph>;
  getHubs(limit?: number): Promise<HubEntry[]>;
  getOrphans(documentNames: string[], mode: OrphanMode): Promise<string[]>;
  getDeadLinks(
    admittedDocuments: Iterable<string>,
    sourceDocumentNames?: readonly string[],
  ): Promise<DeadLinkEntry[]>;
  getLocalTargetAssessmentsForSources(
    sourceDocumentNames?: readonly string[],
  ): Promise<LocalTargetSourceAssessments[]>;
  readLocalTargetGeneration?(): string | number;
  getIndexedDocNames(): Promise<string[]>;
  getRenameSourceInventory(): Promise<
    Array<{ docName: string; wikiTargets: string[]; indexedMtimeMs: number | undefined }>
  >;
  getAllTags(): Promise<TagSummaryEntry[]>;
  getDocsForTagWithMatches(tag: string): Promise<TagDocMatch[]>;
}

export interface DerivedDocumentIndexOptions {
  projectDir: string;
  contentDir: string;
  contentFilter: ContentFilter;
  getGlobalSkillRoots: () => string[];
  signalChannel: (channel: DerivedDocumentRelationChannel) => void;
  getLocalTargetInventory?: () => WatcherLocalTargetInventory | null;
  onRecoveredFileTarget?: (relativePath: string, exists: boolean) => void;
}

interface BranchTransition extends DerivedDocumentIndexBranchTransition {
  barrier: PromiseWithResolvers<void>;
}

class DerivedDocumentIndexClosedError extends Error {
  constructor() {
    super('Derived document index is closed');
    this.name = 'DerivedDocumentIndexClosedError';
  }
}

export function isDerivedDocumentIndexClosedError(err: unknown): boolean {
  return err instanceof DerivedDocumentIndexClosedError;
}

class LocalTargetIndexNotReadyError extends Error {
  constructor() {
    super('Local-target index is not ready');
    this.name = 'LocalTargetIndexNotReadyError';
  }
}

export function isLocalTargetIndexNotReadyError(err: unknown): boolean {
  return err instanceof LocalTargetIndexNotReadyError;
}

export class DerivedDocumentIndex
  implements
    DerivedDocumentIndexApiPort,
    DerivedDocumentIndexLivePort,
    DerivedDocumentIndexPersistencePort
{
  readonly testOnly: DerivedDocumentIndexTestPort;

  private readonly backlinkIndex: BacklinkIndex;
  private readonly tagIndex: TagIndex;
  private readonly localTargetIndex: LocalTargetIndex;
  private readonly contentFilter: ContentFilter;
  private readonly getGlobalSkillRoots: () => string[];
  private readonly getLocalTargetInventory: () => WatcherLocalTargetInventory | null;
  private readonly signalChannel: (channel: DerivedDocumentRelationChannel) => void;
  private readonly onRecoveredFileTarget?: (relativePath: string, exists: boolean) => void;
  private readonly startupAdmission = Promise.withResolvers<void>();
  private readonly readyBarrier = Promise.withResolvers<void>();
  private tail: Promise<void> = Promise.resolve();
  private saveTimer: ReturnType<typeof setTimeout> | null = null;
  private lastNormalSaveAt: number | null = null;
  private localTargetRebuildRetryTimer: ReturnType<typeof setTimeout> | null = null;
  private readonly localTargetFileExistenceTimer: ReturnType<typeof setInterval>;
  private localTargetRebuildRetryAttempt = 0;
  private graphFileTargetsSource: readonly string[] | null = null;
  private graphFileTargets: TargetNamespace<'file'> | null = null;
  private startupBranch: string | null = null;
  private graphBuiltWithoutFileOracle = false;
  private saveTagsOnNextFlush = false;
  private branchTransition: BranchTransition | null = null;
  private startupBegun = false;
  private startupBegunAt = 0;
  private startupSettled = false;
  private ready = false;
  private liveUpdateToken = 0;
  private lastSignaledLocalTargetGeneration = 0;
  private closed = false;
  private closePromise: Promise<void> | null = null;

  constructor(options: DerivedDocumentIndexOptions) {
    this.backlinkIndex = new BacklinkIndex({
      projectDir: options.projectDir,
      contentDir: options.contentDir,
      contentFilter: options.contentFilter,
      getFileOracle: () => this.graphFileOracle(),
    });
    this.tagIndex = new TagIndex({
      projectDir: options.projectDir,
      contentDir: options.contentDir,
      contentFilter: options.contentFilter,
    });
    this.localTargetIndex = new LocalTargetIndex({
      contentDir: options.contentDir,
      contentFilter: options.contentFilter,
    });
    this.contentFilter = options.contentFilter;
    this.getGlobalSkillRoots = options.getGlobalSkillRoots;
    this.getLocalTargetInventory =
      options.getLocalTargetInventory ??
      (() => ({ documentTargets: [], fileTargets: [], folderTargets: [] }));
    this.signalChannel = options.signalChannel;
    this.onRecoveredFileTarget = options.onRecoveredFileTarget;
    this.testOnly = {
      resetDocumentForTest: (documentName) => this.resetDocumentForTest(documentName),
      rescanBacklinksForTest: () => this.rescanBacklinksForTest(),
    };
    this.localTargetFileExistenceTimer = setInterval(() => {
      if (this.closed || !this.startupSettled || !this.localTargetIndex.isReady()) return;
      void this.runCommand(async () => {
        const affected = await this.localTargetIndex.reconcileDependentFileTargetsFromDisk(
          this.onRecoveredFileTarget,
        );
        this.maybeSignalLocalTargets();
        if (affected > 0) this.signalChannel('files');
      }).catch((err) => {
        if (!this.closed) {
          log.warn({ err }, '[local-target] dependent file-existence sweep failed');
        }
      });
    }, LOCAL_TARGET_FILE_EXISTENCE_SWEEP_MS);
    this.localTargetFileExistenceTimer.unref?.();
  }

  beginStartup(branch: string): {
    backlinksReady: Promise<DerivedIndexStartupBacklinksResult>;
    offlineDeletionsReady: Promise<readonly string[]>;
  } {
    this.assertOpen();
    if (this.startupBegun) {
      throw new Error('Derived document index startup has already begun');
    }
    this.startupBegun = true;
    this.startupBegunAt = performance.now();
    this.startupBranch = branch;
    this.backlinkIndex.switchBranch(branch);

    const offlineDeletions = Promise.withResolvers<readonly string[]>();
    const backlinksReady = this.initializeBacklinks(branch, offlineDeletions.resolve);
    const tagsWarm = this.warmTags();
    this.tail = Promise.all([backlinksReady.then(() => undefined), tagsWarm]).then(() => undefined);
    this.startupAdmission.resolve();
    return { backlinksReady, offlineDeletionsReady: offlineDeletions.promise };
  }

  async settleStartupAfterWatcherSeed(): Promise<DerivedIndexStartupSettlement> {
    this.assertOpen();
    if (!this.startupBegun) {
      throw new Error('Derived document index startup has not begun');
    }
    if (this.startupSettled) {
      throw new Error('Derived document index startup has already settled');
    }
    this.startupSettled = true;

    try {
      return await this.enqueueInternal(async () => {
        let tagIndexDegraded = false;
        try {
          const diff = await this.tagIndex.reconcileWithDisk();
          if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
            log.info(diff, '[tag-index] startup reconcile: offline changes applied');
          }
          await this.saveTagsLogOnly('startup');
        } catch (err) {
          log.error(
            { err },
            '[tag-index] startup reconcile failed; tag index updates incrementally via watcher events',
          );
          tagIndexDegraded = true;
        }
        if (await this.rebuildLocalTargets('startup')) this.syncLocalTargetBaseline();
        if (this.startupBranch !== null) {
          await this.reconcileGraphWithFileInventory(this.startupBranch);
        }
        return { tagIndexDegraded };
      });
    } finally {
      this.ready = true;
      this.readyBarrier.resolve();
      const startupMs = Math.round(performance.now() - this.startupBegunAt);
      const logSettled = startupMs >= SLOW_STARTUP_LOG_THRESHOLD_MS ? log.info : log.debug;
      logSettled.call(
        log,
        { startupMs },
        '[derived-index] startup settled; deferred link checks resume',
      );
    }
  }

  async beginBranchSwitch(branch: string): Promise<DerivedDocumentIndexBranchTransition> {
    this.assertOpen();
    if (this.branchTransition) {
      throw new Error(`Derived document index branch switch already active`);
    }
    const transition: BranchTransition = { branch, barrier: Promise.withResolvers<void>() };
    this.liveUpdateToken++;
    this.branchTransition = transition;
    this.cancelScheduledSave();

    try {
      await this.startupAdmission.promise;
      await this.readyBarrier.promise;
      this.assertOpen();
      await this.enqueueInternal(async () => {
        this.backlinkIndex.switchBranch(branch);
      });
      return transition;
    } catch (err) {
      this.releaseBranchTransition(transition);
      throw err;
    }
  }

  async settleBranchFromDisk(transition: DerivedDocumentIndexBranchTransition): Promise<void> {
    this.assertOpen();
    if (this.branchTransition !== transition) {
      throw new Error('No derived document index branch switch is active');
    }

    try {
      await this.enqueueInternal(async () => {
        const branch = transition.branch;
        const loaded = await this.backlinkIndex.loadFromDisk(branch);
        if (loaded) {
          const diff = await this.backlinkIndex.reconcileWithDisk(branch);
          if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
            log.info(diff, `[backlinks] branch-switch reconcile for ${branch}`);
          }
        } else {
          await this.backlinkIndex.rebuildFromDisk(branch);
        }
        let globalIngestFailed = false;
        let globalIngestFailure: unknown;
        try {
          await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
        } catch (err) {
          globalIngestFailed = true;
          globalIngestFailure = err;
        }
        await this.saveBacklinksLogOnly('branch-switch', branch);

        try {
          const tagDiff = await this.tagIndex.reconcileWithDisk();
          if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {
            log.info(tagDiff, `[tag-index] branch-switch reconcile for ${branch}`);
          }
        } catch (err) {
          log.warn(
            { err, branch },
            '[tag-index] branch-switch reconcile failed; falling back to full rebuild',
          );
          await this.tagIndex.init();
        }
        await this.saveTagsLogOnly('branch-switch');
        if (await this.rebuildLocalTargets('branch-switch')) this.syncLocalTargetBaseline();
        if (globalIngestFailed) throw globalIngestFailure;
      });
    } finally {
      this.releaseBranchTransition(transition);
    }
  }

  abortBranchSwitch(transition: DerivedDocumentIndexBranchTransition | undefined): void {
    if (transition) this.releaseBranchTransition(transition);
  }

  captureLiveUpdateToken(): number | null {
    if (this.closed) return null;
    return this.liveUpdateToken;
  }

  recordLiveDocument(documentName: string, markdown: string, token: number): Promise<void> {
    if (this.closed || token !== this.liveUpdateToken) {
      return Promise.resolve();
    }
    return this.runCommand(async () => {
      if (token !== this.liveUpdateToken) return;
      this.updateBoth(documentName, markdown);
      this.signalAllRelations();
    });
  }

  recordDurableStore(documentName: string, markdown: string): Promise<void> {
    return this.runCommand(async () => {
      this.updateBacklinks(documentName, markdown);
      this.scheduleSave(false);
    });
  }

  recordDiskUpsert(documentName: string, markdown: string): Promise<void> {
    return this.runCommand(async () => {
      this.updateBoth(documentName, markdown);
      this.reconcileLocalTargetInventory();
      this.scheduleSave();
      this.signalAllRelations();
    });
  }

  recordDiskDelete(documentName: string): Promise<void> {
    return this.runCommand(async () => {
      this.deleteBoth(documentName);
      this.reconcileLocalTargetInventory();
      this.scheduleSave();
      this.signalAllRelations();
    });
  }

  recordDiskRename(
    oldDocumentName: string,
    newDocumentName: string,
    markdown: string,
  ): Promise<void> {
    return this.runCommand(async () => {
      this.renameBoth(oldDocumentName, newDocumentName, markdown);
      this.reconcileLocalTargetInventory();
      this.scheduleSave();
      this.signalAllRelations();
    });
  }

  recordInventoryReconciled(): Promise<void> {
    return this.runCommand(async () => {
      this.reconcileLocalTargetInventory();
      this.maybeSignalLocalTargets();
    });
  }

  recordFileTargetUpsert(relativePath: string): Promise<void> {
    return this.runCommand(async () => {
      const inventory = this.reconcileLocalTargetInventory();
      if (inventory === null || !inventory.fileTargets.includes(relativePath)) {
        this.localTargetIndex.setFileTarget(relativePath, true);
      }
      this.maybeSignalLocalTargets();
    });
  }

  recordFileTargetDelete(relativePath: string): Promise<void> {
    return this.runCommand(async () => {
      const inventory = this.reconcileLocalTargetInventory();
      if (inventory === null || inventory.fileTargets.includes(relativePath)) {
        this.localTargetIndex.setFileTarget(relativePath, false);
      }
      this.maybeSignalLocalTargets();
    });
  }

  recordDirectDocument(documentName: string, markdown: string): Promise<void> {
    return this.recordDirectMutations([{ kind: 'upsert', documentName, markdown }]);
  }

  recordDirectDelete(documentName: string): Promise<void> {
    return this.recordDirectMutations([{ kind: 'delete', documentName }]);
  }

  recordDirectRename(
    oldDocumentName: string,
    newDocumentName: string,
    markdown: string,
  ): Promise<void> {
    return this.recordDirectMutations([
      { kind: 'rename', oldDocumentName, newDocumentName, markdown },
    ]);
  }

  recordLinkRewrite(documentName: string, markdown: string): Promise<void> {
    return this.runCommand(async () => {
      this.updateBacklinks(documentName, markdown);
      this.scheduleSave(false);
      this.signalBacklinks();
    });
  }

  recordDirectMutations(mutations: readonly DerivedDocumentIndexMutation[]): Promise<void> {
    if (mutations.length === 0) return Promise.resolve();
    return this.runCommand(async () => {
      let tagsChanged = false;
      for (const mutation of mutations) {
        switch (mutation.kind) {
          case 'upsert':
            this.updateBoth(mutation.documentName, mutation.markdown);
            tagsChanged = true;
            break;
          case 'delete':
            this.deleteBoth(mutation.documentName);
            tagsChanged = true;
            break;
          case 'rename':
            this.renameBoth(mutation.oldDocumentName, mutation.newDocumentName, mutation.markdown);
            tagsChanged = true;
            break;
          case 'link-rewrite':
            this.updateBacklinks(mutation.documentName, mutation.markdown);
            break;
        }
      }

      if (tagsChanged) {
        this.reconcileLocalTargetInventory();
        await this.saveMutationCache(true, 'direct-batch');
        this.signalAllRelations();
      } else {
        await this.saveMutationCache(false, 'direct-batch');
        this.signalBacklinks();
      }
    });
  }

  refreshGlobalSkillNodes(): Promise<void> {
    return this.runCommand(async () => {
      await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots());
      this.reconcileLocalTargetInventory();
      this.signalBacklinks();
    });
  }

  refreshContentScope(): Promise<void> {
    return this.runCommand(async () => {
      const branch = this.backlinkIndex.getActiveBranch();
      const scopeDiff = await this.backlinkIndex.reconcileWithDisk(branch);
      let globalIngestFailed = false;
      let globalIngestFailure: unknown;
      try {
        await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
      } catch (err) {
        globalIngestFailed = true;
        globalIngestFailure = err;
      }
      await this.tagIndex.reconcileWithDisk();
      this.reconcileLocalTargetInventory();
      for (const { docName, filePath } of scopeDiff.changedDocs) {
        try {
          this.localTargetIndex.setSource(docName, await readFile(filePath, 'utf-8'));
        } catch {
          this.localTargetIndex.removeSource(docName);
        }
      }
      for (const docName of scopeDiff.deletedDocNames) {
        this.localTargetIndex.removeSource(docName);
      }
      await this.saveBothLogOnly('content-filter', branch);
      this.signalAllRelations();
      if (globalIngestFailed) throw globalIngestFailure;
    });
  }

  announceReadyViews(): void {
    this.assertOpen();
    this.signalAllRelations();
  }

  isReady(): boolean {
    return this.ready && !this.closed;
  }

  getBacklinks(documentName: string): Promise<BacklinkEntry[]> {
    return this.runQuery(() => this.backlinkIndex.getBacklinks(documentName));
  }

  getBacklinkCount(documentName: string): Promise<number> {
    return this.runQuery(() => this.backlinkIndex.getBacklinkCount(documentName));
  }

  getBacklinkCounts(documentNames: readonly string[]): Promise<Record<string, number>> {
    return this.runQuery(() =>
      Object.fromEntries(
        documentNames.map((documentName) => [
          documentName,
          this.backlinkIndex.getBacklinkCount(documentName),
        ]),
      ),
    );
  }

  getForwardLinkEntries(documentName: string): Promise<ForwardLinkEntry[]> {
    return this.runQuery(() => this.backlinkIndex.getForwardLinkEntries(documentName));
  }

  getLinkGraphNeighborhood(documentName: string, degrees: number): Promise<LinkGraph> {
    return this.runQuery(() => this.backlinkIndex.getLinkGraphNeighborhood(documentName, degrees));
  }

  getLinkGraph(): Promise<LinkGraph> {
    return this.runQuery(() => this.backlinkIndex.getLinkGraph());
  }

  getHubs(limit?: number): Promise<HubEntry[]> {
    return this.runQuery(() => this.backlinkIndex.getHubs(limit));
  }

  getOrphans(documentNames: string[], mode: OrphanMode): Promise<string[]> {
    return this.runQuery(() => this.backlinkIndex.getOrphans(documentNames, mode));
  }

  getDeadLinks(
    admittedDocuments: Iterable<string>,
    sourceDocumentNames?: readonly string[],
  ): Promise<DeadLinkEntry[]> {
    return this.runQuery(() =>
      this.backlinkIndex.getDeadLinks(
        admittedDocuments,
        sourceDocumentNames,
        undefined,
        this.getLocalTargetInventory()?.folderTargets,
      ),
    );
  }

  getIndexedDocNames(): Promise<string[]> {
    return this.runQuery(() => this.backlinkIndex.getIndexedDocNames());
  }

  getRenameSourceInventory(): Promise<
    Array<{ docName: string; wikiTargets: string[]; indexedMtimeMs: number | undefined }>
  > {
    return this.runQuery(() => this.backlinkIndex.getRenameSourceInventory());
  }

  getAllTags(): Promise<TagSummaryEntry[]> {
    return this.runQuery(() => this.tagIndex.getAllTags());
  }

  getDocsForTagWithMatches(tag: string): Promise<TagDocMatch[]> {
    return this.runQuery(() => this.tagIndex.getDocsForTagWithMatches(tag));
  }

  getLocalTargetAssessments(documentName: string): Promise<readonly LocalTargetAssessment[]> {
    return this.runLocalTargetQuery(() => this.localTargetIndex.getAssessments(documentName));
  }

  getLocalTargetAssessmentsForSources(
    sourceDocumentNames?: readonly string[],
  ): Promise<LocalTargetSourceAssessments[]> {
    return this.runLocalTargetQuery(() =>
      this.localTargetIndex.getAssessmentsForSources(sourceDocumentNames),
    );
  }

  getLocalTargetGeneration(): Promise<number> {
    return this.runLocalTargetQuery(() => this.localTargetIndex.generation);
  }

  readLocalTargetGeneration(): string {
    return this.localTargetIndex.freshnessToken;
  }

  getLocalTargetStats(): Promise<LocalTargetIndexStats> {
    return this.runLocalTargetQuery(() => this.localTargetIndex.getStats());
  }

  getLocalTargetDocumentDependents(documentName: string): Promise<string[]> {
    return this.runLocalTargetQuery(() =>
      this.localTargetIndex.getDocumentDependents(documentName),
    );
  }

  getLocalTargetFileDependents(relativePath: string): Promise<string[]> {
    return this.runLocalTargetQuery(() => this.localTargetIndex.getFileDependents(relativePath));
  }

  close(): Promise<void> {
    if (this.closePromise) return this.closePromise;
    const pendingSave = this.cancelScheduledSave();
    if (pendingSave !== null) {
      void this.enqueueInternal(() => this.saveMutationCache(pendingSave, 'close-flush'));
    }
    this.closed = true;
    clearInterval(this.localTargetFileExistenceTimer);
    this.cancelLocalTargetRebuildRetry();
    this.localTargetIndex.close();
    this.startupAdmission.resolve();
    this.readyBarrier.resolve();
    this.releaseBranchTransition(this.branchTransition);
    this.closePromise = this.tail
      .then(() => this.tagIndex.close())
      .then(
        () => undefined,
        () => undefined,
      );
    return this.closePromise;
  }

  private resetDocumentForTest(documentName: string): Promise<void> {
    return this.runCommand(async () => {
      this.deleteBoth(documentName);
      await this.saveBothLogOnly('test-reset');
      this.signalAllRelations();
    });
  }

  private rescanBacklinksForTest(): Promise<void> {
    return this.runCommand(async () => {
      const branch = this.backlinkIndex.getActiveBranch();
      await this.backlinkIndex.rebuildFromDisk(branch);
      await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
      this.reconcileLocalTargetInventory();
      await this.saveBacklinksLogOnly('test-rescan', branch);
      this.signalBacklinks();
    });
  }

  private async initializeBacklinks(
    branch: string,
    resolveOfflineDeletions: (deletedDocNames: readonly string[]) => void,
  ): Promise<DerivedIndexStartupBacklinksResult> {
    try {
      this.graphFileOracle();
      if (await this.backlinkIndex.loadFromDisk(branch)) {
        const diff = await this.backlinkIndex.reconcileWithDisk(branch);
        resolveOfflineDeletions(diff.deletedDocNames);
        if (diff.added > 0 || diff.updated > 0 || diff.deleted > 0) {
          log.info(
            {
              added: diff.added,
              updated: diff.updated,
              deleted: diff.deleted,
            },
            '[backlinks] startup reconcile',
          );
        }
      } else {
        resolveOfflineDeletions([]);
        await this.backlinkIndex.rebuildFromDisk(branch);
      }
      await this.ingestGlobalSkillNodesLogOnly(branch, 'startup');
      await this.saveBacklinksLogOnly('startup', branch);
      return { backlinkIndexDegraded: false };
    } catch (err) {
      resolveOfflineDeletions([]);
      log.error(
        { err, branch },
        '[backlinks] startup init failed; index will populate incrementally via watcher',
      );
      return { backlinkIndexDegraded: true };
    }
  }

  private async warmTags(): Promise<void> {
    try {
      if (await this.tagIndex.loadFromDisk()) {
        await this.tagIndex.reconcileWithDisk();
      } else {
        await this.tagIndex.init();
      }
    } catch (err) {
      log.warn(
        { err },
        '[derived-document-index] tag-index warm boot failed; continuing with stale or empty index',
      );
    }
  }

  private updateBoth(documentName: string, markdown: string): void {
    this.updateBacklinks(documentName, markdown);
    if (this.isAdmitted(documentName)) {
      this.tagIndex.updateDocumentFromMarkdown(documentName, markdown);
    } else {
      this.tagIndex.deleteDocument(documentName);
    }
  }

  private updateBacklinks(documentName: string, markdown: string): void {
    if (this.isAdmitted(documentName)) {
      this.backlinkIndex.updateDocumentFromMarkdown(documentName, markdown);
      this.localTargetIndex.setSource(documentName, markdown);
    } else {
      this.backlinkIndex.deleteDocument(documentName);
      this.localTargetIndex.removeSource(documentName);
    }
  }

  private deleteBoth(documentName: string): void {
    this.backlinkIndex.deleteDocument(documentName);
    this.tagIndex.deleteDocument(documentName);
    this.localTargetIndex.removeSource(documentName);
  }

  private renameBoth(oldDocumentName: string, newDocumentName: string, markdown: string): void {
    if (this.isAdmitted(newDocumentName)) {
      this.backlinkIndex.renameDocument(oldDocumentName, newDocumentName, markdown);
      this.tagIndex.renameDocument(oldDocumentName, newDocumentName, markdown);
      this.localTargetIndex.renameSource(oldDocumentName, newDocumentName, markdown);
    } else {
      this.backlinkIndex.deleteDocument(oldDocumentName);
      this.backlinkIndex.deleteDocument(newDocumentName);
      this.tagIndex.deleteDocument(oldDocumentName);
      this.tagIndex.deleteDocument(newDocumentName);
      this.localTargetIndex.removeSource(oldDocumentName);
      this.localTargetIndex.removeSource(newDocumentName);
    }
  }

  private isAdmitted(documentName: string): boolean {
    if (isLinkIndexExcludedDoc(documentName)) return false;
    if (isManagedArtifactDoc(documentName)) return true;
    return !this.contentFilter.isExcluded(docNameToRelativePath(documentName));
  }

  private signalBacklinks(): void {
    this.signalChannel('backlinks');
    this.signalChannel('graph');
    this.maybeSignalLocalTargets();
  }

  private signalAllRelations(): void {
    this.signalBacklinks();
    this.signalChannel('tags');
  }

  private maybeSignalLocalTargets(): void {
    const generation = this.localTargetIndex.generation;
    const lag = generation - this.lastSignaledLocalTargetGeneration;
    if (lag === 0) return;
    recordIndexGenerationLag('local-target', 'signal', lag);
    this.lastSignaledLocalTargetGeneration = generation;
    this.signalChannel('local-targets');
  }

  private graphFileOracle(): { hasFile(path: string): boolean } | undefined {
    const inventory = this.getLocalTargetInventory();
    if (!inventory) {
      this.graphBuiltWithoutFileOracle = true;
      return undefined;
    }
    if (this.graphFileTargetsSource !== inventory.fileTargets) {
      this.graphFileTargetsSource = inventory.fileTargets;
      this.graphFileTargets = createTargetNamespace('file', inventory.fileTargets);
    }
    const files = this.graphFileTargets;
    return files ? { hasFile: (path) => files.resolve(path) !== undefined } : undefined;
  }

  private async reconcileGraphWithFileInventory(branch: string): Promise<void> {
    if (!this.graphBuiltWithoutFileOracle) return;
    if (!this.graphFileOracle()) return;
    this.graphBuiltWithoutFileOracle = false;
    try {
      await this.backlinkIndex.rebuildFromDisk(branch);
      await this.saveBacklinksLogOnly('file-inventory-reconcile', branch);
      this.signalChannel('backlinks');
      this.signalChannel('graph');
    } catch (err) {
      this.graphBuiltWithoutFileOracle = true;
      log.warn(
        { err, branch },
        '[backlinks] file-inventory reconciliation failed; graph retains its startup extraction',
      );
    }
  }

  private async rebuildLocalTargets(reason: string): Promise<boolean> {
    try {
      const watcherInventory = this.getLocalTargetInventory();
      if (watcherInventory === null) {
        this.localTargetIndex.markUnavailable();
        log.warn(
          { reason },
          '[local-target] watcher inventory unavailable; index remains not ready',
        );
        return false;
      }
      const inventory: LocalTargetRebuildInventory = {
        documentTargets: new Set([
          ...this.backlinkIndex.getIndexedDocNames(),
          ...watcherInventory.documentTargets,
        ]),
        fileTargets: watcherInventory.fileTargets,
      };
      await this.localTargetIndex.rebuildFromDisk(inventory);
      this.localTargetRebuildRetryAttempt = 0;
      this.cancelLocalTargetRebuildRetry();
      return true;
    } catch (err) {
      log.warn(
        { err, reason },
        '[local-target] rebuild failed; retaining the last complete snapshot and scheduling recovery',
      );
      this.scheduleLocalTargetRebuildRetry(reason);
      return false;
    }
  }

  private reconcileLocalTargetInventory(): WatcherLocalTargetInventory | null {
    const watcherInventory = this.getLocalTargetInventory();
    if (watcherInventory === null) {
      this.localTargetIndex.markUnavailable();
      return null;
    }
    this.localTargetIndex.reconcileDocumentTargets(
      new Set([...this.backlinkIndex.getIndexedDocNames(), ...watcherInventory.documentTargets]),
    );
    this.localTargetIndex.reconcileFileTargets(watcherInventory.fileTargets);
    this.localTargetIndex.reconcileFolderTargets(watcherInventory.folderTargets);
    return watcherInventory;
  }

  private syncLocalTargetBaseline(): void {
    const generation = this.localTargetIndex.generation;
    const lag = generation - this.lastSignaledLocalTargetGeneration;
    if (lag > 0) recordIndexGenerationLag('local-target', 'baseline', lag);
    this.lastSignaledLocalTargetGeneration = generation;
  }

  private scheduleLocalTargetRebuildRetry(reason: string): void {
    if (this.closed || this.localTargetRebuildRetryTimer !== null) return;
    const delay = Math.min(
      LOCAL_TARGET_REBUILD_RETRY_BASE_MS * 2 ** this.localTargetRebuildRetryAttempt,
      LOCAL_TARGET_REBUILD_RETRY_MAX_MS,
    );
    this.localTargetRebuildRetryAttempt++;
    this.localTargetRebuildRetryTimer = setTimeout(() => {
      this.localTargetRebuildRetryTimer = null;
      void this.runCommand(async () => {
        if (await this.rebuildLocalTargets(`retry:${reason}`)) this.maybeSignalLocalTargets();
      }).catch((err) => {
        if (!this.closed) log.warn({ err, reason }, '[local-target] recovery enqueue failed');
      });
    }, delay);
    this.localTargetRebuildRetryTimer.unref?.();
  }

  private cancelLocalTargetRebuildRetry(): void {
    if (this.localTargetRebuildRetryTimer !== null) {
      clearTimeout(this.localTargetRebuildRetryTimer);
      this.localTargetRebuildRetryTimer = null;
    }
  }

  private scheduleSave(includeTags = true): void {
    if (this.closed || this.branchTransition !== null) return;
    this.saveTagsOnNextFlush ||= includeTags;
    if (this.saveTimer !== null) return;
    const delay =
      this.lastNormalSaveAt === null
        ? DERIVED_INDEX_SAVE_COALESCE_MS
        : Math.max(
            DERIVED_INDEX_SAVE_COALESCE_MS,
            DERIVED_INDEX_SAVE_MIN_INTERVAL_MS - (performance.now() - this.lastNormalSaveAt),
          );
    this.saveTimer = setTimeout(() => {
      this.saveTimer = null;
      const saveTags = this.saveTagsOnNextFlush;
      this.saveTagsOnNextFlush = false;
      void this.enqueueInternal(() => this.saveMutationCache(saveTags, 'coalesced')).catch(
        (err) => {
          log.warn({ err }, 'Failed to enqueue coalesced derived-index cache save');
        },
      );
    }, delay);
  }

  private cancelScheduledSave(): boolean | null {
    const pendingSave = this.saveTimer === null ? null : this.saveTagsOnNextFlush;
    if (this.saveTimer !== null) clearTimeout(this.saveTimer);
    this.saveTimer = null;
    this.saveTagsOnNextFlush = false;
    return pendingSave;
  }

  private async saveMutationCache(includeTags: boolean, reason: string): Promise<void> {
    if (
      !this.closed &&
      this.lastNormalSaveAt !== null &&
      performance.now() - this.lastNormalSaveAt < DERIVED_INDEX_SAVE_MIN_INTERVAL_MS
    ) {
      this.scheduleSave(includeTags);
      return;
    }
    if (includeTags) {
      await this.saveBothLogOnly(reason);
    } else {
      await this.saveBacklinksLogOnly(`${reason}-backlinks`);
    }
    this.lastNormalSaveAt = performance.now();
  }

  private releaseBranchTransition(transition: DerivedDocumentIndexBranchTransition | null): void {
    const active = this.branchTransition;
    if (!transition || active !== transition) return;
    this.branchTransition = null;
    active.barrier.resolve();
  }

  private async saveBothLogOnly(reason: string, branch?: string): Promise<void> {
    await this.saveBacklinksLogOnly(reason, branch);
    await this.saveTagsLogOnly(reason);
  }

  private async saveBacklinksLogOnly(reason: string, branch?: string): Promise<void> {
    try {
      await this.backlinkIndex.saveToDisk(branch);
    } catch (err) {
      log.warn({ err, reason, branch }, 'Failed to persist backlink cache');
    }
  }

  private async saveTagsLogOnly(reason: string): Promise<void> {
    try {
      await this.tagIndex.saveToDisk();
    } catch (err) {
      log.warn({ err, reason }, 'Failed to persist tag snapshot');
    }
  }

  private async ingestGlobalSkillNodesLogOnly(branch: string, reason: string): Promise<void> {
    try {
      await this.backlinkIndex.ingestGlobalSkillBundles(this.getGlobalSkillRoots(), branch);
    } catch (err) {
      log.warn({ err, branch, reason }, 'Global skill bundle ingest failed');
    }
  }

  private async runCommand<T>(operation: () => Promise<T> | T): Promise<T> {
    this.assertOpen();
    await this.startupAdmission.promise;
    this.assertOpen();
    return this.enqueueAfterBranchAdmission(operation);
  }

  private async runQuery<T>(query: () => Promise<T> | T): Promise<T> {
    this.assertOpen();
    await this.startupAdmission.promise;
    await this.readyBarrier.promise;
    this.assertOpen();
    return this.enqueueAfterBranchAdmission(query);
  }

  private runLocalTargetQuery<T>(query: () => Promise<T> | T): Promise<T> {
    return this.runQuery(() => {
      if (!this.localTargetIndex.isReady()) throw new LocalTargetIndexNotReadyError();
      return query();
    });
  }

  private async enqueueAfterBranchAdmission<T>(operation: () => Promise<T> | T): Promise<T> {
    while (true) {
      const transition = this.branchTransition;
      if (!transition) {
        this.assertOpen();
        return this.enqueueInternal(() => {
          this.assertOpen();
          return operation();
        });
      }
      await transition.barrier.promise;
      this.assertOpen();
    }
  }

  private enqueueInternal<T>(operation: () => Promise<T> | T): Promise<T> {
    const run = this.tail.then(operation);
    this.tail = run.then(
      () => undefined,
      (err) => {
        if (isDerivedDocumentIndexClosedError(err)) {
          log.debug({ err }, 'Derived document index operation rejected after close');
        } else {
          log.warn({ err }, 'Derived document index operation failed; queue remains available');
        }
      },
    );
    return run;
  }

  private assertOpen(): void {
    if (this.closed) {
      throw new DerivedDocumentIndexClosedError();
    }
  }
}