Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

perf(server): persist the tag index with mtime reconcile (#2798)

Andrew Mikofalvybf6615d620f9
diff --git a/.changeset/tag-index-persistence.md b/.changeset/tag-index-persistence.md
new file mode 100644
index 000000000..b223599b2
--- /dev/null
+++ b/.changeset/tag-index-persistence.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Persist the tag index with mtime reconcile for faster warm boots and branch switches. The tag index now snapshots to `.ok/local/cache/tags.json` (like the backlink cache) and re-parses only files whose mtime or size changed instead of re-reading the whole content dir on every server start and git branch switch. Backlink and tag index rebuilds also emit OpenTelemetry spans and rebuild counters/duration histograms.
diff --git a/knip.config.ts b/knip.config.ts
index d8a79af2f..6f8bbbc63 100644
--- a/knip.config.ts
+++ b/knip.config.ts
@@ -122,9 +122,7 @@ export default {
'packages/server': {
entry: ['src/**/*.test.ts'],
project: 'src/**',
- ignoreDependencies: [
- '@types/shell-quote',
- ],
+ ignoreDependencies: ['@types/shell-quote'],
},
'packages/cli': {
entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts'],
diff --git a/packages/core/src/bridge/bind-frontmatter-doc.ts b/packages/core/src/bridge/bind-frontmatter-doc.ts
index 8f9d95287..c014b1413 100644
--- a/packages/core/src/bridge/bind-frontmatter-doc.ts
+++ b/packages/core/src/bridge/bind-frontmatter-doc.ts
@@ -1,4 +1,3 @@
-
import type * as Y from 'yjs';
import type { Err, Ok, Result } from '../config/result.ts';
import { type FrontmatterValidationError, toFrontmatterIssue } from '../frontmatter/errors.ts';
diff --git a/packages/core/src/bridge/bridge-invariant.ts b/packages/core/src/bridge/bridge-invariant.ts
index 8bfdb40cd..005627195 100644
--- a/packages/core/src/bridge/bridge-invariant.ts
+++ b/packages/core/src/bridge/bridge-invariant.ts
@@ -1,4 +1,3 @@
-
import { fnv1aDigest } from './hash-util.ts';
export type BridgeInvariantSite = 'observer-b' | 'persistence' | 'test-harness';
diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts
index c18c086d5..8890f6739 100644
--- a/packages/core/src/bridge/doc-boundary-space.ts
+++ b/packages/core/src/bridge/doc-boundary-space.ts
@@ -1,4 +1,3 @@
-
import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';
const LEADING_BOUNDARY_RE = /^(?:\r?\n)+/;
diff --git a/packages/core/src/bridge/growth-detect.ts b/packages/core/src/bridge/growth-detect.ts
index 463e89595..9d9c244b0 100644
--- a/packages/core/src/bridge/growth-detect.ts
+++ b/packages/core/src/bridge/growth-detect.ts
@@ -1,4 +1,3 @@
-
const DEFAULT_MIN_SUBSTANTIVE_LINE_LENGTH = 16;
export const DUPLICATION_GATE_MIN_LINE_LENGTH = 8;
diff --git a/packages/core/src/bridge/merge-three-way.ts b/packages/core/src/bridge/merge-three-way.ts
index 3c927eb47..3ab9cc6ef 100644
--- a/packages/core/src/bridge/merge-three-way.ts
+++ b/packages/core/src/bridge/merge-three-way.ts
@@ -38,7 +38,6 @@ function mergeThreeWayImpl(baseline: string, userText: string, agentText: string
return parts.join('\n');
}
-
export type BridgeMergeContentLossSide = 'user' | 'agent';
export type BridgeMergeContentLossWhich = 'substring' | 'order' | 'growth';
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index 3f0880aa5..4b8890476 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -1,4 +1,3 @@
-
const COMMONMARK_ESCAPE_RE = /\\([!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~])/g;
const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/;
diff --git a/packages/core/src/bridge/pm-structural-equivalence.ts b/packages/core/src/bridge/pm-structural-equivalence.ts
index 0be5203e4..754dc5cc2 100644
--- a/packages/core/src/bridge/pm-structural-equivalence.ts
+++ b/packages/core/src/bridge/pm-structural-equivalence.ts
@@ -32,7 +32,6 @@ const BR_SENTINEL: PmStructuralNode = { type: '__br__' };
const BR_LITERAL_RE = /^<br\s*\/?>$/i;
-
interface DegradeEntry {
readonly label: string;
readonly severity: ToleranceClassSeverity;
@@ -128,7 +127,6 @@ export interface ComparePmStructuralOptions {
ignoreAttrs?: (attrKey: string) => boolean;
}
-
/** Rebuild a node with `fn` applied to each child; identity when childless.
* Shared by the degrade canonicalizers so none re-implements the walk. */
function mapChildren(
@@ -250,7 +248,6 @@ function applyDegrades(tree: PmStructuralNode): {
return { normalized: current, fired };
}
-
export function comparePmStructural(
expected: PmStructuralNode,
actual: PmStructuralNode,
diff --git a/packages/core/src/bridge/subsequence.ts b/packages/core/src/bridge/subsequence.ts
index dca644141..af20de1e2 100644
--- a/packages/core/src/bridge/subsequence.ts
+++ b/packages/core/src/bridge/subsequence.ts
@@ -1,4 +1,3 @@
-
/** True when `needle` is a subsequence of `haystack` (two-pointer, O(n+m)):
* insertions in `haystack` are free; any dropped or substituted `needle` byte
* fails. */
diff --git a/packages/core/src/markdown/callout-transformer.ts b/packages/core/src/markdown/callout-transformer.ts
index a363dfd54..83d47404a 100644
--- a/packages/core/src/markdown/callout-transformer.ts
+++ b/packages/core/src/markdown/callout-transformer.ts
@@ -1,4 +1,3 @@
-
import type { Blockquote, Paragraph, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts
index 47f0d07e4..193e286b0 100644
--- a/packages/core/src/markdown/comment-promoter.ts
+++ b/packages/core/src/markdown/comment-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/dedent-block-jsx-close.ts b/packages/core/src/markdown/dedent-block-jsx-close.ts
index b6299a948..ece5c02a1 100644
--- a/packages/core/src/markdown/dedent-block-jsx-close.ts
+++ b/packages/core/src/markdown/dedent-block-jsx-close.ts
@@ -1,4 +1,3 @@
-
import { findFencedRegions, isInsideFence } from './fence-regions.ts';
const INDENTED_BLOCK_JSX_CLOSE_RE = /^([ ]{1,3})(<\/[A-Z][A-Za-z0-9_]*\s*>)([ \t]*)$/gm;
diff --git a/packages/core/src/markdown/details-accordion-promoter.ts b/packages/core/src/markdown/details-accordion-promoter.ts
index acd398aa6..54774764a 100644
--- a/packages/core/src/markdown/details-accordion-promoter.ts
+++ b/packages/core/src/markdown/details-accordion-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.ts
index fb5ac8e8a..c7518d614 100644
--- a/packages/core/src/markdown/fence-regions.ts
+++ b/packages/core/src/markdown/fence-regions.ts
@@ -1,4 +1,3 @@
-
const FENCE_RE = /^(`{3,}|~{3,})/gm;
export function findFencedRegions(src: string): Array<[number, number]> {
diff --git a/packages/core/src/markdown/fixtures/index.ts b/packages/core/src/markdown/fixtures/index.ts
index 6931623c7..00f5771ee 100644
--- a/packages/core/src/markdown/fixtures/index.ts
+++ b/packages/core/src/markdown/fixtures/index.ts
@@ -1,4 +1,3 @@
-
import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -9,7 +8,6 @@ function fixturePath(...segments: string[]): string {
return resolve(FIXTURES_DIR, ...segments);
}
-
interface GfmExample {
section: string;
markdown: string;
@@ -19,7 +17,6 @@ export function loadGfmExamples(): GfmExample[] {
return JSON.parse(readFileSync(fixturePath('gfm', 'examples.json'), 'utf8')) as GfmExample[];
}
-
interface MdxCrashEntry {
id: string;
input: string;
@@ -71,7 +68,6 @@ export function loadLargeEmbedFixtures(): LargeEmbedFixture[] {
) as LargeEmbedFixture[];
}
-
export function loadPrd6955Before(): string {
return readFileSync(fixturePath('regression', 'prd-6955-before.md'), 'utf8');
}
@@ -80,7 +76,6 @@ export function loadPrd6955CorruptedTriplicated(): string {
return readFileSync(fixturePath('regression', 'prd-6955-corrupted-triplicated.md'), 'utf8');
}
-
export interface NgPinnedCase {
id: string;
name: string;
@@ -97,7 +92,6 @@ export function loadNgPinnedCases(): NgPinnedCase[] {
) as NgPinnedCase[];
}
-
export function loadLargeRealistic(): string {
return readFileSync(fixturePath('perf', 'large-realistic.md'), 'utf8');
}
diff --git a/packages/core/src/markdown/fixtures/perf/generate.ts b/packages/core/src/markdown/fixtures/perf/generate.ts
index daa3ef82b..8b1eb8519 100644
--- a/packages/core/src/markdown/fixtures/perf/generate.ts
+++ b/packages/core/src/markdown/fixtures/perf/generate.ts
@@ -1,4 +1,3 @@
-
import { writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -11,7 +10,6 @@ const BASELINE_ONLY_COUNTS = [500, 2500] as const;
const SEED = 0xf1de1117;
-
function mulberry32(seed: number): () => number {
let a = seed >>> 0;
return () => {
@@ -38,7 +36,6 @@ function pickWeighted<T>(rand: () => number, items: readonly [T, number][]): T {
return items[items.length - 1][0];
}
-
const WORDS = [
'lorem',
'ipsum',
@@ -156,7 +153,6 @@ function mdxBlock(rand: () => number): string {
return `<${name}>\n\n${sentence(rand)}\n\n</${name}>`;
}
-
type BlockKind = 'paragraph' | 'heading' | 'list' | 'code' | 'table' | 'mdx';
const MIX: readonly [BlockKind, number][] = [
@@ -195,7 +191,6 @@ function generateDocument(blockCount: number, seed: number): string {
return `${blocks.join('\n\n')}\n`;
}
-
function main(): void {
for (const count of [...BLOCK_COUNTS, ...BASELINE_ONLY_COUNTS]) {
const doc = generateDocument(count, SEED);
diff --git a/packages/core/src/markdown/guard-flanking-matrix.ts b/packages/core/src/markdown/guard-flanking-matrix.ts
index 32c055c5b..420dc0fe5 100644
--- a/packages/core/src/markdown/guard-flanking-matrix.ts
+++ b/packages/core/src/markdown/guard-flanking-matrix.ts
@@ -7,7 +7,6 @@ import {
import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts';
import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts';
-
export const ATTENTION_DELIMITERS = ['*', '_', '**', '~~', '=='] as const;
export type FlankClass = 'whitespace' | 'punctuation' | 'other';
diff --git a/packages/core/src/markdown/handler-shadow-audit.ts b/packages/core/src/markdown/handler-shadow-audit.ts
index 283913255..2653473da 100644
--- a/packages/core/src/markdown/handler-shadow-audit.ts
+++ b/packages/core/src/markdown/handler-shadow-audit.ts
@@ -1,4 +1,3 @@
-
export interface HandlerShadowWitness {
input: string;
expect: 'byte' | 'byte-and-reparse-type';
diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.ts
index 6319f264f..f55912407 100644
--- a/packages/core/src/markdown/highlight-promoter.ts
+++ b/packages/core/src/markdown/highlight-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Parent, PhrasingContent, Root, Text } from 'mdast';
import type { MdxJsxTextElement } from 'mdast-util-mdx';
import { SKIP, visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.ts
index 8dc3a81b5..b1f252c31 100644
--- a/packages/core/src/markdown/html-to-mdast.ts
+++ b/packages/core/src/markdown/html-to-mdast.ts
@@ -1,4 +1,3 @@
-
import type { Root as HastRoot } from 'hast';
import type { List, Root as MdastRoot } from 'mdast';
import rehypeParse from 'rehype-parse';
@@ -86,8 +85,7 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRo
throw new HtmlPayloadTooLargeError(html.length, maxBytes);
}
- const processor = unified()
- .use(rehypeParse, { fragment: true });
+ const processor = unified().use(rehypeParse, { fragment: true });
for (const plugin of cleanupPlugins) {
processor.use(plugin);
diff --git a/packages/core/src/markdown/image-promoter.ts b/packages/core/src/markdown/image-promoter.ts
index 994eb6917..706fa9afb 100644
--- a/packages/core/src/markdown/image-promoter.ts
+++ b/packages/core/src/markdown/image-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Image, Paragraph, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts
index fc30bda02..8b36807fd 100644
--- a/packages/core/src/markdown/index.ts
+++ b/packages/core/src/markdown/index.ts
@@ -1,4 +1,3 @@
-
import {
type FromProseMirrorOptions,
fromPmMark,
@@ -197,7 +196,6 @@ export class MarkdownManager {
}
}
-
const registry = createRegistry();
function destructureAttrs(
@@ -246,7 +244,6 @@ function destructureAttrs(
return result;
}
-
function hasDirtyDescendant(node: PmNode): boolean {
let found = false;
node.descendants((child) => {
@@ -266,7 +263,6 @@ function effectiveDirty(node: PmNode, freshnessChecker?: StructuralFreshnessChec
return freshnessChecker ? freshnessChecker.isDiverged(node.toJSON()) : false;
}
-
function isEmptyMdastParagraph(node: MdastNodes): boolean {
if (node.type !== 'paragraph') return false;
const children = node.children ?? [];
@@ -335,7 +331,6 @@ function extractTextFromMdastNodes(nodes: MdastNodes[]): string {
return out;
}
-
import {
AUDIO_EXTENSIONS,
FILE_ATTACHMENT_EXTENSIONS,
@@ -582,7 +577,6 @@ function buildMdastToPmHandlers(
}));
}
-
if (m.emphasis) {
handlers.emphasis = toPmMark(m.emphasis, (node: Emphasis) => ({
sourceDelimiter: node.data?.sourceDelimiter ?? '*',
@@ -663,7 +657,6 @@ function buildMdastToPmHandlers(
}));
}
-
if (m.link) {
const sourceLiteralMark = m.sourceLiteral;
handlers.link = (node: Link, _parent: MdastParent, state: MdastToPmState) => {
@@ -942,7 +935,6 @@ function buildMdastToPmHandlers(
};
}
-
const blockUnknownHandler = (node: {
type: string;
position?: { start: { offset: number }; end: { offset: number } };
@@ -1068,7 +1060,6 @@ function buildMdastToPmHandlers(
return handlers as RemarkProseMirrorOptions['handlers'];
}
-
function buildPmToMdastHandlers(
schema: Schema,
freshness: FreshnessCheckerHolder,
diff --git a/packages/core/src/markdown/lint/config-files.ts b/packages/core/src/markdown/lint/config-files.ts
index 94b34dee1..312c9c74b 100644
--- a/packages/core/src/markdown/lint/config-files.ts
+++ b/packages/core/src/markdown/lint/config-files.ts
@@ -1,4 +1,3 @@
-
const MARKDOWNLINT_JSON_CONFIG_FILES: ReadonlySet<string> = new Set([
'.markdownlint.json',
'.markdownlint.jsonc',
diff --git a/packages/core/src/markdown/lint/config-schemas.ts b/packages/core/src/markdown/lint/config-schemas.ts
index 207cd95c2..e78a1381d 100644
--- a/packages/core/src/markdown/lint/config-schemas.ts
+++ b/packages/core/src/markdown/lint/config-schemas.ts
@@ -29,7 +29,6 @@ const fullPluginShape = Object.fromEntries(
LINT_PLUGINS.map((plugin) => [plugin.id, plugin.sliceSchema]),
) as z.ZodRawShape;
-
export const LinterConfigSchema = z.object({
enabled: z.boolean(),
plugins: z.object(fullPluginShape),
diff --git a/packages/core/src/markdown/lint/default-config.ts b/packages/core/src/markdown/lint/default-config.ts
index a51f2ffb0..d48eea5e4 100644
--- a/packages/core/src/markdown/lint/default-config.ts
+++ b/packages/core/src/markdown/lint/default-config.ts
@@ -6,7 +6,6 @@ export const DEFAULT_MARKDOWNLINT_CONFIG: Record<string, MarkdownlintRuleSetting
MD013: false,
};
-
export function resolveMarkdownlintConfig(
rules: Record<string, MarkdownlintRuleSetting> | undefined,
): Configuration {
diff --git a/packages/core/src/markdown/lint/index.ts b/packages/core/src/markdown/lint/index.ts
index d763d8d28..8a154ce30 100644
--- a/packages/core/src/markdown/lint/index.ts
+++ b/packages/core/src/markdown/lint/index.ts
@@ -1,4 +1,3 @@
-
import { LINT_PLUGINS, type LinterConfig } from './plugins.ts';
import type { LintDiagnostic } from './types.ts';
diff --git a/packages/core/src/markdown/lint/plugins.ts b/packages/core/src/markdown/lint/plugins.ts
index 65a1b564c..1cd40ef3a 100644
--- a/packages/core/src/markdown/lint/plugins.ts
+++ b/packages/core/src/markdown/lint/plugins.ts
@@ -1,4 +1,3 @@
-
import { z } from 'zod';
import { DEFAULT_MARKDOWNLINT_CONFIG, resolveMarkdownlintConfig } from './default-config.ts';
import { fixMarkdownText, runMarkdownlint } from './markdownlint-runner.ts';
@@ -9,7 +8,6 @@ import {
type MarkdownlintSlice,
} from './types.ts';
-
const MarkdownlintRuleSettingSchema = z.union([
z.boolean(),
z.enum(MARKDOWNLINT_RULE_SEVERITIES),
diff --git a/packages/core/src/markdown/lint/types.ts b/packages/core/src/markdown/lint/types.ts
index 3651bd7eb..617f6e5b9 100644
--- a/packages/core/src/markdown/lint/types.ts
+++ b/packages/core/src/markdown/lint/types.ts
@@ -1,4 +1,3 @@
-
export const LINT_PLUGIN_IDS = ['markdownlint'] as const;
export type LintPluginId = (typeof LINT_PLUGIN_IDS)[number];
@@ -42,7 +41,6 @@ export interface MarkdownlintSlice {
rules: Record<string, MarkdownlintRuleSetting>;
}
-
interface RuleOptionSpecBase {
key: string;
description: string;
diff --git a/packages/core/src/markdown/math-promoter.ts b/packages/core/src/markdown/math-promoter.ts
index bd79424c7..a5b511f25 100644
--- a/packages/core/src/markdown/math-promoter.ts
+++ b/packages/core/src/markdown/math-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Code, Root } from 'mdast';
import type { Math as MdastMath } from 'mdast-util-math';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.ts
index f84967798..679c72ef7 100644
--- a/packages/core/src/markdown/mdast-to-hast-handlers.ts
+++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts
@@ -1,4 +1,3 @@
-
import type { Comment, Element, ElementContent, Properties } from 'hast';
import type { FootnoteDefinition, FootnoteReference } from 'mdast';
import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';
diff --git a/packages/core/src/markdown/mdast-to-html.ts b/packages/core/src/markdown/mdast-to-html.ts
index df788a80d..8ce2b2492 100644
--- a/packages/core/src/markdown/mdast-to-html.ts
+++ b/packages/core/src/markdown/mdast-to-html.ts
@@ -1,4 +1,3 @@
-
import type { Element, Root as HastRoot, Text as HastText } from 'hast';
import type { Root as MdastRoot, Text as MdastText } from 'mdast';
import type { Handler } from 'mdast-util-to-hast';
diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts
index fa3cfcd86..bfd7e3109 100644
--- a/packages/core/src/markdown/merged-walker.ts
+++ b/packages/core/src/markdown/merged-walker.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Parent, Root } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/mermaid-promoter.ts b/packages/core/src/markdown/mermaid-promoter.ts
index 17ed2b829..1a897fca8 100644
--- a/packages/core/src/markdown/mermaid-promoter.ts
+++ b/packages/core/src/markdown/mermaid-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Code, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.ts
index 7e4a9a48b..9864b3b60 100644
--- a/packages/core/src/markdown/parse-with-fallback.ts
+++ b/packages/core/src/markdown/parse-with-fallback.ts
@@ -168,7 +168,6 @@ export function parseRecursive(
}
}
-
interface VFilePlace {
offset?: number;
start?: { offset?: number };
@@ -187,7 +186,6 @@ function extractErrorOffset(err: unknown): number | undefined {
return undefined;
}
-
interface Region {
start: number;
end: number;
@@ -211,7 +209,6 @@ function nearestBlankLineAfter(src: string, offset: number): number | null {
return null;
}
-
export interface TagEvent {
kind: 'open' | 'close' | 'self-close';
name: string;
@@ -358,7 +355,6 @@ function findFallbackRegion(src: string, errorOffset: number): Region {
return { start: blockStart, end: blockEnd };
}
-
interface SourceBlock {
src: string;
start: number;
@@ -456,7 +452,6 @@ function tryPerBlockFallback(
};
}
-
function wholeDocRawText(source: string): JSONContent {
return {
type: 'doc',
diff --git a/packages/core/src/markdown/parser-drop-closure.ts b/packages/core/src/markdown/parser-drop-closure.ts
index 43d420187..e6ab4fd57 100644
--- a/packages/core/src/markdown/parser-drop-closure.ts
+++ b/packages/core/src/markdown/parser-drop-closure.ts
@@ -1,4 +1,3 @@
-
export type DroppedTokenAdjudication =
| {
kind: 'format-dof-axis';
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index e9e507469..7f2d5adca 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -1,4 +1,3 @@
-
import {
type FromProseMirrorOptions,
fromProseMirror,
diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.ts
index f8f5e8530..5a5e4c42a 100644
--- a/packages/core/src/markdown/position-aware-join.ts
+++ b/packages/core/src/markdown/position-aware-join.ts
@@ -1,4 +1,3 @@
-
import type { Join } from 'mdast-util-to-markdown';
import type { Position } from 'unist';
diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts
index e5b4c9eba..0f399d34c 100644
--- a/packages/core/src/markdown/position-slice.ts
+++ b/packages/core/src/markdown/position-slice.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Root } from 'mdast';
import { visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
@@ -477,8 +476,7 @@ export function applyPositionSliceToNode(
const first = source[startOff];
if (first !== '[' && first !== '<' && !node.title) {
node.data.sourceStyle = 'gfm-autolink';
- }
- else if (first === '[') {
+ } else if (first === '[') {
const slice = source.slice(startOff, endOff);
const closeBracketIdx = slice.lastIndexOf('](');
if (closeBracketIdx !== -1) {
diff --git a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
index ca225ce9b..db10fa54b 100644
--- a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
+++ b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
@@ -1,4 +1,3 @@
-
import type { Comment, Element, Root, Text } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
index 3a2524a9a..34068d103 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
@@ -1,4 +1,3 @@
-
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
index 2279a0189..93151d43b 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
@@ -1,4 +1,3 @@
-
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
index a0f467c42..188f02d83 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
@@ -1,4 +1,3 @@
-
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
index 4e8542a21..4ba2c3418 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
@@ -1,4 +1,3 @@
-
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
index a3c96b24a..1a74b1272 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
@@ -1,4 +1,3 @@
-
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
index e95cf43d6..de35e7b79 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
@@ -1,4 +1,3 @@
-
import type { Comment, Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
index ffdb4d5b5..b4be1b2ea 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
@@ -1,4 +1,3 @@
-
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
index e2a96d1a7..3c8d88b7c 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
@@ -1,4 +1,3 @@
-
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/resolve-image-url.ts b/packages/core/src/markdown/resolve-image-url.ts
index 0389575ac..cf2c9da8e 100644
--- a/packages/core/src/markdown/resolve-image-url.ts
+++ b/packages/core/src/markdown/resolve-image-url.ts
@@ -1,4 +1,3 @@
-
import { isRelativeUrl } from './safe-url.ts';
function isDevDiagnosticContext(): boolean {
diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts
index 8be632ba4..4d3250d3a 100644
--- a/packages/core/src/markdown/safe-url.ts
+++ b/packages/core/src/markdown/safe-url.ts
@@ -1,4 +1,3 @@
-
export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;
const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');
diff --git a/packages/core/src/markdown/serialize-helpers.ts b/packages/core/src/markdown/serialize-helpers.ts
index 542204403..46cc6a074 100644
--- a/packages/core/src/markdown/serialize-helpers.ts
+++ b/packages/core/src/markdown/serialize-helpers.ts
@@ -1,4 +1,3 @@
-
import type { Node as PmNode } from '@tiptap/pm/model';
import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import type { PropDef, SerializeContext } from '../registry/types.ts';
diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.ts
index fe73659e6..be97d4d62 100644
--- a/packages/core/src/markdown/single-dollar-math-promoter.ts
+++ b/packages/core/src/markdown/single-dollar-math-promoter.ts
@@ -1,4 +1,3 @@
-
import type { PhrasingContent, Root, Text } from 'mdast';
import type { InlineMath } from 'mdast-util-math';
import { SKIP, visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts
index 7bce04302..8f4e9a41c 100644
--- a/packages/core/src/markdown/to-markdown-handlers.ts
+++ b/packages/core/src/markdown/to-markdown-handlers.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Parents } from 'mdast';
import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import type { Handle, Info, State } from 'mdast-util-to-markdown';
diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.ts
index c5bfd2386..cd245c38c 100644
--- a/packages/core/src/markdown/unknown-mdast-guard.ts
+++ b/packages/core/src/markdown/unknown-mdast-guard.ts
@@ -1,4 +1,3 @@
-
import type { Root as MdastRoot } from 'mdast';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.ts
index ac9f5239b..d578476a7 100644
--- a/packages/core/src/markdown/whitespace-char-ref.ts
+++ b/packages/core/src/markdown/whitespace-char-ref.ts
@@ -1,4 +1,3 @@
-
const INLINE_WHITESPACE_BY_CODE: ReadonlyMap<number, string> = new Map([
[0x20, ' '],
[0x09, '\t'],
diff --git a/packages/core/src/markdown/wiki-link-micromark.ts b/packages/core/src/markdown/wiki-link-micromark.ts
index afd9936fe..6dad26200 100644
--- a/packages/core/src/markdown/wiki-link-micromark.ts
+++ b/packages/core/src/markdown/wiki-link-micromark.ts
@@ -17,7 +17,6 @@ declare module 'micromark-util-types' {
}
}
-
const CODE_BANG = 33; // !
const CODE_LBRACKET = 91; // [
const CODE_RBRACKET = 93; // ]
@@ -261,7 +260,6 @@ export function wikiLinkSyntax(): Extension {
};
}
-
function enterWikiLink(this: CompileContext, token: Token) {
this.enter(
{
@@ -387,7 +385,6 @@ export const wikiLinkToMarkdown: {
unsafe: [{ character: '[', inConstruct: ['phrasing'] }],
};
-
const MICROMARK_EXT = wikiLinkSyntax();
export function remarkWikiLink(this: Processor) {
diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.ts
index 3709f857a..ff37046f1 100644
--- a/packages/server/src/backlink-index.ts
+++ b/packages/server/src/backlink-index.ts
@@ -23,6 +23,7 @@ import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';
import { getLocalDir } from './config/paths.ts';
import type { ContentFilter } from './content-filter.ts';
import { isSupportedDocFile, stripDocExtension } from './doc-extensions.ts';
+import { instrumentIndexRebuild } from './index-telemetry.ts';
import { readMarkdownLinkAt, readWikiLinkAt } from './link-syntax.ts';
import { getLogger } from './logger.ts';
import { toPosix } from './path-utils.ts';
@@ -1626,6 +1627,10 @@ export class BacklinkIndex {
* `reconcileWithDisk` path takes the same shape.
*/
async rebuildFromDisk(branch = this.activeBranch): Promise<void> {
+ return instrumentIndexRebuild('backlink', 'full', () => this.rebuildFromDiskUnspanned(branch));
+ }
+
+ private async rebuildFromDiskUnspanned(branch: string): Promise<void> {
const state = createEmptyState();
const mtimes = new Map<string, number>();
const rawDocs: Array<{ docName: string; filePath: string }> = [];
@@ -1764,6 +1769,24 @@ export class BacklinkIndex {
* which applies its own doc-kind filters at the populate site.
*/
deletedDocNames: string[];
+ }> {
+ return instrumentIndexRebuild(
+ 'backlink',
+ 'reconcile',
+ () => this.reconcileWithDiskUnspanned(branch),
+ (diff) => ({
+ 'index.added': diff.added,
+ 'index.updated': diff.updated,
+ 'index.deleted': diff.deleted,
+ }),
+ );
+ }
+
+ private async reconcileWithDiskUnspanned(branch: string): Promise<{
+ added: number;
+ updated: number;
+ deleted: number;
+ deletedDocNames: string[];
}> {
if (!existsSync(this.contentDir))
return { added: 0, updated: 0, deleted: 0, deletedDocNames: [] };
diff --git a/packages/server/src/index-telemetry.test.ts b/packages/server/src/index-telemetry.test.ts
new file mode 100644
index 000000000..04e531719
--- /dev/null
+++ b/packages/server/src/index-telemetry.test.ts
@@ -0,0 +1,83 @@
+import { context, metrics, trace } from '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ type ReadableSpan,
+ SimpleSpanProcessor,
+} from '@opentelemetry/sdk-trace-base';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import { __resetIndexTelemetryForTests, instrumentIndexRebuild } from './index-telemetry.ts';
+
+let exporter: InMemorySpanExporter;
+let provider: BasicTracerProvider;
+
+beforeEach(() => {
+ exporter = new InMemorySpanExporter();
+ provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ trace.setGlobalTracerProvider(provider);
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ __resetIndexTelemetryForTests();
+});
+
+afterEach(async () => {
+ await provider.shutdown();
+ trace.disable();
+ metrics.disable();
+ context.disable();
+ __resetIndexTelemetryForTests();
+});
+
+function requireSpan(name: string): ReadableSpan {
+ const spans = exporter.getFinishedSpans().filter((s) => s.name === name);
+ const head = spans[0];
+ if (!head) throw new Error(`requireSpan: no span named '${name}'`);
+ return head;
+}
+
+describe('instrumentIndexRebuild', () => {
+ test('emits ok.index.rebuild with bounded name/mode attributes', async () => {
+ const result = await instrumentIndexRebuild('tag', 'full', async () => 'done');
+ expect(result).toBe('done');
+ const span = requireSpan('ok.index.rebuild');
+ expect(span.attributes['index.name']).toBe('tag');
+ expect(span.attributes['index.mode']).toBe('full');
+ });
+
+ test('maps the result onto span attributes via resultAttrs', async () => {
+ await instrumentIndexRebuild(
+ 'backlink',
+ 'reconcile',
+ async () => ({ added: 3, updated: 1, deleted: 2 }),
+ (diff) => ({
+ 'index.added': diff.added,
+ 'index.updated': diff.updated,
+ 'index.deleted': diff.deleted,
+ }),
+ );
+ const span = requireSpan('ok.index.rebuild');
+ expect(span.attributes['index.name']).toBe('backlink');
+ expect(span.attributes['index.mode']).toBe('reconcile');
+ expect(span.attributes['index.added']).toBe(3);
+ expect(span.attributes['index.updated']).toBe(1);
+ expect(span.attributes['index.deleted']).toBe(2);
+ });
+
+ test('propagates the rebuild error and still ends the span', async () => {
+ await expect(
+ instrumentIndexRebuild('tag', 'full', async () => {
+ throw new Error('walk failed');
+ }),
+ ).rejects.toThrow('walk failed');
+ const span = requireSpan('ok.index.rebuild');
+ expect(span.status.code).not.toBe(0); // ERROR, not UNSET
+ });
+
+ test('metric record path is no-throw under the default (disabled) meter', async () => {
+ // metrics.disable() in afterEach leaves the no-op global meter; the
+ // counter + histogram must never be able to break a rebuild.
+ await expect(instrumentIndexRebuild('tag', 'reconcile', async () => 1)).resolves.toBe(1);
+ });
+});
diff --git a/packages/server/src/index-telemetry.ts b/packages/server/src/index-telemetry.ts
new file mode 100644
index 000000000..d8c8d801f
--- /dev/null
+++ b/packages/server/src/index-telemetry.ts
@@ -0,0 +1,81 @@
+/**
+ * Telemetry primitives for derived-index rebuilds (backlink + tag).
+ *
+ * Lazy-init meter so registration runs against a real provider post-
+ * `initTelemetry` (not the pre-init no-op). Same pattern as
+ * `frontmatter-telemetry.ts`. One span name + one metric family with a
+ * bounded `index.name` / `index.mode` label pair keeps Tempo/Prometheus
+ * cardinality flat regardless of corpus size — NEVER add per-doc labels.
+ */
+import type { Attributes, Counter, Histogram } from '@opentelemetry/api';
+import { getMeter, withSpan } from './telemetry.ts';
+
+/** Bounded label set — extend only for a NEW derived index, never per-doc. */
+export type IndexName = 'backlink' | 'tag';
+
+/** `full` = whole-corpus re-parse; `reconcile` = mtime-gated incremental pass. */
+export type IndexRebuildMode = 'full' | 'reconcile';
+
+let _rebuildCounter: Counter | null = null;
+function rebuildCounter(): Counter {
+ _rebuildCounter ||= getMeter().createCounter('ok.index.rebuild_total', {
+ description:
+ 'Count of derived-index rebuild passes. Bounded labels: index.name ∈ {backlink, tag}, index.mode ∈ {full, reconcile}.',
+ });
+ return _rebuildCounter;
+}
+
+let _rebuildDurationHist: Histogram | null = null;
+function rebuildDurationHist(): Histogram {
+ _rebuildDurationHist ||= getMeter().createHistogram('ok.index.rebuild_duration_ms', {
+ description:
+ 'Duration of derived-index rebuild passes in milliseconds. Same bounded label pair as ok.index.rebuild_total.',
+ unit: 'ms',
+ });
+ return _rebuildDurationHist;
+}
+
+/**
+ * Run a rebuild/reconcile pass inside an `ok.index.rebuild` span and record
+ * the counter + duration histogram. The duration lands even when `fn` throws
+ * (the span records the exception via `withSpan`). `resultAttrs` maps the
+ * result onto extra span attributes — bounded numbers only (counts, not
+ * doc names).
+ *
+ * Zero overhead when OTel is disabled: the no-op tracer/meter make the span
+ * and instruments free beyond a function-call indirection.
+ */
+export async function instrumentIndexRebuild<T>(
+ name: IndexName,
+ mode: IndexRebuildMode,
+ fn: () => Promise<T>,
+ resultAttrs?: (result: T) => Attributes,
+): Promise<T> {
+ return withSpan(
+ 'ok.index.rebuild',
+ { attributes: { 'index.name': name, 'index.mode': mode } },
+ async (span) => {
+ const start = performance.now();
+ try {
+ const result = await fn();
+ if (resultAttrs) span.setAttributes(resultAttrs(result));
+ return result;
+ } finally {
+ const attrs = { 'index.name': name, 'index.mode': mode };
+ rebuildCounter().add(1, attrs);
+ rebuildDurationHist().record(performance.now() - start, attrs);
+ }
+ },
+ );
+}
+
+/**
+ * Drop the cached lazy-init instruments so the next call rebinds against the
+ * currently-registered global MeterProvider. Test-only — production code
+ * never needs this because the global provider is set once via
+ * `initTelemetry()`.
+ */
+export function __resetIndexTelemetryForTests(): void {
+ _rebuildCounter = null;
+ _rebuildDurationHist = null;
+}
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index d7520d703..98a9b91aa 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -966,7 +966,9 @@ export function createServer(options: ServerOptions): ServerInstance {
// Debounced saveToDisk for watcher-event paths. Collapses bursts (e.g. a git
// clone landing many files) into a single write. Startup and branch-switch
// paths call backlinkIndex.saveToDisk() directly — those are deliberate
- // full-state transitions that should not be deferred.
+ // full-state transitions that should not be deferred. The tag snapshot
+ // rides the same debounce: every scheduleSaveToDisk call site pairs with a
+ // tagIndex mutation on the same disk event.
const BACKLINK_SAVE_DEBOUNCE_MS = 2000;
let backlinkSaveTimer: ReturnType<typeof setTimeout> | null = null;
function scheduleSaveToDisk(): void {
@@ -976,6 +978,9 @@ export function createServer(options: ServerOptions): ServerInstance {
void backlinkIndex.saveToDisk().catch((err) => {
getLogger('backlinks').warn({ err }, 'Failed to persist debounced cache');
});
+ void tagIndex.saveToDisk().catch((err) => {
+ getLogger('tag-index').warn({ err }, 'Failed to persist debounced tag snapshot');
+ });
}, BACKLINK_SAVE_DEBOUNCE_MS);
}
@@ -1149,15 +1154,23 @@ export function createServer(options: ServerOptions): ServerInstance {
},
});
backlinkIndex = new BacklinkIndex({ projectDir, contentDir, contentFilter });
- tagIndex = new TagIndex({ contentDir, contentFilter });
+ tagIndex = new TagIndex({ projectDir, contentDir, contentFilter });
// Boot-time scan, fire-and-forget: the factory itself is synchronous.
- // `initAsync` awaits a second `init()` after the watcher starts; TagIndex
- // serializes the two internally, so by the time `ready` resolves the
+ // Warm start restores the persisted snapshot and mtime-reconciles offline
+ // changes; cold start (no/corrupt snapshot) falls back to the full scan.
+ // `initAsync` awaits a reconcile after the watcher starts; TagIndex
+ // serializes the passes internally, so by the time `ready` resolves the
// index reflects disk.
- void tagIndex.init().catch((err) => {
+ void (async () => {
+ if (await tagIndex.loadFromDisk()) {
+ await tagIndex.reconcileWithDisk();
+ } else {
+ await tagIndex.init();
+ }
+ })().catch((err) => {
getLogger('server-factory').warn(
{ err },
- '[server-factory] tag-index init failed; continuing with empty index',
+ '[server-factory] tag-index warm boot failed; continuing with empty index',
);
});
@@ -2609,6 +2622,17 @@ export function createServer(options: ServerOptions): ServerInstance {
backlinkSaveTimer = null;
}
+ // Close tag-index persistence: queued scans/saves become no-ops from
+ // here, so a fire-and-forget snapshot save from the boot path can't
+ // recreate files under `.ok/` while a caller tears the state dir down.
+ // Deliberately NOT awaited: this whole destroy runs under boot.ts's 5s
+ // per-step budget (destroyHocuspocus), and draining a boot-busy chain
+ // here stacks on top of the ready-wait below and blows that budget on
+ // slow machines. The synchronous flag flip is the load-bearing part;
+ // an already-in-flight write finishes long before callers reach their
+ // post-destroy cleanup.
+ void tagIndex.close();
+
// Flush the removal journal synchronously — a tombstone recorded just
// before shutdown must survive the restart or a reconnecting stale
// client resurrects the doc. Failure feeds `phaseErrors`: this flush is
@@ -3583,19 +3607,27 @@ export function createServer(options: ServerOptions): ServerInstance {
);
}
}
- // Re-init tagIndex once the watcher has settled. The earlier `init()`
- // in the synchronous boot path covers the cold-start case; this second
- // pass picks up any disk content that landed between constructor time
- // and watcher startup (rare, but cheap to cover). Isolated in its own
- // try/catch so a failed tag scan degrades tag search only — letting it
- // reach the outer catch would skip the basename-index seed below and
- // misreport a healthy watcher as `file-watcher` in `degraded[]`.
+ // Reconcile tagIndex once the watcher has settled. The warm-boot pass
+ // in the synchronous boot path covers the snapshot restore / cold
+ // scan; this second pass is a cheap stat-only sweep that picks up any
+ // disk content that landed between constructor time and watcher
+ // startup (rare, but cheap to cover), then persists the snapshot for
+ // the next boot. Isolated in its own try/catch so a failed tag scan
+ // degrades tag search only — letting it reach the outer catch would
+ // skip the basename-index seed below and misreport a healthy watcher
+ // as `file-watcher` in `degraded[]`.
try {
- await tagIndex.init();
+ const tagDiff = await tagIndex.reconcileWithDisk();
+ if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {
+ log.info(tagDiff, '[tag-index] startup reconcile: offline changes applied');
+ }
+ void tagIndex.saveToDisk().catch((err) => {
+ getLogger('tag-index').warn({ err }, 'Failed to persist startup tag snapshot');
+ });
} catch (err) {
log.error(
{ err },
- '[tag-index] startup re-init failed; tag index updates incrementally via watcher events',
+ '[tag-index] startup reconcile failed; tag index updates incrementally via watcher events',
);
degraded.push('tag-index');
}
@@ -3980,9 +4012,29 @@ export function createServer(options: ServerOptions): ServerInstance {
);
}
// TagIndex is branch-agnostic but its source-of-truth is the
- // contentDir that just changed underneath it. Re-scanning here
- // is the only way the index reflects the new branch's tags.
- await tagIndex.init();
+ // contentDir that just changed underneath it. The checkout
+ // rewrote changed files' mtimes, so a reconcile re-parses exactly
+ // the docs that differ between branches instead of the whole
+ // corpus; a failed reconcile falls back to the full scan so the
+ // index never silently keeps the old branch's tags.
+ try {
+ const tagDiff = await tagIndex.reconcileWithDisk();
+ if (tagDiff.added > 0 || tagDiff.updated > 0 || tagDiff.deleted > 0) {
+ log.info(tagDiff, `[tag-index] branch-switch reconcile for ${newBranch}`);
+ }
+ } catch (err) {
+ log.warn(
+ { err, branch: newBranch },
+ '[tag-index] branch-switch reconcile failed; falling back to full rebuild',
+ );
+ await tagIndex.init();
+ }
+ void tagIndex.saveToDisk().catch((err) => {
+ getLogger('tag-index').warn(
+ { branch: newBranch, err },
+ 'Failed to persist tag snapshot after branch switch',
+ );
+ });
// Restore parked WIP if exists (three-way merge parked state against current disk)
if (shadowRef.current && info.batchKind === 'cross-branch') {
diff --git a/packages/server/src/tag-index.persistence.test.ts b/packages/server/src/tag-index.persistence.test.ts
new file mode 100644
index 000000000..e780333e6
--- /dev/null
+++ b/packages/server/src/tag-index.persistence.test.ts
@@ -0,0 +1,316 @@
+/**
+ * Persistence + reconcile coverage for the tag-index snapshot
+ * (`.ok/local/cache/tags.json`): cold boot, warm boot on an unchanged tree,
+ * warm boot across offline edits/adds/deletes/renames, the mtime+size
+ * short-circuit, corrupt-snapshot fallback, and the branch-switch-shaped
+ * wholesale content swap.
+ */
+import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { describe, expect, test } from 'vitest';
+import { TagIndex } from './tag-index.ts';
+
+interface Rig {
+ projectDir: string;
+ contentDir: string;
+ snapshotPath: string;
+ cleanup: () => void;
+}
+
+function tempRig(): Rig {
+ const projectDir = mkdtempSync(join(tmpdir(), 'ok-tag-persist-'));
+ const contentDir = join(projectDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ return {
+ projectDir,
+ contentDir,
+ snapshotPath: join(projectDir, '.ok', 'local', 'cache', 'tags.json'),
+ cleanup: () => rmSync(projectDir, { recursive: true, force: true }),
+ };
+}
+
+/** Write a file and pin a deterministic mtime so cross-write mtime deltas
+ * never depend on filesystem timestamp granularity. */
+function writeDoc(contentDir: string, relPath: string, body: string, mtimeSec: number): void {
+ const filePath = join(contentDir, relPath);
+ mkdirSync(dirname(filePath), { recursive: true });
+ writeFileSync(filePath, body);
+ utimesSync(filePath, mtimeSec, mtimeSec);
+}
+
+function newIndex(rig: Rig): TagIndex {
+ return new TagIndex({ projectDir: rig.projectDir, contentDir: rig.contentDir });
+}
+
+describe('TagIndex persistence', () => {
+ test('cold boot: no snapshot on disk reports a cache miss; init + save writes one', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'alpha.md', 'Note on #typescript.\n', 1_000);
+ const idx = newIndex(rig);
+ expect(await idx.loadFromDisk()).toBe(false);
+ await idx.init();
+ expect(idx.getDocsForTag('typescript')).toEqual(['alpha']);
+ await idx.saveToDisk();
+ expect(existsSync(rig.snapshotPath)).toBe(true);
+ const snapshot = JSON.parse(await readFile(rig.snapshotPath, 'utf-8'));
+ expect(snapshot.version).toBe(1);
+ expect(snapshot.docs.alpha).toEqual(['typescript']);
+ expect(snapshot.files.alpha.mtimeMs).toBeGreaterThan(0);
+ expect(snapshot.files.alpha.size).toBeGreaterThan(0);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('warm boot on an unchanged tree: snapshot restores tags, reconcile re-parses nothing', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'alpha.md', '---\ntags: [proj/team]\n---\nBody #extra.\n', 1_000);
+ writeDoc(rig.contentDir, 'beta.md', 'No tags here.\n', 1_000);
+ const first = newIndex(rig);
+ await first.init();
+ await first.saveToDisk();
+
+ const second = newIndex(rig);
+ expect(await second.loadFromDisk()).toBe(true);
+ // Hierarchy expansion is re-derived from the persisted literals.
+ expect(second.getDocsForTag('proj')).toEqual(['alpha']);
+ expect(second.getDocsForTag('proj/team')).toEqual(['alpha']);
+ expect(second.getDocsForTag('extra')).toEqual(['alpha']);
+ const diff = await second.reconcileWithDisk();
+ expect(diff).toEqual({ added: 0, updated: 0, deleted: 0 });
+ expect(second.getDocsForTag('proj/team')).toEqual(['alpha']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('warm boot with offline edits, adds, and deletes reconciles all three', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'edited.md', 'Was #old-tag.\n', 1_000);
+ writeDoc(rig.contentDir, 'gone.md', 'Carried #doomed.\n', 1_000);
+ const first = newIndex(rig);
+ await first.init();
+ await first.saveToDisk();
+
+ // Offline: edit one doc's tags, delete one doc, add one doc.
+ writeDoc(rig.contentDir, 'edited.md', 'Now #new-tag.\n', 2_000);
+ rmSync(join(rig.contentDir, 'gone.md'));
+ writeDoc(rig.contentDir, 'fresh.md', 'Brand #minty.\n', 2_000);
+
+ const second = newIndex(rig);
+ expect(await second.loadFromDisk()).toBe(true);
+ const diff = await second.reconcileWithDisk();
+ expect(diff).toEqual({ added: 1, updated: 1, deleted: 1 });
+ expect(second.getDocsForTag('old-tag')).toEqual([]);
+ expect(second.getDocsForTag('new-tag')).toEqual(['edited']);
+ expect(second.getDocsForTag('doomed')).toEqual([]);
+ expect(second.getDocsForTag('minty')).toEqual(['fresh']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('warm boot across an offline rename moves the doc under its tags', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'before.md', 'Tagged #stable.\n', 1_000);
+ const first = newIndex(rig);
+ await first.init();
+ await first.saveToDisk();
+
+ // Offline rename = old path gone + new path present.
+ rmSync(join(rig.contentDir, 'before.md'));
+ writeDoc(rig.contentDir, 'after.md', 'Tagged #stable.\n', 2_000);
+
+ const second = newIndex(rig);
+ expect(await second.loadFromDisk()).toBe(true);
+ const diff = await second.reconcileWithDisk();
+ expect(diff).toEqual({ added: 1, updated: 0, deleted: 1 });
+ expect(second.getDocsForTag('stable')).toEqual(['after']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('mtime+size short-circuit: matching witnesses skip the re-parse', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'alpha.md', 'Tag #before-x.\n', 1_000);
+ const first = newIndex(rig);
+ await first.init();
+ await first.saveToDisk();
+
+ // Rewrite with DIFFERENT tags but identical byte length, then pin the
+ // mtime back — the witness pair matches, so reconcile must skip the
+ // file and keep the snapshot's tags. This pins the short-circuit
+ // actually short-circuiting (a full re-parse would surface #after-y).
+ writeFileSync(join(rig.contentDir, 'alpha.md'), 'Tag #after-yy.\n');
+ utimesSync(join(rig.contentDir, 'alpha.md'), 1_000, 1_000);
+
+ const second = newIndex(rig);
+ expect(await second.loadFromDisk()).toBe(true);
+ const diff = await second.reconcileWithDisk();
+ expect(diff).toEqual({ added: 0, updated: 0, deleted: 0 });
+ expect(second.getDocsForTag('before-x')).toEqual(['alpha']);
+ expect(second.getDocsForTag('after-yy')).toEqual([]);
+
+ // A size change alone (same mtime) must break the short-circuit.
+ writeFileSync(join(rig.contentDir, 'alpha.md'), 'Tag #after-y-longer.\n');
+ utimesSync(join(rig.contentDir, 'alpha.md'), 1_000, 1_000);
+ const diff2 = await second.reconcileWithDisk();
+ expect(diff2).toEqual({ added: 0, updated: 1, deleted: 0 });
+ expect(second.getDocsForTag('after-y-longer')).toEqual(['alpha']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('corrupt snapshot (invalid JSON) reports a miss so callers full-rebuild', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'alpha.md', 'Tag #real.\n', 1_000);
+ mkdirSync(join(rig.projectDir, '.ok', 'local', 'cache'), { recursive: true });
+ writeFileSync(rig.snapshotPath, '{ not json');
+
+ const idx = newIndex(rig);
+ expect(await idx.loadFromDisk()).toBe(false);
+ await idx.init();
+ expect(idx.getDocsForTag('real')).toEqual(['alpha']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('off-shape snapshot (valid JSON, wrong structure) also reports a miss', async () => {
+ const rig = tempRig();
+ try {
+ mkdirSync(join(rig.projectDir, '.ok', 'local', 'cache'), { recursive: true });
+ for (const bad of [
+ '[]',
+ '{"version":2,"docs":{},"files":{}}',
+ '{"version":1,"docs":{"a":"not-an-array"},"files":{}}',
+ '{"version":1,"docs":{"a":[""]},"files":{}}',
+ '{"version":1,"docs":{},"files":{"a":{"mtimeMs":"nan","size":1}}}',
+ ]) {
+ writeFileSync(rig.snapshotPath, bad);
+ const idx = newIndex(rig);
+ expect(await idx.loadFromDisk()).toBe(false);
+ }
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('snapshot restore refuses synthetic doc names', async () => {
+ const rig = tempRig();
+ try {
+ mkdirSync(join(rig.projectDir, '.ok', 'local', 'cache'), { recursive: true });
+ writeFileSync(
+ rig.snapshotPath,
+ JSON.stringify({
+ version: 1,
+ docs: { __system__: ['smuggled'], alpha: ['legit'] },
+ files: {},
+ }),
+ );
+ const idx = newIndex(rig);
+ expect(await idx.loadFromDisk()).toBe(true);
+ expect(idx.getDocsForTag('smuggled')).toEqual([]);
+ expect(idx.getDocsForTag('legit')).toEqual(['alpha']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('branch-switch-shaped swap: reconcile re-parses the rewritten corpus in place', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'shared.md', 'On main: #main-only.\n', 1_000);
+ writeDoc(rig.contentDir, 'main-doc.md', 'Tag #on-main.\n', 1_000);
+ const idx = newIndex(rig);
+ await idx.init();
+ await idx.saveToDisk();
+
+ // Simulate `git checkout feature`: shared file rewritten (new mtime),
+ // a main-only file removed, a feature-only file added.
+ writeDoc(rig.contentDir, 'shared.md', 'On feature: #feature-only.\n', 2_000);
+ rmSync(join(rig.contentDir, 'main-doc.md'));
+ writeDoc(rig.contentDir, 'feature-doc.md', 'Tag #on-feature.\n', 2_000);
+
+ const diff = await idx.reconcileWithDisk();
+ expect(diff).toEqual({ added: 1, updated: 1, deleted: 1 });
+ expect(idx.getDocsForTag('main-only')).toEqual([]);
+ expect(idx.getDocsForTag('on-main')).toEqual([]);
+ expect(idx.getDocsForTag('feature-only')).toEqual(['shared']);
+ expect(idx.getDocsForTag('on-feature')).toEqual(['feature-doc']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('reconcile drops runtime-indexed docs whose file never existed on disk', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'real.md', 'Tag #kept.\n', 1_000);
+ const idx = newIndex(rig);
+ await idx.init();
+ // Live update for a doc that has no file (e.g. CRDT-only doc later
+ // deleted on disk before persistence) — reconcile prunes it.
+ idx.updateDocumentFromMarkdown('phantom', 'Tag #ghost.\n');
+ expect(idx.getDocsForTag('ghost')).toEqual(['phantom']);
+ const diff = await idx.reconcileWithDisk();
+ expect(diff.deleted).toBe(1);
+ expect(idx.getDocsForTag('ghost')).toEqual([]);
+ expect(idx.getDocsForTag('kept')).toEqual(['real']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('close() drains the chain and makes later persistence calls no-ops', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'alpha.md', 'Tag #closing.\n', 1_000);
+ const idx = newIndex(rig);
+ await idx.init();
+ await idx.close();
+ // A save issued after close must not recreate anything under .ok/ —
+ // callers rm -rf the state dir right after destroy.
+ await idx.saveToDisk();
+ expect(existsSync(rig.snapshotPath)).toBe(false);
+ await expect(idx.reconcileWithDisk()).resolves.toEqual({
+ added: 0,
+ updated: 0,
+ deleted: 0,
+ });
+ expect(await idx.loadFromDisk()).toBe(false);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
+ test('persistence disabled without projectDir: load misses, save is a no-op', async () => {
+ const rig = tempRig();
+ try {
+ writeDoc(rig.contentDir, 'alpha.md', 'Tag #memory-only.\n', 1_000);
+ const idx = new TagIndex({ contentDir: rig.contentDir });
+ expect(await idx.loadFromDisk()).toBe(false);
+ await idx.init();
+ await idx.saveToDisk();
+ expect(existsSync(rig.snapshotPath)).toBe(false);
+ // Reconcile still works purely in-memory (branch-switch callers don't
+ // need persistence wired to get the incremental pass).
+ writeDoc(rig.contentDir, 'alpha.md', 'Tag #updated-now.\n', 2_000);
+ const diff = await idx.reconcileWithDisk();
+ expect(diff).toEqual({ added: 0, updated: 1, deleted: 0 });
+ expect(idx.getDocsForTag('updated-now')).toEqual(['alpha']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+});
diff --git a/packages/server/src/tag-index.ts b/packages/server/src/tag-index.ts
index 43cf7f585..14e396f88 100644
--- a/packages/server/src/tag-index.ts
+++ b/packages/server/src/tag-index.ts
@@ -13,23 +13,25 @@
* `extractFrontmatterTags` so the inline + YAML surfaces share one
* regex and one drop-on-invalid policy.
*
- * Persistence is intentionally NOT implemented. Tag extraction is cheap
- * (single regex scan + YAML parse), `init()` walks the content dir on boot,
- * and the live-derived-index extension feeds incremental updates. Mirrors
- * the boot-time rebuild path that backlinks have for cold caches without
- * the on-disk cache cost.
+ * Persistence mirrors the backlink cache: a JSON snapshot of each doc's
+ * literal authored tags plus per-file mtime+size, at
+ * `<projectDir>/.ok/local/cache/tags.json`. Warm boots load the snapshot and
+ * `reconcileWithDisk()` re-parses only files whose mtime or size changed;
+ * cold boots (or a corrupt snapshot) fall back to the full `init()` scan.
+ * Only the literal tag sets are persisted — hierarchy expansion is
+ * re-derived on load, so the snapshot can never disagree with the
+ * expansion rules.
*
- * Similar surface to BacklinkIndex (contentDir + contentFilter only; no
- * projectDir since TagIndex has no on-disk persistence cache). Same
- * `updateDocumentFromMarkdown` / `deleteDocument` method names. Branch-
- * agnostic single-state simplification (tag rollup is not branch-aware
- * today; multi-branch derivation can layer on later via the same
- * `BranchGraphState` shape backlinks use).
+ * Similar surface to BacklinkIndex. Same `updateDocumentFromMarkdown` /
+ * `deleteDocument` method names. Branch-agnostic single-state
+ * simplification (tag rollup is not branch-aware today), so the snapshot is
+ * a single file rather than per-branch: a branch switch rewrites changed
+ * files' mtimes, which the same reconcile pass picks up.
*/
import { type Dirent, existsSync } from 'node:fs';
-import { readdir, readFile } from 'node:fs/promises';
-import { join, relative } from 'node:path';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { dirname, join, relative, resolve } from 'node:path';
import {
createTagInTextRegex,
expandTagToHierarchy,
@@ -39,8 +41,11 @@ import {
unwrapFrontmatterFences,
} from '@inkeep/open-knowledge-core';
import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';
+import { getLocalDir } from './config/paths.ts';
import type { ContentFilter } from './content-filter.ts';
import { isSupportedDocFile, stripDocExtension } from './doc-extensions.ts';
+import { tracedMkdir, tracedWriteFile } from './fs-traced.ts';
+import { instrumentIndexRebuild } from './index-telemetry.ts';
import { getLogger } from './logger.ts';
import { toPosix } from './path-utils.ts';
@@ -66,6 +71,54 @@ export interface TagSummaryEntry {
export interface TagIndexOptions {
contentDir: string;
contentFilter?: ContentFilter;
+ /**
+ * Project root (where `.ok/` lives). Enables the on-disk snapshot at
+ * `<projectDir>/.ok/local/cache/tags.json`. When omitted (in-memory-only
+ * usage, most unit tests), `saveToDisk` is a no-op and `loadFromDisk`
+ * reports a cache miss so callers fall back to the full scan.
+ */
+ projectDir?: string;
+}
+
+/** Per-file freshness witness captured at rebuild/reconcile time. */
+interface FileMeta {
+ mtimeMs: number;
+ size: number;
+}
+
+interface SerializedTagIndexSnapshot {
+ version: 1;
+ /** docName → sorted literal authored tags (hierarchy re-expanded on load). */
+ docs: Record<string, string[]>;
+ /** docName → mtime+size at the last rebuild/reconcile; the boot short-circuit witness. */
+ files: Record<string, FileMeta>;
+}
+
+const SNAPSHOT_VERSION = 1;
+
+/**
+ * Structural validation of a parsed snapshot. Anything off-shape (wrong
+ * version, non-string tags, non-numeric file meta) reports corrupt so the
+ * caller falls back to a full rebuild — a partial restore from a bad
+ * snapshot would surface stale or garbage tags to the user.
+ */
+function isValidSnapshot(data: unknown): data is SerializedTagIndexSnapshot {
+ if (typeof data !== 'object' || data === null) return false;
+ const snapshot = data as Record<string, unknown>;
+ if (snapshot.version !== SNAPSHOT_VERSION) return false;
+ const { docs, files } = snapshot;
+ if (typeof docs !== 'object' || docs === null || Array.isArray(docs)) return false;
+ if (typeof files !== 'object' || files === null || Array.isArray(files)) return false;
+ for (const tags of Object.values(docs)) {
+ if (!Array.isArray(tags)) return false;
+ if (!tags.every((tag) => typeof tag === 'string' && tag.length > 0)) return false;
+ }
+ for (const meta of Object.values(files)) {
+ if (typeof meta !== 'object' || meta === null) return false;
+ const { mtimeMs, size } = meta as Record<string, unknown>;
+ if (typeof mtimeMs !== 'number' || typeof size !== 'number') return false;
+ }
+ return true;
}
interface TagIndexState {
@@ -177,12 +230,60 @@ function extractInlineTagsFromBody(body: string): string[] {
export class TagIndex {
private readonly contentDir: string;
private readonly contentFilter?: ContentFilter;
+ private readonly projectDir?: string;
private state: TagIndexState = createEmptyState();
+ /**
+ * Per-doc freshness witnesses from the last full scan or reconcile.
+ * Persisted in the snapshot; runtime incremental updates deliberately do
+ * NOT refresh entries here, so the next boot's reconcile re-parses any
+ * file whose bytes moved under a stale witness (same contract as
+ * BacklinkIndex's mtime map).
+ */
+ private fileMeta = new Map<string, FileMeta>();
private initChain: Promise<void> = Promise.resolve();
+ private closed = false;
constructor(options: TagIndexOptions) {
this.contentDir = options.contentDir;
this.contentFilter = options.contentFilter;
+ this.projectDir = options.projectDir;
+ }
+
+ private snapshotPath(): string | null {
+ if (!this.projectDir) return null;
+ return resolve(getLocalDir(this.projectDir), 'cache', 'tags.json');
+ }
+
+ /**
+ * Serialize a task onto the internal chain shared with `init()`. Every
+ * operation that reads or replaces whole-index state (full scan,
+ * reconcile, snapshot load/save) runs here so a snapshot save can never
+ * observe a half-populated rebuild and two scans can never interleave
+ * their per-doc updates. Failures clear the chain link (same shape as
+ * `init()`'s original chaining) so one failed task doesn't poison the
+ * next.
+ */
+ private enqueue<T>(fn: () => Promise<T>): Promise<T> {
+ const run = this.initChain.then(fn);
+ this.initChain = run.then(
+ () => undefined,
+ (err) => {
+ log.warn({ err }, 'queued tag-index task failed (chain cleared for next task)');
+ },
+ );
+ return run;
+ }
+
+ /**
+ * Stop all future queued work (scans, reconciles, snapshot saves become
+ * no-ops) and resolve once in-flight chain tasks have drained. Server
+ * destroy calls this so a fire-and-forget snapshot save can never land
+ * inside `.ok/` after teardown starts — test rigs `rm -rf` the state dir
+ * right after destroy, and a late write turns that into ENOTEMPTY.
+ */
+ close(): Promise<void> {
+ this.closed = true;
+ return this.initChain;
}
/**
@@ -319,10 +420,11 @@ export class TagIndex {
}
/**
- * Boot-time scan. Walks `contentDir` (respecting `contentFilter`) and
- * indexes every supported markdown file. Mirrors
+ * Full cold-start scan. Walks `contentDir` (respecting `contentFilter`),
+ * indexes every supported markdown file, and collects the freshness
+ * witnesses a later `saveToDisk` persists. Mirrors
* `BacklinkIndex.rebuildFromDisk` — same recursion shape, same exclusion
- * gates, no on-disk cache.
+ * gates.
*
* Reads files via the absolute path discovered during the walk rather than
* `getDocExtension(docName)` so the indexer doesn't have to mutate the
@@ -331,21 +433,13 @@ export class TagIndex {
* test order across surfaces).
*/
init(): Promise<void> {
- // Serialize concurrent init calls. The boot path fires one init from the
- // synchronous constructor phase and another after the watcher starts; with
- // an async scan those would otherwise interleave their per-doc updates.
- const run = this.initChain.then(() => this.initOnce());
- // Catch on the chain link (not the returned promise) so one failed scan
- // doesn't poison every subsequent init() call. The breadcrumb keeps a
- // trace when a caller drops the returned promise without handling it.
- this.initChain = run.catch((err) => {
- log.warn({ err }, 'init failed (chain cleared for next init)');
- });
- return run;
+ return this.enqueue(() => instrumentIndexRebuild('tag', 'full', () => this.initOnce()));
}
private async initOnce(): Promise<void> {
+ if (this.closed) return;
this.state = createEmptyState();
+ this.fileMeta = new Map();
if (!existsSync(this.contentDir)) return;
const entries = await this.listDocsWithPaths();
// Bounded read batches keep the event loop responsive on large content
@@ -356,7 +450,11 @@ export class TagIndex {
const results = await Promise.all(
batch.map(async ({ docName, filePath }) => {
try {
- return { docName, markdown: await readFile(filePath, 'utf-8') };
+ const [fileStat, markdown] = await Promise.all([
+ stat(filePath),
+ readFile(filePath, 'utf-8'),
+ ]);
+ return { docName, markdown, mtimeMs: fileStat.mtimeMs, size: fileStat.size };
} catch (err) {
log.warn({ docName, err }, `Failed to read ${docName} during init`);
return null;
@@ -367,6 +465,7 @@ export class TagIndex {
if (!result) continue;
try {
this.updateDocumentFromMarkdown(result.docName, result.markdown);
+ this.fileMeta.set(result.docName, { mtimeMs: result.mtimeMs, size: result.size });
} catch (err) {
log.warn(
{ docName: result.docName, err },
@@ -377,6 +476,170 @@ export class TagIndex {
}
}
+ /**
+ * Restore the persisted snapshot into memory. Returns `false` on cache
+ * miss, disabled persistence (no `projectDir`), or ANY corruption —
+ * callers must treat `false` as "run the full `init()` scan". A `true`
+ * return means the in-memory state now reflects the snapshot; follow with
+ * `reconcileWithDisk()` to fold in offline changes.
+ */
+ loadFromDisk(): Promise<boolean> {
+ return this.enqueue(() => this.loadOnce());
+ }
+
+ private async loadOnce(): Promise<boolean> {
+ if (this.closed) return false;
+ const filePath = this.snapshotPath();
+ if (!filePath || !existsSync(filePath)) return false;
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(await readFile(filePath, 'utf-8'));
+ } catch (err) {
+ log.warn({ err }, 'Failed to load tag snapshot; falling back to full rebuild');
+ return false;
+ }
+ if (!isValidSnapshot(parsed)) {
+ log.warn({}, 'Tag snapshot failed validation; falling back to full rebuild');
+ return false;
+ }
+ this.state = createEmptyState();
+ this.fileMeta = new Map(Object.entries(parsed.files));
+ for (const [docName, tags] of Object.entries(parsed.docs)) {
+ // Same synthetic-doc gate as the live update path — a hand-edited or
+ // poisoned snapshot must not smuggle system docs into the index.
+ if (isLinkIndexExcludedDoc(docName)) continue;
+ const authoredTags = new Set(tags);
+ const expanded = new Set<string>();
+ for (const tag of authoredTags) {
+ for (const prefix of expandTagToHierarchy(tag)) {
+ expanded.add(prefix);
+ }
+ }
+ this.applyDocSnapshot(docName, authoredTags, expanded);
+ }
+ return true;
+ }
+
+ /**
+ * Persist the current literal tag sets + freshness witnesses. No-op when
+ * persistence is disabled. Enqueued on the internal chain so a save never
+ * captures a half-populated rebuild.
+ */
+ saveToDisk(): Promise<void> {
+ return this.enqueue(() => this.saveOnce());
+ }
+
+ private async saveOnce(): Promise<void> {
+ if (this.closed) return;
+ const filePath = this.snapshotPath();
+ if (!filePath) return;
+ const snapshot: SerializedTagIndexSnapshot = {
+ version: SNAPSHOT_VERSION,
+ docs: Object.fromEntries(
+ [...this.state.byDocLiteral.entries()].map(([docName, tags]) => [
+ docName,
+ [...tags].sort((a, b) => a.localeCompare(b)),
+ ]),
+ ),
+ files: Object.fromEntries(this.fileMeta),
+ };
+ await tracedMkdir(dirname(filePath), { recursive: true });
+ await tracedWriteFile(filePath, JSON.stringify(snapshot, null, 2), 'utf-8');
+ }
+
+ /**
+ * Incremental startup / branch-switch reconcile. Walks the content dir,
+ * short-circuits files whose mtime AND size match the stored witness, and
+ * re-parses only changed or new files (bounded 50-file read batches).
+ * Docs whose file vanished are dropped. Requires `loadFromDisk()` (or a
+ * prior `init()`) to have populated the witnesses — with none stored,
+ * every file classifies as new and the pass degrades gracefully to a full
+ * scan that also prunes stale in-memory entries.
+ */
+ reconcileWithDisk(): Promise<{ added: number; updated: number; deleted: number }> {
+ return this.enqueue(() =>
+ instrumentIndexRebuild(
+ 'tag',
+ 'reconcile',
+ () => this.reconcileOnce(),
+ (diff) => ({
+ 'index.added': diff.added,
+ 'index.updated': diff.updated,
+ 'index.deleted': diff.deleted,
+ }),
+ ),
+ );
+ }
+
+ private async reconcileOnce(): Promise<{ added: number; updated: number; deleted: number }> {
+ if (this.closed || !existsSync(this.contentDir)) return { added: 0, updated: 0, deleted: 0 };
+ const docs = await this.listDocsWithPaths();
+ const currentDocSet = new Set(docs.map((d) => d.docName));
+ const newMeta = new Map<string, FileMeta>();
+ let added = 0;
+ let updated = 0;
+
+ // Phase 1: stat all files concurrently to find changed/new ones.
+ const toProcess: Array<{ docName: string; filePath: string; meta: FileMeta; isNew: boolean }> =
+ [];
+ const statResults = await Promise.allSettled(
+ docs.map(async ({ docName, filePath }) => {
+ const fileStat = await stat(filePath);
+ return { docName, filePath, meta: { mtimeMs: fileStat.mtimeMs, size: fileStat.size } };
+ }),
+ );
+ for (const result of statResults) {
+ if (result.status === 'rejected') continue; // inaccessible; skip
+ const { docName, filePath, meta } = result.value;
+ const stored = this.fileMeta.get(docName);
+ if (stored !== undefined && stored.mtimeMs === meta.mtimeMs && stored.size === meta.size) {
+ newMeta.set(docName, meta);
+ continue;
+ }
+ toProcess.push({ docName, filePath, meta, isNew: stored === undefined });
+ }
+
+ // Phase 2: read + parse changed/new files in bounded batches.
+ const BATCH_SIZE = 50;
+ for (let i = 0; i < toProcess.length; i += BATCH_SIZE) {
+ const batch = toProcess.slice(i, i + BATCH_SIZE);
+ const settled = await Promise.allSettled(
+ batch.map(async ({ docName, meta, isNew, filePath }) => ({
+ docName,
+ meta,
+ isNew,
+ markdown: await readFile(filePath, 'utf-8'),
+ })),
+ );
+ for (const result of settled) {
+ if (result.status === 'rejected') {
+ log.warn({ err: result.reason }, 'Failed to reconcile file');
+ continue;
+ }
+ const { docName, meta, isNew, markdown } = result.value;
+ this.updateDocumentFromMarkdown(docName, markdown);
+ newMeta.set(docName, meta);
+ if (isNew) added++;
+ else updated++;
+ }
+ }
+
+ // Phase 3: drop docs whose file is gone. Union of stored witnesses and
+ // live doc keys covers docs indexed at runtime (live-derived updates)
+ // that never got a witness entry.
+ let deleted = 0;
+ const allKnownDocs = new Set([...this.fileMeta.keys(), ...this.state.byDoc.keys()]);
+ for (const docName of allKnownDocs) {
+ if (!currentDocSet.has(docName)) {
+ this.deleteDocument(docName);
+ deleted++;
+ }
+ }
+
+ this.fileMeta = newMeta;
+ return { added, updated, deleted };
+ }
+
private applyDocSnapshot(
docName: string,
authoredTags: Set<string>,