Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

refactor(server): consolidate link recognition on one grammar (#2802)

Andrew Mikofalvy4b9b52063be4
diff --git a/.changeset/link-syntax-consolidation.md b/.changeset/link-syntax-consolidation.md
new file mode 100644
index 000000000..e4ac018e5
--- /dev/null
+++ b/.changeset/link-syntax-consolidation.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Consolidate the server's inline link recognizers (backlink index, rename rewriting, link suggestions, referenced-asset scanning) onto one shared grammar, so a link that counts for backlinks can no longer be silently missed by rename rewriting or the asset sidebar. The backlink/rename grammar is the canonical one; the asset scanner picks up small correctness fixes from the alignment: links with parenthesized titles (`[doc](file.pdf (title))`) and titles containing the other quote character (`[doc](file.pdf "it's here")`) now count as asset references, malformed mismatched-quote titles no longer do, and wiki-link targets are trimmed before resolution.
diff --git a/knip.config.ts b/knip.config.ts
index 6f8bbbc63..d8a79af2f 100644
--- a/knip.config.ts
+++ b/knip.config.ts
@@ -122,7 +122,9 @@ export default {
'packages/server': {
entry: ['src/**/*.test.ts'],
project: 'src/**',
- ignoreDependencies: ['@types/shell-quote'],
+ ignoreDependencies: [
+ '@types/shell-quote',
+ ],
},
'packages/cli': {
entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts'],
diff --git a/packages/core/src/bridge/bind-frontmatter-doc.ts b/packages/core/src/bridge/bind-frontmatter-doc.ts
index c014b1413..8f9d95287 100644
--- a/packages/core/src/bridge/bind-frontmatter-doc.ts
+++ b/packages/core/src/bridge/bind-frontmatter-doc.ts
@@ -1,3 +1,4 @@
+
import type * as Y from 'yjs';
import type { Err, Ok, Result } from '../config/result.ts';
import { type FrontmatterValidationError, toFrontmatterIssue } from '../frontmatter/errors.ts';
diff --git a/packages/core/src/bridge/bridge-invariant.ts b/packages/core/src/bridge/bridge-invariant.ts
index 005627195..8bfdb40cd 100644
--- a/packages/core/src/bridge/bridge-invariant.ts
+++ b/packages/core/src/bridge/bridge-invariant.ts
@@ -1,3 +1,4 @@
+
import { fnv1aDigest } from './hash-util.ts';
export type BridgeInvariantSite = 'observer-b' | 'persistence' | 'test-harness';
diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts
index 8890f6739..c18c086d5 100644
--- a/packages/core/src/bridge/doc-boundary-space.ts
+++ b/packages/core/src/bridge/doc-boundary-space.ts
@@ -1,3 +1,4 @@
+
import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';
const LEADING_BOUNDARY_RE = /^(?:\r?\n)+/;
diff --git a/packages/core/src/bridge/growth-detect.ts b/packages/core/src/bridge/growth-detect.ts
index 9d9c244b0..463e89595 100644
--- a/packages/core/src/bridge/growth-detect.ts
+++ b/packages/core/src/bridge/growth-detect.ts
@@ -1,3 +1,4 @@
+
const DEFAULT_MIN_SUBSTANTIVE_LINE_LENGTH = 16;
export const DUPLICATION_GATE_MIN_LINE_LENGTH = 8;
diff --git a/packages/core/src/bridge/merge-three-way.ts b/packages/core/src/bridge/merge-three-way.ts
index 3ab9cc6ef..3c927eb47 100644
--- a/packages/core/src/bridge/merge-three-way.ts
+++ b/packages/core/src/bridge/merge-three-way.ts
@@ -38,6 +38,7 @@ function mergeThreeWayImpl(baseline: string, userText: string, agentText: string
return parts.join('\n');
}
+
export type BridgeMergeContentLossSide = 'user' | 'agent';
export type BridgeMergeContentLossWhich = 'substring' | 'order' | 'growth';
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index 4b8890476..3f0880aa5 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -1,3 +1,4 @@
+
const COMMONMARK_ESCAPE_RE = /\\([!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~])/g;
const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/;
diff --git a/packages/core/src/bridge/pm-structural-equivalence.ts b/packages/core/src/bridge/pm-structural-equivalence.ts
index 754dc5cc2..0be5203e4 100644
--- a/packages/core/src/bridge/pm-structural-equivalence.ts
+++ b/packages/core/src/bridge/pm-structural-equivalence.ts
@@ -32,6 +32,7 @@ const BR_SENTINEL: PmStructuralNode = { type: '__br__' };
const BR_LITERAL_RE = /^<br\s*\/?>$/i;
+
interface DegradeEntry {
readonly label: string;
readonly severity: ToleranceClassSeverity;
@@ -127,6 +128,7 @@ export interface ComparePmStructuralOptions {
ignoreAttrs?: (attrKey: string) => boolean;
}
+
/** Rebuild a node with `fn` applied to each child; identity when childless.
* Shared by the degrade canonicalizers so none re-implements the walk. */
function mapChildren(
@@ -248,6 +250,7 @@ function applyDegrades(tree: PmStructuralNode): {
return { normalized: current, fired };
}
+
export function comparePmStructural(
expected: PmStructuralNode,
actual: PmStructuralNode,
diff --git a/packages/core/src/bridge/subsequence.ts b/packages/core/src/bridge/subsequence.ts
index af20de1e2..dca644141 100644
--- a/packages/core/src/bridge/subsequence.ts
+++ b/packages/core/src/bridge/subsequence.ts
@@ -1,3 +1,4 @@
+
/** True when `needle` is a subsequence of `haystack` (two-pointer, O(n+m)):
* insertions in `haystack` are free; any dropped or substituted `needle` byte
* fails. */
diff --git a/packages/core/src/markdown/callout-transformer.ts b/packages/core/src/markdown/callout-transformer.ts
index 83d47404a..a363dfd54 100644
--- a/packages/core/src/markdown/callout-transformer.ts
+++ b/packages/core/src/markdown/callout-transformer.ts
@@ -1,3 +1,4 @@
+
import type { Blockquote, Paragraph, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts
index 193e286b0..47f0d07e4 100644
--- a/packages/core/src/markdown/comment-promoter.ts
+++ b/packages/core/src/markdown/comment-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/dedent-block-jsx-close.ts b/packages/core/src/markdown/dedent-block-jsx-close.ts
index ece5c02a1..b6299a948 100644
--- a/packages/core/src/markdown/dedent-block-jsx-close.ts
+++ b/packages/core/src/markdown/dedent-block-jsx-close.ts
@@ -1,3 +1,4 @@
+
import { findFencedRegions, isInsideFence } from './fence-regions.ts';
const INDENTED_BLOCK_JSX_CLOSE_RE = /^([ ]{1,3})(<\/[A-Z][A-Za-z0-9_]*\s*>)([ \t]*)$/gm;
diff --git a/packages/core/src/markdown/details-accordion-promoter.ts b/packages/core/src/markdown/details-accordion-promoter.ts
index 54774764a..acd398aa6 100644
--- a/packages/core/src/markdown/details-accordion-promoter.ts
+++ b/packages/core/src/markdown/details-accordion-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.ts
index c7518d614..fb5ac8e8a 100644
--- a/packages/core/src/markdown/fence-regions.ts
+++ b/packages/core/src/markdown/fence-regions.ts
@@ -1,3 +1,4 @@
+
const FENCE_RE = /^(`{3,}|~{3,})/gm;
export function findFencedRegions(src: string): Array<[number, number]> {
diff --git a/packages/core/src/markdown/fixtures/index.ts b/packages/core/src/markdown/fixtures/index.ts
index 00f5771ee..6931623c7 100644
--- a/packages/core/src/markdown/fixtures/index.ts
+++ b/packages/core/src/markdown/fixtures/index.ts
@@ -1,3 +1,4 @@
+
import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -8,6 +9,7 @@ function fixturePath(...segments: string[]): string {
return resolve(FIXTURES_DIR, ...segments);
}
+
interface GfmExample {
section: string;
markdown: string;
@@ -17,6 +19,7 @@ export function loadGfmExamples(): GfmExample[] {
return JSON.parse(readFileSync(fixturePath('gfm', 'examples.json'), 'utf8')) as GfmExample[];
}
+
interface MdxCrashEntry {
id: string;
input: string;
@@ -68,6 +71,7 @@ export function loadLargeEmbedFixtures(): LargeEmbedFixture[] {
) as LargeEmbedFixture[];
}
+
export function loadPrd6955Before(): string {
return readFileSync(fixturePath('regression', 'prd-6955-before.md'), 'utf8');
}
@@ -76,6 +80,7 @@ export function loadPrd6955CorruptedTriplicated(): string {
return readFileSync(fixturePath('regression', 'prd-6955-corrupted-triplicated.md'), 'utf8');
}
+
export interface NgPinnedCase {
id: string;
name: string;
@@ -92,6 +97,7 @@ export function loadNgPinnedCases(): NgPinnedCase[] {
) as NgPinnedCase[];
}
+
export function loadLargeRealistic(): string {
return readFileSync(fixturePath('perf', 'large-realistic.md'), 'utf8');
}
diff --git a/packages/core/src/markdown/fixtures/perf/generate.ts b/packages/core/src/markdown/fixtures/perf/generate.ts
index 8b1eb8519..daa3ef82b 100644
--- a/packages/core/src/markdown/fixtures/perf/generate.ts
+++ b/packages/core/src/markdown/fixtures/perf/generate.ts
@@ -1,3 +1,4 @@
+
import { writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -10,6 +11,7 @@ const BASELINE_ONLY_COUNTS = [500, 2500] as const;
const SEED = 0xf1de1117;
+
function mulberry32(seed: number): () => number {
let a = seed >>> 0;
return () => {
@@ -36,6 +38,7 @@ function pickWeighted<T>(rand: () => number, items: readonly [T, number][]): T {
return items[items.length - 1][0];
}
+
const WORDS = [
'lorem',
'ipsum',
@@ -153,6 +156,7 @@ function mdxBlock(rand: () => number): string {
return `<${name}>\n\n${sentence(rand)}\n\n</${name}>`;
}
+
type BlockKind = 'paragraph' | 'heading' | 'list' | 'code' | 'table' | 'mdx';
const MIX: readonly [BlockKind, number][] = [
@@ -191,6 +195,7 @@ function generateDocument(blockCount: number, seed: number): string {
return `${blocks.join('\n\n')}\n`;
}
+
function main(): void {
for (const count of [...BLOCK_COUNTS, ...BASELINE_ONLY_COUNTS]) {
const doc = generateDocument(count, SEED);
diff --git a/packages/core/src/markdown/guard-flanking-matrix.ts b/packages/core/src/markdown/guard-flanking-matrix.ts
index 420dc0fe5..32c055c5b 100644
--- a/packages/core/src/markdown/guard-flanking-matrix.ts
+++ b/packages/core/src/markdown/guard-flanking-matrix.ts
@@ -7,6 +7,7 @@ import {
import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts';
import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts';
+
export const ATTENTION_DELIMITERS = ['*', '_', '**', '~~', '=='] as const;
export type FlankClass = 'whitespace' | 'punctuation' | 'other';
diff --git a/packages/core/src/markdown/handler-shadow-audit.ts b/packages/core/src/markdown/handler-shadow-audit.ts
index 2653473da..283913255 100644
--- a/packages/core/src/markdown/handler-shadow-audit.ts
+++ b/packages/core/src/markdown/handler-shadow-audit.ts
@@ -1,3 +1,4 @@
+
export interface HandlerShadowWitness {
input: string;
expect: 'byte' | 'byte-and-reparse-type';
diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.ts
index f55912407..6319f264f 100644
--- a/packages/core/src/markdown/highlight-promoter.ts
+++ b/packages/core/src/markdown/highlight-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Parent, PhrasingContent, Root, Text } from 'mdast';
import type { MdxJsxTextElement } from 'mdast-util-mdx';
import { SKIP, visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.ts
index b1f252c31..8dc3a81b5 100644
--- a/packages/core/src/markdown/html-to-mdast.ts
+++ b/packages/core/src/markdown/html-to-mdast.ts
@@ -1,3 +1,4 @@
+
import type { Root as HastRoot } from 'hast';
import type { List, Root as MdastRoot } from 'mdast';
import rehypeParse from 'rehype-parse';
@@ -85,7 +86,8 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRo
throw new HtmlPayloadTooLargeError(html.length, maxBytes);
}
- const processor = unified().use(rehypeParse, { fragment: true });
+ const processor = unified()
+ .use(rehypeParse, { fragment: true });
for (const plugin of cleanupPlugins) {
processor.use(plugin);
diff --git a/packages/core/src/markdown/image-promoter.ts b/packages/core/src/markdown/image-promoter.ts
index 706fa9afb..994eb6917 100644
--- a/packages/core/src/markdown/image-promoter.ts
+++ b/packages/core/src/markdown/image-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Image, Paragraph, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts
index 8b36807fd..fc30bda02 100644
--- a/packages/core/src/markdown/index.ts
+++ b/packages/core/src/markdown/index.ts
@@ -1,3 +1,4 @@
+
import {
type FromProseMirrorOptions,
fromPmMark,
@@ -196,6 +197,7 @@ export class MarkdownManager {
}
}
+
const registry = createRegistry();
function destructureAttrs(
@@ -244,6 +246,7 @@ function destructureAttrs(
return result;
}
+
function hasDirtyDescendant(node: PmNode): boolean {
let found = false;
node.descendants((child) => {
@@ -263,6 +266,7 @@ function effectiveDirty(node: PmNode, freshnessChecker?: StructuralFreshnessChec
return freshnessChecker ? freshnessChecker.isDiverged(node.toJSON()) : false;
}
+
function isEmptyMdastParagraph(node: MdastNodes): boolean {
if (node.type !== 'paragraph') return false;
const children = node.children ?? [];
@@ -331,6 +335,7 @@ function extractTextFromMdastNodes(nodes: MdastNodes[]): string {
return out;
}
+
import {
AUDIO_EXTENSIONS,
FILE_ATTACHMENT_EXTENSIONS,
@@ -577,6 +582,7 @@ function buildMdastToPmHandlers(
}));
}
+
if (m.emphasis) {
handlers.emphasis = toPmMark(m.emphasis, (node: Emphasis) => ({
sourceDelimiter: node.data?.sourceDelimiter ?? '*',
@@ -657,6 +663,7 @@ function buildMdastToPmHandlers(
}));
}
+
if (m.link) {
const sourceLiteralMark = m.sourceLiteral;
handlers.link = (node: Link, _parent: MdastParent, state: MdastToPmState) => {
@@ -935,6 +942,7 @@ function buildMdastToPmHandlers(
};
}
+
const blockUnknownHandler = (node: {
type: string;
position?: { start: { offset: number }; end: { offset: number } };
@@ -1060,6 +1068,7 @@ function buildMdastToPmHandlers(
return handlers as RemarkProseMirrorOptions['handlers'];
}
+
function buildPmToMdastHandlers(
schema: Schema,
freshness: FreshnessCheckerHolder,
diff --git a/packages/core/src/markdown/lint/config-files.ts b/packages/core/src/markdown/lint/config-files.ts
index 312c9c74b..94b34dee1 100644
--- a/packages/core/src/markdown/lint/config-files.ts
+++ b/packages/core/src/markdown/lint/config-files.ts
@@ -1,3 +1,4 @@
+
const MARKDOWNLINT_JSON_CONFIG_FILES: ReadonlySet<string> = new Set([
'.markdownlint.json',
'.markdownlint.jsonc',
diff --git a/packages/core/src/markdown/lint/config-schemas.ts b/packages/core/src/markdown/lint/config-schemas.ts
index e78a1381d..207cd95c2 100644
--- a/packages/core/src/markdown/lint/config-schemas.ts
+++ b/packages/core/src/markdown/lint/config-schemas.ts
@@ -29,6 +29,7 @@ const fullPluginShape = Object.fromEntries(
LINT_PLUGINS.map((plugin) => [plugin.id, plugin.sliceSchema]),
) as z.ZodRawShape;
+
export const LinterConfigSchema = z.object({
enabled: z.boolean(),
plugins: z.object(fullPluginShape),
diff --git a/packages/core/src/markdown/lint/default-config.ts b/packages/core/src/markdown/lint/default-config.ts
index d48eea5e4..a51f2ffb0 100644
--- a/packages/core/src/markdown/lint/default-config.ts
+++ b/packages/core/src/markdown/lint/default-config.ts
@@ -6,6 +6,7 @@ export const DEFAULT_MARKDOWNLINT_CONFIG: Record<string, MarkdownlintRuleSetting
MD013: false,
};
+
export function resolveMarkdownlintConfig(
rules: Record<string, MarkdownlintRuleSetting> | undefined,
): Configuration {
diff --git a/packages/core/src/markdown/lint/index.ts b/packages/core/src/markdown/lint/index.ts
index 8a154ce30..d763d8d28 100644
--- a/packages/core/src/markdown/lint/index.ts
+++ b/packages/core/src/markdown/lint/index.ts
@@ -1,3 +1,4 @@
+
import { LINT_PLUGINS, type LinterConfig } from './plugins.ts';
import type { LintDiagnostic } from './types.ts';
diff --git a/packages/core/src/markdown/lint/plugins.ts b/packages/core/src/markdown/lint/plugins.ts
index 1cd40ef3a..65a1b564c 100644
--- a/packages/core/src/markdown/lint/plugins.ts
+++ b/packages/core/src/markdown/lint/plugins.ts
@@ -1,3 +1,4 @@
+
import { z } from 'zod';
import { DEFAULT_MARKDOWNLINT_CONFIG, resolveMarkdownlintConfig } from './default-config.ts';
import { fixMarkdownText, runMarkdownlint } from './markdownlint-runner.ts';
@@ -8,6 +9,7 @@ import {
type MarkdownlintSlice,
} from './types.ts';
+
const MarkdownlintRuleSettingSchema = z.union([
z.boolean(),
z.enum(MARKDOWNLINT_RULE_SEVERITIES),
diff --git a/packages/core/src/markdown/lint/types.ts b/packages/core/src/markdown/lint/types.ts
index 617f6e5b9..3651bd7eb 100644
--- a/packages/core/src/markdown/lint/types.ts
+++ b/packages/core/src/markdown/lint/types.ts
@@ -1,3 +1,4 @@
+
export const LINT_PLUGIN_IDS = ['markdownlint'] as const;
export type LintPluginId = (typeof LINT_PLUGIN_IDS)[number];
@@ -41,6 +42,7 @@ export interface MarkdownlintSlice {
rules: Record<string, MarkdownlintRuleSetting>;
}
+
interface RuleOptionSpecBase {
key: string;
description: string;
diff --git a/packages/core/src/markdown/math-promoter.ts b/packages/core/src/markdown/math-promoter.ts
index a5b511f25..bd79424c7 100644
--- a/packages/core/src/markdown/math-promoter.ts
+++ b/packages/core/src/markdown/math-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Code, Root } from 'mdast';
import type { Math as MdastMath } from 'mdast-util-math';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.ts
index 679c72ef7..f84967798 100644
--- a/packages/core/src/markdown/mdast-to-hast-handlers.ts
+++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts
@@ -1,3 +1,4 @@
+
import type { Comment, Element, ElementContent, Properties } from 'hast';
import type { FootnoteDefinition, FootnoteReference } from 'mdast';
import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';
diff --git a/packages/core/src/markdown/mdast-to-html.ts b/packages/core/src/markdown/mdast-to-html.ts
index 8ce2b2492..df788a80d 100644
--- a/packages/core/src/markdown/mdast-to-html.ts
+++ b/packages/core/src/markdown/mdast-to-html.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root as HastRoot, Text as HastText } from 'hast';
import type { Root as MdastRoot, Text as MdastText } from 'mdast';
import type { Handler } from 'mdast-util-to-hast';
diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts
index bfd7e3109..fa3cfcd86 100644
--- a/packages/core/src/markdown/merged-walker.ts
+++ b/packages/core/src/markdown/merged-walker.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Parent, Root } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/mermaid-promoter.ts b/packages/core/src/markdown/mermaid-promoter.ts
index 1a897fca8..17ed2b829 100644
--- a/packages/core/src/markdown/mermaid-promoter.ts
+++ b/packages/core/src/markdown/mermaid-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Code, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.ts
index 9864b3b60..7e4a9a48b 100644
--- a/packages/core/src/markdown/parse-with-fallback.ts
+++ b/packages/core/src/markdown/parse-with-fallback.ts
@@ -168,6 +168,7 @@ export function parseRecursive(
}
}
+
interface VFilePlace {
offset?: number;
start?: { offset?: number };
@@ -186,6 +187,7 @@ function extractErrorOffset(err: unknown): number | undefined {
return undefined;
}
+
interface Region {
start: number;
end: number;
@@ -209,6 +211,7 @@ function nearestBlankLineAfter(src: string, offset: number): number | null {
return null;
}
+
export interface TagEvent {
kind: 'open' | 'close' | 'self-close';
name: string;
@@ -355,6 +358,7 @@ function findFallbackRegion(src: string, errorOffset: number): Region {
return { start: blockStart, end: blockEnd };
}
+
interface SourceBlock {
src: string;
start: number;
@@ -452,6 +456,7 @@ function tryPerBlockFallback(
};
}
+
function wholeDocRawText(source: string): JSONContent {
return {
type: 'doc',
diff --git a/packages/core/src/markdown/parser-drop-closure.ts b/packages/core/src/markdown/parser-drop-closure.ts
index e6ab4fd57..43d420187 100644
--- a/packages/core/src/markdown/parser-drop-closure.ts
+++ b/packages/core/src/markdown/parser-drop-closure.ts
@@ -1,3 +1,4 @@
+
export type DroppedTokenAdjudication =
| {
kind: 'format-dof-axis';
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index 7f2d5adca..e9e507469 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -1,3 +1,4 @@
+
import {
type FromProseMirrorOptions,
fromProseMirror,
diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.ts
index 5a5e4c42a..f8f5e8530 100644
--- a/packages/core/src/markdown/position-aware-join.ts
+++ b/packages/core/src/markdown/position-aware-join.ts
@@ -1,3 +1,4 @@
+
import type { Join } from 'mdast-util-to-markdown';
import type { Position } from 'unist';
diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts
index 0f399d34c..e5b4c9eba 100644
--- a/packages/core/src/markdown/position-slice.ts
+++ b/packages/core/src/markdown/position-slice.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Root } from 'mdast';
import { visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
@@ -476,7 +477,8 @@ export function applyPositionSliceToNode(
const first = source[startOff];
if (first !== '[' && first !== '<' && !node.title) {
node.data.sourceStyle = 'gfm-autolink';
- } else if (first === '[') {
+ }
+ else if (first === '[') {
const slice = source.slice(startOff, endOff);
const closeBracketIdx = slice.lastIndexOf('](');
if (closeBracketIdx !== -1) {
diff --git a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
index db10fa54b..ca225ce9b 100644
--- a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
+++ b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
@@ -1,3 +1,4 @@
+
import type { Comment, Element, Root, Text } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
index 34068d103..3a2524a9a 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
@@ -1,3 +1,4 @@
+
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
index 93151d43b..2279a0189 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
index 188f02d83..a0f467c42 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
index 4ba2c3418..4e8542a21 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
index 1a74b1272..a3c96b24a 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
@@ -1,3 +1,4 @@
+
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
index de35e7b79..e95cf43d6 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
@@ -1,3 +1,4 @@
+
import type { Comment, Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
index b4be1b2ea..ffdb4d5b5 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
@@ -1,3 +1,4 @@
+
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
index 3c8d88b7c..e2a96d1a7 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/resolve-image-url.ts b/packages/core/src/markdown/resolve-image-url.ts
index cf2c9da8e..0389575ac 100644
--- a/packages/core/src/markdown/resolve-image-url.ts
+++ b/packages/core/src/markdown/resolve-image-url.ts
@@ -1,3 +1,4 @@
+
import { isRelativeUrl } from './safe-url.ts';
function isDevDiagnosticContext(): boolean {
diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts
index 4d3250d3a..8be632ba4 100644
--- a/packages/core/src/markdown/safe-url.ts
+++ b/packages/core/src/markdown/safe-url.ts
@@ -1,3 +1,4 @@
+
export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;
const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');
diff --git a/packages/core/src/markdown/serialize-helpers.ts b/packages/core/src/markdown/serialize-helpers.ts
index 46cc6a074..542204403 100644
--- a/packages/core/src/markdown/serialize-helpers.ts
+++ b/packages/core/src/markdown/serialize-helpers.ts
@@ -1,3 +1,4 @@
+
import type { Node as PmNode } from '@tiptap/pm/model';
import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import type { PropDef, SerializeContext } from '../registry/types.ts';
diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.ts
index be97d4d62..fe73659e6 100644
--- a/packages/core/src/markdown/single-dollar-math-promoter.ts
+++ b/packages/core/src/markdown/single-dollar-math-promoter.ts
@@ -1,3 +1,4 @@
+
import type { PhrasingContent, Root, Text } from 'mdast';
import type { InlineMath } from 'mdast-util-math';
import { SKIP, visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts
index 8f4e9a41c..7bce04302 100644
--- a/packages/core/src/markdown/to-markdown-handlers.ts
+++ b/packages/core/src/markdown/to-markdown-handlers.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Parents } from 'mdast';
import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import type { Handle, Info, State } from 'mdast-util-to-markdown';
diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.ts
index cd245c38c..c5bfd2386 100644
--- a/packages/core/src/markdown/unknown-mdast-guard.ts
+++ b/packages/core/src/markdown/unknown-mdast-guard.ts
@@ -1,3 +1,4 @@
+
import type { Root as MdastRoot } from 'mdast';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.ts
index d578476a7..ac9f5239b 100644
--- a/packages/core/src/markdown/whitespace-char-ref.ts
+++ b/packages/core/src/markdown/whitespace-char-ref.ts
@@ -1,3 +1,4 @@
+
const INLINE_WHITESPACE_BY_CODE: ReadonlyMap<number, string> = new Map([
[0x20, ' '],
[0x09, '\t'],
diff --git a/packages/core/src/markdown/wiki-link-micromark.ts b/packages/core/src/markdown/wiki-link-micromark.ts
index 6dad26200..afd9936fe 100644
--- a/packages/core/src/markdown/wiki-link-micromark.ts
+++ b/packages/core/src/markdown/wiki-link-micromark.ts
@@ -17,6 +17,7 @@ declare module 'micromark-util-types' {
}
}
+
const CODE_BANG = 33; // !
const CODE_LBRACKET = 91; // [
const CODE_RBRACKET = 93; // ]
@@ -260,6 +261,7 @@ export function wikiLinkSyntax(): Extension {
};
}
+
function enterWikiLink(this: CompileContext, token: Token) {
this.enter(
{
@@ -385,6 +387,7 @@ export const wikiLinkToMarkdown: {
unsafe: [{ character: '[', inConstruct: ['phrasing'] }],
};
+
const MICROMARK_EXT = wikiLinkSyntax();
export function remarkWikiLink(this: Processor) {
diff --git a/packages/server/src/asset-references.ts b/packages/server/src/asset-references.ts
index f769448d1..82152a427 100644
--- a/packages/server/src/asset-references.ts
+++ b/packages/server/src/asset-references.ts
@@ -8,6 +8,7 @@ import {
resolveAssetProjectPath,
} from '@inkeep/open-knowledge-core';
import type { FileIndexEntry } from './file-watcher.ts';
+import { matchMarkdownLinks, matchWikiLinks } from './link-syntax.ts';
import { getLogger } from './logger.ts';
import { isWithinContentDir } from './persistence.ts';
@@ -23,9 +24,9 @@ interface ReferencedAssetEntry {
referencedBy: string[];
}
-const MARKDOWN_LINK_OR_IMAGE_RE =
- /!?\[[^\]\n]*(?:\][^[\]\n]*)?\]\((?:<([^>\n]+)>|([^)\s]+))(?:\s+['"][^'"]*['"])?\)/g;
-const WIKI_LINK_OR_EMBED_RE = /!?\[\[([^[\]|#]+?)(?:#[^\]|]+?)?(?:\|[^\]]+?)?\]\]/g;
+// HTML `href`/`src` attributes are a different syntax family from the
+// markdown/wiki link grammar in link-syntax.ts, so this recognizer stays
+// local.
const HTML_LINK_ATTR_RE =
/<[\w:-]+\b[^>]*?\s+(?:href|src)\s*=\s*(?:"([^"\n]*)"|'([^'\n]*)'|“([^”\n]*)”|([^\s"'=<>`]+))/gi;
@@ -85,13 +86,14 @@ function errnoCode(err: unknown): string | null {
}
function collectHrefsFromLine(line: string, hrefs: Set<string>): void {
- for (const match of line.matchAll(MARKDOWN_LINK_OR_IMAGE_RE)) {
- const href = match[1] ?? match[2];
- if (href) hrefs.add(href);
+ // nestedBracketLabels keeps the pre-consolidation preference for the
+ // OUTER destination of badge-style `[![alt](img)](file.pdf)` links —
+ // the linked file counts as the referenced asset, not the inner image.
+ for (const match of matchMarkdownLinks(line, { nestedBracketLabels: true })) {
+ if (match.href) hrefs.add(match.href);
}
- for (const match of line.matchAll(WIKI_LINK_OR_EMBED_RE)) {
- const target = match[1];
- if (target) hrefs.add(target);
+ for (const match of matchWikiLinks(line)) {
+ hrefs.add(match.target);
}
for (const match of line.matchAll(HTML_LINK_ATTR_RE)) {
const href = match[1] ?? match[2] ?? match[3] ?? match[4];
diff --git a/packages/server/src/backlink-index.ts b/packages/server/src/backlink-index.ts
index 68c887a58..3709f857a 100644
--- a/packages/server/src/backlink-index.ts
+++ b/packages/server/src/backlink-index.ts
@@ -23,21 +23,12 @@ import { isLinkIndexExcludedDoc } from './cc1-broadcast.ts';
import { getLocalDir } from './config/paths.ts';
import type { ContentFilter } from './content-filter.ts';
import { isSupportedDocFile, stripDocExtension } from './doc-extensions.ts';
+import { readMarkdownLinkAt, readWikiLinkAt } from './link-syntax.ts';
import { getLogger } from './logger.ts';
import { toPosix } from './path-utils.ts';
const log = getLogger('backlinks');
-// Line-oriented variant: excludes \n since lines are pre-split.
-// cf. packages/core/src/extensions/wiki-link.ts WIKI_LINK_PATTERN (no \n exclusion).
-// Sticky flag ('y') enables position-based matching via lastIndex.
-const WIKI_LINK_RE = /\[\[([^\n#[\]|]+)(?:#([^\n[\]|]+))?(?:\|([^\n[\]]+))?\]\]/y;
-
-// Inline link form: [text](href) with an optional CommonMark title.
-// Sticky flag for position-based matching. Does NOT match reference-style [text][ref].
-const MD_LINK_RE =
- /\[([^\]\n]*)\]\((<[^>\n]+>|[^)\s\n]+)(?:\s+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?\)/y;
-
interface InlineWikiLinkOccurrence {
target: string;
anchor: string | null;
@@ -298,23 +289,15 @@ function readWikiLink(
line: string,
start: number,
): { target: string; alias: string | null; anchor: string | null; nextIndex: number } | null {
- // Uses sticky flag for position-based matching via lastIndex.
// core's parseWikiLink expects the string to start with '[[' (^ anchor) and
// cannot be used here where start may be mid-line.
- WIKI_LINK_RE.lastIndex = start;
- const match = WIKI_LINK_RE.exec(line);
+ const match = readWikiLinkAt(line, start);
if (!match) return null;
-
- const target = match[1]?.trim();
- const anchor = match[2]?.trim() || null;
- const alias = match[3]?.trim() || null;
- if (!target) return null;
-
return {
- target,
- alias,
- anchor,
- nextIndex: start + match[0].length,
+ target: match.target,
+ alias: match.alias,
+ anchor: match.anchor,
+ nextIndex: match.end,
};
}
@@ -439,21 +422,16 @@ export function resolveMarkdownHref(href: string, sourceDocName: string): string
return resolveInternalHref(href, sourceDocName)?.docName ?? null;
}
-function normalizeMarkdownHref(rawHref: string): string {
- return rawHref.startsWith('<') && rawHref.endsWith('>') ? rawHref.slice(1, -1) : rawHref;
-}
-
function readMarkdownLink(
line: string,
start: number,
): { text: string; href: string; nextIndex: number } | null {
- MD_LINK_RE.lastIndex = start;
- const match = MD_LINK_RE.exec(line);
+ const match = readMarkdownLinkAt(line, start);
if (!match) return null;
return {
- text: match[1] ?? '',
- href: normalizeMarkdownHref(match[2] ?? ''),
- nextIndex: start + match[0].length,
+ text: match.label,
+ href: match.href,
+ nextIndex: match.end,
};
}
diff --git a/packages/server/src/link-syntax.test.ts b/packages/server/src/link-syntax.test.ts
new file mode 100644
index 000000000..06093d8d9
--- /dev/null
+++ b/packages/server/src/link-syntax.test.ts
@@ -0,0 +1,293 @@
+import { describe, expect, test } from 'vitest';
+import { extractLocalAssetHrefs } from './asset-references.ts';
+import {
+ extractMarkdownLinksFromMarkdown,
+ extractWikiLinksFromMarkdown,
+} from './backlink-index.ts';
+import {
+ matchMarkdownLinks,
+ matchWikiLinks,
+ readMarkdownLinkAt,
+ readWikiLinkAt,
+} from './link-syntax.ts';
+import { rewriteWikiLinksForDocumentRename } from './managed-rename-rewrite.ts';
+
+describe('readWikiLinkAt', () => {
+ test('plain target', () => {
+ expect(readWikiLinkAt('[[Wiki Page]]', 0)).toMatchObject({
+ embed: false,
+ target: 'Wiki Page',
+ anchor: null,
+ alias: null,
+ start: 0,
+ end: 13,
+ });
+ });
+
+ test('target with anchor and alias', () => {
+ expect(readWikiLinkAt('[[wiki#anchor|Alias]]', 0)).toMatchObject({
+ target: 'wiki',
+ anchor: 'anchor',
+ alias: 'Alias',
+ });
+ });
+
+ test('anchor may contain further # chars; alias may contain further | chars', () => {
+ expect(readWikiLinkAt('[[a#b#c]]', 0)).toMatchObject({ target: 'a', anchor: 'b#c' });
+ expect(readWikiLinkAt('[[a|b|c]]', 0)).toMatchObject({ target: 'a', alias: 'b|c' });
+ });
+
+ test('embed form at the ! position', () => {
+ expect(readWikiLinkAt('![[embed.png]]', 0)).toMatchObject({
+ embed: true,
+ target: 'embed.png',
+ end: 14,
+ });
+ });
+
+ test('called at the [[ of an embed, the ! is outside the match', () => {
+ expect(readWikiLinkAt('![[embed.png]]', 1)).toMatchObject({
+ embed: false,
+ target: 'embed.png',
+ start: 1,
+ end: 14,
+ });
+ });
+
+ test('target is trimmed; raw capture is preserved', () => {
+ expect(readWikiLinkAt('[[ spaced target ]]', 0)).toMatchObject({
+ target: 'spaced target',
+ targetRaw: ' spaced target ',
+ });
+ });
+
+ test('whitespace-only target rejects the match', () => {
+ expect(readWikiLinkAt('[[ ]]', 0)).toBeNull();
+ });
+
+ test('empty target with only anchor or alias rejects', () => {
+ expect(readWikiLinkAt('[[#only-anchor]]', 0)).toBeNull();
+ expect(readWikiLinkAt('[[|only-alias]]', 0)).toBeNull();
+ });
+
+ test('unterminated form does not match', () => {
+ expect(readWikiLinkAt('[[unterminated', 0)).toBeNull();
+ });
+
+ test('nested opener: outer fails, inner matches at its own position', () => {
+ expect(readWikiLinkAt('[[outer [[inner]] ]]', 0)).toBeNull();
+ expect(readWikiLinkAt('[[outer [[inner]] ]]', 8)).toMatchObject({ target: 'inner' });
+ });
+
+ test('unicode targets pass through', () => {
+ expect(readWikiLinkAt('[[héllo wörld]]', 0)).toMatchObject({ target: 'héllo wörld' });
+ expect(readWikiLinkAt('[[emoji 🎉 target]]', 0)).toMatchObject({ target: 'emoji 🎉 target' });
+ });
+
+ test('only matches at exactly the given position', () => {
+ expect(readWikiLinkAt('x [[a]]', 0)).toBeNull();
+ expect(readWikiLinkAt('x [[a]]', 2)).toMatchObject({ target: 'a' });
+ });
+});
+
+describe('readMarkdownLinkAt', () => {
+ test('plain link', () => {
+ expect(readMarkdownLinkAt('[doc](./target.md)', 0)).toMatchObject({
+ image: false,
+ label: 'doc',
+ href: './target.md',
+ hrefRaw: './target.md',
+ titleSuffix: '',
+ end: 18,
+ });
+ });
+
+ test('image form at the ! position', () => {
+ expect(readMarkdownLinkAt('![img](./photo.png)', 0)).toMatchObject({
+ image: true,
+ label: 'img',
+ href: './photo.png',
+ });
+ });
+
+ test('angle destination admits spaces and is unwrapped in href', () => {
+ expect(readMarkdownLinkAt('[doc](<./my file.md>)', 0)).toMatchObject({
+ href: './my file.md',
+ hrefRaw: '<./my file.md>',
+ });
+ });
+
+ test('empty angle destination falls to the bare-form alternative', () => {
+ // `<...>` requires one char, so `<>` matches as a bare destination and
+ // unwraps to ''.
+ expect(readMarkdownLinkAt('[doc](<>)', 0)).toMatchObject({ hrefRaw: '<>', href: '' });
+ });
+
+ test('all three CommonMark title forms, captured as authored suffix', () => {
+ expect(readMarkdownLinkAt('[doc](./t.md "title")', 0)).toMatchObject({
+ href: './t.md',
+ titleSuffix: ' "title"',
+ });
+ expect(readMarkdownLinkAt("[doc](./t.md 'title')", 0)).toMatchObject({
+ titleSuffix: " 'title'",
+ });
+ expect(readMarkdownLinkAt('[doc](./t.md (title))', 0)).toMatchObject({
+ titleSuffix: ' (title)',
+ });
+ expect(readMarkdownLinkAt('[doc](./t.md "it\'s here")', 0)).toMatchObject({
+ titleSuffix: ' "it\'s here"',
+ });
+ expect(readMarkdownLinkAt('[doc](./t.md "ti)tle")', 0)).toMatchObject({
+ href: './t.md',
+ titleSuffix: ' "ti)tle"',
+ });
+ });
+
+ test('mismatched title quotes reject the match', () => {
+ expect(readMarkdownLinkAt('[a](./x.md "t\')', 0)).toBeNull();
+ });
+
+ test('bare destination stops at whitespace or )', () => {
+ expect(readMarkdownLinkAt('[a](two words)', 0)).toBeNull();
+ expect(readMarkdownLinkAt('[a](./x.md extra)', 0)).toBeNull();
+ expect(readMarkdownLinkAt('[empty]()', 0)).toBeNull();
+ });
+
+ test('unterminated form does not match', () => {
+ expect(readMarkdownLinkAt('[unterminated](./x.md', 0)).toBeNull();
+ });
+
+ test('empty label is allowed', () => {
+ expect(readMarkdownLinkAt('[](./empty-label.md)', 0)).toMatchObject({
+ label: '',
+ href: './empty-label.md',
+ });
+ });
+
+ test('strict label stops at the first ]', () => {
+ expect(readMarkdownLinkAt('[a [b] c](./x.md)', 0)).toBeNull();
+ expect(readMarkdownLinkAt('[a]b](./x.md)', 0)).toBeNull();
+ });
+
+ test('badge nesting at position 0 matches the inner image destination', () => {
+ // The label runs `![alt` up to the inner `]`, so the match carries the
+ // INNER destination. matchMarkdownLinks({ nestedBracketLabels }) is the
+ // outer-destination variant.
+ expect(readMarkdownLinkAt('[![alt](./inner.png)](./outer.pdf)', 0)).toMatchObject({
+ image: false,
+ label: '![alt',
+ href: './inner.png',
+ });
+ });
+});
+
+describe('matchWikiLinks', () => {
+ test('finds plain links and embeds across a line', () => {
+ expect(matchWikiLinks('a [[x]] b ![[y.png]] c').map((m) => [m.target, m.embed])).toEqual([
+ ['x', false],
+ ['y.png', true],
+ ]);
+ });
+
+ test('drops whitespace-only targets but keeps scanning past them', () => {
+ expect(matchWikiLinks('[[ ]] [[real]]').map((m) => m.target)).toEqual(['real']);
+ });
+
+ test('skips a doubled opener and matches the inner form', () => {
+ expect(matchWikiLinks('[[outer [[inner]] ]]').map((m) => m.target)).toEqual(['inner']);
+ });
+});
+
+describe('matchMarkdownLinks', () => {
+ test('finds links and images across a line', () => {
+ expect(
+ matchMarkdownLinks('[a](./x.md) and ![b](./y.png)').map((m) => [m.href, m.image]),
+ ).toEqual([
+ ['./x.md', false],
+ ['./y.png', true],
+ ]);
+ });
+
+ test('strict scan of badge nesting yields the inner destination', () => {
+ expect(matchMarkdownLinks('[![alt](./inner.png)](./outer.pdf)').map((m) => m.href)).toEqual([
+ './inner.png',
+ ]);
+ });
+
+ test('nestedBracketLabels scan of badge nesting yields the outer destination', () => {
+ expect(
+ matchMarkdownLinks('[![alt](./inner.png)](./outer.pdf)', { nestedBracketLabels: true }).map(
+ (m) => m.href,
+ ),
+ ).toEqual(['./outer.pdf']);
+ });
+
+ test('nestedBracketLabels admits one bracketed run inside the label', () => {
+ expect(
+ matchMarkdownLinks('[a [b] c](./x.md)', { nestedBracketLabels: true }).map((m) => m.href),
+ ).toEqual(['./x.md']);
+ expect(matchMarkdownLinks('[a [b] c](./x.md)').map((m) => m.href)).toEqual([]);
+ });
+});
+
+/**
+ * Characterization ledger for the pre-consolidation recognizer divergences.
+ * Each entry pins how the shared grammar decided a case where the four
+ * original per-consumer recognizers disagreed, exercised through the
+ * consumers' own exported surfaces so a regression in either direction
+ * fails here first.
+ */
+describe('cross-consumer divergence ledger', () => {
+ test('paren and apostrophe titles: asset extraction now agrees with the backlink indexer', () => {
+ // Pre-consolidation asset-references missed both title forms entirely.
+ for (const md of ['[doc](./t.pdf (title))', '[doc](./t.pdf "it\'s here")']) {
+ expect(extractLocalAssetHrefs(md)).toEqual(['./t.pdf']);
+ expect(extractMarkdownLinksFromMarkdown(md.replace('.pdf', '.md'), 'src')).toHaveLength(1);
+ }
+ });
+
+ test('mismatched title quotes: asset extraction now rejects, agreeing with the indexer', () => {
+ // Pre-consolidation asset-references matched `"t'` as a title.
+ expect(extractLocalAssetHrefs('[a](./x.pdf "t\')')).toEqual([]);
+ expect(extractMarkdownLinksFromMarkdown('[a](./x.md "t\')', 'src')).toEqual([]);
+ });
+
+ test('wiki targets in asset extraction are now trimmed and whitespace-only targets drop', () => {
+ // Pre-consolidation asset-references collected the raw untrimmed target;
+ // downstream href decoration stripping trimmed it anyway, so resolution
+ // is unchanged.
+ expect(extractLocalAssetHrefs('[[ spaced.png ]]')).toEqual(['spaced.png']);
+ expect(extractLocalAssetHrefs('[[ ]]')).toEqual([]);
+ });
+
+ test('badge-style image-in-link: asset extraction keeps the outer destination', () => {
+ expect(extractLocalAssetHrefs('[![alt](./inner.png)](./outer.pdf)')).toEqual(['./outer.pdf']);
+ });
+
+ test('escape handling is a scanner concern, not a grammar concern', () => {
+ // Cursor-based consumers (backlink indexer, rename rewriter) honor a
+ // leading backslash escape before dispatching into the grammar; the
+ // regex-scan consumer (asset-references) never did and still does not.
+ const md = '\\[[not-wiki.png]]';
+ expect(extractWikiLinksFromMarkdown(md, 'src')).toEqual([]);
+ expect(rewriteWikiLinksForDocumentRename(md, 'not-wiki.png', 'renamed')).toMatchObject({
+ rewrites: 0,
+ });
+ expect(extractLocalAssetHrefs(md)).toEqual(['not-wiki.png']);
+ });
+
+ test('deliberate residual: the observer bare-text strip is looser than the grammar', () => {
+ // Characterization copy of `markdownBareText`'s strip regex in
+ // server-observers.ts. That reduction deliberately over-matches
+ // (whitespace destinations, no title handling) because attribution
+ // only needs a stable common form — see link-syntax.ts. WARN: if the
+ // strip regex in server-observers.ts changes, re-pin here.
+ const observerStrip = (line: string) => line.replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1');
+ // The loose strip reduces a whitespace destination the grammar rejects.
+ expect(observerStrip('[a](two words)')).toBe('a');
+ expect(readMarkdownLinkAt('[a](two words)', 0)).toBeNull();
+ // The loose strip truncates at the first `)` inside a quoted title.
+ expect(observerStrip('[doc](./t.md "ti)tle")')).toBe('doctle")');
+ expect(readMarkdownLinkAt('[doc](./t.md "ti)tle")', 0)).toMatchObject({ href: './t.md' });
+ });
+});
diff --git a/packages/server/src/link-syntax.ts b/packages/server/src/link-syntax.ts
new file mode 100644
index 000000000..dd59b3361
--- /dev/null
+++ b/packages/server/src/link-syntax.ts
@@ -0,0 +1,191 @@
+/**
+ * Single line-oriented grammar for the server's inline link recognizers:
+ * `[[wiki links]]` / `![[wiki embeds]]` and `[markdown](links)` /
+ * `![markdown](images)`. Consumers: `backlink-index.ts`,
+ * `managed-rename-rewrite.ts`, `suggest-links.ts`, `asset-references.ts`.
+ * Before consolidation each site owned a subtly different regex, so a link
+ * could count for backlinks yet be missed by rename rewriting — the
+ * divergence class the canonical link contract (precedent #56) exists to
+ * prevent. Recognition is consolidated here; resolution stays in
+ * `resolveInternalHref` / `classifyMarkdownHref` (core).
+ *
+ * Line-oriented: callers pre-split into lines, so the character classes
+ * exclude `\n` defensively rather than relying on multiline flags.
+ * cf. packages/core/src/extensions/wiki-link.ts WIKI_LINK_PATTERN — the
+ * editor-side pattern (no `\n` exclusion, `^`-anchored) is a sibling grammar
+ * that must stay observationally aligned with this one on shared inputs.
+ *
+ * Known deliberate residual: `server-observers.ts` `markdownBareText` strips
+ * link syntax with a LOOSER regex (`/!?\[([^\]]*)\]\([^)]*\)/g`) that admits
+ * whitespace in destinations and ignores titles/angle wrappers. That is a
+ * reduction for carrier attribution, not link recognition — over-stripping
+ * is acceptable there, under-stripping is not — so it intentionally does not
+ * share this grammar. See the divergence pins in `link-syntax.test.ts`.
+ */
+
+// Wiki form: [[target]], [[target#anchor]], [[target|alias]],
+// [[target#anchor|alias]]. Target excludes `[ ] | #`; anchor excludes
+// `[ ] |`; alias excludes `[ ]` (so aliases may contain `#` and later `|`
+// chars fold into the alias, matching how `[[a|b|c]]` has alias `b|c`).
+const WIKI_BODY_SOURCE = String.raw`\[\[([^\n#[\]|]+)(?:#([^\n[\]|]+))?(?:\|([^\n[\]]+))?\]\]`;
+
+// Inline destination + optional CommonMark title: `<angle form>` admits
+// spaces; the bare form runs to the first `)` or whitespace. Titles come in
+// the three CommonMark forms ("…", '…', (…)); the authored whitespace+title
+// suffix is captured verbatim so rewriters can re-emit it byte-identically.
+// Does NOT match reference-style `[text][ref]`.
+const DEST_AND_TITLE_SOURCE = String.raw`\((<[^>\n]+>|[^)\s\n]+)((?:\s+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?)\)`;
+
+// Strict labels stop at the first `]`. The nested variant additionally
+// admits one `]…` run inside the label so a bracketed construct nested in
+// the label — badge-style `[![alt](img)](target)` — matches as ONE link
+// carrying the outermost destination instead of stopping at the inner image.
+const LABEL_STRICT_SOURCE = String.raw`([^\]\n]*)`;
+const LABEL_NESTED_SOURCE = String.raw`([^\]\n]*(?:\][^[\]\n]*)?)`;
+
+const wikiPattern = (flags: string) => new RegExp(`(!?)${WIKI_BODY_SOURCE}`, flags);
+const markdownPattern = (label: string, flags: string) =>
+ new RegExp(`(!?)\\[${label}\\]${DEST_AND_TITLE_SOURCE}`, flags);
+
+// Sticky ('y') for position-based matching via lastIndex — no per-call
+// line.slice allocation. Module-level regexes are shared mutable state
+// (lastIndex); every use sets lastIndex immediately before exec.
+const WIKI_AT_RE = wikiPattern('y');
+const WIKI_SCAN_RE = wikiPattern('g');
+const MD_AT_RE = markdownPattern(LABEL_STRICT_SOURCE, 'y');
+const MD_SCAN_STRICT_RE = markdownPattern(LABEL_STRICT_SOURCE, 'g');
+const MD_SCAN_NESTED_RE = markdownPattern(LABEL_NESTED_SOURCE, 'g');
+
+export interface WikiLinkMatch {
+ /** True for the `![[…]]` embed form. */
+ embed: boolean;
+ /** Trimmed target; matches with a whitespace-only target are rejected. */
+ target: string;
+ /** Target capture as authored (untrimmed), for label-offset math. */
+ targetRaw: string;
+ /** Trimmed anchor; null when absent or whitespace-only. */
+ anchor: string | null;
+ anchorRaw: string | null;
+ /** Trimmed alias; null when absent or whitespace-only. */
+ alias: string | null;
+ aliasRaw: string | null;
+ start: number;
+ /** Index just past the closing `]]` — the caller's next cursor position. */
+ end: number;
+}
+
+export interface MarkdownLinkMatch {
+ /** True for the `![…](…)` image form. */
+ image: boolean;
+ label: string;
+ /** Destination as authored, including any `<…>` wrapper. */
+ hrefRaw: string;
+ /** Destination with a `<…>` wrapper removed. */
+ href: string;
+ /** Authored whitespace+title suffix, '' when no title. */
+ titleSuffix: string;
+ start: number;
+ /** Index just past the closing `)` — the caller's next cursor position. */
+ end: number;
+}
+
+function toWikiLinkMatch(match: RegExpExecArray, start: number): WikiLinkMatch | null {
+ const targetRaw = match[2] ?? '';
+ const target = targetRaw.trim();
+ if (!target) return null;
+ const anchorRaw = match[3] ?? null;
+ const aliasRaw = match[4] ?? null;
+ return {
+ embed: match[1] === '!',
+ target,
+ targetRaw,
+ anchor: anchorRaw?.trim() || null,
+ anchorRaw,
+ alias: aliasRaw?.trim() || null,
+ aliasRaw,
+ start,
+ end: start + match[0].length,
+ };
+}
+
+function unwrapAngleHref(rawHref: string): string {
+ return rawHref.startsWith('<') && rawHref.endsWith('>') ? rawHref.slice(1, -1) : rawHref;
+}
+
+function toMarkdownLinkMatch(match: RegExpExecArray, start: number): MarkdownLinkMatch {
+ const hrefRaw = match[3] ?? '';
+ return {
+ image: match[1] === '!',
+ label: match[2] ?? '',
+ hrefRaw,
+ href: unwrapAngleHref(hrefRaw),
+ titleSuffix: match[4] ?? '',
+ start,
+ end: start + match[0].length,
+ };
+}
+
+/**
+ * Match a wiki link or embed starting exactly at `start`. `![[…]]` at
+ * `start` is an embed; `[[…]]` is a plain wiki link. Callers that treat a
+ * preceding `!` as ordinary text simply call at the `[[` position.
+ */
+export function readWikiLinkAt(line: string, start: number): WikiLinkMatch | null {
+ WIKI_AT_RE.lastIndex = start;
+ const match = WIKI_AT_RE.exec(line);
+ if (!match) return null;
+ return toWikiLinkMatch(match, start);
+}
+
+/**
+ * Match a markdown inline link or image starting exactly at `start`.
+ * `![…](…)` at `start` is an image; `[…](…)` is a link — dispatch on
+ * `.image` when the two forms need different handling.
+ */
+export function readMarkdownLinkAt(line: string, start: number): MarkdownLinkMatch | null {
+ MD_AT_RE.lastIndex = start;
+ const match = MD_AT_RE.exec(line);
+ if (!match) return null;
+ return toMarkdownLinkMatch(match, start);
+}
+
+/**
+ * All wiki links and embeds on a line, left to right, non-overlapping.
+ * Whitespace-only targets are dropped (the scan still advances past them).
+ */
+export function matchWikiLinks(line: string): WikiLinkMatch[] {
+ const matches: WikiLinkMatch[] = [];
+ WIKI_SCAN_RE.lastIndex = 0;
+ for (const match of line.matchAll(WIKI_SCAN_RE)) {
+ const wikiLink = toWikiLinkMatch(match, match.index);
+ if (wikiLink) matches.push(wikiLink);
+ }
+ return matches;
+}
+
+export interface MatchMarkdownLinksOptions {
+ /**
+ * Admit one `]…` run inside the label so badge-style image-in-link
+ * (`[![alt](img)](target)`) matches as one link with the OUTER
+ * destination. Without it the scan matches the nested image and yields
+ * the inner destination instead. Neither mode yields both destinations.
+ */
+ nestedBracketLabels?: boolean;
+}
+
+/**
+ * All markdown inline links AND images on a line, left to right,
+ * non-overlapping. Filter on `.image` when only one form is wanted.
+ */
+export function matchMarkdownLinks(
+ line: string,
+ options?: MatchMarkdownLinksOptions,
+): MarkdownLinkMatch[] {
+ const scanRe = options?.nestedBracketLabels ? MD_SCAN_NESTED_RE : MD_SCAN_STRICT_RE;
+ scanRe.lastIndex = 0;
+ const matches: MarkdownLinkMatch[] = [];
+ for (const match of line.matchAll(scanRe)) {
+ matches.push(toMarkdownLinkMatch(match, match.index));
+ }
+ return matches;
+}
diff --git a/packages/server/src/managed-rename-rewrite.ts b/packages/server/src/managed-rename-rewrite.ts
index a07c4c386..ba48ade5e 100644
--- a/packages/server/src/managed-rename-rewrite.ts
+++ b/packages/server/src/managed-rename-rewrite.ts
@@ -1,5 +1,6 @@
import { posix } from 'node:path';
import { resolveAssetProjectPath, resolveInternalHref } from '@inkeep/open-knowledge-core';
+import { readMarkdownLinkAt, readWikiLinkAt } from './link-syntax.ts';
interface FenceState {
char: '`' | '~';
@@ -60,21 +61,13 @@ function readWikiLink(
line: string,
start: number,
): { target: string; alias: string | null; anchor: string | null; nextIndex: number } | null {
- const match = /^\[\[([^\n#[\]|]+)(?:#([^\n[\]|]+))?(?:\|([^\n[\]]+))?\]\]/.exec(
- line.slice(start),
- );
+ const match = readWikiLinkAt(line, start);
if (!match) return null;
-
- const target = match[1]?.trim();
- const anchor = match[2]?.trim() || null;
- const alias = match[3]?.trim() || null;
- if (!target) return null;
-
return {
- target,
- alias,
- anchor,
- nextIndex: start + match[0].length,
+ target: match.target,
+ alias: match.alias,
+ anchor: match.anchor,
+ nextIndex: match.end,
};
}
@@ -87,15 +80,15 @@ interface WikiLinkOrEmbed {
}
function readWikiLinkOrEmbed(line: string, start: number): WikiLinkOrEmbed | null {
- if (line[start] === '!' && line[start + 1] === '[' && line[start + 2] === '[') {
- const link = readWikiLink(line, start + 1);
- return link ? { ...link, embed: true } : null;
- }
- if (line[start] === '[' && line[start + 1] === '[') {
- const link = readWikiLink(line, start);
- return link ? { ...link, embed: false } : null;
- }
- return null;
+ const match = readWikiLinkAt(line, start);
+ if (!match) return null;
+ return {
+ target: match.target,
+ alias: match.alias,
+ anchor: match.anchor,
+ nextIndex: match.end,
+ embed: match.embed,
+ };
}
function readMarkdownLink(
@@ -108,25 +101,20 @@ function readMarkdownLink(
titleSuffix: string;
nextIndex: number;
} | null {
- const match =
- /^\[([^\]\n]*)\]\((<[^>\n]+>|[^)\s\n]+)((?:\s+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?)\)/.exec(
- line.slice(start),
- );
- if (!match) return null;
-
- const hrefRaw = match[2] ?? '';
+ const match = readMarkdownLinkAt(line, start);
+ if (!match || match.image) return null;
return {
- text: match[1] ?? '',
- hrefRaw,
- href: hrefRaw.startsWith('<') && hrefRaw.endsWith('>') ? hrefRaw.slice(1, -1) : hrefRaw,
- titleSuffix: match[3] ?? '',
- nextIndex: start + match[0].length,
+ text: match.label,
+ hrefRaw: match.hrefRaw,
+ href: match.href,
+ titleSuffix: match.titleSuffix,
+ nextIndex: match.end,
};
}
// Matches `![alt](src "optional title")`. Wiki-embeds (`![[file.ext]]`)
-// fail this pattern because the second char after `!` is `[` not
-// `]`-then-`(`, so they flow through untouched.
+// fail the shared grammar's label because the char after `![` is `[`,
+// so they flow through untouched.
function readImageRef(
line: string,
start: number,
@@ -137,19 +125,14 @@ function readImageRef(
titleSuffix: string;
nextIndex: number;
} | null {
- const match =
- /^!\[([^\]\n]*)\]\((<[^>\n]+>|[^)\s\n]+)((?:\s+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?)\)/.exec(
- line.slice(start),
- );
- if (!match) return null;
-
- const hrefRaw = match[2] ?? '';
+ const match = readMarkdownLinkAt(line, start);
+ if (!match?.image) return null;
return {
- alt: match[1] ?? '',
- hrefRaw,
- href: hrefRaw.startsWith('<') && hrefRaw.endsWith('>') ? hrefRaw.slice(1, -1) : hrefRaw,
- titleSuffix: match[3] ?? '',
- nextIndex: start + match[0].length,
+ alt: match.label,
+ hrefRaw: match.hrefRaw,
+ href: match.href,
+ titleSuffix: match.titleSuffix,
+ nextIndex: match.end,
};
}
diff --git a/packages/server/src/suggest-links.ts b/packages/server/src/suggest-links.ts
index 37ebe425f..c394c95e7 100644
--- a/packages/server/src/suggest-links.ts
+++ b/packages/server/src/suggest-links.ts
@@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises';
import type { Document, Hocuspocus } from '@hocuspocus/server';
import { resolveInternalHref, stripFrontmatter, toWikiLinkSlug } from '@inkeep/open-knowledge-core';
import type { FileIndexEntry } from './file-watcher.ts';
+import { readMarkdownLinkAt, readWikiLinkAt } from './link-syntax.ts';
import { getLogger } from './logger.ts';
import { extractPageIdentity, type PageIdentity } from './page-identity.ts';
@@ -9,13 +10,6 @@ const log = getLogger('suggest-links');
const WORD_CHAR_RE = /[\p{L}\p{N}]/u;
-// Line-oriented variants: excludes \n since lines are pre-split.
-const WIKI_LINK_RE = /\[\[([^\n#[\]|]+)(?:#([^\n[\]|]+))?(?:\|([^\n[\]]+))?\]\]/y;
-const MD_LINK_RE =
- /\[([^\]\n]*)\]\((<[^>\n]+>|[^)\s\n]+)(?:\s+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?\)/y;
-const MD_IMAGE_RE =
- /!\[([^\]\n]*)\]\((<[^>\n]+>|[^)\s\n]+)(?:\s+(?:"[^"\n]*"|'[^'\n]*'|\([^)\n]*\)))?\)/y;
-
interface FenceState {
char: '`' | '~';
length: number;
@@ -173,58 +167,47 @@ function readWikiLink(
labelStart: number;
nextIndex: number;
} | null {
- WIKI_LINK_RE.lastIndex = start;
- const match = WIKI_LINK_RE.exec(line);
+ const match = readWikiLinkAt(line, start);
if (!match) return null;
- const targetRaw = match[1] ?? '';
- const target = targetRaw.trim();
- const anchor = match[2]?.trim() || null;
- const aliasRaw = match[3] ?? null;
- const alias = aliasRaw?.trim() || null;
- if (!target) return null;
-
- const label = alias ?? target;
- const rawLabel = alias ? aliasRaw : targetRaw;
- const labelIndexInMatch = alias ? match[0].lastIndexOf(aliasRaw ?? '') : 2;
+ const label = match.alias ?? match.target;
+ const rawLabel = match.alias ? match.aliasRaw : match.targetRaw;
+ // Alias raw capture ends right before the closing `]]`; target raw
+ // capture starts right after the opening `[[`.
+ const labelIndexInMatch =
+ match.alias && match.aliasRaw ? match.end - match.start - 2 - match.aliasRaw.length : 2;
const labelTrimOffset = rawLabel?.indexOf(label) ?? 0;
return {
- target,
- alias,
- anchor,
+ target: match.target,
+ alias: match.alias,
+ anchor: match.anchor,
label,
labelStart: start + labelIndexInMatch + Math.max(labelTrimOffset, 0),
- nextIndex: start + match[0].length,
+ nextIndex: match.end,
};
}
-function normalizeMarkdownHref(rawHref: string): string {
- return rawHref.startsWith('<') && rawHref.endsWith('>') ? rawHref.slice(1, -1) : rawHref;
-}
-
function readMarkdownLink(
line: string,
start: number,
): { text: string; href: string; nextIndex: number } | null {
- MD_LINK_RE.lastIndex = start;
- const match = MD_LINK_RE.exec(line);
- if (!match) return null;
+ const match = readMarkdownLinkAt(line, start);
+ if (!match || match.image) return null;
return {
- text: match[1] ?? '',
- href: normalizeMarkdownHref(match[2] ?? ''),
- nextIndex: start + match[0].length,
+ text: match.label,
+ href: match.href,
+ nextIndex: match.end,
};
}
function readMarkdownImage(line: string, start: number): { alt: string; nextIndex: number } | null {
- MD_IMAGE_RE.lastIndex = start;
- const match = MD_IMAGE_RE.exec(line);
- if (!match) return null;
+ const match = readMarkdownLinkAt(line, start);
+ if (!match?.image) return null;
return {
- alt: match[1] ?? '',
- nextIndex: start + match[0].length,
+ alt: match.label,
+ nextIndex: match.end,
};
}