A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(markdown): linear-time parser and composer caches (#3628)
Hampusb6d02820d55d
diff --git a/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts b/fluxer_app/src/features/expressions/utils/UnicodeEmojis.tsindex bfdcbea64..4dc7045e9 100644--- a/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts+++ b/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts@@ -66,6 +66,7 @@ const HAIR_COMPONENT_NAMES: Record<string, string> = {'\u{1F9B3}': 'white_hair','\u{1F9B2}': 'bald',};+const REGIONAL_INDICATOR_HIGH_SURROGATE = 0xd83c;const REGIONAL_INDICATOR_PAIR_PATTERN = '\\uD83C[\\uDDE6-\\uDDFF]\\uD83C[\\uDDE6-\\uDDFF]';let defaultSkinTone: string = '';@@ -178,6 +179,9 @@ interface EmojiIndex {skinToneSpriteCount: number;baseSpriteCount: number;emojiSurrogateRegex: RegExp;+ emojiSurrogateStickyRegex: RegExp;+ emojiTriggerCodeUnits: Uint8Array;+ maxEmojiTriggerOffset: number;emojiShortcutRegex: RegExp;}@@ -284,10 +288,16 @@ function buildEmojiIndex(): EmojiIndex {shortcutToName[shortcut] = emoji;});- const surrogateAlternation = Object.keys(surrogateToName)- .sort((a, b) => b.length - a.length)- .map(RegexUtils.escapeRegex)- .join('|');+ const surrogates = Object.keys(surrogateToName).sort((a, b) => b.length - a.length);+ const surrogatePattern = `(${REGIONAL_INDICATOR_PAIR_PATTERN}|${surrogates.map(RegexUtils.escapeRegex).join('|')})`;+ const emojiTriggerCodeUnits = new Uint8Array(0x10000);+ emojiTriggerCodeUnits[REGIONAL_INDICATOR_HIGH_SURROGATE] = 1;+ let maxEmojiTriggerOffset = 0;+ for (const surrogate of surrogates) {+ const offset = findEmojiTriggerOffset(surrogate);+ emojiTriggerCodeUnits[surrogate.charCodeAt(offset)] = 1;+ maxEmojiTriggerOffset = Math.max(maxEmojiTriggerOffset, offset);+ }const shortcutAlternation = Object.keys(shortcutToName).map(RegexUtils.escapeRegex).join('|');return {@@ -302,7 +312,10 @@ function buildEmojiIndex(): EmojiIndex {emojis,skinToneSpriteCount,baseSpriteCount,- emojiSurrogateRegex: new RegExp(`(${REGIONAL_INDICATOR_PAIR_PATTERN}|${surrogateAlternation})`, 'g'),+ emojiSurrogateRegex: new RegExp(surrogatePattern, 'g'),+ emojiSurrogateStickyRegex: new RegExp(surrogatePattern, 'y'),+ emojiTriggerCodeUnits,+ maxEmojiTriggerOffset,emojiShortcutRegex: new RegExp(`^(${shortcutAlternation})`),};}@@ -317,7 +330,9 @@ const lookupSurrogateName = (surrogate: string): string | null =>const isRegionalIndicatorAt = (text: string, index: number): boolean => {const lowSurrogate = text.charCodeAt(index + 1);- return text.charCodeAt(index) === 0xd83c && lowSurrogate >= 0xdde6 && lowSurrogate <= 0xddff;+ return (+ text.charCodeAt(index) === REGIONAL_INDICATOR_HIGH_SURROGATE && lowSurrogate >= 0xdde6 && lowSurrogate <= 0xddff+ );};const isInsideRegionalIndicatorPair = (text: string, index: number): boolean => {@@ -327,6 +342,19 @@ const isInsideRegionalIndicatorPair = (text: string, index: number): boolean =>return (index - runStart) % 4 === 2;};+function findEmojiTriggerOffset(surrogate: string): number {+ let offset = 0;+ while (offset < surrogate.length - 1 && surrogate.charCodeAt(offset) <= 0x7f) offset += 1;+ return offset;+}++function findEmojiTrigger(text: string, from: number, triggers: Uint8Array): number {+ for (let index = from; index < text.length; index += 1) {+ if (triggers[text.charCodeAt(index)] === 1) return index;+ }+ return -1;+}+const toEmojiSurrogateMatch = (text: string, start: number, end: number): EmojiSurrogateMatch => ({start,end,@@ -334,17 +362,24 @@ const toEmojiSurrogateMatch = (text: string, start: number, end: number): EmojiS});function* matchEmojiSurrogates(text: string, startIndex = 0): Generator<EmojiSurrogateMatch, void> {- const regex = getEmojiIndex().emojiSurrogateRegex;+ const {emojiSurrogateStickyRegex: regex, emojiTriggerCodeUnits, maxEmojiTriggerOffset} = getEmojiIndex();let index = startIndex;if (isInsideRegionalIndicatorPair(text, index)) {yield toEmojiSurrogateMatch(text, index, index + 2);index += 2;}- while (true) {+ let trigger = findEmojiTrigger(text, index, emojiTriggerCodeUnits);+ while (trigger >= 0) {+ index = Math.max(index, trigger - maxEmojiTriggerOffset);regex.lastIndex = index;const match = regex.exec(text);- if (match == null) return;+ if (match == null) {+ index += 1;+ if (index > trigger) trigger = findEmojiTrigger(text, index, emojiTriggerCodeUnits);+ continue;+ }index = match.index + match[0].length;+ trigger = findEmojiTrigger(text, index, emojiTriggerCodeUnits);const emojiMatch = toEmojiSurrogateMatch(text, match.index, index);const isUnknownRegionalIndicatorPair =emojiMatch.name == null &&diff --git a/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts b/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.tsindex bf3a7e6e0..c0128c46b 100644--- a/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts+++ b/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts@@ -14,6 +14,7 @@ import {type MarkdownHlFormat,type MarkdownSpan,markSilentMessagePrefix,+ splitSpansByLine,} from '@app/features/lexical/composer/markdownSpans';import {$createComposerBlockquoteMarkerNode,@@ -134,22 +135,12 @@ function $reconcileParagraph(return;}const spans = silentMessagePrefix ? markSilentMessagePrefix(markdownSpans, source) : markdownSpans;- let lineStart = 0;+ const lineSpans = splitSpansByLine(spans, lineSources);for (let index = 0; index < lines.length; index += 1) {const nodes = lines[index]!.nodes;- const lineSource = lineSources[index]!;- const lineEnd = lineStart + lineSource.length;if (nodes.length > 0) {- const lineSpans = spans- .map((span) => ({- ...span,- start: Math.max(span.start, lineStart) - lineStart,- end: Math.min(span.end, lineEnd) - lineStart,- }))- .filter((span) => span.end > span.start);- $reconcileLine(nodes, parserFlags, lineSource, lineSpans, quoteMarkerEnds[index]);+ $reconcileLine(nodes, parserFlags, lineSources[index]!, lineSpans[index]!, quoteMarkerEnds[index]);}- lineStart = lineEnd + 1;}$syncComposerBlockquoteLines(paragraph,diff --git a/fluxer_app/src/features/lexical/composer/blockquoteLines.ts b/fluxer_app/src/features/lexical/composer/blockquoteLines.tsindex 1aabea3e7..ed86a9618 100644--- a/fluxer_app/src/features/lexical/composer/blockquoteLines.ts+++ b/fluxer_app/src/features/lexical/composer/blockquoteLines.ts@@ -98,12 +98,18 @@ function scanMarkerEnds(markerEndByStart.set(span.start, span.end);}}- const blocks = scanAllowedCodeBlocks(source, parserFlags);+ const blocks = scanAllowedCodeBlocks(source, parserFlags).sort((left, right) => left.start - right.start);const ends: Array<number> = [];let lineStart = 0;+ let nextBlock = 0;+ let coveredUntil = -1;for (const line of lines) {+ while (nextBlock < blocks.length && blocks[nextBlock]!.start <= lineStart) {+ coveredUntil = Math.max(coveredUntil, blocks[nextBlock]!.end);+ nextBlock += 1;+ }const markerEnd = markerEndByStart.get(lineStart);- const suppressed = isInsideCodeBlock(blocks, lineStart) || MULTILINE_ALERT_RE.test(line);+ const suppressed = coveredUntil >= lineStart || MULTILINE_ALERT_RE.test(line);ends.push(markerEnd == null || suppressed ? 0 : Math.min(markerEnd, lineStart + line.length) - lineStart);lineStart += line.length + 1;}diff --git a/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts b/fluxer_app/src/features/lexical/composer/codeBlockIndent.tsindex ad8ebcd7d..7c00295be 100644--- a/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts+++ b/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts@@ -59,6 +59,11 @@ interface ScanScope {rendered: boolean;}+interface ClosingLines {+ validFrom: number;+ nextByFence: Map<number, Int32Array>;+}+interface PhysicalLine {start: number;end: number;@@ -170,12 +175,48 @@ function opensUnclosedBlock(line: SourceLine, column: number, info: string): boo);}+const closingLinesByLines = new WeakMap<Array<SourceLine>, ClosingLines>();++function findClosingLine(lines: Array<SourceLine>, from: number, fence: string): number {+ let memo = closingLinesByLines.get(lines);+ if (memo == null || from < memo.validFrom) {+ memo = {validFrom: 0, nextByFence: new Map()};+ closingLinesByLines.set(lines, memo);+ }+ let next = memo.nextByFence.get(fence.length);+ if (next == null) {+ next = new Int32Array(lines.length + 1);+ memo.nextByFence.set(fence.length, next);+ }+ let position = from;+ let found = lines.length;+ while (position < lines.length) {+ const known = next[position]!;+ if (known !== 0) {+ found = known - 1;+ break;+ }+ if (findCodeBlockClosing(lines[position]!.text, fence, fence.length) != null) {+ found = position;+ next[position] = found + 1;+ break;+ }+ position += 1;+ }+ next.fill(found + 1, from, position);+ return found;+}+function resumeAt(lines: Array<SourceLine>, {index, column}: LinePoint): number {const line = lines[index]!;if (column >= line.text.length) {return index + 1;}lines[index] = {text: line.text.slice(column), offset: line.offset + column};+ const memo = closingLinesByLines.get(lines);+ if (memo != null) {+ memo.validFrom = Math.max(memo.validFrom, index + 1);+ }return index;}@@ -200,19 +241,19 @@ function parseCodeBlock(lines: Array<SourceLine>, index: number, column: number,resume: visible ? {index, column: infoStart + inlineEnd + fence.length} : null,};}- let content = FENCE_LANGUAGE_RE.test(info) ? '' : info;- for (let next = index + 1; next < lines.length; next += 1) {- const text = lines[next]!.text;- const closing = findCodeBlockClosing(text, fence, fence.length);- if (closing == null) {- content += text;- continue;+ const closingLine = findClosingLine(lines, index + 1, fence);+ if (closingLine < lines.length) {+ let content = FENCE_LANGUAGE_RE.test(info) ? '' : info;+ for (let between = index + 1; between < closingLine; between += 1) {+ content += lines[between]!.text;}+ const {text, offset} = lines[closingLine]!;+ const closing = findCodeBlockClosing(text, fence, fence.length)!;const visible = hasVisibleContent(content + text.slice(0, closing.fenceIndex));const trailingStart = closing.fenceIndex + (closing.trailing.length > 0 ? closing.runLength : fence.length);return {- body: {start, end: lines[next]!.offset + closing.fenceIndex, rendered: visible && scope.rendered},- resume: visible ? {index: next, column: trailingStart} : null,+ body: {start, end: offset + closing.fenceIndex, rendered: visible && scope.rendered},+ resume: visible ? {index: closingLine, column: trailingStart} : null,};}if (!opensUnclosedBlock(line, column, info)) {diff --git a/fluxer_app/src/features/lexical/composer/markdownSpans.ts b/fluxer_app/src/features/lexical/composer/markdownSpans.tsindex 74f83e518..a8087a3a7 100644--- a/fluxer_app/src/features/lexical/composer/markdownSpans.ts+++ b/fluxer_app/src/features/lexical/composer/markdownSpans.ts@@ -7,6 +7,7 @@ import type {Node} from '@app/features/messaging/utils/markdown/parser/Nodes';import {normalizeUrl} from '@app/features/messaging/utils/markdown/parser/UrlUtils';import {findUrlEnd} from '@app/features/messaging/utils/markdown/UrlSpanUtils';import {parseSilentMessagePrefix} from '@app/features/messaging/utils/SilentMessagePrefix';+import {LRUMap} from '@app/lib/list/ListLruMap';export const MarkdownHl = {none: 0,@@ -76,12 +77,49 @@ type AlignAttempt = {type: 'aligned'; spans: Array<MarkdownSpan>} | {type: 'desyconst MAX_ALIGN_ATTEMPTS = 64;const MAX_URL_REALIGN_STEPS = 256;+const HIGHLIGHT_CACHE_CAPACITY = 32;++const highlightCache = new LRUMap<string, ReadonlyArray<MarkdownSpan>>(HIGHLIGHT_CACHE_CAPACITY);export function computeMarkdownHighlightSpans(source: string,parserFlags = DEFAULT_COMPOSER_MARKDOWN_FLAGS,-): Array<MarkdownSpan> {- return computeMarkdownHighlightResult(source, parserFlags).spans;+): ReadonlyArray<MarkdownSpan> {+ const key = `${parserFlags}\u0000${source}`;+ const cached = highlightCache.get(key);+ if (cached !== undefined) {+ return cached;+ }+ const spans = computeMarkdownHighlightResult(source, parserFlags).spans;+ highlightCache.set(key, spans);+ return spans;+}++export function splitSpansByLine(+ spans: ReadonlyArray<MarkdownSpan>,+ lineSources: ReadonlyArray<string>,+): Array<Array<MarkdownSpan>> {+ const result: Array<Array<MarkdownSpan>> = [];+ let first = 0;+ let lineStart = 0;+ for (const lineSource of lineSources) {+ const lineEnd = lineStart + lineSource.length;+ while (first < spans.length && spans[first]!.end <= lineStart) {+ first += 1;+ }+ const lineSpans: Array<MarkdownSpan> = [];+ for (let index = first; index < spans.length && spans[index]!.start < lineEnd; index += 1) {+ const span = spans[index]!;+ const start = Math.max(span.start, lineStart) - lineStart;+ const end = Math.min(span.end, lineEnd) - lineStart;+ if (end > start) {+ lineSpans.push({...span, start, end});+ }+ }+ result.push(lineSpans);+ lineStart = lineEnd + 1;+ }+ return result;}function computeMarkdownHighlightResult(@@ -126,7 +164,10 @@ function computeMarkdownHighlightResult(return {spans: coalesce(spans), recovered};}-export function markSilentMessagePrefix(spans: Array<MarkdownSpan>, source: string): Array<MarkdownSpan> {+export function markSilentMessagePrefix(+ spans: ReadonlyArray<MarkdownSpan>,+ source: string,+): ReadonlyArray<MarkdownSpan> {const prefix = parseSilentMessagePrefix(source);if (prefix == null) {return spans;diff --git a/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts b/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.tsindex 45b073efb..f08535988 100644--- a/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts+++ b/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts@@ -7,6 +7,7 @@ import {MARKDOWN_PARSER_WASM_BASE64} from '@app/features/messaging/utils/markdowimport type {Node} from '@app/features/messaging/utils/markdown/parser/Nodes';import * as StringUtils from '@app/features/messaging/utils/markdown/parser/StringUtils';import * as URLUtils from '@app/features/messaging/utils/markdown/parser/UrlUtils';+import {LRUMap} from '@app/lib/list/ListLruMap';interface WasmExports {memory: {buffer: ArrayBuffer};@@ -42,6 +43,7 @@ type CleanableNode = Node & {const RESULT_HEADER_BYTES = 16;const MAX_RETAINED_WASM_MEMORY_BYTES = 64 * 1024 * 1024;+const LINK_URL_CACHE_CAPACITY = 512;const textEncoder = new TextEncoder();const lenientTextDecoder = new TextDecoder('utf-8', {ignoreBOM: true, fatal: false});@@ -320,6 +322,38 @@ function shouldTreatAsMaskedLink(trimmedLinkText: string, url: string): boolean}}+const linkUrlCache = new LRUMap<string, string | null>(LINK_URL_CACHE_CAPACITY);++function cleanLinkUrl(rawUrl: string, linkText: string): string | null {+ const key = `${rawUrl.length}:${rawUrl}${linkText}`;+ const cached = linkUrlCache.get(key);+ if (cached !== undefined) return cached;+ const url = resolveLinkUrl(rawUrl, linkText);+ linkUrlCache.set(key, url);+ return url;+}++function resolveLinkUrl(rawUrl: string, linkText: string): string | null {+ try {+ const normalizedUrl = URLUtils.normalizeUrl(rawUrl);+ if (+ !URLUtils.isValidUrl(normalizedUrl) ||+ (rawUrl.startsWith('/') && !rawUrl.startsWith('//')) ||+ (StringUtils.startsWithUrl(linkText.trim()) && shouldTreatAsMaskedLink(linkText, rawUrl))+ ) {+ return null;+ }+ if (normalizedUrl.startsWith('tel:') || normalizedUrl.startsWith('sms:')) {+ const protocol = normalizedUrl.substring(0, normalizedUrl.indexOf(':') + 1);+ const phoneNumber = normalizedUrl.substring(normalizedUrl.indexOf(':') + 1);+ return phoneNumber.startsWith('+') ? protocol + URLUtils.normalizePhoneNumber(phoneNumber) : normalizedUrl;+ }+ return URLUtils.convertToAsciiUrl(normalizedUrl);+ } catch {+ return null;+ }+}+function makeCleanedTextNode(content: string): CleanedTextNode {return {type: 'Text', content, __cleanedLinkText: true};}@@ -335,34 +369,16 @@ function cleanNode(node: CleanableNode): Node {if (node.type === 'Link') {const rawUrl = coerceWasmString(node.rawUrl ?? node.url);const source = coerceWasmString(node.source ?? rawUrl);- const linkText = textFromNode(node.text);- try {- const normalizedUrl = URLUtils.normalizeUrl(rawUrl);- if (- !URLUtils.isValidUrl(normalizedUrl) ||- (rawUrl.startsWith('/') && !rawUrl.startsWith('//')) ||- (StringUtils.startsWithUrl(linkText.trim()) && shouldTreatAsMaskedLink(linkText, rawUrl))- ) {- return makeCleanedTextNode(source);- }- let finalUrl = normalizedUrl;- if (finalUrl.startsWith('tel:') || finalUrl.startsWith('sms:')) {- const protocol = finalUrl.substring(0, finalUrl.indexOf(':') + 1);- const phoneNumber = finalUrl.substring(finalUrl.indexOf(':') + 1);- if (phoneNumber.startsWith('+')) finalUrl = protocol + URLUtils.normalizePhoneNumber(phoneNumber);- } else {- finalUrl = URLUtils.convertToAsciiUrl(finalUrl);- }- const cleaned: Extract<Node, {type: 'Link'}> = {- type: 'Link',- url: finalUrl,- escaped: Boolean(node.escaped),- };- cleaned.text = node.text ? cleanNode(node.text) : undefined;- return cleaned;- } catch {+ const url = cleanLinkUrl(rawUrl, textFromNode(node.text));+ if (url == null) {return makeCleanedTextNode(source);}+ return {+ type: 'Link',+ url,+ escaped: Boolean(node.escaped),+ text: node.text ? cleanNode(node.text) : undefined,+ };}if (node.type === 'CodeBlock' && !('language' in node)) {node.language = undefined;diff --git a/packages/markdown_parser/rust/src/block.rs b/packages/markdown_parser/rust/src/block.rsindex f3cc23e0c..bce40a700 100644--- a/packages/markdown_parser/rust/src/block.rs+++ b/packages/markdown_parser/rust/src/block.rs@@ -1,14 +1,15 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::ast::{AlertType, ListItem, Node, ParserFlags, TableAlignment};-use crate::constants::{CODE_FENCE_LENGTH, MAX_AST_NODES, MAX_LINE_LENGTH};+use crate::constants::{+ CODE_FENCE_LENGTH, MAX_AST_NODES, MAX_CODE_BLOCK_SEARCH_LINES, MAX_LINE_LENGTH,+};use crate::links::{has_open_inline_code, has_valid_code_fence_language};use crate::normalize::{normalize_nodes, replace_trailing_whitespace_with_newline};use crate::parser::{MarkdownParser, ParseError, RuntimeState};use crate::text::{- Line, bounded_line_text, byte_at, concat, concat3, has_visible_content, line_as_text,- lines_to_text, starts_with, trim, trim_line_window, trim_right, trim_start,- trim_start_newline_whitespace,+ Line, bounded_line_text, byte_at, concat, has_visible_content, line_as_text, lines_to_text,+ starts_with, trim, trim_line_window, trim_right, trim_start, trim_start_newline_whitespace,};#[derive(Clone, Debug)]@@ -73,6 +74,12 @@ struct TableCellText {offset: usize,}+#[derive(Clone, Copy, Debug)]+struct TableLine<'a> {+ text: &'a str,+ offset: usize,+}+pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResult, ParseError> {let current = state.current_line;let line = state.lines[current].text.clone();@@ -110,14 +117,20 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResuif let Some(list_match) = match_list_item(&line) {if ParserFlags::has(state.parser.flags(), ParserFlags::ALLOW_LISTS) {- let list = parse_list(- state.parser,- &state.lines,+ let node_count = state.parser.node_count;+ let list = ListContext {+ parser: state.parser,+ lines: &state.lines,+ fences: &mut state.fences,+ }+ .parse_list(current,- list_match.ordered,- list_match.indent_level,- 1,- state.parser.node_count,+ ListLevel {+ ordered: list_match.ordered,+ indent_level: list_match.indent_level,+ depth: 1,+ },+ node_count,)?;return Ok(BlockParseResult {node: Some(list.node),@@ -137,9 +150,8 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResuif starts_with(trimmed, "||") && !trimmed[2..].contains("||") {if ParserFlags::has(state.parser.flags(), ParserFlags::ALLOW_SPOILERS) {let result = parse_block_spoiler(state.parser, &state.lines, current)?;- if let Some(extra) = &result.extra_content {- state.lines[result.new_line_index].text = extra.content.clone();- state.lines[result.new_line_index].offset = extra.offset;+ if let Some(extra) = result.extra_content {+ state.replace_line(result.new_line_index, extra.content, extra.offset);}return Ok(BlockParseResult {node: Some(result.node),@@ -161,10 +173,11 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResulet starts_with_fence =starts_with(trimmed, "```") && fence_pos == line.len() - trimmed.len();if starts_with_fence {- if let Some(result) = parse_code_block(&state.lines, current)? {- if let Some(extra) = &result.extra_content {- state.lines[result.new_line_index].text = extra.content.clone();- state.lines[result.new_line_index].offset = extra.offset;+ if let Some(result) =+ parse_code_block(&state.lines, current, &line, line_offset, &mut state.fences)+ {+ if let Some(extra) = result.extra_content {+ state.replace_line(result.new_line_index, extra.content, extra.offset);}return Ok(BlockParseResult {node: Some(result.node),@@ -179,25 +192,37 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResuif has_open_inline_code(prefix) {return Ok(no_block(current, state.parser.node_count));}- let inline_nodes = crate::inline::parse_inline(state.parser, prefix, line_offset)?;- let code_lines = slice_lines_from_fence(&state.lines, current, fence_pos);- if let Some(code_result) = parse_code_block(&code_lines, 0)? {- let new_line_index = current + code_result.new_line_index;- let mut extra_nodes = Vec::new();- if inline_nodes.len() > 1 {- extra_nodes.extend(inline_nodes[1..].iter().cloned());- }- extra_nodes.push(code_result.node.clone());- if let Some(extra) = &code_result.extra_content {- state.lines[new_line_index].text = extra.content.clone();- state.lines[new_line_index].offset = extra.offset;+ if let Some(code_result) = parse_code_block(+ &state.lines,+ current,+ &line[fence_pos..],+ line_offset + fence_pos,+ &mut state.fences,+ ) {+ let inline_nodes = crate::inline::parse_inline(state.parser, prefix, line_offset)?;+ let new_node_count = state.parser.node_count + inline_nodes.len() + 1;+ let new_line_index = code_result.new_line_index;+ if let Some(extra) = code_result.extra_content {+ state.replace_line(new_line_index, extra.content, extra.offset);}- let first_node = inline_nodes.first().cloned().unwrap_or(code_result.node);+ let mut inline_nodes = inline_nodes.into_iter();+ let first_node = inline_nodes.next();+ let mut extra_nodes: Vec<Node> = inline_nodes.collect();+ let first_node = match first_node {+ Some(first_node) => {+ extra_nodes.push(code_result.node);+ first_node+ }+ None => {+ extra_nodes.push(code_result.node.clone());+ code_result.node+ }+ };return Ok(BlockParseResult {node: Some(first_node),extra_nodes: Some(extra_nodes),new_line_index,- new_node_count: state.parser.node_count + inline_nodes.len() + 1,+ new_node_count,});}}@@ -205,7 +230,8 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResulet mut content = line;let mut end_index = current + 1;while end_index < state.lines.len() {- content = concat3(&content, "\n", &state.lines[end_index].text);+ content.push('\n');+ content.push_str(&state.lines[end_index].text);if trim(&state.lines[end_index].text) == "```" {end_index += 1;break;@@ -279,30 +305,37 @@ fn no_block(current: usize, node_count: usize) -> BlockParseResult {}}-pub(crate) fn parse_heading_node(- parser: &mut MarkdownParser,- trimmed_line: &str,- offset: usize,-) -> Result<Option<Node>, ParseError> {+pub(crate) fn heading_content(trimmed_line: &str) -> Option<(usize, &str)> {let mut level = 0usize;while level < trimmed_line.len() && level < 4 && byte_at(trimmed_line, level) == b'#' {level += 1;}- if (1..=4).contains(&level)- && level < trimmed_line.len()- && byte_at(trimmed_line, level) == b' '+ if !(1..=4).contains(&level)+ || level >= trimmed_line.len()+ || byte_at(trimmed_line, level) != b' '{- let content = &trimmed_line[level + 1..];- if !has_visible_content(content) && trim(content).is_empty() {- return Ok(None);- }- let children = crate::inline::parse_inline(parser, content, offset + level + 1)?;- return Ok(Some(Node::Heading {- level: level as u8,- children,- }));+ return None;+ }+ let content = &trimmed_line[level + 1..];+ if !has_visible_content(content) && trim(content).is_empty() {+ return None;}- Ok(None)+ Some((level, content))+}++fn parse_heading_node(+ parser: &mut MarkdownParser,+ trimmed_line: &str,+ offset: usize,+) -> Result<Option<Node>, ParseError> {+ let Some((level, content)) = heading_content(trimmed_line) else {+ return Ok(None);+ };+ let children = crate::inline::parse_inline(parser, content, offset + level + 1)?;+ Ok(Some(Node::Heading {+ level: level as u8,+ children,+ }))}fn parse_subtext_node(@@ -365,7 +398,7 @@ fn parse_blockquote(return Ok(no_block(index, node_count));}if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_ALERTS)- && let Some(alert) = parse_alert_lines(parser, child_lines.clone())?+ && let Some(alert) = parse_alert_lines(parser, &child_lines)?{return Ok(BlockParseResult {node: Some(alert),@@ -447,11 +480,60 @@ fn parse_multiline_blockquote(})}-fn parse_code_block(lines: &[Line], current: usize) -> Result<Option<CodeBlockResult>, ParseError> {- if current >= lines.len() {- return Ok(None);+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum CodeBlockMode {+ Build,+ Probe,+}++#[derive(Clone, Debug)]+struct CodeBlockContent {+ mode: CodeBlockMode,+ text: String,+ len: usize,+ visible: bool,+}++impl CodeBlockContent {+ fn push_line(&mut self, line: &str) {+ self.len += line.len() + 1;+ self.visible = self.visible || has_visible_content(line);+ if self.mode == CodeBlockMode::Build {+ self.text.push_str(line);+ self.text.push('\n');+ }}- let line = &lines[current].text;++ fn settled(&self) -> bool {+ self.mode == CodeBlockMode::Probe && self.visible+ }+}++fn parse_code_block(+ lines: &[Line],+ current: usize,+ line: &str,+ line_offset: usize,+ fences: &mut ClosingFenceLines,+) -> Option<CodeBlockResult> {+ scan_code_block(+ lines,+ current,+ line,+ line_offset,+ fences,+ CodeBlockMode::Build,+ )+}++fn scan_code_block(+ lines: &[Line],+ current: usize,+ line: &str,+ line_offset: usize,+ fences: &mut ClosingFenceLines,+ mode: CodeBlockMode,+) -> Option<CodeBlockResult> {let trimmed_line = trim_start(line);let indent_spaces = line.len() - trimmed_line.len();let list_indent = &line[..indent_spaces];@@ -460,17 +542,17 @@ fn parse_code_block(lines: &[Line], current: usize) -> Result<Option<CodeBlockRefence_length += 1;}if fence_length < CODE_FENCE_LENGTH {- return Ok(None);+ return None;}let language_part = &trimmed_line[fence_length..];let closing_fence = "`".repeat(fence_length);if let Some(closing_index) = language_part.find(&closing_fence) {let inline_content = &language_part[..closing_index];if !has_visible_content(inline_content) {- return Ok(None);+ return None;}let trailing = &language_part[closing_index + fence_length..];- return Ok(Some(CodeBlockResult {+ return Some(CodeBlockResult {node: Node::CodeBlock {language: None,content: inline_content.to_owned(),@@ -482,94 +564,142 @@ fn parse_code_block(lines: &[Line], current: usize) -> Result<Option<CodeBlockRe},extra_content: (!trailing.is_empty()).then(|| ExtraContent {content: trailing.to_owned(),- offset: lines[current].offset + (line.len() - trailing.len()),+ offset: line_offset + (line.len() - trailing.len()),}),- }));+ });}let treat_opening_as_language = has_valid_code_fence_language(language_part);let language = treat_opening_as_language.then(|| trim(language_part).to_owned());- let mut temp = current + 1;- let mut line_count = 0usize;- let mut has_closing = false;- while temp < lines.len() {- if find_closing_fence(trim_start(&lines[temp].text), &closing_fence, fence_length).is_some()- {- has_closing = true;- break;- }- line_count += 1;- if line_count > 1000 {- break;- }- temp += 1;- }- if !has_closing {- return Ok(None);+ let closing_line = fences.next(lines, current + 1, &closing_fence)?;+ if closing_line - current - 1 > MAX_CODE_BLOCK_SEARCH_LINES {+ return None;}- let mut content = String::new();+ let mut content = CodeBlockContent {+ mode,+ text: String::new(),+ len: 0,+ visible: false,+ };if !treat_opening_as_language && !language_part.is_empty() {- content.push_str(language_part);- content.push('\n');+ content.push_line(language_part);}let mut index = current + 1;- while index < lines.len() {+ while index < closing_line && !content.settled() {let current_line = &lines[index].text;- let trimmed = trim_start(current_line);- if let Some(closing) = find_closing_fence(trimmed, &closing_fence, fence_length) {- let absolute_fence = current_line.find(&closing_fence).unwrap_or(0);- let prefix = ¤t_line[..absolute_fence];- let content_line = if indent_spaces > 0 && starts_with(prefix, list_indent) {- &prefix[indent_spaces..]- } else {- prefix- };- if !content_line.is_empty() {- content.push_str(content_line);- content.push('\n');- }- let extra_text = if !closing.trailing_text.is_empty() {- closing.trailing_text- } else if closing.backtick_count > fence_length {- &trimmed[closing.fence_index + fence_length..]- } else {- ""- };- if !extra_text.is_empty() {- if !has_visible_content(&content) {- return Ok(None);- }- return Ok(Some(CodeBlockResult {- node: Node::CodeBlock { language, content },- new_line_index: index,- extra_content: Some(ExtraContent {- content: extra_text.to_owned(),- offset: lines[index].offset + (current_line.len() - extra_text.len()),- }),- }));- }- index += 1;- break;- }let content_line = if indent_spaces > 0 && starts_with(current_line, list_indent) {¤t_line[indent_spaces..]} else {current_line};- content.push_str(content_line);- content.push('\n');- if content.len() > MAX_LINE_LENGTH * 100 {+ content.push_line(content_line);+ if content.len > MAX_LINE_LENGTH * 100 {break;}index += 1;}- if !has_visible_content(&content) {- return Ok(None);+ let mut extra_content = None;+ if index == closing_line && !content.settled() {+ let current_line = &lines[index].text;+ let trimmed = trim_start(current_line);+ let closing = find_closing_fence(trimmed, &closing_fence, fence_length)?;+ let absolute_fence = current_line.find(&closing_fence).unwrap_or(0);+ let prefix = ¤t_line[..absolute_fence];+ let content_line = if indent_spaces > 0 && starts_with(prefix, list_indent) {+ &prefix[indent_spaces..]+ } else {+ prefix+ };+ if !content_line.is_empty() {+ content.push_line(content_line);+ }+ let extra_text = if !closing.trailing_text.is_empty() {+ closing.trailing_text+ } else if closing.backtick_count > fence_length {+ &trimmed[closing.fence_index + fence_length..]+ } else {+ ""+ };+ if extra_text.is_empty() {+ index += 1;+ } else {+ extra_content = Some(ExtraContent {+ content: extra_text.to_owned(),+ offset: lines[index].offset + (current_line.len() - extra_text.len()),+ });+ }}- Ok(Some(CodeBlockResult {- node: Node::CodeBlock { language, content },+ if !content.visible {+ return None;+ }+ Some(CodeBlockResult {+ node: Node::CodeBlock {+ language,+ content: content.text,+ },new_line_index: index,- extra_content: None,- }))+ extra_content,+ })+}++#[derive(Clone, Debug, Default)]+pub(crate) struct ClosingFenceLines {+ next_by_fence: Vec<(usize, Vec<u32>)>,+ valid_from: usize,+}++impl ClosingFenceLines {+ pub(crate) fn line_replaced(&mut self, index: usize) {+ self.valid_from = self.valid_from.max(index + 1);+ }++ fn next(&mut self, lines: &[Line], from: usize, closing_fence: &str) -> Option<usize> {+ if from >= lines.len() {+ return None;+ }+ if from < self.valid_from {+ self.next_by_fence.clear();+ self.valid_from = 0;+ }+ let fence_length = closing_fence.len();+ let index = match self+ .next_by_fence+ .iter()+ .position(|(length, _)| *length == fence_length)+ {+ Some(index) => index,+ None => {+ self.next_by_fence+ .push((fence_length, vec![0; lines.len()]));+ self.next_by_fence.len() - 1+ }+ };+ let next = &mut self.next_by_fence[index].1;+ let mut position = from;+ let found = loop {+ if position == lines.len() {+ break position;+ }+ if next[position] != 0 {+ break next[position] as usize - 1;+ }+ if find_closing_fence(+ trim_start(&lines[position].text),+ closing_fence,+ fence_length,+ )+ .is_some()+ {+ break position;+ }+ position += 1;+ };+ let stored = found as u32 + 1;+ next[from..position].fill(stored);+ if found == position && found < lines.len() {+ next[found] = stored;+ }+ (found < lines.len()).then_some(found)+ }}fn find_closing_fence<'a>(@@ -681,7 +811,7 @@ fn parse_block_spoiler(fn parse_alert_lines(parser: &mut MarkdownParser,- lines: Vec<Line>,+ lines: &[Line],) -> Result<Option<Node>, ParseError> {if lines.is_empty() || !starts_with(&lines[0].text, "[!") {return Ok(None);@@ -764,220 +894,231 @@ fn post_process_alert_nodes(nodes: Vec<Node>) -> Vec<Node> {return nodes;}let mut out = Vec::new();- let mut index = 0usize;- while index < nodes.len() {- let node = nodes[index].clone();- if let Node::Text { content } = &node- && index + 1 < nodes.len()- && matches!(nodes[index + 1], Node::List { .. })- {- let content = replace_trailing_whitespace_with_newline(content);- if !content.is_empty() {- out.push(Node::Text { content });- }- index += 1;- continue;- }- if matches!(node, Node::List { .. }) && index + 1 < nodes.len() {- out.push(node);- if let Node::Text { content } = &nodes[index + 1] {- let content = trim(content);+ let mut nodes = nodes.into_iter().peekable();+ while let Some(node) = nodes.next() {+ match node {+ Node::Text { content } if matches!(nodes.peek(), Some(Node::List { .. })) => {+ let content = replace_trailing_whitespace_with_newline(&content);if !content.is_empty() {- out.push(Node::Text {- content: concat("\n", content),- });- index += 2;- continue;+ out.push(Node::Text { content });}}- index += 1;- continue;+ node @ Node::List { .. } if nodes.peek().is_some() => {+ out.push(node);+ let trailing = match nodes.peek() {+ Some(Node::Text { content }) if !trim(content).is_empty() => {+ Some(concat("\n", trim(content)))+ }+ _ => None,+ };+ if let Some(content) = trailing {+ nodes.next();+ out.push(Node::Text { content });+ }+ }+ node => out.push(node),}- out.push(node);- index += 1;}out}-fn parse_list(- parser: &mut MarkdownParser,- lines: &[Line],- current: usize,+#[derive(Clone, Copy, Debug)]+struct ListLevel {ordered: bool,indent_level: usize,depth: usize,- node_count: usize,-) -> Result<ListResult, ParseError> {- let mut items = Vec::new();- let mut index = current;- let mut new_count = node_count;- while index < lines.len() {- if new_count > MAX_AST_NODES {- break;- }- let current_line = &lines[index].text;- let trimmed_line = trim_start(current_line);- if starts_with(trimmed_line, "#") || is_blockquote_start(trimmed_line, parser.flags()) {- break;- }- if let Some(item) = match_list_item(current_line) {- let ordinal = normalise_ordinal(&items, item.ordinal, ordered);- if item.indent_level < indent_level {+}++struct ListContext<'a> {+ parser: &'a mut MarkdownParser,+ lines: &'a [Line],+ fences: &'a mut ClosingFenceLines,+}++impl ListContext<'_> {+ fn parse_list(+ &mut self,+ current: usize,+ level: ListLevel,+ node_count: usize,+ ) -> Result<ListResult, ParseError> {+ let ListLevel {+ ordered,+ indent_level,+ depth,+ } = level;+ let lines = self.lines;+ let mut items = Vec::new();+ let mut index = current;+ let mut new_count = node_count;+ while index < lines.len() {+ if new_count > MAX_AST_NODES {break;}- if item.indent_level == indent_level {- if item.ordered != ordered {+ let current_line = &lines[index].text;+ let trimmed_line = trim_start(current_line);+ if starts_with(trimmed_line, "#")+ || is_blockquote_start(trimmed_line, self.parser.flags())+ {+ break;+ }+ if let Some(item) = match_list_item(current_line) {+ let ordinal = normalise_ordinal(&items, item.ordinal, ordered);+ if item.indent_level < indent_level {break;}- let mut children = Vec::new();- if let Some(inline_item) = match_list_item(item.content) {- let mut nested_items = Vec::new();- let inline_nodes = crate::inline::parse_inline(- parser,- inline_item.content,- lines[index].offset + (current_line.len() - inline_item.content.len()),- )?;- nested_items.push(ListItem {- children: inline_nodes,- ordinal: None,- });- if let Some(nested) = try_parse_nested_list(- parser,- lines,- index + 1,- indent_level,- depth,- new_count,- )? {- if let Node::List { items, .. } = nested.node {- nested_items.extend(items);- index = nested.new_line_index;- new_count = nested.new_node_count;+ if item.indent_level == indent_level {+ if item.ordered != ordered {+ break;+ }+ let mut children = Vec::new();+ if let Some(inline_item) = match_list_item(item.content) {+ let mut nested_items = Vec::new();+ let inline_nodes = crate::inline::parse_inline(+ self.parser,+ inline_item.content,+ lines[index].offset + (current_line.len() - inline_item.content.len()),+ )?;+ nested_items.push(ListItem {+ children: inline_nodes,+ ordinal: None,+ });+ if let Some(nested) =+ self.try_parse_nested_list(index + 1, indent_level, depth, new_count)?+ {+ if let Node::List { items, .. } = nested.node {+ nested_items.extend(items);+ index = nested.new_line_index;+ new_count = nested.new_node_count;+ } else {+ index += 1;+ new_count += 1;+ }} else {index += 1;new_count += 1;}+ children.push(Node::List {+ ordered: inline_item.ordered,+ items: nested_items,+ });} else {- index += 1;- new_count += 1;+ let content_nodes = crate::inline::parse_inline(+ self.parser,+ item.content,+ lines[index].offset + (current_line.len() - item.content.len()),+ )?;+ let content_node_count = content_nodes.len();+ children.extend(content_nodes);+ if let Some(nested) =+ self.try_parse_nested_list(index + 1, indent_level, depth, new_count)?+ {+ children.push(nested.node);+ index = nested.new_line_index;+ new_count = nested.new_node_count;+ } else {+ index += 1;+ new_count += content_node_count;+ }}- children.push(Node::List {- ordered: inline_item.ordered,- items: nested_items,- });- } else {- let content_nodes = crate::inline::parse_inline(- parser,- item.content,- lines[index].offset + (current_line.len() - item.content.len()),- )?;- children.extend(content_nodes.iter().cloned());- if let Some(nested) = try_parse_nested_list(- parser,- lines,- index + 1,- indent_level,- depth,+ items.push(ListItem { children, ordinal });+ } else if item.indent_level == indent_level + 1 && depth < 9 {+ let nested = self.parse_list(+ index,+ ListLevel {+ ordered: item.ordered,+ indent_level: item.indent_level,+ depth: depth + 1,+ },new_count,- )? {- children.push(nested.node);- index = nested.new_line_index;- new_count = nested.new_node_count;- } else {- index += 1;- new_count += content_nodes.len();+ )?;+ if let Some(last) = items.last_mut() {+ last.children.push(nested.node);}+ index = nested.new_line_index;+ new_count = nested.new_node_count;+ } else {+ break;}- items.push(ListItem { children, ordinal });- } else if item.indent_level == indent_level + 1 && depth < 9 {- let nested = parse_list(- parser,- lines,- index,- item.ordered,- item.indent_level,- depth + 1,- new_count,- )?;+ } else if is_bullet_point_text(current_line) {if let Some(last) = items.last_mut() {- last.children.push(nested.node);+ last.children.push(Node::Text {+ content: trim(current_line).to_owned(),+ });+ }+ index += 1;+ new_count += 1;+ } else if is_list_continuation(current_line, indent_level) {+ if let Some(last) = items.last_mut() {+ let trimmed = trim_start(current_line);+ let parsed = crate::inline::parse_inline(+ self.parser,+ trimmed,+ lines[index].offset + (current_line.len() - trimmed.len()),+ )?;+ new_count += parsed.len();+ last.children.extend(parsed);}- index = nested.new_line_index;- new_count = nested.new_node_count;+ index += 1;} else {break;}- } else if is_bullet_point_text(current_line) {- if let Some(last) = items.last_mut() {- last.children.push(Node::Text {- content: trim(current_line).to_owned(),- });- }- index += 1;- new_count += 1;- } else if is_list_continuation(current_line, indent_level) {- if let Some(last) = items.last_mut() {- let trimmed = trim_start(current_line);- let parsed = crate::inline::parse_inline(- parser,- trimmed,- lines[index].offset + (current_line.len() - trimmed.len()),- )?;- last.children.extend(parsed.iter().cloned());- new_count += parsed.len();+ if items.len() > MAX_AST_NODES {+ break;}- index += 1;- } else {- break;- }- if items.len() > MAX_AST_NODES {- break;}+ Ok(ListResult {+ node: Node::List { ordered, items },+ new_line_index: index,+ new_node_count: new_count,+ })}- Ok(ListResult {- node: Node::List { ordered, items },- new_line_index: index,- new_node_count: new_count,- })-}-fn try_parse_nested_list(- parser: &mut MarkdownParser,- lines: &[Line],- current: usize,- parent_indent: usize,- depth: usize,- node_count: usize,-) -> Result<Option<ListResult>, ParseError> {- if current >= lines.len() {- return Ok(None);- }- let trimmed = trim_start(&lines[current].text);- if starts_with(trimmed, "```")- && let Some(result) = parse_code_block(lines, current)?- {- return Ok(Some(ListResult {- node: result.node,- new_line_index: result.new_line_index,- new_node_count: node_count + 1,- }));+ fn try_parse_nested_list(+ &mut self,+ current: usize,+ parent_indent: usize,+ depth: usize,+ node_count: usize,+ ) -> Result<Option<ListResult>, ParseError> {+ let lines = self.lines;+ if current >= lines.len() {+ return Ok(None);+ }+ let trimmed = trim_start(&lines[current].text);+ if starts_with(trimmed, "```")+ && let Some(result) = parse_code_block(+ lines,+ current,+ &lines[current].text,+ lines[current].offset,+ self.fences,+ )+ {+ return Ok(Some(ListResult {+ node: result.node,+ new_line_index: result.new_line_index,+ new_node_count: node_count + 1,+ }));+ }+ if let Some(item) = match_list_item(&lines[current].text)+ && item.indent_level > parent_indent+ && depth < 9+ {+ return self+ .parse_list(+ current,+ ListLevel {+ ordered: item.ordered,+ indent_level: item.indent_level,+ depth: depth + 1,+ },+ node_count,+ )+ .map(Some);+ }+ Ok(None)}- if let Some(item) = match_list_item(&lines[current].text)- && item.indent_level > parent_indent- && depth < 9- {- return parse_list(- parser,- lines,- current,- item.ordered,- item.indent_level,- depth + 1,- node_count,- )- .map(Some);- }- Ok(None)}pub(crate) fn match_list_item(line: &str) -> Option<ListMatch<'_>> {@@ -1028,41 +1169,22 @@ fn parse_table(lines: &[Line],current: usize,) -> Result<Option<TableResult>, ParseError> {- if current + 2 >= lines.len() {- return Ok(None);- }- let header_line = trim_table_line(&lines[current]);- let align_line = trim_table_line(&lines[current + 1]);- if !header_line.text.contains('|') || !align_line.text.contains('|') {- return Ok(None);- }- let header_cells = split_table_cells(&header_line);- if header_cells.is_empty() || !cells_have_content(&header_cells) {- return Ok(None);- }- let align_cells = split_table_cells(&align_line);- let Some(alignments) = parse_alignments(&align_cells) else {+ let Some((header_cells, alignments)) = table_header(lines, current) else {return Ok(None);};- if header_cells.len() != alignments.len() {- return Ok(None);- }let header_row = create_table_row(parser, &header_cells)?;let column_count = header_cells.len();let mut rows = Vec::new();let mut index = current + 2;while index < lines.len() {let line = trim_table_line(&lines[index]);- if !line.text.contains('|') || is_table_block_break(&line.text) {+ if !is_table_body_line(line.text) {break;}- let cells = normalize_column_count(split_table_cells(&line), column_count);+ let cells = normalize_column_count(split_table_cells(line), column_count);rows.push(create_table_row(parser, &cells)?);index += 1;}- if rows.is_empty() {- return Ok(None);- }Ok(Some(TableResult {node: Node::Table {header: Box::new(header_row),@@ -1073,31 +1195,52 @@ fn parse_table(}))}-fn trim_table_line(line: &Line) -> TableCellText {- let left = trim_start(&line.text);- let leading = line.text.len() - left.len();- TableCellText {- text: trim_right(left).to_owned(),- offset: line.offset + leading,+fn table_header(+ lines: &[Line],+ current: usize,+) -> Option<(Vec<TableCellText>, Vec<TableAlignment>)> {+ if current + 2 >= lines.len() {+ return None;}+ let header_line = trim_table_line(&lines[current]);+ let align_line = trim_table_line(&lines[current + 1]);+ if !header_line.text.contains('|')+ || !align_line.text.contains('|')+ || !align_line.text.bytes().all(|byte| {+ matches!(+ byte,+ b' ' | b':' | b'-' | b'|' | b'\\' | b'\t' | b'\r' | b'\n'+ )+ })+ || !is_table_body_line(trim_table_line(&lines[current + 2]).text)+ {+ return None;+ }+ let header_cells = split_table_cells(header_line);+ if header_cells.is_empty() || !cells_have_content(&header_cells) {+ return None;+ }+ let alignments = parse_alignments(&split_table_cells(align_line))?;+ (header_cells.len() == alignments.len()).then_some((header_cells, alignments))+}++fn is_table_body_line(text: &str) -> bool {+ text.contains('|') && !is_table_block_break(text)}-fn trim_table_cell(cell: &TableCellText) -> TableCellText {- let left = trim_start(&cell.text);- let leading = cell.text.len() - left.len();- TableCellText {- text: trim_right(left).to_owned(),- offset: cell.offset + leading,+fn trim_table_line(line: &Line) -> TableLine<'_> {+ let left = trim_start(&line.text);+ TableLine {+ text: trim_right(left),+ offset: line.offset + (line.text.len() - left.len()),}}-fn split_table_cells(source: &TableCellText) -> Vec<TableCellText> {- let line = &source.text;- let mut start = 0usize;+fn split_table_cells(source: TableLine<'_>) -> Vec<TableCellText> {+ let line = source.text;+ let bytes = line.as_bytes();+ let start = usize::from(line.starts_with('|'));let mut end = line.len();- if line.starts_with('|') {- start = 1;- }if end > start && line.ends_with('|') {end -= 1;}@@ -1107,26 +1250,30 @@ fn split_table_cells(source: &TableCellText) -> Vec<TableCellText> {}let mut cell = String::new();let mut cell_offset = source.offset + start;+ let mut segment_start = start;let mut i = start;while i < end {- if byte_at(line, i) == b'\\' && i + 1 < end && byte_at(line, i + 1) == b'|' {+ if bytes[i] == b'\\' && i + 1 < end && bytes[i + 1] == b'|' {+ cell.push_str(&line[segment_start..i]);cell.push('|');i += 2;+ segment_start = i;continue;}- if byte_at(line, i) == b'|' {+ if bytes[i] == b'|' {+ cell.push_str(&line[segment_start..i]);cells.push(TableCellText {text: std::mem::take(&mut cell),offset: cell_offset,});i += 1;+ segment_start = i;cell_offset = source.offset + i;continue;}- let ch = line[i..].chars().next().unwrap_or('\0');- cell.push(ch);- i += ch.len_utf8();+ i += 1;}+ cell.push_str(&line[segment_start..end]);cells.push(TableCellText {text: cell,offset: cell_offset,@@ -1171,11 +1318,14 @@ fn create_table_row() -> Result<Node, ParseError> {let mut out = Vec::new();for cell in cells {- let trimmed_cell = trim_table_cell(cell);- let value = trimmed_cell.text;- let parsed = crate::inline::parse_inline(parser, &value, trimmed_cell.offset)?;+ let left = trim_start(&cell.text);+ let value = trim_right(left);+ let offset = cell.offset + (cell.text.len() - left.len());+ let parsed = crate::inline::parse_inline(parser, value, offset)?;let children = if parsed.is_empty() {- vec![Node::Text { content: value }]+ vec![Node::Text {+ content: value.to_owned(),+ }]} else {parsed};@@ -1192,7 +1342,12 @@ pub(crate) fn is_block_start(line: &str, flags: u32) -> bool {|| is_blockquote_start(line, flags)}-pub(crate) fn opens_code_block_midline(lines: &[Line], index: usize, flags: u32) -> bool {+pub(crate) fn opens_code_block_midline(+ lines: &[Line],+ index: usize,+ flags: u32,+ fences: &mut ClosingFenceLines,+) -> bool {if !ParserFlags::has(flags, ParserFlags::ALLOW_CODE_BLOCKS) || index >= lines.len() {return false;}@@ -1208,31 +1363,19 @@ pub(crate) fn opens_code_block_midline(lines: &[Line], index: usize, flags: u32)if has_open_inline_code(prefix) {return false;}- let code_lines = slice_lines_from_fence(lines, index, fence_pos);- matches!(parse_code_block(&code_lines, 0), Ok(Some(_)))+ scan_code_block(+ lines,+ index,+ &line[fence_pos..],+ lines[index].offset + fence_pos,+ fences,+ CodeBlockMode::Probe,+ )+ .is_some()}pub(crate) fn is_table_start(lines: &[Line], index: usize, flags: u32) -> bool {- if !ParserFlags::has(flags, ParserFlags::ALLOW_TABLES) || index + 2 >= lines.len() {- return false;- }- let header_line = trim_table_line(&lines[index]);- let align_line = trim_table_line(&lines[index + 1]);- if !header_line.text.contains('|') || !align_line.text.contains('|') {- return false;- }- let header_cells = split_table_cells(&header_line);- if header_cells.is_empty() || !cells_have_content(&header_cells) {- return false;- }- let Some(alignments) = parse_alignments(&split_table_cells(&align_line)) else {- return false;- };- if header_cells.len() != alignments.len() {- return false;- }- let body_line = trim_table_line(&lines[index + 2]);- body_line.text.contains('|') && !is_table_block_break(&body_line.text)+ ParserFlags::has(flags, ParserFlags::ALLOW_TABLES) && table_header(lines, index).is_some()}pub(crate) fn is_heading_start(line: &str, flags: u32) -> bool {@@ -1289,16 +1432,6 @@ fn is_escaped_fence(line: &str, fence_pos: usize) -> bool {backslashes % 2 == 1}-fn slice_lines_from_fence(lines: &[Line], current: usize, fence_pos: usize) -> Vec<Line> {- let mut out = Vec::with_capacity(lines.len() - current);- out.push(Line {- text: lines[current].text[fence_pos..].to_owned(),- offset: lines[current].offset + fence_pos,- });- out.extend(lines[current + 1..].iter().cloned());- out-}-fn normalise_ordinal(items: &[ListItem], ordinal: Option<usize>, ordered: bool) -> Option<usize> {if !ordered {return None;@@ -1334,23 +1467,20 @@ fn normalize_column_count(cells: Vec<TableCellText>, expected: usize) -> Vec<Tabif cells.len() == expected {return cells;}- let mut out = Vec::with_capacity(expected);- for index in 0..expected {- if let Some(cell) = cells.get(index) {- out.push(cell.clone());- } else {- out.push(TableCellText {+ let mut cells = cells.into_iter();+ let mut out: Vec<TableCellText> = (0..expected)+ .map(|_| {+ cells.next().unwrap_or(TableCellText {text: String::new(),offset: 0,- });- }- }- if cells.len() > expected && expected > 0 {- let mut combined = out[expected - 1].clone();- for extra in &cells[expected..] {- combined.text = concat3(&combined.text, "|", &extra.text);+ })+ })+ .collect();+ if let Some(last) = out.last_mut() {+ for extra in cells {+ last.text.push('|');+ last.text.push_str(&extra.text);}- out[expected - 1] = combined;}out}diff --git a/packages/markdown_parser/rust/src/constants.rs b/packages/markdown_parser/rust/src/constants.rsindex 33b4a168c..a5fe8aad6 100644--- a/packages/markdown_parser/rust/src/constants.rs+++ b/packages/markdown_parser/rust/src/constants.rs@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterpub const CODE_FENCE_LENGTH: usize = 3;+pub const MAX_CODE_BLOCK_SEARCH_LINES: usize = 1000;pub const MAX_AST_NODES: usize = 10_000;pub const MAX_INLINE_DEPTH: usize = 10;pub const MAX_LINES: usize = 10_000;diff --git a/packages/markdown_parser/rust/src/emoji.rs b/packages/markdown_parser/rust/src/emoji.rsindex 96c73f9b1..61fc9401b 100644--- a/packages/markdown_parser/rust/src/emoji.rs+++ b/packages/markdown_parser/rust/src/emoji.rs@@ -34,6 +34,18 @@ pub struct SkinEmoji {impl EmojiContext {pub fn parse(bytes: &str) -> Self {let mut context = Self::default();+ let mut counts = [0usize; 3];+ for line in bytes.lines() {+ match line.as_bytes().first() {+ Some(b'S') => counts[0] += 1,+ Some(b'C') => counts[1] += 1,+ Some(b'K') => counts[2] += 1,+ _ => {}+ }+ }+ context.standard.reserve_exact(counts[0]);+ context.shortcodes.reserve_exact(counts[1]);+ context.skins.reserve_exact(counts[2]);for line in bytes.lines() {if line.is_empty() {continue;@@ -93,20 +105,70 @@ impl EmojiContext {_ => {}}}+ context.standard.sort_by_key(|record| record.offset);+ context+ .shortcodes+ .sort_by(|left, right| left.name.cmp(&right.name));+ context+ .skins+ .sort_by(|left, right| (&left.name, left.tone).cmp(&(&right.name, right.tone)));context}- pub fn standard_at(&self, offset: usize) -> Option<&StandardEmoji> {- self.standard.iter().find(|record| record.offset == offset)+ pub(crate) fn standard_cursor(&self) -> StandardCursor<'_> {+ StandardCursor {+ emojis: &self.standard,+ index: 0,+ }}pub fn shortcode(&self, name: &str) -> Option<&ShortcodeEmoji> {- self.shortcodes.iter().find(|record| record.name == name)+ let index = self+ .shortcodes+ .partition_point(|record| record.name.as_str() < name);+ self.shortcodes+ .get(index)+ .filter(|record| record.name == name)}pub fn skin(&self, name: &str, tone: u8) -> Option<&SkinEmoji> {+ let index = self+ .skins+ .partition_point(|record| (record.name.as_str(), record.tone) < (name, tone));self.skins- .iter()- .find(|record| record.tone == tone && record.name == name)+ .get(index)+ .filter(|record| record.name == name && record.tone == tone)+ }+}++pub(crate) struct StandardCursor<'a> {+ emojis: &'a [StandardEmoji],+ index: usize,+}++impl<'a> StandardCursor<'a> {+ pub(crate) fn next_offset(&mut self, offset: usize) -> Option<usize> {+ self.seek(offset);+ self.emojis.get(self.index).map(|emoji| emoji.offset)+ }++ pub(crate) fn at(&mut self, offset: usize) -> Option<&'a StandardEmoji> {+ self.seek(offset);+ self.emojis+ .get(self.index)+ .filter(|emoji| emoji.offset == offset)+ }++ fn seek(&mut self, offset: usize) {+ while self+ .emojis+ .get(self.index)+ .is_some_and(|emoji| emoji.offset < offset)+ {+ self.index += 1;+ }+ while self.index > 0 && self.emojis[self.index - 1].offset >= offset {+ self.index -= 1;+ }}}diff --git a/packages/markdown_parser/rust/src/inline.rs b/packages/markdown_parser/rust/src/inline.rsindex 5b0544913..979e95ce7 100644--- a/packages/markdown_parser/rust/src/inline.rs+++ b/packages/markdown_parser/rust/src/inline.rs@@ -2,19 +2,20 @@use crate::ast::{EmojiKind, Node, ParserFlags, ParserResult};use crate::constants::{CODE_FENCE_LENGTH, MAX_INLINE_DEPTH, MAX_LINE_LENGTH};-use crate::emoji::{EmojiContext, StandardEmoji};+use crate::emoji::{StandardCursor, StandardEmoji};use crate::links;use crate::normalize::{combine_adjacent_text, compact_empty_text_nodes, flatten_top_level_formatting,merge_adjacent_text_simple, normalize_nodes,};use crate::parser::{MarkdownParser, ParseError};+use crate::scan::{Closing, Lookahead, Pattern, Token, step};use crate::text::{- advance_one, append_text, byte_at, ends_with, has_visible_content, is_alpha_numeric,- is_escapable_character, string_from_lossy,+ append_text, byte_at, has_visible_content, is_alpha_numeric, is_escapable_character,+ string_from_lossy,};-#[derive(Clone, Copy, Debug, Eq, PartialEq)]+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]struct FormattingEntry {delimiter: u8,is_double: bool,@@ -22,26 +23,28 @@ struct FormattingEntry {#[derive(Clone, Debug, Default)]struct FormattingContext {- entries: Vec<FormattingEntry>,+ entries: [FormattingEntry; MAX_INLINE_DEPTH],+ len: usize,}impl FormattingContext {fn can_enter(&self, delimiter: u8, is_double: bool) -> bool {- self.entries.len() < MAX_INLINE_DEPTH && !self.is_active(delimiter, is_double)+ self.len < MAX_INLINE_DEPTH && !self.is_active(delimiter, is_double)}fn is_active(&self, delimiter: u8, is_double: bool) -> bool {- self.entries+ self.entries[..self.len].iter().any(|entry| entry.delimiter == delimiter && entry.is_double == is_double)}fn push(&mut self, delimiter: u8, is_double: bool) {- if self.entries.len() < MAX_INLINE_DEPTH {- self.entries.push(FormattingEntry {+ if self.len < MAX_INLINE_DEPTH {+ self.entries[self.len] = FormattingEntry {delimiter,is_double,- });+ };+ self.len += 1;}}}@@ -57,15 +60,34 @@ enum FormattingNodeType {}#[derive(Clone, Copy, Debug)]-struct FormattingMarkerInfo<'a> {- marker: &'a str,+struct FormattingMarkerInfo {+ delimiter: u8,node_type: FormattingNodeType,marker_length: usize,}-#[derive(Clone, Copy, Debug)]-struct FormattingEnd {- end_position: usize,+impl FormattingMarkerInfo {+ fn is_double(self) -> bool {+ self.marker_length > 1+ }++ fn is_triple_emphasis(self) -> bool {+ self.marker_length == 3 && self.node_type == FormattingNodeType::Emphasis+ }++ fn scan_index(self) -> usize {+ match (self.delimiter, self.marker_length) {+ (b'*', 2) => 0,+ (b'_', 2) => 1,+ (b'~', _) => 2,+ (b'|', _) => 3,+ (b'*', 3) => 4,+ (b'_', 3) => 5,+ (b'*', _) => 6,+ (b'_', _) => 7,+ _ => 8,+ }+ }}pub fn parse_inline(@@ -95,15 +117,39 @@ fn parse_inline_with_context(context: &mut FormattingContext,) -> Result<Vec<Node>, ParseError> {let mut nodes = Vec::new();- let mut accumulated = Vec::<u8>::new();+ let mut accumulated = String::new();+ let mut scan = Lookahead::new(text);+ let emoji_context = parser.shared_emoji_context();+ let mut emojis = emoji_context.standard_cursor();let mut position = 0usize;while position < text.len() {let current = byte_at(text, position);+ if is_plain_text_byte(current) {+ let run_limit = emojis+ .next_offset(base_offset + position)+ .map_or(text.len(), |offset| (offset - base_offset).min(text.len()));+ if run_limit > position {+ let run_end = text.as_bytes()[position..run_limit]+ .iter()+ .position(|byte| !is_plain_text_byte(*byte))+ .map_or(run_limit, |offset| position + offset);+ let room = (MAX_LINE_LENGTH + 1)+ .saturating_sub(accumulated.len())+ .max(1);+ let end = run_end.min(position + room);+ accumulated.push_str(&text[position..end]);+ position = end;+ if accumulated.len() > MAX_LINE_LENGTH {+ flush_bounded_accumulated_text(&mut nodes, &mut accumulated, MAX_LINE_LENGTH);+ break;+ }+ continue;+ }+ }if current == b'\\' && position + 1 < text.len() {let next = byte_at(text, position + 1);if next == b'_' && position > 0 && byte_at(text, position - 1) == 0xaf {- accumulated.push(current);- accumulated.push(next);+ accumulated.push_str("\\_");position += 2;continue;}@@ -113,23 +159,21 @@ fn parse_inline_with_context(kind: EmojiKind::Standard { raw, .. },} = &result.node{- accumulated.extend_from_slice(raw.as_bytes());+ accumulated.push_str(raw);position += 1 + result.advance;continue;}- if let Some(emoji) = parser- .emoji_context()- .standard_at(base_offset + position + 1)+ if let Some(emoji) = emojis.at(base_offset + position + 1)&& text[position + 1..].starts_with(&emoji.raw){- accumulated.extend_from_slice(emoji.raw.as_bytes());+ accumulated.push_str(&emoji.raw);position += 1 + emoji.len;continue;}if next == b'`' {let run = backtick_run_length(text, position + 1);if run >= CODE_FENCE_LENGTH {- accumulated.extend(std::iter::repeat_n(b'`', run));+ accumulated.extend(std::iter::repeat_n('`', run));position += 1 + run;continue;}@@ -138,18 +182,18 @@ fn parse_inline_with_context(|| is_ordered_list_marker_dot_escape(text, position)|| is_word_dot_escape(text, position){- accumulated.push(next);+ accumulated.push(char::from(next));position += 2;continue;}}let remaining = &text[position..];- if !ends_with(&accumulated, b"<\"")- && !ends_with(&accumulated, b"<'")+ if !accumulated.ends_with("<\"")+ && !accumulated.ends_with("<'")&& ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)&& links::starts_with_url(remaining)- && let Some(result) = links::parse_url_segment(remaining, parser.flags())+ && let Some(result) = links::parse_url_segment(&mut scan, position, parser.flags()){flush_accumulated_text(&mut nodes, &mut accumulated);nodes.push(result.node);@@ -162,7 +206,7 @@ fn parse_inline_with_context(&& byte_at(text, position + 1) != b'_'&& links::is_word_underscore(text, position){- accumulated.push(b'_');+ accumulated.push('_');position += 1;continue;}@@ -171,7 +215,7 @@ fn parse_inline_with_context(flush_accumulated_text(&mut nodes, &mut accumulated);push_regional_indicator_pair(&mut nodes,- parser.emoji_context(),+ &mut emojis,base_offset + position,&remaining[..result.advance],result.node,@@ -180,7 +224,7 @@ fn parse_inline_with_context(continue;}- if let Some(emoji) = parser.emoji_context().standard_at(base_offset + position)+ if let Some(emoji) = emojis.at(base_offset + position)&& remaining.starts_with(&emoji.raw){flush_accumulated_text(&mut nodes, &mut accumulated);@@ -195,9 +239,8 @@ fn parse_inline_with_context(continue;}- if current == b'<'- && position + 2 < text.len()- && let Some(result) = links::parse_custom_emoji(remaining)+ if current == b':'+ && let Some(result) = links::parse_emoji_shortcode(parser, remaining){flush_accumulated_text(&mut nodes, &mut accumulated);nodes.push(result.node);@@ -206,8 +249,8 @@ fn parse_inline_with_context(}if current == b'<'- && remaining.starts_with("<t:")- && let Some(result) = links::parse_timestamp(remaining)+ && let Some(close) = scan.next(Pattern::AngleClose, position, text.len())+ && let Some(result) = parse_angle_bracket(parser, &mut scan, position, close){flush_accumulated_text(&mut nodes, &mut accumulated);nodes.push(result.node);@@ -215,59 +258,6 @@ fn parse_inline_with_context(continue;}- if current == b':'- && let Some(result) = links::parse_emoji_shortcode(parser, remaining)- {- flush_accumulated_text(&mut nodes, &mut accumulated);- nodes.push(result.node);- position += result.advance;- continue;- }-- if current == b'<' && position + 1 < text.len() {- let next = byte_at(text, position + 1);- if next == b'+'- && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)- && let Some(result) = links::parse_phone_link(parser, remaining)- {- flush_accumulated_text(&mut nodes, &mut accumulated);- nodes.push(result.node);- position += result.advance;- continue;- }- if remaining.starts_with("<sms:")- && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)- && let Some(result) = links::parse_sms_link(parser, remaining)- {- flush_accumulated_text(&mut nodes, &mut accumulated);- nodes.push(result.node);- position += result.advance;- continue;- }- if matches!(next, b'@' | b'#' | b'/' | b'i')- && let Some(result) = links::parse_mention(remaining, parser.flags())- {- flush_accumulated_text(&mut nodes, &mut accumulated);- nodes.push(result.node);- position += result.advance;- continue;- }- if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS) {- if let Some(result) = links::parse_autolink(remaining, parser.flags()) {- flush_accumulated_text(&mut nodes, &mut accumulated);- nodes.push(result.node);- position += result.advance;- continue;- }- if let Some(result) = links::parse_email_link(parser, remaining) {- flush_accumulated_text(&mut nodes, &mut accumulated);- nodes.push(result.node);- position += result.advance;- continue;- }- }- }-if current == b'@' && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_EVERYONE_MENTIONS){let is_escaped = position > 0 && byte_at(text, position - 1) == b'\\';@@ -291,7 +281,7 @@ fn parse_inline_with_context(if current >= 0x80 {let char_len = remaining.chars().next().map_or(1, char::len_utf8);- accumulated.extend_from_slice(&text.as_bytes()[position..position + char_len]);+ accumulated.push_str(&text[position..position + char_len]);position += char_len;continue;}@@ -302,7 +292,7 @@ fn parse_inline_with_context(&& (is_double_underscore|| !(current == b'_'&& !accumulated.is_empty()- && is_alpha_numeric(*accumulated.last().unwrap_or(&0))))+ && is_alpha_numeric(*accumulated.as_bytes().last().unwrap_or(&0)))){let previous = if position > 0 {byte_at(text, position - 1)@@ -311,7 +301,8 @@ fn parse_inline_with_context(};if let Some(result) = parse_special_sequence(parser,- remaining,+ &mut scan,+ position,base_offset + position,context,previous,@@ -323,7 +314,7 @@ fn parse_inline_with_context(}}- accumulated.push(current);+ accumulated.push(char::from(current));position += 1;if accumulated.len() > MAX_LINE_LENGTH {flush_bounded_accumulated_text(&mut nodes, &mut accumulated, MAX_LINE_LENGTH);@@ -358,17 +349,17 @@ fn parse_regional_indicator_flag(text: &str) -> Option<ParserResult> {fn push_regional_indicator_pair(nodes: &mut Vec<Node>,- emoji_context: &EmojiContext,+ emojis: &mut StandardCursor<'_>,offset: usize,pair: &str,flag: Node,) {let (first, second) = pair.split_at(pair.len() / 2);- match emoji_context.standard_at(offset) {+ match emojis.at(offset) {Some(emoji) if emoji.raw == pair => nodes.push(standard_emoji_node(emoji)),Some(emoji) if emoji.raw == first => {nodes.push(standard_emoji_node(emoji));- match emoji_context.standard_at(offset + first.len()) {+ match emojis.at(offset + first.len()) {Some(emoji) if emoji.raw == second => nodes.push(standard_emoji_node(emoji)),_ => nodes.extend(regional_indicator_emoji(second)),}@@ -429,13 +420,11 @@ fn is_ordered_list_marker_dot_escape(text: &str, backslash_position: usize) -> bif digit_start == backslash_position {return false;}- let line_start = text[..digit_start]- .rfind('\n')- .map_or(0, |position| position + 1);- if text.as_bytes()[line_start..digit_start]- .iter()- .any(|byte| *byte != b' ')- {+ let mut line_start = digit_start;+ while line_start > 0 && byte_at(text, line_start - 1) == b' ' {+ line_start -= 1;+ }+ if line_start > 0 && byte_at(text, line_start - 1) != b'\n' {return false;}let indent = digit_start - line_start;@@ -450,80 +439,52 @@ fn is_word_dot_escape(text: &str, backslash_position: usize) -> bool {previous.is_ascii_alphanumeric() || previous == b'.'}+fn parse_angle_bracket(+ parser: &MarkdownParser,+ scan: &mut Lookahead<'_>,+ position: usize,+ close: usize,+) -> Option<ParserResult> {+ let tag = &scan.text[position..=close];+ links::parse_custom_emoji(tag)+ .or_else(|| links::parse_timestamp(tag))+ .or_else(|| links::parse_phone_link(parser, tag))+ .or_else(|| links::parse_sms_link(parser, tag))+ .or_else(|| links::parse_mention(tag, parser.flags()))+ .or_else(|| links::parse_autolink(scan, position, close, parser.flags()))+ .or_else(|| links::parse_email_link(parser, tag))+}+fn parse_special_sequence(parser: &mut MarkdownParser,- text: &str,+ scan: &mut Lookahead<'_>,+ position: usize,base_offset: usize,context: &mut FormattingContext,previous: u8,) -> Result<Option<ParserResult>, ParseError> {- if text.is_empty() {- return Ok(None);- }- match byte_at(text, 0) {- b'*' | b'_' | b'~' | b'|' | b'`' => {- if let Some(result) = parse_formatting(parser, text, base_offset, context, previous)? {- return Ok(Some(result));- }- }- b'@' => {- if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_EVERYONE_MENTIONS) {- if text.starts_with("@everyone") {- return Ok(Some(ParserResult {- node: Node::Mention {- kind: crate::ast::MentionKind::Everyone,- },- advance: 9,- }));- }- if text.starts_with("@here") {- return Ok(Some(ParserResult {- node: Node::Mention {- kind: crate::ast::MentionKind::Here,- },- advance: 5,- }));- }- }- }- b'[' => {- if let Some(result) = links::parse_timestamp(text) {- return Ok(Some(result));- }- if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_MASKED_LINKS)- && let Some(result) = links::parse_masked_link(parser, text, base_offset)?- {- return Ok(Some(result));- }- }- _ => {}- }- if byte_at(text, 0) != b'['- && let Some(result) = links::parse_timestamp(text)- {- return Ok(Some(result));+ if byte_at(scan.text, position) != b'[' {+ return parse_formatting(parser, scan, position, base_offset, context, previous);}- if byte_at(text, 0) != b'<'- && byte_at(text, 0) != b'['- && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_MASKED_LINKS)- && let Some(result) = links::parse_masked_link(parser, text, base_offset)?- {- return Ok(Some(result));+ if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_MASKED_LINKS) {+ return Ok(None);}- Ok(None)+ links::parse_masked_link(parser, scan, position, base_offset)}fn parse_formatting(parser: &mut MarkdownParser,- text: &str,+ scan: &mut Lookahead<'_>,+ position: usize,base_offset: usize,context: &mut FormattingContext,previous: u8,) -> Result<Option<ParserResult>, ParseError> {+ let text = &scan.text[position..];if text.len() < 2 {return Ok(None);}- let Some(marker) = get_formatting_marker_info(text) else {+ let Some(marker) = get_formatting_marker_info(scan, position) else {return Ok(None);};if marker.node_type == FormattingNodeType::Spoiler@@ -531,23 +492,23 @@ fn parse_formatting({return Ok(None);}- if !context.can_enter(byte_at(marker.marker, 0), marker.marker.len() > 1) {+ if !context.can_enter(marker.delimiter, marker.is_double()) {return Ok(None);}if marker.marker_length == 1 && marker.node_type == FormattingNodeType::Emphasis {let next = byte_at(text, marker.marker_length);- if !links::can_open_single_emphasis(byte_at(marker.marker, 0), previous, next) {+ if !links::can_open_single_emphasis(marker.delimiter, previous, next) {return Ok(None);}}- let Some(end) = find_formatting_end(text, marker) else {+ let Some(end_position) = find_formatting_end(scan, position, marker) else {return Ok(None);};- let inner = &text[marker.marker_length..end.end_position];- if !formatting_inner_has_visible_content(marker, inner) {+ let inner = &text[marker.marker_length..end_position];+ if !has_visible_content(inner) {return Ok(None);}- let is_block = context.is_active(byte_at(marker.marker, 0), marker.marker.len() > 1);+ let is_block = context.is_active(marker.delimiter, marker.is_double());let node = create_formatting_node(parser,marker,@@ -557,262 +518,142 @@ fn parse_formatting()?;Ok(Some(ParserResult {node,- advance: end.end_position + marker.marker_length,+ advance: end_position + marker.marker_length,}))}-fn formatting_inner_has_visible_content(marker: FormattingMarkerInfo<'_>, inner: &str) -> bool {- if marker.node_type == FormattingNodeType::InlineCode {- return has_visible_content(&unescape_inline_code(inner));- }- has_visible_content(inner)-}--fn get_formatting_marker_info(text: &str) -> Option<FormattingMarkerInfo<'_>> {- if text.is_empty() || !is_formatting_char(byte_at(text, 0)) {+fn get_formatting_marker_info(+ scan: &mut Lookahead<'_>,+ position: usize,+) -> Option<FormattingMarkerInfo> {+ let text = scan.text;+ let first = byte_at(text, position);+ if !is_formatting_char(first) {return None;}- let first = byte_at(text, 0);- let second = byte_at(text, 1);- let third = byte_at(text, 2);- if first == b'*' && second == b'*' && third == b'*' {- return Some(marker("***", FormattingNodeType::Emphasis, 3));- }- if first == b'_' && second == b'_' && third == b'_' {- return Some(marker("___", FormattingNodeType::Emphasis, 3));- }- if first == b'|' && second == b'|' {- return Some(marker("||", FormattingNodeType::Spoiler, 2));- }- if first == b'~' && second == b'~' {- return Some(marker("~~", FormattingNodeType::Strikethrough, 2));- }- if first == b'*' && second == b'*' {- return Some(marker("**", FormattingNodeType::Strong, 2));- }- if first == b'_' && second == b'_' {- return Some(marker("__", FormattingNodeType::Underline, 2));- }- if first == b'`' {- let mut count = 1;- while count < text.len() && byte_at(text, count) == b'`' {- count += 1;- }- return Some(FormattingMarkerInfo {- marker: &text[..count],- node_type: FormattingNodeType::InlineCode,- marker_length: count,- });- }- if first == b'*' {- return Some(marker("*", FormattingNodeType::Emphasis, 1));- }- if first == b'_' {- return Some(marker("_", FormattingNodeType::Emphasis, 1));- }- None-}--fn marker(- marker: &'static str,- node_type: FormattingNodeType,- marker_length: usize,-) -> FormattingMarkerInfo<'static> {- FormattingMarkerInfo {- marker,+ let second = byte_at(text, position + 1);+ let third = byte_at(text, position + 2);+ let (node_type, marker_length) = match (first, second, third) {+ (b'*', b'*', b'*') | (b'_', b'_', b'_') => (FormattingNodeType::Emphasis, 3),+ (b'|', b'|', _) => (FormattingNodeType::Spoiler, 2),+ (b'~', b'~', _) => (FormattingNodeType::Strikethrough, 2),+ (b'*', b'*', _) => (FormattingNodeType::Strong, 2),+ (b'_', b'_', _) => (FormattingNodeType::Underline, 2),+ (b'`', _, _) => (+ FormattingNodeType::InlineCode,+ scan.run_end(position, b'`') - position,+ ),+ (b'*' | b'_', _, _) => (FormattingNodeType::Emphasis, 1),+ _ => return None,+ };+ Some(FormattingMarkerInfo {+ delimiter: first,node_type,marker_length,- }+ })}-fn find_formatting_end(text: &str, marker: FormattingMarkerInfo<'_>) -> Option<FormattingEnd> {- let mut position = marker.marker_length;- let mut nested_level = 0usize;- if text.len() < marker.marker_length * 2 {+fn find_formatting_end(+ scan: &mut Lookahead<'_>,+ opener: usize,+ marker: FormattingMarkerInfo,+) -> Option<usize> {+ let len = scan.text.len();+ if len - opener < marker.marker_length * 2 {return None;}+ let mut start = opener + marker.marker_length;if marker.node_type == FormattingNodeType::InlineCode && marker.marker_length > 1 {- while position < text.len() {- let current = byte_at(text, position);- if is_inline_code_line_break(current) {- return None;- }- if current == b'`' {- let mut count = 0usize;- let mut check = position;- while check < text.len() && byte_at(text, check) == b'`' {- count += 1;- check += 1;- }- if count >= marker.marker_length {- return Some(FormattingEnd {- end_position: position + count - marker.marker_length,- });- }- position = check;- continue;- }- position += advance_one(text, position);- if position > MAX_LINE_LENGTH {- break;- }- }- return None;+ return scan.code_span_end(opener, marker.marker_length, MAX_LINE_LENGTH);}- if marker.marker_length == 1 && marker.node_type == FormattingNodeType::InlineCode {- let marker_char = byte_at(marker.marker, 0);- while position < text.len() {- let current = byte_at(text, position);- if is_inline_code_line_break(current) {- return None;- }- if current == b'\\' && position + 1 < text.len() {- position += 2;- continue;- }- if current == marker_char {- if marker_char == b'`'- && position + 1 < text.len()- && byte_at(text, position + 1) == b'`'- {- while position < text.len() && byte_at(text, position) == b'`' {- position += 1;- }- continue;- }- return Some(FormattingEnd {- end_position: position,- });- }- position += advance_one(text, position);- if position > MAX_LINE_LENGTH {- break;- }- }- return None;- }- if marker.marker_length == 1 && marker.node_type == FormattingNodeType::Emphasis {- let marker_char = byte_at(marker.marker, 0);- while position < text.len() {- let current = byte_at(text, position);- if current == b'\\' && position + 1 < text.len() {- position += 2;- continue;- }- if current == marker_char {- if marker_char == b'*'- && let Some(nested_end) = find_nested_asterisk_run_end(text, position)- {- position = nested_end;- continue;- }- if marker_char == b'_'- && position + 1 < text.len()- && byte_at(text, position + 1) == b'_'- {- position += 2;- continue;- }- let prev = if position > 0 {- byte_at(text, position - 1)- } else {- 0- };- let next = byte_at(text, position + 1);- if links::can_close_single_emphasis(marker_char, prev, next) {- return Some(FormattingEnd {- end_position: position,- });- }- }- position += advance_one(text, position);- if position > MAX_LINE_LENGTH {- break;+ if marker.node_type == FormattingNodeType::Spoiler+ && scan.text.as_bytes()[start..].starts_with(b"||")+ && start + 2 < len+ {+ start += 1;+ }+ let close_len = if marker.is_triple_emphasis() { 3 } else { 1 };+ let closer = scan.closer(+ Closing::Formatting(marker.scan_index()),+ start,+ close_len,+ |scan, position| classify_formatting(scan, marker, position),+ );+ closer?+ .within(opener, MAX_LINE_LENGTH)+ .map(|end| end - opener)+}++fn classify_formatting(+ scan: &mut Lookahead<'_>,+ marker: FormattingMarkerInfo,+ position: usize,+) -> Token {+ let text = scan.text;+ let bytes = text.as_bytes();+ let current = bytes[position];+ let delimiter = marker.delimiter;+ if marker.node_type == FormattingNodeType::InlineCode && is_inline_code_line_break(current) {+ return Token::Fail;+ }+ if current == b'\\' && position + 1 < bytes.len() {+ return Token::Skip(2);+ }+ let repeats = |count: usize| {+ bytes.len() - position >= count+ && bytes[position..position + count]+ .iter()+ .all(|byte| *byte == delimiter)+ };+ match (marker.node_type, marker.marker_length) {+ (FormattingNodeType::InlineCode, _) if current == b'`' => {+ if repeats(2) {+ Token::Skip(scan.run_end(position, b'`') - position)+ } else {+ Token::Close}}- return None;- }- if marker.node_type == FormattingNodeType::InlineCode {- while position < text.len() {- let current = byte_at(text, position);- if is_inline_code_line_break(current) {- return None;- }- if current == b'`' {- return Some(FormattingEnd {- end_position: position,- });+ (FormattingNodeType::Emphasis, 1) if current == delimiter => {+ if delimiter == b'*'+ && let Some(nested_end) = find_nested_asterisk_run_end(scan, position)+ {+ return Token::Skip(nested_end - position);}- position += advance_one(text, position);- if position > MAX_LINE_LENGTH {- break;+ if delimiter == b'_' && repeats(2) {+ return Token::Skip(2);}- }- return None;- }- let first_marker_char = byte_at(marker.marker, 0);- let is_double = marker.marker.len() > 1;- while position < text.len() {- if byte_at(text, position) == b'\\' && position + 1 < text.len() {- position += 2;- continue;- }- if text.as_bytes()[position..].starts_with(marker.marker.as_bytes()) {- if nested_level == 0 {- if marker.node_type == FormattingNodeType::Spoiler- && position == marker.marker_length- && position + marker.marker.len() < text.len()- {- position += 1;- continue;- }- return Some(FormattingEnd {- end_position: position,- });+ let next = byte_at(text, position + 1);+ if links::can_close_single_emphasis(delimiter, bytes[position - 1], next) {+ return Token::Close;}- nested_level -= 1;- position += marker.marker.len();- continue;- }- if is_double- && position + 1 < text.len()- && byte_at(text, position) == first_marker_char- && byte_at(text, position + 1) == first_marker_char- {- nested_level += 1;- }- position += advance_one(text, position);- if position > MAX_LINE_LENGTH {- break;+ step(text, position)}+ (FormattingNodeType::Emphasis, 3) if repeats(3) => Token::Close,+ (FormattingNodeType::Emphasis, 3) if repeats(2) => Token::Open(1),+ (FormattingNodeType::InlineCode | FormattingNodeType::Emphasis, _) => step(text, position),+ _ if repeats(2) => Token::Close,+ _ => step(text, position),}- None}fn is_inline_code_line_break(byte: u8) -> bool {matches!(byte, b'\n' | b'\r')}-fn find_nested_asterisk_run_end(text: &str, position: usize) -> Option<usize> {- let run_length = links::count_repeated_byte(text, position, b'*');+fn find_nested_asterisk_run_end(scan: &mut Lookahead<'_>, position: usize) -> Option<usize> {+ let text = scan.text;+ let run_length = scan.run_end(position, b'*') - position;if run_length < 2 {return None;}- let previous = if position > 0 {- byte_at(text, position - 1)- } else {- 0- };+ let previous = byte_at(text, position - 1);let next = byte_at(text, position + run_length);if !links::is_left_flanking_delimiter(previous, next) {return None;}- let nested_marker = get_formatting_marker_info(&text[position..])?;- if byte_at(nested_marker.marker, 0) != b'*' || nested_marker.marker_length < 2 {- return None;- }- let nested_end = find_formatting_end(&text[position..], nested_marker)?;- let end_position = position + nested_end.end_position + nested_marker.marker_length;+ let nested_marker = get_formatting_marker_info(scan, position)?;+ let nested_end = find_formatting_end(scan, position, nested_marker)?;+ let end_position = position + nested_end + nested_marker.marker_length;if end_position <= position || end_position > text.len() {return None;}@@ -821,7 +662,7 @@ fn find_nested_asterisk_run_end(text: &str, position: usize) -> Option<usize> {fn create_formatting_node(parser: &mut MarkdownParser,- marker: FormattingMarkerInfo<'_>,+ marker: FormattingMarkerInfo,inner: &str,inner_offset: usize,is_block: bool,@@ -831,7 +672,7 @@ fn create_formatting_node(content: unescape_inline_code(inner),});}- if marker.marker == "***" || marker.marker == "___" {+ if marker.is_triple_emphasis() {let mut strong_context = FormattingContext::default();strong_context.push(b'*', true);let inner_nodes =@@ -843,7 +684,7 @@ fn create_formatting_node(});}let mut new_context = FormattingContext::default();- new_context.push(byte_at(marker.marker, 0), marker.marker.len() > 1);+ new_context.push(marker.delimiter, marker.is_double());let children = parse_inline_with_context(parser, inner, inner_offset, &mut new_context)?;Ok(match marker.node_type {FormattingNodeType::Strong => Node::Strong { children },@@ -887,31 +728,26 @@ fn unescape_inline_code(content: &str) -> String {string_from_lossy(&out)}-fn flush_accumulated_text(nodes: &mut Vec<Node>, accumulated: &mut Vec<u8>) {- if accumulated.is_empty() {- return;- }- let content = string_from_lossy(accumulated);- append_text(nodes, content);+fn flush_accumulated_text(nodes: &mut Vec<Node>, accumulated: &mut String) {+ append_text(nodes, accumulated);accumulated.clear();}-fn flush_bounded_accumulated_text(- nodes: &mut Vec<Node>,- accumulated: &mut Vec<u8>,- max_len: usize,-) {- if accumulated.is_empty() {- return;- }+fn flush_bounded_accumulated_text(nodes: &mut Vec<Node>, accumulated: &mut String, max_len: usize) {let mut prefix_len = accumulated.len().min(max_len);- while prefix_len > 0 && std::str::from_utf8(&accumulated[..prefix_len]).is_err() {+ while !accumulated.is_char_boundary(prefix_len) {prefix_len -= 1;}- if prefix_len > 0 {- append_text(nodes, string_from_lossy(&accumulated[..prefix_len]));- }- accumulated.clear();+ accumulated.truncate(prefix_len);+ flush_accumulated_text(nodes, accumulated);+}++fn is_plain_text_byte(byte: u8) -> bool {+ byte.is_ascii()+ && !matches!(+ byte,+ b'\\' | b'h' | b'f' | b'_' | b'<' | b':' | b'@' | b'*' | b'~' | b'|' | b'`' | b'['+ )}fn is_formatting_char(char: u8) -> bool {diff --git a/packages/markdown_parser/rust/src/json.rs b/packages/markdown_parser/rust/src/json.rsindex f4e74decc..8b6921908 100644--- a/packages/markdown_parser/rust/src/json.rs+++ b/packages/markdown_parser/rust/src/json.rs@@ -1,15 +1,310 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use serde::Serialize;+use std::fmt::Write;-use crate::ast::Node;-use crate::parser::ParseError;+use crate::ast::{EmojiKind, ListItem, MentionKind, Node};-#[derive(Serialize)]-struct AstEnvelope<'a> {- nodes: &'a [Node],+macro_rules! key {+ ($name:literal) => {+ concat!(",\"", $name, "\":")+ };}-pub fn write_ast_json(nodes: &[Node]) -> Result<String, ParseError> {- Ok(serde_json::to_string(&AstEnvelope { nodes })?)+pub fn write_ast_json(nodes: &[Node], capacity: usize) -> String {+ let mut out = String::with_capacity(capacity);+ out.push_str("{\"nodes\":");+ write_nodes(&mut out, nodes);+ out.push('}');+ out}++fn write_nodes(out: &mut String, nodes: &[Node]) {+ out.push('[');+ for (index, node) in nodes.iter().enumerate() {+ if index > 0 {+ out.push(',');+ }+ write_node(out, node);+ }+ out.push(']');+}++fn write_node(out: &mut String, node: &Node) {+ out.push_str("{\"type\":\"");+ out.push_str(node.tag_name());+ out.push('"');+ match node {+ Node::Text { content } | Node::InlineCode { content } => {+ out.push_str(key!("content"));+ write_str(out, content);+ }+ Node::Blockquote {+ children,+ blank_lines,+ } => {+ write_children(out, children);+ if let Some(blank_lines) = blank_lines {+ out.push_str(key!("blankLines"));+ write_number(out, *blank_lines as u64);+ }+ }+ Node::Strong { children }+ | Node::Emphasis { children }+ | Node::Underline { children }+ | Node::Strikethrough { children }+ | Node::Subtext { children }+ | Node::Sequence { children } => write_children(out, children),+ Node::Spoiler { children, is_block } => {+ write_children(out, children);+ if let Some(is_block) = is_block {+ out.push_str(key!("isBlock"));+ write_bool(out, *is_block);+ }+ }+ Node::Heading { level, children } => {+ out.push_str(key!("level"));+ write_number(out, u64::from(*level));+ write_children(out, children);+ }+ Node::List { ordered, items } => {+ out.push_str(key!("ordered"));+ write_bool(out, *ordered);+ out.push_str(key!("items"));+ out.push('[');+ for (index, ListItem { children, ordinal }) in items.iter().enumerate() {+ if index > 0 {+ out.push(',');+ }+ out.push_str("{\"children\":");+ write_nodes(out, children);+ if let Some(ordinal) = ordinal {+ out.push_str(key!("ordinal"));+ write_number(out, *ordinal as u64);+ }+ out.push('}');+ }+ out.push(']');+ }+ Node::CodeBlock { language, content } => {+ if let Some(language) = language {+ out.push_str(key!("language"));+ write_str(out, language);+ }+ out.push_str(key!("content"));+ write_str(out, content);+ }+ Node::Link {+ text,+ url,+ escaped,+ raw_url,+ source,+ } => {+ if let Some(text) = text {+ out.push_str(key!("text"));+ write_node(out, text);+ }+ out.push_str(key!("url"));+ write_str(out, url);+ out.push_str(key!("escaped"));+ write_bool(out, *escaped);+ out.push_str(key!("rawUrl"));+ write_str(out, raw_url);+ out.push_str(key!("source"));+ write_str(out, source);+ }+ Node::Mention { kind } => {+ out.push_str(key!("kind"));+ write_mention_kind(out, kind);+ }+ Node::Timestamp { timestamp, style } => {+ out.push_str(key!("timestamp"));+ write_number(out, *timestamp);+ out.push_str(key!("style"));+ let _ = write!(out, "\"{style:?}\"");+ }+ Node::Emoji { kind } => {+ out.push_str(key!("kind"));+ write_emoji_kind(out, kind);+ }+ Node::Table {+ header,+ alignments,+ rows,+ } => {+ out.push_str(key!("header"));+ write_node(out, header);+ out.push_str(key!("alignments"));+ out.push('[');+ for (index, alignment) in alignments.iter().enumerate() {+ if index > 0 {+ out.push(',');+ }+ let _ = write!(out, "\"{alignment:?}\"");+ }+ out.push(']');+ out.push_str(key!("rows"));+ write_nodes(out, rows);+ }+ Node::TableRow { cells } => {+ out.push_str(key!("cells"));+ write_nodes(out, cells);+ }+ Node::TableCell { children } => write_children(out, children),+ Node::Alert {+ alert_type,+ children,+ } => {+ out.push_str(key!("alertType"));+ let _ = write!(out, "\"{alert_type:?}\"");+ write_children(out, children);+ }+ }+ out.push('}');+}++fn write_mention_kind(out: &mut String, kind: &MentionKind) {+ match kind {+ MentionKind::User { id } => write_tagged_id(out, "User", id),+ MentionKind::Channel { id } => write_tagged_id(out, "Channel", id),+ MentionKind::Role { id } => write_tagged_id(out, "Role", id),+ MentionKind::Command {+ name,+ subcommand_group,+ subcommand,+ id,+ } => {+ out.push_str("{\"kind\":\"Command\"");+ out.push_str(key!("name"));+ write_str(out, name);+ if let Some(subcommand_group) = subcommand_group {+ out.push_str(key!("subcommandGroup"));+ write_str(out, subcommand_group);+ }+ if let Some(subcommand) = subcommand {+ out.push_str(key!("subcommand"));+ write_str(out, subcommand);+ }+ out.push_str(key!("id"));+ write_str(out, id);+ out.push('}');+ }+ MentionKind::GuildNavigation {+ navigation_type,+ id,+ } => {+ out.push_str("{\"kind\":\"GuildNavigation\"");+ out.push_str(key!("navigationType"));+ let _ = write!(out, "\"{navigation_type:?}\"");+ if let Some(id) = id {+ out.push_str(key!("id"));+ write_str(out, id);+ }+ out.push('}');+ }+ MentionKind::Everyone => out.push_str("{\"kind\":\"Everyone\"}"),+ MentionKind::Here => out.push_str("{\"kind\":\"Here\"}"),+ }+}++fn write_tagged_id(out: &mut String, kind: &str, id: &str) {+ out.push_str("{\"kind\":\"");+ out.push_str(kind);+ out.push('"');+ out.push_str(key!("id"));+ write_str(out, id);+ out.push('}');+}++fn write_emoji_kind(out: &mut String, kind: &EmojiKind) {+ match kind {+ EmojiKind::Standard {+ raw,+ codepoints,+ name,+ } => {+ out.push_str("{\"kind\":\"Standard\"");+ out.push_str(key!("raw"));+ write_str(out, raw);+ out.push_str(key!("codepoints"));+ write_str(out, codepoints);+ out.push_str(key!("name"));+ write_str(out, name);+ }+ EmojiKind::Custom { name, id, animated } => {+ out.push_str("{\"kind\":\"Custom\"");+ out.push_str(key!("name"));+ write_str(out, name);+ out.push_str(key!("id"));+ write_str(out, id);+ out.push_str(key!("animated"));+ write_bool(out, *animated);+ }+ }+ out.push('}');+}++fn write_children(out: &mut String, children: &[Node]) {+ out.push_str(key!("children"));+ write_nodes(out, children);+}++fn write_bool(out: &mut String, value: bool) {+ out.push_str(if value { "true" } else { "false" });+}++fn write_number(out: &mut String, value: u64) {+ let _ = write!(out, "{value}");+}++fn write_str(out: &mut String, value: &str) {+ out.push('"');+ let bytes = value.as_bytes();+ let mut start = 0;+ let mut index = next_escape(bytes, 0);+ while index < bytes.len() {+ out.push_str(&value[start..index]);+ let byte = bytes[index];+ match byte {+ b'"' => out.push_str("\\\""),+ b'\\' => out.push_str("\\\\"),+ b'\n' => out.push_str("\\n"),+ b'\r' => out.push_str("\\r"),+ b'\t' => out.push_str("\\t"),+ 0x08 => out.push_str("\\b"),+ 0x0c => out.push_str("\\f"),+ _ => {+ out.push_str("\\u00");+ out.push(char::from(HEX_DIGITS[usize::from(byte >> 4)]));+ out.push(char::from(HEX_DIGITS[usize::from(byte & 0xf)]));+ }+ }+ start = index + 1;+ index = next_escape(bytes, start);+ }+ out.push_str(&value[start..]);+ out.push('"');+}++fn next_escape(bytes: &[u8], from: usize) -> usize {+ const LOW_BITS: u64 = u64::from_ne_bytes([0x01; 8]);+ const HIGH_BITS: u64 = u64::from_ne_bytes([0x80; 8]);+ let has_zero = |word: u64| word.wrapping_sub(LOW_BITS) & !word & HIGH_BITS != 0;+ let mut index = from;+ while let Some(chunk) = bytes.get(index..index + 8) {+ let word = u64::from_ne_bytes(chunk.try_into().unwrap_or_default());+ if word.wrapping_sub(LOW_BITS * 0x20) & !word & HIGH_BITS != 0+ || has_zero(word ^ (LOW_BITS * u64::from(b'"')))+ || has_zero(word ^ (LOW_BITS * u64::from(b'\\')))+ {+ break;+ }+ index += 8;+ }+ bytes[index..]+ .iter()+ .position(|byte| *byte < 0x20 || matches!(byte, b'"' | b'\\'))+ .map_or(bytes.len(), |offset| index + offset)+}++const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef";diff --git a/packages/markdown_parser/rust/src/lib.rs b/packages/markdown_parser/rust/src/lib.rsindex e7e8ccf96..b2eebb33c 100644--- a/packages/markdown_parser/rust/src/lib.rs+++ b/packages/markdown_parser/rust/src/lib.rs@@ -12,6 +12,7 @@ pub mod native;pub mod normalize;pub mod parser;pub mod plaintext;+mod scan;pub mod text;pub mod wasm;@@ -27,5 +28,5 @@ pub fn parse_markdown_json(let context = EmojiContext::parse(emoji_context);let mut parser = MarkdownParser::new(flags, context);let nodes = parser.parse(input)?;- json::write_ast_json(&nodes)+ Ok(json::write_ast_json(&nodes, input.len() * 8 + 64))}diff --git a/packages/markdown_parser/rust/src/links.rs b/packages/markdown_parser/rust/src/links.rsindex 5fd1f9583..80001a918 100644--- a/packages/markdown_parser/rust/src/links.rs+++ b/packages/markdown_parser/rust/src/links.rs@@ -5,22 +5,21 @@ use crate::ast::{};use crate::constants::MAX_LINK_URL_LENGTH;use crate::parser::{MarkdownParser, ParseError};+use crate::scan::{ANGLE_SYNTAX_PREFIXES, Lookahead, Pattern, http_scheme_len};use crate::text::{- advance_one, bounded_url_end, byte_at, find_from, has_visible_content, is_alpha_numeric,+ bounded_utf8_prefix_length, byte_at, find_from, has_visible_content, is_alpha_numeric,is_digit, is_digit_only, is_whitespace, starts_with, trim,};const APP_PROTOCOL_SCHEME: &str = "fluxer:";--#[derive(Clone, Debug)]-struct BracketResult<'a> {- bracket_position: usize,- link_text: &'a str,-}+const MAX_COMMAND_NAME_LENGTH: usize = 32;+const MAX_COMMAND_SEGMENTS: usize = 3;+const MAX_COMMAND_COLON_INDEX: usize = MAX_COMMAND_SEGMENTS * (MAX_COMMAND_NAME_LENGTH + 1);#[derive(Clone, Debug)]struct UrlInfo<'a> {url: &'a str,+ start: usize,escaped: bool,advance_by: usize,}@@ -33,11 +32,11 @@ struct ComparableUrl<'a> {hash: &'a str,}-pub fn parse_custom_emoji(text: &str) -> Option<ParserResult> {+pub(crate) fn parse_custom_emoji(text: &str) -> Option<ParserResult> {if !(starts_with(text, "<:") || starts_with(text, "<a:")) {return None;}- let end = find_from(text, 0, b'>')?;+ let end = text.len() - 1;if end < 4 {return None;}@@ -136,11 +135,11 @@ pub fn parse_emoji_shortcode(parser: &MarkdownParser, text: &str) -> Option<Pars})}-pub fn parse_timestamp(text: &str) -> Option<ParserResult> {+pub(crate) fn parse_timestamp(text: &str) -> Option<ParserResult> {if !starts_with(text, "<t:") {return None;}- let end = find_from(text, 0, b'>')?;+ let end = text.len() - 1;let inner = &text[3..end];if inner.is_empty() {return None;@@ -179,11 +178,11 @@ pub fn parse_timestamp(text: &str) -> Option<ParserResult> {})}-pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {+pub(crate) fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {if text.len() < 2 || byte_at(text, 0) != b'<' {return None;}- let end = find_from(text, 0, b'>')?;+ let end = text.len() - 1;if end < 2 {return None;}@@ -219,7 +218,10 @@ pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {if !ParserFlags::has(flags, ParserFlags::ALLOW_COMMAND_MENTIONS) {return None;}- let colon = find_from(inner, 0, b':')?;+ let colon = inner+ .bytes()+ .take(MAX_COMMAND_COLON_INDEX + 1)+ .position(|byte| byte == b':')?;let id = &inner[colon + 1..];if !is_digit_only(id) {return None;@@ -227,7 +229,10 @@ pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {let command = &inner[1..colon];let mut segments = Vec::new();for segment in command.split(' ') {- if segment.is_empty() || !is_valid_command_name(segment) || segments.len() == 3 {+ if segment.is_empty()+ || !is_valid_command_name(segment)+ || segments.len() == MAX_COMMAND_SEGMENTS+ {return None;}segments.push(segment);@@ -238,7 +243,8 @@ pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {return mention(MentionKind::Command {name: segments[0].to_owned(),- subcommand_group: (segments.len() == 3).then(|| segments[1].to_owned()),+ subcommand_group: (segments.len() == MAX_COMMAND_SEGMENTS)+ .then(|| segments[1].to_owned()),subcommand: (segments.len() >= 2).then(|| segments[segments.len() - 1].to_owned()),id: id.to_owned(),@@ -281,23 +287,28 @@ fn mention(kind: MentionKind, end: usize) -> Option<ParserResult> {})}-pub fn parse_masked_link(+pub(crate) fn parse_masked_link(parser: &mut MarkdownParser,- text: &str,+ scan: &mut Lookahead<'_>,+ position: usize,base_offset: usize,) -> Result<Option<ParserResult>, ParseError> {+ let text = &scan.text[position..];if text.is_empty() || byte_at(text, 0) != b'[' {return Ok(None);}- let Some(closing) = find_closing_bracket(text) else {+ let Some(bracket_position) = scan+ .closing_bracket(position)+ .map(|closing| closing - position)+ .filter(|distance| *distance <= MAX_LINK_URL_LENGTH)+ else {return Ok(None);};- let link_text = closing.link_text;- let trimmed_link_text = trim(link_text);- if contains_link_syntax(link_text) {- if closing.bracket_position + 1 < text.len()- && byte_at(text, closing.bracket_position + 1) == b'('- && let Some(url_info) = extract_url(text, closing.bracket_position + 2)+ let link_text = &text[1..bracket_position];+ if scan.has_link_syntax(position + 1, position + bracket_position) {+ if bracket_position + 1 < text.len()+ && byte_at(text, bracket_position + 1) == b'('+ && let Some(url_info) = extract_url(scan, position, bracket_position + 2){return Ok(Some(ParserResult {node: Node::Text {@@ -308,34 +319,33 @@ pub fn parse_masked_link(}return Ok(Some(ParserResult {node: Node::Text {- content: text[..closing.bracket_position + 1].to_owned(),+ content: text[..bracket_position + 1].to_owned(),},- advance: closing.bracket_position + 1,+ advance: bracket_position + 1,}));}- if closing.bracket_position + 1 >= text.len()- || byte_at(text, closing.bracket_position + 1) != b'('- {+ if bracket_position + 1 >= text.len() || byte_at(text, bracket_position + 1) != b'(' {return Ok(None);}- let Some(url_info) = extract_url(text, closing.bracket_position + 2) else {- if let Some(mention) = parse_mention(trimmed_link_text, parser.flags())- && mention.advance == trimmed_link_text.len()+ let trimmed_link_text = trim(link_text);+ let Some(url_info) = extract_url(scan, position, bracket_position + 2) else {+ if trimmed_link_text.find('>') == Some(trimmed_link_text.len().wrapping_sub(1))+ && parse_mention(trimmed_link_text, parser.flags()).is_some(){return Ok(None);}return Ok(Some(ParserResult {node: Node::Text {- content: text[..closing.bracket_position + 1].to_owned(),+ content: text[..bracket_position + 1].to_owned(),},- advance: closing.bracket_position + 1,+ advance: bracket_position + 1,}));};if !has_visible_content(link_text)- || is_slash_command_masked_link_text(trim(link_text))- || is_email_like(trim(link_text))+ || is_slash_command_masked_link_text(trimmed_link_text)+ || is_email_like(trimmed_link_text)|| url_info.url.len() > MAX_LINK_URL_LENGTH- || !is_valid_masked_link_url(url_info.url)+ || !is_valid_masked_link_url(scan, position + url_info.start, url_info.url.len())|| (starts_with_url(trimmed_link_text)&& should_treat_as_masked_link(trimmed_link_text, url_info.url)){@@ -367,95 +377,42 @@ pub fn parse_masked_link(}))}-fn find_closing_bracket(text: &str) -> Option<BracketResult<'_>> {- let mut position = 1;- let mut nested = 0usize;- while position < text.len() {- match byte_at(text, position) {- b'[' => {- nested += 1;- position += 1;- }- b']' if nested > 0 => {- nested -= 1;- position += 1;- }- b']' => {- return Some(BracketResult {- bracket_position: position,- link_text: &text[1..position],- });- }- b'\\' => position += if position + 1 < text.len() { 2 } else { 1 },- _ => position += advance_one(text, position),- }- if position > MAX_LINK_URL_LENGTH {- break;- }- }- None-}--fn extract_url(text: &str, start: usize) -> Option<UrlInfo<'_>> {+fn extract_url<'a>(scan: &mut Lookahead<'a>, position: usize, start: usize) -> Option<UrlInfo<'a>> {+ let text = &scan.text[position..];if start >= text.len() {return None;}if byte_at(text, start) == b'<' {- return extract_escaped_url(text, start + 1);- }- extract_unescaped_url(text, start)-}--fn extract_escaped_url(text: &str, start: usize) -> Option<UrlInfo<'_>> {- let mut pos = start;- while pos < text.len() {- if byte_at(text, pos) == b'>' {- let url = &text[start..pos];- pos += 1;- if pos >= text.len() || byte_at(text, pos) != b')' {- return None;- }- return Some(UrlInfo {- url,- escaped: true,- advance_by: pos + 1,- });- }- pos += advance_one(text, pos);- }- None-}--fn extract_unescaped_url(text: &str, start: usize) -> Option<UrlInfo<'_>> {- let mut pos = start;- let mut nested = 0usize;- while pos < text.len() {- match byte_at(text, pos) {- b'(' => {- nested += 1;- pos += 1;- }- b')' if nested > 0 => {- nested -= 1;- pos += 1;- }- b')' => {- return Some(UrlInfo {- url: &text[start..pos],- escaped: false,- advance_by: pos + 1,- });- }- _ => pos += advance_one(text, pos),+ let close =+ scan.next(Pattern::AngleClose, position + start + 1, scan.text.len())? - position;+ if close + 1 >= text.len() || byte_at(text, close + 1) != b')' {+ return None;}+ return Some(UrlInfo {+ url: &text[start + 1..close],+ start: start + 1,+ escaped: true,+ advance_by: close + 2,+ });}- None+ let close = scan.closing_paren(position + start)? - position;+ Some(UrlInfo {+ url: &text[start..close],+ start,+ escaped: false,+ advance_by: close + 1,+ })}-pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {+pub(crate) fn parse_url_segment(+ scan: &mut Lookahead<'_>,+ position: usize,+ flags: u32,+) -> Option<ParserResult> {if !ParserFlags::has(flags, ParserFlags::ALLOW_AUTOLINKS) {return None;}+ let text = &scan.text[position..];let prefix_len = if starts_with(text, "https://") {8} else if starts_with(text, "http://") {@@ -465,30 +422,12 @@ pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {} else {return None;};- let mut end = prefix_len;- let mut paren_depth = 0usize;- while end < text.len() {- let c = byte_at(text, end);- if c == b'(' {- paren_depth += 1;- end += 1;- } else if c == b')' {- if paren_depth > 0 {- paren_depth -= 1;- end += 1;- } else {- break;- }- } else if is_url_termination_char(c) {- break;- } else {- end += advance_one(text, end);- }- if end - prefix_len > MAX_LINK_URL_LENGTH {- end = bounded_url_end(text, end, prefix_len);- break;- }- }+ let stop = scan.url_stop(position + prefix_len) - position;+ let mut end = if stop - prefix_len <= MAX_LINK_URL_LENGTH {+ stop+ } else {+ bounded_utf8_prefix_length(text, prefix_len + MAX_LINK_URL_LENGTH)+ };while end > 0&& is_trimmed_autolink_punctuation(byte_at(text, end - 1))&& !has_terminal_tld(&text[..end])@@ -498,16 +437,17 @@ pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {if end <= prefix_len {return None;}- let url = &text[..end];- if (starts_with_app_protocol_url(url) && !looks_like_valid_app_protocol_url(url))- || contains_nested_scheme_before_path(url, prefix_len)- || contains_formatting_marker_before_path(url, prefix_len)- || contains_unparenthesized_nested_http_url(url)- || contains_angle_bracket_syntax(url)- || contains_angle_email_like(url)+ let url_end = position + end;+ if (starts_with_app_protocol_url(&text[..end])+ && !is_valid_app_protocol_url(scan, position, url_end))+ || contains_nested_scheme_before_path(scan, position + prefix_len, url_end)+ || contains_formatting_marker_before_path(scan, position + prefix_len, url_end)+ || contains_angle_bracket_markup(scan, position, url_end){return None;}+ let text = &scan.text[position..];+ let url = &text[..end];let escaped = matches!(byte_at(text, 0), b'"' | b'\'')|| (end < text.len() && matches!(byte_at(text, end), b'"' | b'\''));Some(ParserResult {@@ -522,14 +462,20 @@ pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {})}-pub fn parse_autolink(text: &str, flags: u32) -> Option<ParserResult> {+pub(crate) fn parse_autolink(+ scan: &mut Lookahead<'_>,+ position: usize,+ close: usize,+ flags: u32,+) -> Option<ParserResult> {+ let text = &scan.text[position..=close];if !ParserFlags::has(flags, ParserFlags::ALLOW_AUTOLINKS) || byte_at(text, 0) != b'<' {return None;}if matches!(byte_at(text, 1), b'"' | b'\'') || !starts_with_url(&text[1..]) {return None;}- let end = find_from(text, 1, b'>')?;+ let end = text.len() - 1;let url = &text[1..end];if url.len() > MAX_LINK_URL_LENGTH {return None;@@ -544,10 +490,10 @@ pub fn parse_autolink(text: &str, flags: u32) -> Option<ParserResult> {return None;};if starts_with_app_protocol_url(url) {- if !looks_like_valid_app_protocol_url(url) {+ if !is_valid_app_protocol_url(scan, position + 1, close) {return None;}- } else if !looks_like_valid_http_url_prefix(url, prefix_len) {+ } else if !looks_like_valid_http_url_prefix(scan, position + 1, close, prefix_len, true) {return None;}Some(ParserResult {@@ -562,11 +508,11 @@ pub fn parse_autolink(text: &str, flags: u32) -> Option<ParserResult> {})}-pub fn parse_email_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {+pub(crate) fn parse_email_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS) || byte_at(text, 0) != b'<' {return None;}- let end = find_from(text, 1, b'>')?;+ let end = text.len() - 1;let content = &text[1..end];if starts_with(content, "http://")|| starts_with(content, "https://")@@ -593,11 +539,11 @@ pub fn parse_email_link(parser: &MarkdownParser, text: &str) -> Option<ParserRes})}-pub fn parse_phone_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {+pub(crate) fn parse_phone_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS) || byte_at(text, 0) != b'<' {return None;}- let end = find_from(text, 1, b'>')?;+ let end = text.len() - 1;let content = &text[1..end];if !is_valid_phone_number(content) {return None;@@ -618,13 +564,13 @@ pub fn parse_phone_link(parser: &MarkdownParser, text: &str) -> Option<ParserRes})}-pub fn parse_sms_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {+pub(crate) fn parse_sms_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)|| !starts_with(text, "<sms:"){return None;}- let end = find_from(text, 1, b'>')?;+ let end = text.len() - 1;let content = &text[1..end];let phone = &content[4..];if !is_valid_phone_number(phone) {@@ -662,30 +608,18 @@ fn starts_with_app_protocol_url(text: &str) -> bool {)}-pub fn looks_like_valid_http_url_prefix(url: &str, prefix_len: usize) -> bool {- looks_like_valid_http_url_prefix_with_userinfo(url, prefix_len, true)-}--fn looks_like_valid_http_url_prefix_with_userinfo(- url: &str,+fn looks_like_valid_http_url_prefix(+ scan: &mut Lookahead<'_>,+ start: usize,+ end: usize,prefix_len: usize,allow_userinfo: bool,) -> bool {- if url.len() <= prefix_len- || contains_unparenthesized_nested_http_url(url)- || contains_angle_bracket_syntax(url)- || contains_angle_email_like(url)- {+ if end - start <= prefix_len || contains_angle_bracket_markup(scan, start, end) {return false;}- let mut authority_end = prefix_len;- while authority_end < url.len() {- let char = byte_at(url, authority_end);- if matches!(char, b'/' | b'?' | b'#') {- break;- }- authority_end += 1;- }+ let url = &scan.text[start..end];+ let authority_end = path_start(scan, start + prefix_len, end) - start;let colon_before_path_slash = authority_end < url.len()&& byte_at(url, authority_end) == b'/'&& authority_end > prefix_len@@ -697,6 +631,10 @@ fn looks_like_valid_http_url_prefix_with_userinfo()}+fn path_start(scan: &mut Lookahead<'_>, from: usize, end: usize) -> usize {+ scan.next(Pattern::PathDelimiter, from, end).unwrap_or(end)+}+fn is_valid_http_authority(authority: &str,allow_userinfo: bool,@@ -751,52 +689,78 @@ fn is_valid_http_authority(true}-fn contains_angle_bracket_syntax(value: &str) -> bool {- ["<:", "<a:", "<id:", "<@", "<#", "</", "<t:", "<+", "<sms:"]- .iter()- .any(|needle| value.contains(needle))+fn contains_angle_bracket_markup(scan: &mut Lookahead<'_>, start: usize, end: usize) -> bool {+ scan.next(Pattern::AngleOpen, start, end).is_some()+ && (contains_unparenthesized_nested_http_url(scan, start, end)+ || contains_angle_bracket_syntax(scan, start, end)+ || scan.has_angle_email(start, end))}-fn contains_unparenthesized_nested_http_url(url: &str) -> bool {- for needle in ["<http://", "<https://"] {- if let Some(pos) = url.find(needle)- && !url[..pos].contains('(')+fn contains_angle_bracket_syntax(scan: &mut Lookahead<'_>, start: usize, end: usize) -> bool {+ let mut from = start;+ while let Some(position) = scan.next(Pattern::AngleSyntax, from, end) {+ let window = &scan.text.as_bytes()[position..end];+ if ANGLE_SYNTAX_PREFIXES+ .iter()+ .any(|prefix| window.starts_with(prefix.as_bytes())){return true;}+ from = position + 1;}false}-fn contains_nested_scheme_before_path(url: &str, prefix_len: usize) -> bool {- let rest = &url[prefix_len..];- let first_path_delimiter = rest- .bytes()- .position(|char| matches!(char, b'/' | b'?' | b'#'))- .unwrap_or(rest.len());- for needle in ["http://", "https://"] {- if let Some(pos) = rest.find(needle)- && pos < first_path_delimiter- && pos > 0- && byte_at(rest, pos - 1) == b'('- {- return true;+fn contains_unparenthesized_nested_http_url(+ scan: &mut Lookahead<'_>,+ start: usize,+ end: usize,+) -> bool {+ let bytes = scan.text.as_bytes();+ let Some(position) = scan+ .next(Pattern::NestedHttpUrl, start, end)+ .filter(|position| {+ http_scheme_len(&bytes[position + 1..]).is_some_and(|len| position + 1 + len <= end)+ })+ else {+ return false;+ };+ scan.next(Pattern::OpenParen, start, position).is_none()+}++fn contains_nested_scheme_before_path(+ scan: &mut Lookahead<'_>,+ rest_start: usize,+ end: usize,+) -> bool {+ let bytes = scan.text.as_bytes();+ let path = path_start(scan, rest_start, end);+ let mut checked = [false; 2];+ let mut from = rest_start;+ while !(checked[0] && checked[1])+ && let Some(position) = scan.next(Pattern::HttpScheme, from, path)+ {+ let len = http_scheme_len(&bytes[position..]).unwrap_or_default();+ let kind = usize::from(len == 8);+ if !checked[kind] {+ checked[kind] = true;+ if position + len <= end && position > rest_start && bytes[position - 1] == b'(' {+ return true;+ }}+ from = position + 1;}false}-fn contains_formatting_marker_before_path(url: &str, prefix_len: usize) -> bool {- let rest = &url[prefix_len..];- let authority_end = rest- .bytes()- .position(|char| matches!(char, b'/' | b'?' | b'#'))- .unwrap_or(rest.len());- let authority = &rest[..authority_end];- authority.contains("__")- || authority.contains("**")- || authority.contains("~~")- || authority.contains("||")+fn contains_formatting_marker_before_path(+ scan: &mut Lookahead<'_>,+ rest_start: usize,+ end: usize,+) -> bool {+ let path = path_start(scan, rest_start, end);+ scan.next(Pattern::DoubledMarker, rest_start, path)+ .is_some_and(|position| position + 2 <= path)}fn is_valid_bracketed_ip_literal(value: &str) -> bool {@@ -807,15 +771,16 @@ fn is_valid_bracketed_ip_literal(value: &str) -> bool {.all(|char| is_digit(char) || char.is_ascii_hexdigit() || matches!(char, b':' | b'.'))}-pub fn is_valid_masked_link_url(url: &str) -> bool {+fn is_valid_masked_link_url(scan: &mut Lookahead<'_>, start: usize, len: usize) -> bool {+ let url = &scan.text[start..start + len];if url.is_empty() || url.contains('"') {return false;}if starts_with(url, "https://") {- return looks_like_valid_http_url_prefix_with_userinfo(url, 8, false);+ return looks_like_valid_http_url_prefix(scan, start, start + len, 8, false);}if starts_with(url, "http://") {- return looks_like_valid_http_url_prefix_with_userinfo(url, 7, false);+ return looks_like_valid_http_url_prefix(scan, start, start + len, 7, false);}if starts_with(url, "mailto:") {return is_valid_email(&url["mailto:".len()..]);@@ -826,30 +791,7 @@ pub fn is_valid_masked_link_url(url: &str) -> bool {if starts_with(url, "sms:") {return is_valid_phone_number(&url["sms:".len()..]);}- if starts_with_app_protocol_url(url) {- return looks_like_valid_app_protocol_url(url);- }- false-}--fn contains_angle_email_like(value: &str) -> bool {- let mut offset = 0;- while offset < value.len() {- let Some(open_relative) = value[offset..].find('<') else {- return false;- };- let open = offset + open_relative;- let Some(close_relative) = value[open + 1..].find('>') else {- return false;- };- let close = open + 1 + close_relative;- let inner = &value[open + 1..close];- if !inner.is_empty() && byte_at(inner, 0) != b'@' && inner.contains('@') {- return true;- }- offset = close + 1;- }- false+ starts_with_app_protocol_url(url) && is_valid_app_protocol_url(scan, start, start + len)}pub fn should_treat_as_masked_link(link_text: &str, url: &str) -> bool {@@ -925,20 +867,13 @@ fn equivalent_url_path(left: &str, right: &str) -> bool {normalized_left == normalized_right}-fn looks_like_valid_app_protocol_url(url: &str) -> bool {- if !starts_with_app_protocol_url(url) {- return false;- }- url[APP_PROTOCOL_SCHEME.len()..].bytes().all(|char| {- !matches!(- char,- b'"' | b'\'' | b'<' | b'>' | b'\\' | b'|' | b' ' | b'\t' | b'\r' | b'\n'- )- })-}--pub fn is_url_termination_char(char: u8) -> bool {- matches!(char, b' ' | b'\t' | b'\n' | b'\r' | b')' | b'"')+fn is_valid_app_protocol_url(scan: &mut Lookahead<'_>, start: usize, end: usize) -> bool {+ scan.next(+ Pattern::AppProtocolInvalid,+ start + APP_PROTOCOL_SCHEME.len(),+ end,+ )+ .is_none()}pub fn is_word_underscore(text: &str, pos: usize) -> bool {@@ -963,30 +898,13 @@ pub fn is_valid_emoji_name(name: &str) -> bool {fn is_valid_command_name(name: &str) -> bool {!name.is_empty()- && name.len() <= 32+ && name.len() <= MAX_COMMAND_NAME_LENGTH&& name.bytes().all(|char| {!char.is_ascii_uppercase()&& (char >= 0x80 || is_alpha_numeric(char) || matches!(char, b'-' | b'_' | b'\''))})}-fn contains_link_syntax(text: &str) -> bool {- let mut search_offset = 0;- while let Some(bracket) = text[search_offset..].find('[') {- let bracket = search_offset + bracket;- let after = &text[bracket + 1..];- let Some(close) = after.find(']') else {- return false;- };- let close_abs = bracket + 1 + close;- if close_abs + 1 < text.len() && byte_at(text, close_abs + 1) == b'(' {- return true;- }- search_offset = close_abs + 1;- }- false-}-fn is_slash_command_masked_link_text(text: &str) -> bool {starts_with(text, "</") && text.ends_with('>') && text.contains(':')}@@ -1008,25 +926,23 @@ fn is_email_like(text: &str) -> bool {}pub fn is_valid_email(text: &str) -> bool {- if !text.contains('@') || text.contains(' ') {- return false;- }- let Some(at) = text.find('@') else {+ let Some(at) = text+ .bytes()+ .position(|char| {+ !(is_alpha_numeric(char) || matches!(char, b'.' | b'_' | b'%' | b'+' | b'-'))+ })+ .filter(|at| byte_at(text, *at) == b'@')+ else {return false;};if at == 0 || at + 1 >= text.len() {return false;}- for char in text[..at].bytes() {- if !(is_alpha_numeric(char) || matches!(char, b'.' | b'_' | b'%' | b'+' | b'-')) {- return false;- }- }let domain = &text[at + 1..];- let Some(dot) = domain.rfind('.') else {- return false;- };- dot > 0 && dot + 2 <= domain.len() && all_ascii_domain(domain)+ all_ascii_domain(domain)+ && domain+ .rfind('.')+ .is_some_and(|dot| dot > 0 && dot + 2 <= domain.len())}fn all_ascii_domain(domain: &str) -> bool {@@ -1392,11 +1308,3 @@ pub fn is_right_flanking_delimiter(previous: u8, next: u8) -> bool {|| crate::text::is_whitespace(next)|| crate::text::is_punctuation(next)}--pub fn count_repeated_byte(text: &str, position: usize, byte: u8) -> usize {- let mut count = 0;- while position + count < text.len() && byte_at(text, position + count) == byte {- count += 1;- }- count-}diff --git a/packages/markdown_parser/rust/src/normalize.rs b/packages/markdown_parser/rust/src/normalize.rsindex 46dfcf3ef..1cc16cda5 100644--- a/packages/markdown_parser/rust/src/normalize.rs+++ b/packages/markdown_parser/rust/src/normalize.rs@@ -18,7 +18,7 @@ pub fn normalize_child_nodes(nodes: &mut Vec<Node>, inside_blockquote: bool) {pub fn normalize_node(node: &mut Node) {match node {- Node::Text { content } => *content = remove_text_presentation(content),+ Node::Text { content } => remove_text_presentation(content),Node::Strong { children }| Node::Emphasis { children }| Node::Underline { children }@@ -61,7 +61,7 @@ pub fn normalize_node(node: &mut Node) {pub fn apply_text_presentation_node(node: &mut Node) {match node {- Node::Text { content } => *content = remove_text_presentation(content),+ Node::Text { content } => remove_text_presentation(content),Node::Strong { children }| Node::Emphasis { children }| Node::Underline { children }diff --git a/packages/markdown_parser/rust/src/parser.rs b/packages/markdown_parser/rust/src/parser.rsindex 98385be56..851685899 100644--- a/packages/markdown_parser/rust/src/parser.rs+++ b/packages/markdown_parser/rust/src/parser.rs@@ -1,8 +1,10 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse std::fmt;+use std::sync::Arc;use crate::ast::Node;+use crate::block::ClosingFenceLines;use crate::constants::{MAX_AST_NODES, MAX_LINE_LENGTH};use crate::emoji::EmojiContext;use crate::normalize::{@@ -35,7 +37,7 @@ impl From<serde_json::Error> for ParseError {#[derive(Clone, Debug)]pub struct MarkdownParser {flags: u32,- emoji_context: EmojiContext,+ emoji_context: Arc<EmojiContext>,pub(crate) node_count: usize,}@@ -43,7 +45,7 @@ impl MarkdownParser {pub fn new(flags: u32, emoji_context: EmojiContext) -> Self {Self {flags,- emoji_context,+ emoji_context: Arc::new(emoji_context),node_count: 0,}}@@ -56,6 +58,10 @@ impl MarkdownParser {&self.emoji_context}+ pub(crate) fn shared_emoji_context(&self) -> Arc<EmojiContext> {+ Arc::clone(&self.emoji_context)+ }+pub fn parse(&mut self, input: &str) -> Result<Vec<Node>, ParseError> {let lines = text::split_lines(input);self.parse_lines(lines)@@ -66,12 +72,17 @@ impl MarkdownParser {parser: self,lines,current_line: 0,+ fences: ClosingFenceLines::default(),}.parse()}pub(crate) fn child_with_flags(&self, flags: u32) -> Self {- Self::new(flags, self.emoji_context.clone())+ Self {+ flags,+ emoji_context: Arc::clone(&self.emoji_context),+ node_count: 0,+ }}}@@ -79,9 +90,15 @@ pub(crate) struct RuntimeState<'a> {pub(crate) parser: &'a mut MarkdownParser,pub(crate) lines: Vec<Line>,pub(crate) current_line: usize,+ pub(crate) fences: ClosingFenceLines,}impl RuntimeState<'_> {+ pub(crate) fn replace_line(&mut self, index: usize, text: String, offset: usize) {+ self.lines[index] = Line { text, offset };+ self.fences.line_replaced(index);+ }+fn parse(&mut self) -> Result<Vec<Node>, ParseError> {let mut ast = Vec::new();if self.lines.is_empty() {@@ -93,18 +110,12 @@ impl RuntimeState<'_> {self.lines[self.current_line].text =text::bounded_line_text(&self.lines[self.current_line].text).to_owned();}- let line = self.lines[self.current_line].text.clone();- let trimmed = trim_start(&line);- if trim(trimmed).is_empty() {+ if trim(&self.lines[self.current_line].text).is_empty() {let blank_count = self.count_blank_lines();if !ast.is_empty() && self.current_line + blank_count < self.lines.len() {- let next_line = self.lines[self.current_line + blank_count].clone();- let next_trimmed = trim_start(&next_line.text);- let next_offset =- next_line.offset + (next_line.text.len() - next_trimmed.len());- let is_next_heading =- crate::block::parse_heading_node(self.parser, next_trimmed, next_offset)?- .is_some();+ let next_trimmed =+ trim_start(&self.lines[self.current_line + blank_count].text);+ let is_next_heading = crate::block::heading_content(next_trimmed).is_some();let is_prev_heading = matches!(ast.last(), Some(Node::Heading { .. }));if !is_next_heading && !is_prev_heading {ast.push(Node::Text {@@ -168,6 +179,7 @@ impl RuntimeState<'_> {&self.lines,self.current_line + consumed,self.parser.flags(),+ &mut self.fences,)|| trimmed_next.is_empty(){diff --git a/packages/markdown_parser/rust/src/scan.rs b/packages/markdown_parser/rust/src/scan.rsnew file mode 100644index 000000000..e91659e42--- /dev/null+++ b/packages/markdown_parser/rust/src/scan.rs@@ -0,0 +1,668 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use crate::text::advance_one;++#[derive(Clone, Copy, Debug)]+pub(crate) enum Token {+ Close,+ Fail,+ Open(usize),+ Skip(usize),+ Step(usize),+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) struct Closer {+ pub(crate) at: usize,+ last_checked: Option<usize>,+}++impl Closer {+ fn or_checked(self, landing: Option<usize>) -> Self {+ Self {+ at: self.at,+ last_checked: self.last_checked.or(landing),+ }+ }++ pub(crate) fn within(self, opener: usize, limit: usize) -> Option<usize> {+ self.last_checked+ .is_none_or(|landing| landing - opener <= limit)+ .then_some(self.at)+ }+}++#[derive(Clone, Copy, Debug)]+enum Frame {+ Then { at: usize, landing: Option<usize> },+ Inner { at: usize, inner: usize },+ Outer { at: usize, last_checked: usize },+}++#[derive(Clone, Copy, Debug, Default)]+struct Slot {+ state: u32,+ last_checked: u32,+}++#[derive(Clone, Debug, Default)]+struct ClosingMemo {+ slots: Vec<Slot>,+ frames: Vec<Frame>,+}++impl ClosingMemo {+ fn get(&self, position: usize) -> Option<Option<Closer>> {+ let slot = self.slots[position];+ match slot.state {+ 0 => None,+ 1 => Some(None),+ state => Some(Some(Closer {+ at: state as usize - 2,+ last_checked: slot.last_checked.checked_sub(1).map(|value| value as usize),+ })),+ }+ }++ fn set(&mut self, position: usize, value: Option<Closer>) {+ let slot = match value {+ None => Slot {+ state: 1,+ last_checked: 0,+ },+ Some(closer) => Slot {+ state: closer.at as u32 + 2,+ last_checked: closer.last_checked.map_or(0, |value| value as u32 + 1),+ },+ };+ self.slots[position] = slot;+ }++ fn resolve(+ &mut self,+ len: usize,+ start: usize,+ close_len: usize,+ mut classify: impl FnMut(usize) -> Token,+ ) -> Option<Closer> {+ if self.slots.is_empty() {+ self.slots = vec![Slot::default(); len + 1];+ }+ let mut frames = std::mem::take(&mut self.frames);+ let mut position = start;+ let mut result;+ 'descend: loop {+ result = loop {+ if position >= len {+ break None;+ }+ if let Some(known) = self.get(position) {+ break known;+ }+ match classify(position) {+ Token::Close => {+ let closer = Some(Closer {+ at: position,+ last_checked: None,+ });+ self.set(position, closer);+ break closer;+ }+ Token::Fail => {+ self.set(position, None);+ break None;+ }+ Token::Skip(advance) => {+ frames.push(Frame::Then {+ at: position,+ landing: None,+ });+ position += advance;+ }+ Token::Step(advance) => {+ frames.push(Frame::Then {+ at: position,+ landing: Some(position + advance),+ });+ position += advance;+ }+ Token::Open(advance) => {+ frames.push(Frame::Inner {+ at: position,+ inner: position + advance,+ });+ position += advance;+ }+ }+ };+ while let Some(frame) = frames.pop() {+ match frame {+ Frame::Then { at, landing } => {+ result = result.map(|closer| closer.or_checked(landing));+ self.set(at, result);+ }+ Frame::Inner { at, inner } => match result {+ None => self.set(at, None),+ Some(closer) => {+ frames.push(Frame::Outer {+ at,+ last_checked: closer.last_checked.unwrap_or(inner),+ });+ position = closer.at + close_len;+ continue 'descend;+ }+ },+ Frame::Outer { at, last_checked } => {+ result = result.map(|closer| closer.or_checked(Some(last_checked)));+ self.set(at, result);+ }+ }+ }+ break;+ }+ self.frames = frames;+ result+ }+}++const MIN_UNINDEXED_STEPS: usize = 256;++fn scan_briefly(+ len: usize,+ start: usize,+ close_len: usize,+ steps: &mut usize,+ classify: &mut impl FnMut(usize) -> Token,+) -> Option<Option<Closer>> {+ let mut position = start;+ let mut depth = 0usize;+ let mut last_checked = None;+ while *steps > 0 {+ *steps -= 1;+ if position >= len {+ return Some(None);+ }+ match classify(position) {+ Token::Close if depth == 0 => {+ return Some(Some(Closer {+ at: position,+ last_checked,+ }));+ }+ Token::Close => {+ depth -= 1;+ position += close_len;+ }+ Token::Fail => return Some(None),+ Token::Skip(advance) => position += advance,+ Token::Open(advance) => {+ depth += 1;+ position += advance;+ last_checked = Some(position);+ }+ Token::Step(advance) => {+ position += advance;+ last_checked = Some(position);+ }+ }+ }+ None+}++#[derive(Clone, Copy, Debug)]+pub(crate) enum Pattern {+ AngleOpen,+ AngleClose,+ At,+ OpenBracket,+ CloseBracket,+ OpenParen,+ Paren,+ UrlTerminator,+ PathDelimiter,+ AppProtocolInvalid,+ HttpScheme,+ NestedHttpUrl,+ AngleSyntax,+ DoubledMarker,+}++const PATTERNS: usize = 14;++pub(crate) const ANGLE_SYNTAX_PREFIXES: [&str; 9] =+ ["<:", "<a:", "<id:", "<@", "<#", "</", "<t:", "<+", "<sms:"];++impl Pattern {+ fn find(self, bytes: &[u8], from: usize, end: usize) -> Option<usize> {+ let mut window = bytes[from..end].iter();+ let offset = match self {+ Self::AngleOpen => window.position(|byte| *byte == b'<'),+ Self::AngleClose => window.position(|byte| *byte == b'>'),+ Self::At => window.position(|byte| *byte == b'@'),+ Self::OpenBracket => window.position(|byte| *byte == b'['),+ Self::CloseBracket => window.position(|byte| *byte == b']'),+ Self::OpenParen => window.position(|byte| *byte == b'('),+ Self::Paren => window.position(|byte| matches!(byte, b'(' | b')')),+ Self::UrlTerminator => window.position(|byte| is_url_terminator(*byte)),+ Self::PathDelimiter => window.position(|byte| matches!(byte, b'/' | b'?' | b'#')),+ _ => (from..end).position(|position| self.matches(bytes, position)),+ };+ offset.map(|offset| from + offset)+ }++ fn matches(self, bytes: &[u8], position: usize) -> bool {+ let byte = bytes[position];+ let rest = &bytes[position..];+ match self {+ Self::AngleOpen => byte == b'<',+ Self::AngleClose => byte == b'>',+ Self::At => byte == b'@',+ Self::OpenBracket => byte == b'[',+ Self::CloseBracket => byte == b']',+ Self::OpenParen => byte == b'(',+ Self::Paren => matches!(byte, b'(' | b')'),+ Self::UrlTerminator => is_url_terminator(byte),+ Self::PathDelimiter => matches!(byte, b'/' | b'?' | b'#'),+ Self::AppProtocolInvalid => matches!(+ byte,+ b'"' | b'\'' | b'<' | b'>' | b'\\' | b'|' | b' ' | b'\t' | b'\r' | b'\n'+ ),+ Self::HttpScheme => byte == b'h' && http_scheme_len(rest).is_some(),+ Self::NestedHttpUrl => byte == b'<' && http_scheme_len(&rest[1..]).is_some(),+ Self::AngleSyntax => {+ byte == b'<'+ && ANGLE_SYNTAX_PREFIXES+ .iter()+ .any(|prefix| rest.starts_with(prefix.as_bytes()))+ }+ Self::DoubledMarker => {+ matches!(byte, b'_' | b'*' | b'~' | b'|') && rest.get(1) == Some(&byte)+ }+ }+ }+}++pub(crate) fn http_scheme_len(bytes: &[u8]) -> Option<usize> {+ if bytes.starts_with(b"http://") {+ Some(7)+ } else if bytes.starts_with(b"https://") {+ Some(8)+ } else {+ None+ }+}++#[derive(Clone, Debug, Default)]+struct NextMatch {+ next: Vec<u32>,+}++impl NextMatch {+ fn find(&mut self, bytes: &[u8], from: usize, limit: usize, pattern: Pattern) -> Option<usize> {+ let len = bytes.len();+ if self.next.is_empty() {+ self.next = vec![0; len];+ }+ let mut position = from;+ let found = loop {+ if position == len {+ break len;+ }+ let known = self.next[position];+ if known != 0 {+ break known as usize - 1;+ }+ if pattern.matches(bytes, position) {+ break position;+ }+ position += 1;+ };+ let stored = found as u32 + 1;+ self.next[from..position].fill(stored);+ if found == position && found < len {+ self.next[found] = stored;+ }+ (found < limit).then_some(found)+ }+}++enum ChainLink {+ Hit(usize),+ Next(usize),+ End,+}++#[derive(Clone, Debug, Default)]+struct ChainMemo {+ hits: Vec<u32>,+ chain: Vec<usize>,+}++impl ChainMemo {+ fn resolve(+ &mut self,+ len: usize,+ first: usize,+ mut link: impl FnMut(usize) -> ChainLink,+ ) -> Option<usize> {+ if self.hits.is_empty() {+ self.hits = vec![0; len];+ }+ self.chain.clear();+ let mut node = first;+ let found = loop {+ match self.hits[node] {+ 0 => {}+ 1 => break None,+ known => break Some(known as usize - 2),+ }+ self.chain.push(node);+ match link(node) {+ ChainLink::Hit(at) => break Some(at),+ ChainLink::Next(next) => node = next,+ ChainLink::End => break None,+ }+ };+ let stored = found.map_or(1, |at| at as u32 + 2);+ for &node in &self.chain {+ self.hits[node] = stored;+ }+ found+ }+}++fn is_url_terminator(byte: u8) -> bool {+ matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'"')+}++pub(crate) fn step(text: &str, position: usize) -> Token {+ Token::Step(advance_one(text, position))+}++pub(crate) const FORMATTING_SCANS: usize = 9;++#[derive(Clone, Copy, Debug)]+pub(crate) enum Closing {+ Formatting(usize),+ Bracket,+ Paren,+ UrlParen,+}++impl Closing {+ fn index(self) -> usize {+ match self {+ Self::Formatting(index) => index,+ Self::Bracket => FORMATTING_SCANS,+ Self::Paren => FORMATTING_SCANS + 1,+ Self::UrlParen => FORMATTING_SCANS + 2,+ }+ }+}++const CLOSINGS: usize = FORMATTING_SCANS + 3;++#[derive(Clone, Debug, Default)]+struct Memos {+ formatting: [ClosingMemo; FORMATTING_SCANS],+ brackets: ClosingMemo,+ parens: ClosingMemo,+ url_parens: ClosingMemo,+ next: [NextMatch; PATTERNS],+ link_syntax: ChainMemo,+ angle_email: ChainMemo,+}++impl Memos {+ fn closing(&mut self, closing: Closing) -> &mut ClosingMemo {+ match closing {+ Closing::Formatting(index) => &mut self.formatting[index],+ Closing::Bracket => &mut self.brackets,+ Closing::Paren => &mut self.parens,+ Closing::UrlParen => &mut self.url_parens,+ }+ }+}++#[derive(Clone, Debug, Default)]+struct CodeSpanScan {+ start: usize,+ frontier: usize,+ ended: bool,+ longest_runs: Vec<(usize, usize)>,+}++#[derive(Clone, Debug)]+pub(crate) struct Lookahead<'a> {+ pub(crate) text: &'a str,+ backtick_run: (usize, usize),+ asterisk_run: (usize, usize),+ code_span: CodeSpanScan,+ closing_steps: [usize; CLOSINGS],+ pattern_steps: [usize; PATTERNS],+ memos: Option<Box<Memos>>,+}++impl<'a> Lookahead<'a> {+ pub(crate) fn new(text: &'a str) -> Self {+ Self {+ text,+ backtick_run: (0, 0),+ asterisk_run: (0, 0),+ code_span: CodeSpanScan::default(),+ closing_steps: [text.len() + MIN_UNINDEXED_STEPS; CLOSINGS],+ pattern_steps: [text.len() + MIN_UNINDEXED_STEPS; PATTERNS],+ memos: None,+ }+ }++ fn memos(&mut self) -> &mut Memos {+ self.memos.get_or_insert_with(Box::default)+ }++ pub(crate) fn next(&mut self, pattern: Pattern, from: usize, limit: usize) -> Option<usize> {+ if from >= limit {+ return None;+ }+ let bytes = self.text.as_bytes();+ let steps = &mut self.pattern_steps[pattern as usize];+ if *steps > 0 {+ let brief_end = limit.min(from + *steps);+ let found = pattern.find(bytes, from, brief_end);+ *steps -= found.map_or(brief_end, |found| found + 1) - from;+ if found.is_some() || brief_end == limit {+ return found;+ }+ }+ self.memos().next[pattern as usize].find(bytes, from, limit, pattern)+ }++ pub(crate) fn closer(+ &mut self,+ closing: Closing,+ start: usize,+ close_len: usize,+ mut classify: impl FnMut(&mut Self, usize) -> Token,+ ) -> Option<Closer> {+ let len = self.text.len();+ let mut steps = self.closing_steps[closing.index()];+ if steps > 0 {+ let result = scan_briefly(len, start, close_len, &mut steps, &mut |position| {+ classify(self, position)+ });+ self.closing_steps[closing.index()] = steps;+ if let Some(result) = result {+ return result;+ }+ }+ let mut memo = std::mem::take(self.memos().closing(closing));+ let result = memo.resolve(len, start, close_len, |position| classify(self, position));+ *self.memos().closing(closing) = memo;+ result+ }++ pub(crate) fn run_end(&mut self, position: usize, byte: u8) -> usize {+ let cache = if byte == b'`' {+ &mut self.backtick_run+ } else {+ &mut self.asterisk_run+ };+ let (start, end) = *cache;+ if start <= position && position < end {+ return end;+ }+ let bytes = self.text.as_bytes();+ let mut end = position;+ while end < bytes.len() && bytes[end] == byte {+ end += 1;+ }+ *cache = (position, end);+ end+ }++ pub(crate) fn code_span_end(+ &mut self,+ opener: usize,+ marker_length: usize,+ limit: usize,+ ) -> Option<usize> {+ let start = opener + marker_length;+ if self.code_span.start != start || self.code_span.frontier < start {+ self.code_span = CodeSpanScan {+ start,+ frontier: start,+ ended: false,+ longest_runs: Vec::new(),+ };+ }+ let bytes = self.text.as_bytes();+ loop {+ let scan = &mut self.code_span;+ let index = scan+ .longest_runs+ .partition_point(|(count, _)| *count < marker_length);+ if let Some(&(count, run_start)) = scan.longest_runs.get(index) {+ return (run_start - opener <= limit)+ .then(|| run_start + count - marker_length - opener);+ }+ if scan.ended || scan.frontier - opener > limit {+ return None;+ }+ let Some(offset) = bytes[scan.frontier..]+ .iter()+ .position(|byte| matches!(byte, b'`' | b'\n' | b'\r'))+ else {+ scan.ended = true;+ continue;+ };+ let run_start = scan.frontier + offset;+ if bytes[run_start] != b'`' {+ scan.ended = true;+ continue;+ }+ let mut run_end = run_start;+ while run_end < bytes.len() && bytes[run_end] == b'`' {+ run_end += 1;+ }+ let count = run_end - run_start;+ if scan+ .longest_runs+ .last()+ .is_none_or(|(longest, _)| count > *longest)+ {+ scan.longest_runs.push((count, run_start));+ }+ scan.frontier = run_end;+ }+ }++ pub(crate) fn closing_bracket(&mut self, opener: usize) -> Option<usize> {+ let text = self.text;+ let bytes = text.as_bytes();+ self.closer(Closing::Bracket, opener + 1, 1, |_, position| {+ match bytes[position] {+ b'[' => Token::Open(1),+ b']' => Token::Close,+ b'\\' if position + 1 < bytes.len() => Token::Skip(2),+ _ => step(text, position),+ }+ })+ .map(|closer| closer.at)+ }++ pub(crate) fn closing_paren(&mut self, start: usize) -> Option<usize> {+ let bytes = self.text.as_bytes();+ self.closer(Closing::Paren, start, 1, |_, position| {+ match bytes[position] {+ b'(' => Token::Open(1),+ b')' => Token::Close,+ _ => Token::Step(1),+ }+ })+ .map(|closer| closer.at)+ }++ pub(crate) fn url_stop(&mut self, start: usize) -> usize {+ let bytes = self.text.as_bytes();+ let terminator = self+ .next(Pattern::UrlTerminator, start, bytes.len())+ .unwrap_or(bytes.len());+ if self.next(Pattern::Paren, start, terminator).is_none() {+ return terminator;+ }+ self.closer(Closing::UrlParen, start, 1, |_, position| {+ match bytes[position] {+ b'(' => Token::Open(1),+ b')' => Token::Close,+ byte if is_url_terminator(byte) => Token::Fail,+ _ => Token::Step(1),+ }+ })+ .map_or(terminator, |closer| closer.at)+ }++ pub(crate) fn has_link_syntax(&mut self, start: usize, end: usize) -> bool {+ let Some(first) = self.next(Pattern::OpenBracket, start, end) else {+ return false;+ };+ let bytes = self.text.as_bytes();+ let len = bytes.len();+ let mut link_syntax = std::mem::take(&mut self.memos().link_syntax);+ let found = link_syntax.resolve(len, first, |open| {+ let Some(close) = self.next(Pattern::CloseBracket, open + 1, len) else {+ return ChainLink::End;+ };+ if bytes.get(close + 1) == Some(&b'(') {+ return ChainLink::Hit(close);+ }+ self.next(Pattern::OpenBracket, close + 1, len)+ .map_or(ChainLink::End, ChainLink::Next)+ });+ self.memos().link_syntax = link_syntax;+ found.is_some_and(|close| close + 1 < end)+ }++ pub(crate) fn has_angle_email(&mut self, start: usize, end: usize) -> bool {+ let Some(first) = self.next(Pattern::AngleOpen, start, end) else {+ return false;+ };+ let bytes = self.text.as_bytes();+ let len = bytes.len();+ let mut angle_email = std::mem::take(&mut self.memos().angle_email);+ let found = angle_email.resolve(len, first, |open| {+ let Some(close) = self.next(Pattern::AngleClose, open + 1, len) else {+ return ChainLink::End;+ };+ if close > open + 1+ && bytes[open + 1] != b'@'+ && self.next(Pattern::At, open + 1, close).is_some()+ {+ return ChainLink::Hit(close);+ }+ self.next(Pattern::AngleOpen, close + 1, len)+ .map_or(ChainLink::End, ChainLink::Next)+ });+ self.memos().angle_email = angle_email;+ found.is_some_and(|close| close < end)+ }+}diff --git a/packages/markdown_parser/rust/src/text.rs b/packages/markdown_parser/rust/src/text.rsindex 5e9b3fc6b..55c08abc2 100644--- a/packages/markdown_parser/rust/src/text.rs+++ b/packages/markdown_parser/rust/src/text.rs@@ -1,7 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::ast::Node;-use crate::constants::{MAX_LINE_LENGTH, MAX_LINK_URL_LENGTH};+use crate::constants::MAX_LINE_LENGTH;#[derive(Clone, Debug, Eq, PartialEq)]pub struct Line {@@ -75,24 +75,17 @@ pub fn concat(left: &str, right: &str) -> String {out}-pub fn concat3(a: &str, b: &str, c: &str) -> String {- let mut out = String::with_capacity(a.len() + b.len() + c.len());- out.push_str(a);- out.push_str(b);- out.push_str(c);- out-}--pub fn append_text(nodes: &mut Vec<Node>, content: impl Into<String>) {- let content = content.into();+pub fn append_text(nodes: &mut Vec<Node>, content: &str) {if content.is_empty() {return;}if let Some(Node::Text { content: previous }) = nodes.last_mut() {- previous.push_str(&content);+ previous.push_str(content);return;}- nodes.push(Node::Text { content });+ nodes.push(Node::Text {+ content: content.to_owned(),+ });}pub fn line_as_text(lines: &[Line], current: usize) -> String {@@ -141,11 +134,6 @@ pub fn starts_with(value: &str, prefix: &str) -> bool {value.as_bytes().starts_with(prefix.as_bytes())}-#[inline]-pub fn ends_with(value: &[u8], suffix: &[u8]) -> bool {- value.ends_with(suffix)-}-#[inline]pub fn is_digit(byte: u8) -> bool {byte.is_ascii_digit()@@ -241,17 +229,13 @@ pub fn find_from(value: &str, start: usize, needle: u8) -> Option<usize> {.map(|pos| start + pos)}-pub fn bounded_url_end(value: &str, end: usize, prefix_len: usize) -> usize {- bounded_utf8_prefix_length(&value[..end], prefix_len + MAX_LINK_URL_LENGTH)-}--pub fn remove_text_presentation(text: &str) -> String {- if !text.contains('\u{fe0f}') {- return text.to_owned();+pub fn remove_text_presentation(text: &mut String) {+ if !text.is_ascii() && text.contains('\u{fe0f}') {+ *text = text+ .replace("™️", "™")+ .replace("©️", "©")+ .replace("®️", "®");}- text.replace("™️", "™")- .replace("©️", "©")- .replace("®️", "®")}pub fn trim_start_newline_whitespace(value: &str) -> &str {diff --git a/packages/markdown_parser/rust/tests/fixtures.rs b/packages/markdown_parser/rust/tests/fixtures.rsindex 5ab3d637e..ea9a6c0dd 100644--- a/packages/markdown_parser/rust/tests/fixtures.rs+++ b/packages/markdown_parser/rust/tests/fixtures.rs@@ -6,7 +6,14 @@ use serde_json::json;fn parse(input: &str, flags: u32, emoji_context: &str) -> serde_json::Value {let mut parser = MarkdownParser::new(flags, EmojiContext::parse(emoji_context));let nodes = parser.parse(input).expect("parse succeeds");- serde_json::to_value(json!({"nodes": nodes})).expect("serialize succeeds")+ let value = serde_json::to_value(json!({"nodes": nodes})).expect("serialize succeeds");+ let written = fluxer_markdown_parser::parse_markdown_json(input, flags, emoji_context)+ .expect("parse succeeds");+ assert_eq!(+ serde_json::from_str::<serde_json::Value>(&written).expect("written json parses"),+ value+ );+ value}#[test]diff --git a/packages/markdown_parser/rust/tests/linear_time.rs b/packages/markdown_parser/rust/tests/linear_time.rsnew file mode 100644index 000000000..582542390--- /dev/null+++ b/packages/markdown_parser/rust/tests/linear_time.rs@@ -0,0 +1,21 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use std::time::{Duration, Instant};++use fluxer_markdown_parser::{EmojiContext, MarkdownParser, ParserFlags};++#[test]+fn unclosed_masked_links_parse_in_linear_time() {+ let line = "[a](".repeat(1024);+ let input = format!("{line}\n").repeat(400);+ let mut parser = MarkdownParser::new(ParserFlags::ALL, EmojiContext::default());+ let started = Instant::now();+ let nodes = parser.parse(&input).expect("parse succeeds");+ let elapsed = started.elapsed();+ assert_eq!(nodes.len(), 1);+ assert!(+ elapsed < Duration::from_secs(20),+ "parsing {} bytes took {elapsed:?}",+ input.len()+ );+}