Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(markdown): linear-time parser and composer caches (#3628)

Hampusb6d02820d55d
diff --git a/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts b/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts
index bfdcbea64..4dc7045e9 100644
--- a/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts
+++ b/fluxer_app/src/features/expressions/utils/UnicodeEmojis.ts
@@ -66,6 +66,7 @@ const HAIR_COMPONENT_NAMES: Record<string, string> = {
'\u{1F9B3}': 'white_hair',
'\u{1F9B2}': 'bald',
};
+const REGIONAL_INDICATOR_HIGH_SURROGATE = 0xd83c;
const REGIONAL_INDICATOR_PAIR_PATTERN = '\\uD83C[\\uDDE6-\\uDDFF]\\uD83C[\\uDDE6-\\uDDFF]';
let defaultSkinTone: string = '';
@@ -178,6 +179,9 @@ interface EmojiIndex {
skinToneSpriteCount: number;
baseSpriteCount: number;
emojiSurrogateRegex: RegExp;
+ emojiSurrogateStickyRegex: RegExp;
+ emojiTriggerCodeUnits: Uint8Array;
+ maxEmojiTriggerOffset: number;
emojiShortcutRegex: RegExp;
}
@@ -284,10 +288,16 @@ function buildEmojiIndex(): EmojiIndex {
shortcutToName[shortcut] = emoji;
});
- const surrogateAlternation = Object.keys(surrogateToName)
- .sort((a, b) => b.length - a.length)
- .map(RegexUtils.escapeRegex)
- .join('|');
+ const surrogates = Object.keys(surrogateToName).sort((a, b) => b.length - a.length);
+ const surrogatePattern = `(${REGIONAL_INDICATOR_PAIR_PATTERN}|${surrogates.map(RegexUtils.escapeRegex).join('|')})`;
+ const emojiTriggerCodeUnits = new Uint8Array(0x10000);
+ emojiTriggerCodeUnits[REGIONAL_INDICATOR_HIGH_SURROGATE] = 1;
+ let maxEmojiTriggerOffset = 0;
+ for (const surrogate of surrogates) {
+ const offset = findEmojiTriggerOffset(surrogate);
+ emojiTriggerCodeUnits[surrogate.charCodeAt(offset)] = 1;
+ maxEmojiTriggerOffset = Math.max(maxEmojiTriggerOffset, offset);
+ }
const shortcutAlternation = Object.keys(shortcutToName).map(RegexUtils.escapeRegex).join('|');
return {
@@ -302,7 +312,10 @@ function buildEmojiIndex(): EmojiIndex {
emojis,
skinToneSpriteCount,
baseSpriteCount,
- emojiSurrogateRegex: new RegExp(`(${REGIONAL_INDICATOR_PAIR_PATTERN}|${surrogateAlternation})`, 'g'),
+ emojiSurrogateRegex: new RegExp(surrogatePattern, 'g'),
+ emojiSurrogateStickyRegex: new RegExp(surrogatePattern, 'y'),
+ emojiTriggerCodeUnits,
+ maxEmojiTriggerOffset,
emojiShortcutRegex: new RegExp(`^(${shortcutAlternation})`),
};
}
@@ -317,7 +330,9 @@ const lookupSurrogateName = (surrogate: string): string | null =>
const isRegionalIndicatorAt = (text: string, index: number): boolean => {
const lowSurrogate = text.charCodeAt(index + 1);
- return text.charCodeAt(index) === 0xd83c && lowSurrogate >= 0xdde6 && lowSurrogate <= 0xddff;
+ return (
+ text.charCodeAt(index) === REGIONAL_INDICATOR_HIGH_SURROGATE && lowSurrogate >= 0xdde6 && lowSurrogate <= 0xddff
+ );
};
const isInsideRegionalIndicatorPair = (text: string, index: number): boolean => {
@@ -327,6 +342,19 @@ const isInsideRegionalIndicatorPair = (text: string, index: number): boolean =>
return (index - runStart) % 4 === 2;
};
+function findEmojiTriggerOffset(surrogate: string): number {
+ let offset = 0;
+ while (offset < surrogate.length - 1 && surrogate.charCodeAt(offset) <= 0x7f) offset += 1;
+ return offset;
+}
+
+function findEmojiTrigger(text: string, from: number, triggers: Uint8Array): number {
+ for (let index = from; index < text.length; index += 1) {
+ if (triggers[text.charCodeAt(index)] === 1) return index;
+ }
+ return -1;
+}
+
const toEmojiSurrogateMatch = (text: string, start: number, end: number): EmojiSurrogateMatch => ({
start,
end,
@@ -334,17 +362,24 @@ const toEmojiSurrogateMatch = (text: string, start: number, end: number): EmojiS
});
function* matchEmojiSurrogates(text: string, startIndex = 0): Generator<EmojiSurrogateMatch, void> {
- const regex = getEmojiIndex().emojiSurrogateRegex;
+ const {emojiSurrogateStickyRegex: regex, emojiTriggerCodeUnits, maxEmojiTriggerOffset} = getEmojiIndex();
let index = startIndex;
if (isInsideRegionalIndicatorPair(text, index)) {
yield toEmojiSurrogateMatch(text, index, index + 2);
index += 2;
}
- while (true) {
+ let trigger = findEmojiTrigger(text, index, emojiTriggerCodeUnits);
+ while (trigger >= 0) {
+ index = Math.max(index, trigger - maxEmojiTriggerOffset);
regex.lastIndex = index;
const match = regex.exec(text);
- if (match == null) return;
+ if (match == null) {
+ index += 1;
+ if (index > trigger) trigger = findEmojiTrigger(text, index, emojiTriggerCodeUnits);
+ continue;
+ }
index = match.index + match[0].length;
+ trigger = findEmojiTrigger(text, index, emojiTriggerCodeUnits);
const emojiMatch = toEmojiSurrogateMatch(text, match.index, index);
const isUnknownRegionalIndicatorPair =
emojiMatch.name == null &&
diff --git a/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts b/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts
index bf3a7e6e0..c0128c46b 100644
--- a/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts
+++ b/fluxer_app/src/features/lexical/composer/ComposerMarkdownHighlight.ts
@@ -14,6 +14,7 @@ import {
type MarkdownHlFormat,
type MarkdownSpan,
markSilentMessagePrefix,
+ splitSpansByLine,
} from '@app/features/lexical/composer/markdownSpans';
import {
$createComposerBlockquoteMarkerNode,
@@ -134,22 +135,12 @@ function $reconcileParagraph(
return;
}
const spans = silentMessagePrefix ? markSilentMessagePrefix(markdownSpans, source) : markdownSpans;
- let lineStart = 0;
+ const lineSpans = splitSpansByLine(spans, lineSources);
for (let index = 0; index < lines.length; index += 1) {
const nodes = lines[index]!.nodes;
- const lineSource = lineSources[index]!;
- const lineEnd = lineStart + lineSource.length;
if (nodes.length > 0) {
- const lineSpans = spans
- .map((span) => ({
- ...span,
- start: Math.max(span.start, lineStart) - lineStart,
- end: Math.min(span.end, lineEnd) - lineStart,
- }))
- .filter((span) => span.end > span.start);
- $reconcileLine(nodes, parserFlags, lineSource, lineSpans, quoteMarkerEnds[index]);
+ $reconcileLine(nodes, parserFlags, lineSources[index]!, lineSpans[index]!, quoteMarkerEnds[index]);
}
- lineStart = lineEnd + 1;
}
$syncComposerBlockquoteLines(
paragraph,
diff --git a/fluxer_app/src/features/lexical/composer/blockquoteLines.ts b/fluxer_app/src/features/lexical/composer/blockquoteLines.ts
index 1aabea3e7..ed86a9618 100644
--- a/fluxer_app/src/features/lexical/composer/blockquoteLines.ts
+++ b/fluxer_app/src/features/lexical/composer/blockquoteLines.ts
@@ -98,12 +98,18 @@ function scanMarkerEnds(
markerEndByStart.set(span.start, span.end);
}
}
- const blocks = scanAllowedCodeBlocks(source, parserFlags);
+ const blocks = scanAllowedCodeBlocks(source, parserFlags).sort((left, right) => left.start - right.start);
const ends: Array<number> = [];
let lineStart = 0;
+ let nextBlock = 0;
+ let coveredUntil = -1;
for (const line of lines) {
+ while (nextBlock < blocks.length && blocks[nextBlock]!.start <= lineStart) {
+ coveredUntil = Math.max(coveredUntil, blocks[nextBlock]!.end);
+ nextBlock += 1;
+ }
const markerEnd = markerEndByStart.get(lineStart);
- const suppressed = isInsideCodeBlock(blocks, lineStart) || MULTILINE_ALERT_RE.test(line);
+ const suppressed = coveredUntil >= lineStart || MULTILINE_ALERT_RE.test(line);
ends.push(markerEnd == null || suppressed ? 0 : Math.min(markerEnd, lineStart + line.length) - lineStart);
lineStart += line.length + 1;
}
diff --git a/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts b/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts
index ad8ebcd7d..7c00295be 100644
--- a/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts
+++ b/fluxer_app/src/features/lexical/composer/codeBlockIndent.ts
@@ -59,6 +59,11 @@ interface ScanScope {
rendered: boolean;
}
+interface ClosingLines {
+ validFrom: number;
+ nextByFence: Map<number, Int32Array>;
+}
+
interface PhysicalLine {
start: number;
end: number;
@@ -170,12 +175,48 @@ function opensUnclosedBlock(line: SourceLine, column: number, info: string): boo
);
}
+const closingLinesByLines = new WeakMap<Array<SourceLine>, ClosingLines>();
+
+function findClosingLine(lines: Array<SourceLine>, from: number, fence: string): number {
+ let memo = closingLinesByLines.get(lines);
+ if (memo == null || from < memo.validFrom) {
+ memo = {validFrom: 0, nextByFence: new Map()};
+ closingLinesByLines.set(lines, memo);
+ }
+ let next = memo.nextByFence.get(fence.length);
+ if (next == null) {
+ next = new Int32Array(lines.length + 1);
+ memo.nextByFence.set(fence.length, next);
+ }
+ let position = from;
+ let found = lines.length;
+ while (position < lines.length) {
+ const known = next[position]!;
+ if (known !== 0) {
+ found = known - 1;
+ break;
+ }
+ if (findCodeBlockClosing(lines[position]!.text, fence, fence.length) != null) {
+ found = position;
+ next[position] = found + 1;
+ break;
+ }
+ position += 1;
+ }
+ next.fill(found + 1, from, position);
+ return found;
+}
+
function resumeAt(lines: Array<SourceLine>, {index, column}: LinePoint): number {
const line = lines[index]!;
if (column >= line.text.length) {
return index + 1;
}
lines[index] = {text: line.text.slice(column), offset: line.offset + column};
+ const memo = closingLinesByLines.get(lines);
+ if (memo != null) {
+ memo.validFrom = Math.max(memo.validFrom, index + 1);
+ }
return index;
}
@@ -200,19 +241,19 @@ function parseCodeBlock(lines: Array<SourceLine>, index: number, column: number,
resume: visible ? {index, column: infoStart + inlineEnd + fence.length} : null,
};
}
- let content = FENCE_LANGUAGE_RE.test(info) ? '' : info;
- for (let next = index + 1; next < lines.length; next += 1) {
- const text = lines[next]!.text;
- const closing = findCodeBlockClosing(text, fence, fence.length);
- if (closing == null) {
- content += text;
- continue;
+ const closingLine = findClosingLine(lines, index + 1, fence);
+ if (closingLine < lines.length) {
+ let content = FENCE_LANGUAGE_RE.test(info) ? '' : info;
+ for (let between = index + 1; between < closingLine; between += 1) {
+ content += lines[between]!.text;
}
+ const {text, offset} = lines[closingLine]!;
+ const closing = findCodeBlockClosing(text, fence, fence.length)!;
const visible = hasVisibleContent(content + text.slice(0, closing.fenceIndex));
const trailingStart = closing.fenceIndex + (closing.trailing.length > 0 ? closing.runLength : fence.length);
return {
- body: {start, end: lines[next]!.offset + closing.fenceIndex, rendered: visible && scope.rendered},
- resume: visible ? {index: next, column: trailingStart} : null,
+ body: {start, end: offset + closing.fenceIndex, rendered: visible && scope.rendered},
+ resume: visible ? {index: closingLine, column: trailingStart} : null,
};
}
if (!opensUnclosedBlock(line, column, info)) {
diff --git a/fluxer_app/src/features/lexical/composer/markdownSpans.ts b/fluxer_app/src/features/lexical/composer/markdownSpans.ts
index 74f83e518..a8087a3a7 100644
--- a/fluxer_app/src/features/lexical/composer/markdownSpans.ts
+++ b/fluxer_app/src/features/lexical/composer/markdownSpans.ts
@@ -7,6 +7,7 @@ import type {Node} from '@app/features/messaging/utils/markdown/parser/Nodes';
import {normalizeUrl} from '@app/features/messaging/utils/markdown/parser/UrlUtils';
import {findUrlEnd} from '@app/features/messaging/utils/markdown/UrlSpanUtils';
import {parseSilentMessagePrefix} from '@app/features/messaging/utils/SilentMessagePrefix';
+import {LRUMap} from '@app/lib/list/ListLruMap';
export const MarkdownHl = {
none: 0,
@@ -76,12 +77,49 @@ type AlignAttempt = {type: 'aligned'; spans: Array<MarkdownSpan>} | {type: 'desy
const MAX_ALIGN_ATTEMPTS = 64;
const MAX_URL_REALIGN_STEPS = 256;
+const HIGHLIGHT_CACHE_CAPACITY = 32;
+
+const highlightCache = new LRUMap<string, ReadonlyArray<MarkdownSpan>>(HIGHLIGHT_CACHE_CAPACITY);
export function computeMarkdownHighlightSpans(
source: string,
parserFlags = DEFAULT_COMPOSER_MARKDOWN_FLAGS,
-): Array<MarkdownSpan> {
- return computeMarkdownHighlightResult(source, parserFlags).spans;
+): ReadonlyArray<MarkdownSpan> {
+ const key = `${parserFlags}\u0000${source}`;
+ const cached = highlightCache.get(key);
+ if (cached !== undefined) {
+ return cached;
+ }
+ const spans = computeMarkdownHighlightResult(source, parserFlags).spans;
+ highlightCache.set(key, spans);
+ return spans;
+}
+
+export function splitSpansByLine(
+ spans: ReadonlyArray<MarkdownSpan>,
+ lineSources: ReadonlyArray<string>,
+): Array<Array<MarkdownSpan>> {
+ const result: Array<Array<MarkdownSpan>> = [];
+ let first = 0;
+ let lineStart = 0;
+ for (const lineSource of lineSources) {
+ const lineEnd = lineStart + lineSource.length;
+ while (first < spans.length && spans[first]!.end <= lineStart) {
+ first += 1;
+ }
+ const lineSpans: Array<MarkdownSpan> = [];
+ for (let index = first; index < spans.length && spans[index]!.start < lineEnd; index += 1) {
+ const span = spans[index]!;
+ const start = Math.max(span.start, lineStart) - lineStart;
+ const end = Math.min(span.end, lineEnd) - lineStart;
+ if (end > start) {
+ lineSpans.push({...span, start, end});
+ }
+ }
+ result.push(lineSpans);
+ lineStart = lineEnd + 1;
+ }
+ return result;
}
function computeMarkdownHighlightResult(
@@ -126,7 +164,10 @@ function computeMarkdownHighlightResult(
return {spans: coalesce(spans), recovered};
}
-export function markSilentMessagePrefix(spans: Array<MarkdownSpan>, source: string): Array<MarkdownSpan> {
+export function markSilentMessagePrefix(
+ spans: ReadonlyArray<MarkdownSpan>,
+ source: string,
+): ReadonlyArray<MarkdownSpan> {
const prefix = parseSilentMessagePrefix(source);
if (prefix == null) {
return spans;
diff --git a/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts b/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts
index 45b073efb..f08535988 100644
--- a/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts
+++ b/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasm.ts
@@ -7,6 +7,7 @@ import {MARKDOWN_PARSER_WASM_BASE64} from '@app/features/messaging/utils/markdow
import type {Node} from '@app/features/messaging/utils/markdown/parser/Nodes';
import * as StringUtils from '@app/features/messaging/utils/markdown/parser/StringUtils';
import * as URLUtils from '@app/features/messaging/utils/markdown/parser/UrlUtils';
+import {LRUMap} from '@app/lib/list/ListLruMap';
interface WasmExports {
memory: {buffer: ArrayBuffer};
@@ -42,6 +43,7 @@ type CleanableNode = Node & {
const RESULT_HEADER_BYTES = 16;
const MAX_RETAINED_WASM_MEMORY_BYTES = 64 * 1024 * 1024;
+const LINK_URL_CACHE_CAPACITY = 512;
const textEncoder = new TextEncoder();
const lenientTextDecoder = new TextDecoder('utf-8', {ignoreBOM: true, fatal: false});
@@ -320,6 +322,38 @@ function shouldTreatAsMaskedLink(trimmedLinkText: string, url: string): boolean
}
}
+const linkUrlCache = new LRUMap<string, string | null>(LINK_URL_CACHE_CAPACITY);
+
+function cleanLinkUrl(rawUrl: string, linkText: string): string | null {
+ const key = `${rawUrl.length}:${rawUrl}${linkText}`;
+ const cached = linkUrlCache.get(key);
+ if (cached !== undefined) return cached;
+ const url = resolveLinkUrl(rawUrl, linkText);
+ linkUrlCache.set(key, url);
+ return url;
+}
+
+function resolveLinkUrl(rawUrl: string, linkText: string): string | null {
+ try {
+ const normalizedUrl = URLUtils.normalizeUrl(rawUrl);
+ if (
+ !URLUtils.isValidUrl(normalizedUrl) ||
+ (rawUrl.startsWith('/') && !rawUrl.startsWith('//')) ||
+ (StringUtils.startsWithUrl(linkText.trim()) && shouldTreatAsMaskedLink(linkText, rawUrl))
+ ) {
+ return null;
+ }
+ if (normalizedUrl.startsWith('tel:') || normalizedUrl.startsWith('sms:')) {
+ const protocol = normalizedUrl.substring(0, normalizedUrl.indexOf(':') + 1);
+ const phoneNumber = normalizedUrl.substring(normalizedUrl.indexOf(':') + 1);
+ return phoneNumber.startsWith('+') ? protocol + URLUtils.normalizePhoneNumber(phoneNumber) : normalizedUrl;
+ }
+ return URLUtils.convertToAsciiUrl(normalizedUrl);
+ } catch {
+ return null;
+ }
+}
+
function makeCleanedTextNode(content: string): CleanedTextNode {
return {type: 'Text', content, __cleanedLinkText: true};
}
@@ -335,34 +369,16 @@ function cleanNode(node: CleanableNode): Node {
if (node.type === 'Link') {
const rawUrl = coerceWasmString(node.rawUrl ?? node.url);
const source = coerceWasmString(node.source ?? rawUrl);
- const linkText = textFromNode(node.text);
- try {
- const normalizedUrl = URLUtils.normalizeUrl(rawUrl);
- if (
- !URLUtils.isValidUrl(normalizedUrl) ||
- (rawUrl.startsWith('/') && !rawUrl.startsWith('//')) ||
- (StringUtils.startsWithUrl(linkText.trim()) && shouldTreatAsMaskedLink(linkText, rawUrl))
- ) {
- return makeCleanedTextNode(source);
- }
- let finalUrl = normalizedUrl;
- if (finalUrl.startsWith('tel:') || finalUrl.startsWith('sms:')) {
- const protocol = finalUrl.substring(0, finalUrl.indexOf(':') + 1);
- const phoneNumber = finalUrl.substring(finalUrl.indexOf(':') + 1);
- if (phoneNumber.startsWith('+')) finalUrl = protocol + URLUtils.normalizePhoneNumber(phoneNumber);
- } else {
- finalUrl = URLUtils.convertToAsciiUrl(finalUrl);
- }
- const cleaned: Extract<Node, {type: 'Link'}> = {
- type: 'Link',
- url: finalUrl,
- escaped: Boolean(node.escaped),
- };
- cleaned.text = node.text ? cleanNode(node.text) : undefined;
- return cleaned;
- } catch {
+ const url = cleanLinkUrl(rawUrl, textFromNode(node.text));
+ if (url == null) {
return makeCleanedTextNode(source);
}
+ return {
+ type: 'Link',
+ url,
+ escaped: Boolean(node.escaped),
+ text: node.text ? cleanNode(node.text) : undefined,
+ };
}
if (node.type === 'CodeBlock' && !('language' in node)) {
node.language = undefined;
diff --git a/packages/markdown_parser/rust/src/block.rs b/packages/markdown_parser/rust/src/block.rs
index f3cc23e0c..bce40a700 100644
--- a/packages/markdown_parser/rust/src/block.rs
+++ b/packages/markdown_parser/rust/src/block.rs
@@ -1,14 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::ast::{AlertType, ListItem, Node, ParserFlags, TableAlignment};
-use crate::constants::{CODE_FENCE_LENGTH, MAX_AST_NODES, MAX_LINE_LENGTH};
+use crate::constants::{
+ CODE_FENCE_LENGTH, MAX_AST_NODES, MAX_CODE_BLOCK_SEARCH_LINES, MAX_LINE_LENGTH,
+};
use crate::links::{has_open_inline_code, has_valid_code_fence_language};
use crate::normalize::{normalize_nodes, replace_trailing_whitespace_with_newline};
use crate::parser::{MarkdownParser, ParseError, RuntimeState};
use crate::text::{
- Line, bounded_line_text, byte_at, concat, concat3, has_visible_content, line_as_text,
- lines_to_text, starts_with, trim, trim_line_window, trim_right, trim_start,
- trim_start_newline_whitespace,
+ Line, bounded_line_text, byte_at, concat, has_visible_content, line_as_text, lines_to_text,
+ starts_with, trim, trim_line_window, trim_right, trim_start, trim_start_newline_whitespace,
};
#[derive(Clone, Debug)]
@@ -73,6 +74,12 @@ struct TableCellText {
offset: usize,
}
+#[derive(Clone, Copy, Debug)]
+struct TableLine<'a> {
+ text: &'a str,
+ offset: usize,
+}
+
pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResult, ParseError> {
let current = state.current_line;
let line = state.lines[current].text.clone();
@@ -110,14 +117,20 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResu
if let Some(list_match) = match_list_item(&line) {
if ParserFlags::has(state.parser.flags(), ParserFlags::ALLOW_LISTS) {
- let list = parse_list(
- state.parser,
- &state.lines,
+ let node_count = state.parser.node_count;
+ let list = ListContext {
+ parser: state.parser,
+ lines: &state.lines,
+ fences: &mut state.fences,
+ }
+ .parse_list(
current,
- list_match.ordered,
- list_match.indent_level,
- 1,
- state.parser.node_count,
+ ListLevel {
+ ordered: list_match.ordered,
+ indent_level: list_match.indent_level,
+ depth: 1,
+ },
+ node_count,
)?;
return Ok(BlockParseResult {
node: Some(list.node),
@@ -137,9 +150,8 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResu
if starts_with(trimmed, "||") && !trimmed[2..].contains("||") {
if ParserFlags::has(state.parser.flags(), ParserFlags::ALLOW_SPOILERS) {
let result = parse_block_spoiler(state.parser, &state.lines, current)?;
- if let Some(extra) = &result.extra_content {
- state.lines[result.new_line_index].text = extra.content.clone();
- state.lines[result.new_line_index].offset = extra.offset;
+ if let Some(extra) = result.extra_content {
+ state.replace_line(result.new_line_index, extra.content, extra.offset);
}
return Ok(BlockParseResult {
node: Some(result.node),
@@ -161,10 +173,11 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResu
let starts_with_fence =
starts_with(trimmed, "```") && fence_pos == line.len() - trimmed.len();
if starts_with_fence {
- if let Some(result) = parse_code_block(&state.lines, current)? {
- if let Some(extra) = &result.extra_content {
- state.lines[result.new_line_index].text = extra.content.clone();
- state.lines[result.new_line_index].offset = extra.offset;
+ if let Some(result) =
+ parse_code_block(&state.lines, current, &line, line_offset, &mut state.fences)
+ {
+ if let Some(extra) = result.extra_content {
+ state.replace_line(result.new_line_index, extra.content, extra.offset);
}
return Ok(BlockParseResult {
node: Some(result.node),
@@ -179,25 +192,37 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResu
if has_open_inline_code(prefix) {
return Ok(no_block(current, state.parser.node_count));
}
- let inline_nodes = crate::inline::parse_inline(state.parser, prefix, line_offset)?;
- let code_lines = slice_lines_from_fence(&state.lines, current, fence_pos);
- if let Some(code_result) = parse_code_block(&code_lines, 0)? {
- let new_line_index = current + code_result.new_line_index;
- let mut extra_nodes = Vec::new();
- if inline_nodes.len() > 1 {
- extra_nodes.extend(inline_nodes[1..].iter().cloned());
- }
- extra_nodes.push(code_result.node.clone());
- if let Some(extra) = &code_result.extra_content {
- state.lines[new_line_index].text = extra.content.clone();
- state.lines[new_line_index].offset = extra.offset;
+ if let Some(code_result) = parse_code_block(
+ &state.lines,
+ current,
+ &line[fence_pos..],
+ line_offset + fence_pos,
+ &mut state.fences,
+ ) {
+ let inline_nodes = crate::inline::parse_inline(state.parser, prefix, line_offset)?;
+ let new_node_count = state.parser.node_count + inline_nodes.len() + 1;
+ let new_line_index = code_result.new_line_index;
+ if let Some(extra) = code_result.extra_content {
+ state.replace_line(new_line_index, extra.content, extra.offset);
}
- let first_node = inline_nodes.first().cloned().unwrap_or(code_result.node);
+ let mut inline_nodes = inline_nodes.into_iter();
+ let first_node = inline_nodes.next();
+ let mut extra_nodes: Vec<Node> = inline_nodes.collect();
+ let first_node = match first_node {
+ Some(first_node) => {
+ extra_nodes.push(code_result.node);
+ first_node
+ }
+ None => {
+ extra_nodes.push(code_result.node.clone());
+ code_result.node
+ }
+ };
return Ok(BlockParseResult {
node: Some(first_node),
extra_nodes: Some(extra_nodes),
new_line_index,
- new_node_count: state.parser.node_count + inline_nodes.len() + 1,
+ new_node_count,
});
}
}
@@ -205,7 +230,8 @@ pub(crate) fn parse_block(state: &mut RuntimeState<'_>) -> Result<BlockParseResu
let mut content = line;
let mut end_index = current + 1;
while end_index < state.lines.len() {
- content = concat3(&content, "\n", &state.lines[end_index].text);
+ content.push('\n');
+ content.push_str(&state.lines[end_index].text);
if trim(&state.lines[end_index].text) == "```" {
end_index += 1;
break;
@@ -279,30 +305,37 @@ fn no_block(current: usize, node_count: usize) -> BlockParseResult {
}
}
-pub(crate) fn parse_heading_node(
- parser: &mut MarkdownParser,
- trimmed_line: &str,
- offset: usize,
-) -> Result<Option<Node>, ParseError> {
+pub(crate) fn heading_content(trimmed_line: &str) -> Option<(usize, &str)> {
let mut level = 0usize;
while level < trimmed_line.len() && level < 4 && byte_at(trimmed_line, level) == b'#' {
level += 1;
}
- if (1..=4).contains(&level)
- && level < trimmed_line.len()
- && byte_at(trimmed_line, level) == b' '
+ if !(1..=4).contains(&level)
+ || level >= trimmed_line.len()
+ || byte_at(trimmed_line, level) != b' '
{
- let content = &trimmed_line[level + 1..];
- if !has_visible_content(content) && trim(content).is_empty() {
- return Ok(None);
- }
- let children = crate::inline::parse_inline(parser, content, offset + level + 1)?;
- return Ok(Some(Node::Heading {
- level: level as u8,
- children,
- }));
+ return None;
+ }
+ let content = &trimmed_line[level + 1..];
+ if !has_visible_content(content) && trim(content).is_empty() {
+ return None;
}
- Ok(None)
+ Some((level, content))
+}
+
+fn parse_heading_node(
+ parser: &mut MarkdownParser,
+ trimmed_line: &str,
+ offset: usize,
+) -> Result<Option<Node>, ParseError> {
+ let Some((level, content)) = heading_content(trimmed_line) else {
+ return Ok(None);
+ };
+ let children = crate::inline::parse_inline(parser, content, offset + level + 1)?;
+ Ok(Some(Node::Heading {
+ level: level as u8,
+ children,
+ }))
}
fn parse_subtext_node(
@@ -365,7 +398,7 @@ fn parse_blockquote(
return Ok(no_block(index, node_count));
}
if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_ALERTS)
- && let Some(alert) = parse_alert_lines(parser, child_lines.clone())?
+ && let Some(alert) = parse_alert_lines(parser, &child_lines)?
{
return Ok(BlockParseResult {
node: Some(alert),
@@ -447,11 +480,60 @@ fn parse_multiline_blockquote(
})
}
-fn parse_code_block(lines: &[Line], current: usize) -> Result<Option<CodeBlockResult>, ParseError> {
- if current >= lines.len() {
- return Ok(None);
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum CodeBlockMode {
+ Build,
+ Probe,
+}
+
+#[derive(Clone, Debug)]
+struct CodeBlockContent {
+ mode: CodeBlockMode,
+ text: String,
+ len: usize,
+ visible: bool,
+}
+
+impl CodeBlockContent {
+ fn push_line(&mut self, line: &str) {
+ self.len += line.len() + 1;
+ self.visible = self.visible || has_visible_content(line);
+ if self.mode == CodeBlockMode::Build {
+ self.text.push_str(line);
+ self.text.push('\n');
+ }
}
- let line = &lines[current].text;
+
+ fn settled(&self) -> bool {
+ self.mode == CodeBlockMode::Probe && self.visible
+ }
+}
+
+fn parse_code_block(
+ lines: &[Line],
+ current: usize,
+ line: &str,
+ line_offset: usize,
+ fences: &mut ClosingFenceLines,
+) -> Option<CodeBlockResult> {
+ scan_code_block(
+ lines,
+ current,
+ line,
+ line_offset,
+ fences,
+ CodeBlockMode::Build,
+ )
+}
+
+fn scan_code_block(
+ lines: &[Line],
+ current: usize,
+ line: &str,
+ line_offset: usize,
+ fences: &mut ClosingFenceLines,
+ mode: CodeBlockMode,
+) -> Option<CodeBlockResult> {
let trimmed_line = trim_start(line);
let indent_spaces = line.len() - trimmed_line.len();
let list_indent = &line[..indent_spaces];
@@ -460,17 +542,17 @@ fn parse_code_block(lines: &[Line], current: usize) -> Result<Option<CodeBlockRe
fence_length += 1;
}
if fence_length < CODE_FENCE_LENGTH {
- return Ok(None);
+ return None;
}
let language_part = &trimmed_line[fence_length..];
let closing_fence = "`".repeat(fence_length);
if let Some(closing_index) = language_part.find(&closing_fence) {
let inline_content = &language_part[..closing_index];
if !has_visible_content(inline_content) {
- return Ok(None);
+ return None;
}
let trailing = &language_part[closing_index + fence_length..];
- return Ok(Some(CodeBlockResult {
+ return Some(CodeBlockResult {
node: Node::CodeBlock {
language: None,
content: inline_content.to_owned(),
@@ -482,94 +564,142 @@ fn parse_code_block(lines: &[Line], current: usize) -> Result<Option<CodeBlockRe
},
extra_content: (!trailing.is_empty()).then(|| ExtraContent {
content: trailing.to_owned(),
- offset: lines[current].offset + (line.len() - trailing.len()),
+ offset: line_offset + (line.len() - trailing.len()),
}),
- }));
+ });
}
let treat_opening_as_language = has_valid_code_fence_language(language_part);
let language = treat_opening_as_language.then(|| trim(language_part).to_owned());
- let mut temp = current + 1;
- let mut line_count = 0usize;
- let mut has_closing = false;
- while temp < lines.len() {
- if find_closing_fence(trim_start(&lines[temp].text), &closing_fence, fence_length).is_some()
- {
- has_closing = true;
- break;
- }
- line_count += 1;
- if line_count > 1000 {
- break;
- }
- temp += 1;
- }
- if !has_closing {
- return Ok(None);
+ let closing_line = fences.next(lines, current + 1, &closing_fence)?;
+ if closing_line - current - 1 > MAX_CODE_BLOCK_SEARCH_LINES {
+ return None;
}
- let mut content = String::new();
+ let mut content = CodeBlockContent {
+ mode,
+ text: String::new(),
+ len: 0,
+ visible: false,
+ };
if !treat_opening_as_language && !language_part.is_empty() {
- content.push_str(language_part);
- content.push('\n');
+ content.push_line(language_part);
}
let mut index = current + 1;
- while index < lines.len() {
+ while index < closing_line && !content.settled() {
let current_line = &lines[index].text;
- let trimmed = trim_start(current_line);
- if let Some(closing) = find_closing_fence(trimmed, &closing_fence, fence_length) {
- let absolute_fence = current_line.find(&closing_fence).unwrap_or(0);
- let prefix = &current_line[..absolute_fence];
- let content_line = if indent_spaces > 0 && starts_with(prefix, list_indent) {
- &prefix[indent_spaces..]
- } else {
- prefix
- };
- if !content_line.is_empty() {
- content.push_str(content_line);
- content.push('\n');
- }
- let extra_text = if !closing.trailing_text.is_empty() {
- closing.trailing_text
- } else if closing.backtick_count > fence_length {
- &trimmed[closing.fence_index + fence_length..]
- } else {
- ""
- };
- if !extra_text.is_empty() {
- if !has_visible_content(&content) {
- return Ok(None);
- }
- return Ok(Some(CodeBlockResult {
- node: Node::CodeBlock { language, content },
- new_line_index: index,
- extra_content: Some(ExtraContent {
- content: extra_text.to_owned(),
- offset: lines[index].offset + (current_line.len() - extra_text.len()),
- }),
- }));
- }
- index += 1;
- break;
- }
let content_line = if indent_spaces > 0 && starts_with(current_line, list_indent) {
&current_line[indent_spaces..]
} else {
current_line
};
- content.push_str(content_line);
- content.push('\n');
- if content.len() > MAX_LINE_LENGTH * 100 {
+ content.push_line(content_line);
+ if content.len > MAX_LINE_LENGTH * 100 {
break;
}
index += 1;
}
- if !has_visible_content(&content) {
- return Ok(None);
+ let mut extra_content = None;
+ if index == closing_line && !content.settled() {
+ let current_line = &lines[index].text;
+ let trimmed = trim_start(current_line);
+ let closing = find_closing_fence(trimmed, &closing_fence, fence_length)?;
+ let absolute_fence = current_line.find(&closing_fence).unwrap_or(0);
+ let prefix = &current_line[..absolute_fence];
+ let content_line = if indent_spaces > 0 && starts_with(prefix, list_indent) {
+ &prefix[indent_spaces..]
+ } else {
+ prefix
+ };
+ if !content_line.is_empty() {
+ content.push_line(content_line);
+ }
+ let extra_text = if !closing.trailing_text.is_empty() {
+ closing.trailing_text
+ } else if closing.backtick_count > fence_length {
+ &trimmed[closing.fence_index + fence_length..]
+ } else {
+ ""
+ };
+ if extra_text.is_empty() {
+ index += 1;
+ } else {
+ extra_content = Some(ExtraContent {
+ content: extra_text.to_owned(),
+ offset: lines[index].offset + (current_line.len() - extra_text.len()),
+ });
+ }
}
- Ok(Some(CodeBlockResult {
- node: Node::CodeBlock { language, content },
+ if !content.visible {
+ return None;
+ }
+ Some(CodeBlockResult {
+ node: Node::CodeBlock {
+ language,
+ content: content.text,
+ },
new_line_index: index,
- extra_content: None,
- }))
+ extra_content,
+ })
+}
+
+#[derive(Clone, Debug, Default)]
+pub(crate) struct ClosingFenceLines {
+ next_by_fence: Vec<(usize, Vec<u32>)>,
+ valid_from: usize,
+}
+
+impl ClosingFenceLines {
+ pub(crate) fn line_replaced(&mut self, index: usize) {
+ self.valid_from = self.valid_from.max(index + 1);
+ }
+
+ fn next(&mut self, lines: &[Line], from: usize, closing_fence: &str) -> Option<usize> {
+ if from >= lines.len() {
+ return None;
+ }
+ if from < self.valid_from {
+ self.next_by_fence.clear();
+ self.valid_from = 0;
+ }
+ let fence_length = closing_fence.len();
+ let index = match self
+ .next_by_fence
+ .iter()
+ .position(|(length, _)| *length == fence_length)
+ {
+ Some(index) => index,
+ None => {
+ self.next_by_fence
+ .push((fence_length, vec![0; lines.len()]));
+ self.next_by_fence.len() - 1
+ }
+ };
+ let next = &mut self.next_by_fence[index].1;
+ let mut position = from;
+ let found = loop {
+ if position == lines.len() {
+ break position;
+ }
+ if next[position] != 0 {
+ break next[position] as usize - 1;
+ }
+ if find_closing_fence(
+ trim_start(&lines[position].text),
+ closing_fence,
+ fence_length,
+ )
+ .is_some()
+ {
+ break position;
+ }
+ position += 1;
+ };
+ let stored = found as u32 + 1;
+ next[from..position].fill(stored);
+ if found == position && found < lines.len() {
+ next[found] = stored;
+ }
+ (found < lines.len()).then_some(found)
+ }
}
fn find_closing_fence<'a>(
@@ -681,7 +811,7 @@ fn parse_block_spoiler(
fn parse_alert_lines(
parser: &mut MarkdownParser,
- lines: Vec<Line>,
+ lines: &[Line],
) -> Result<Option<Node>, ParseError> {
if lines.is_empty() || !starts_with(&lines[0].text, "[!") {
return Ok(None);
@@ -764,220 +894,231 @@ fn post_process_alert_nodes(nodes: Vec<Node>) -> Vec<Node> {
return nodes;
}
let mut out = Vec::new();
- let mut index = 0usize;
- while index < nodes.len() {
- let node = nodes[index].clone();
- if let Node::Text { content } = &node
- && index + 1 < nodes.len()
- && matches!(nodes[index + 1], Node::List { .. })
- {
- let content = replace_trailing_whitespace_with_newline(content);
- if !content.is_empty() {
- out.push(Node::Text { content });
- }
- index += 1;
- continue;
- }
- if matches!(node, Node::List { .. }) && index + 1 < nodes.len() {
- out.push(node);
- if let Node::Text { content } = &nodes[index + 1] {
- let content = trim(content);
+ let mut nodes = nodes.into_iter().peekable();
+ while let Some(node) = nodes.next() {
+ match node {
+ Node::Text { content } if matches!(nodes.peek(), Some(Node::List { .. })) => {
+ let content = replace_trailing_whitespace_with_newline(&content);
if !content.is_empty() {
- out.push(Node::Text {
- content: concat("\n", content),
- });
- index += 2;
- continue;
+ out.push(Node::Text { content });
}
}
- index += 1;
- continue;
+ node @ Node::List { .. } if nodes.peek().is_some() => {
+ out.push(node);
+ let trailing = match nodes.peek() {
+ Some(Node::Text { content }) if !trim(content).is_empty() => {
+ Some(concat("\n", trim(content)))
+ }
+ _ => None,
+ };
+ if let Some(content) = trailing {
+ nodes.next();
+ out.push(Node::Text { content });
+ }
+ }
+ node => out.push(node),
}
- out.push(node);
- index += 1;
}
out
}
-fn parse_list(
- parser: &mut MarkdownParser,
- lines: &[Line],
- current: usize,
+#[derive(Clone, Copy, Debug)]
+struct ListLevel {
ordered: bool,
indent_level: usize,
depth: usize,
- node_count: usize,
-) -> Result<ListResult, ParseError> {
- let mut items = Vec::new();
- let mut index = current;
- let mut new_count = node_count;
- while index < lines.len() {
- if new_count > MAX_AST_NODES {
- break;
- }
- let current_line = &lines[index].text;
- let trimmed_line = trim_start(current_line);
- if starts_with(trimmed_line, "#") || is_blockquote_start(trimmed_line, parser.flags()) {
- break;
- }
- if let Some(item) = match_list_item(current_line) {
- let ordinal = normalise_ordinal(&items, item.ordinal, ordered);
- if item.indent_level < indent_level {
+}
+
+struct ListContext<'a> {
+ parser: &'a mut MarkdownParser,
+ lines: &'a [Line],
+ fences: &'a mut ClosingFenceLines,
+}
+
+impl ListContext<'_> {
+ fn parse_list(
+ &mut self,
+ current: usize,
+ level: ListLevel,
+ node_count: usize,
+ ) -> Result<ListResult, ParseError> {
+ let ListLevel {
+ ordered,
+ indent_level,
+ depth,
+ } = level;
+ let lines = self.lines;
+ let mut items = Vec::new();
+ let mut index = current;
+ let mut new_count = node_count;
+ while index < lines.len() {
+ if new_count > MAX_AST_NODES {
break;
}
- if item.indent_level == indent_level {
- if item.ordered != ordered {
+ let current_line = &lines[index].text;
+ let trimmed_line = trim_start(current_line);
+ if starts_with(trimmed_line, "#")
+ || is_blockquote_start(trimmed_line, self.parser.flags())
+ {
+ break;
+ }
+ if let Some(item) = match_list_item(current_line) {
+ let ordinal = normalise_ordinal(&items, item.ordinal, ordered);
+ if item.indent_level < indent_level {
break;
}
- let mut children = Vec::new();
- if let Some(inline_item) = match_list_item(item.content) {
- let mut nested_items = Vec::new();
- let inline_nodes = crate::inline::parse_inline(
- parser,
- inline_item.content,
- lines[index].offset + (current_line.len() - inline_item.content.len()),
- )?;
- nested_items.push(ListItem {
- children: inline_nodes,
- ordinal: None,
- });
- if let Some(nested) = try_parse_nested_list(
- parser,
- lines,
- index + 1,
- indent_level,
- depth,
- new_count,
- )? {
- if let Node::List { items, .. } = nested.node {
- nested_items.extend(items);
- index = nested.new_line_index;
- new_count = nested.new_node_count;
+ if item.indent_level == indent_level {
+ if item.ordered != ordered {
+ break;
+ }
+ let mut children = Vec::new();
+ if let Some(inline_item) = match_list_item(item.content) {
+ let mut nested_items = Vec::new();
+ let inline_nodes = crate::inline::parse_inline(
+ self.parser,
+ inline_item.content,
+ lines[index].offset + (current_line.len() - inline_item.content.len()),
+ )?;
+ nested_items.push(ListItem {
+ children: inline_nodes,
+ ordinal: None,
+ });
+ if let Some(nested) =
+ self.try_parse_nested_list(index + 1, indent_level, depth, new_count)?
+ {
+ if let Node::List { items, .. } = nested.node {
+ nested_items.extend(items);
+ index = nested.new_line_index;
+ new_count = nested.new_node_count;
+ } else {
+ index += 1;
+ new_count += 1;
+ }
} else {
index += 1;
new_count += 1;
}
+ children.push(Node::List {
+ ordered: inline_item.ordered,
+ items: nested_items,
+ });
} else {
- index += 1;
- new_count += 1;
+ let content_nodes = crate::inline::parse_inline(
+ self.parser,
+ item.content,
+ lines[index].offset + (current_line.len() - item.content.len()),
+ )?;
+ let content_node_count = content_nodes.len();
+ children.extend(content_nodes);
+ if let Some(nested) =
+ self.try_parse_nested_list(index + 1, indent_level, depth, new_count)?
+ {
+ children.push(nested.node);
+ index = nested.new_line_index;
+ new_count = nested.new_node_count;
+ } else {
+ index += 1;
+ new_count += content_node_count;
+ }
}
- children.push(Node::List {
- ordered: inline_item.ordered,
- items: nested_items,
- });
- } else {
- let content_nodes = crate::inline::parse_inline(
- parser,
- item.content,
- lines[index].offset + (current_line.len() - item.content.len()),
- )?;
- children.extend(content_nodes.iter().cloned());
- if let Some(nested) = try_parse_nested_list(
- parser,
- lines,
- index + 1,
- indent_level,
- depth,
+ items.push(ListItem { children, ordinal });
+ } else if item.indent_level == indent_level + 1 && depth < 9 {
+ let nested = self.parse_list(
+ index,
+ ListLevel {
+ ordered: item.ordered,
+ indent_level: item.indent_level,
+ depth: depth + 1,
+ },
new_count,
- )? {
- children.push(nested.node);
- index = nested.new_line_index;
- new_count = nested.new_node_count;
- } else {
- index += 1;
- new_count += content_nodes.len();
+ )?;
+ if let Some(last) = items.last_mut() {
+ last.children.push(nested.node);
}
+ index = nested.new_line_index;
+ new_count = nested.new_node_count;
+ } else {
+ break;
}
- items.push(ListItem { children, ordinal });
- } else if item.indent_level == indent_level + 1 && depth < 9 {
- let nested = parse_list(
- parser,
- lines,
- index,
- item.ordered,
- item.indent_level,
- depth + 1,
- new_count,
- )?;
+ } else if is_bullet_point_text(current_line) {
if let Some(last) = items.last_mut() {
- last.children.push(nested.node);
+ last.children.push(Node::Text {
+ content: trim(current_line).to_owned(),
+ });
+ }
+ index += 1;
+ new_count += 1;
+ } else if is_list_continuation(current_line, indent_level) {
+ if let Some(last) = items.last_mut() {
+ let trimmed = trim_start(current_line);
+ let parsed = crate::inline::parse_inline(
+ self.parser,
+ trimmed,
+ lines[index].offset + (current_line.len() - trimmed.len()),
+ )?;
+ new_count += parsed.len();
+ last.children.extend(parsed);
}
- index = nested.new_line_index;
- new_count = nested.new_node_count;
+ index += 1;
} else {
break;
}
- } else if is_bullet_point_text(current_line) {
- if let Some(last) = items.last_mut() {
- last.children.push(Node::Text {
- content: trim(current_line).to_owned(),
- });
- }
- index += 1;
- new_count += 1;
- } else if is_list_continuation(current_line, indent_level) {
- if let Some(last) = items.last_mut() {
- let trimmed = trim_start(current_line);
- let parsed = crate::inline::parse_inline(
- parser,
- trimmed,
- lines[index].offset + (current_line.len() - trimmed.len()),
- )?;
- last.children.extend(parsed.iter().cloned());
- new_count += parsed.len();
+ if items.len() > MAX_AST_NODES {
+ break;
}
- index += 1;
- } else {
- break;
- }
- if items.len() > MAX_AST_NODES {
- break;
}
+ Ok(ListResult {
+ node: Node::List { ordered, items },
+ new_line_index: index,
+ new_node_count: new_count,
+ })
}
- Ok(ListResult {
- node: Node::List { ordered, items },
- new_line_index: index,
- new_node_count: new_count,
- })
-}
-fn try_parse_nested_list(
- parser: &mut MarkdownParser,
- lines: &[Line],
- current: usize,
- parent_indent: usize,
- depth: usize,
- node_count: usize,
-) -> Result<Option<ListResult>, ParseError> {
- if current >= lines.len() {
- return Ok(None);
- }
- let trimmed = trim_start(&lines[current].text);
- if starts_with(trimmed, "```")
- && let Some(result) = parse_code_block(lines, current)?
- {
- return Ok(Some(ListResult {
- node: result.node,
- new_line_index: result.new_line_index,
- new_node_count: node_count + 1,
- }));
+ fn try_parse_nested_list(
+ &mut self,
+ current: usize,
+ parent_indent: usize,
+ depth: usize,
+ node_count: usize,
+ ) -> Result<Option<ListResult>, ParseError> {
+ let lines = self.lines;
+ if current >= lines.len() {
+ return Ok(None);
+ }
+ let trimmed = trim_start(&lines[current].text);
+ if starts_with(trimmed, "```")
+ && let Some(result) = parse_code_block(
+ lines,
+ current,
+ &lines[current].text,
+ lines[current].offset,
+ self.fences,
+ )
+ {
+ return Ok(Some(ListResult {
+ node: result.node,
+ new_line_index: result.new_line_index,
+ new_node_count: node_count + 1,
+ }));
+ }
+ if let Some(item) = match_list_item(&lines[current].text)
+ && item.indent_level > parent_indent
+ && depth < 9
+ {
+ return self
+ .parse_list(
+ current,
+ ListLevel {
+ ordered: item.ordered,
+ indent_level: item.indent_level,
+ depth: depth + 1,
+ },
+ node_count,
+ )
+ .map(Some);
+ }
+ Ok(None)
}
- if let Some(item) = match_list_item(&lines[current].text)
- && item.indent_level > parent_indent
- && depth < 9
- {
- return parse_list(
- parser,
- lines,
- current,
- item.ordered,
- item.indent_level,
- depth + 1,
- node_count,
- )
- .map(Some);
- }
- Ok(None)
}
pub(crate) fn match_list_item(line: &str) -> Option<ListMatch<'_>> {
@@ -1028,41 +1169,22 @@ fn parse_table(
lines: &[Line],
current: usize,
) -> Result<Option<TableResult>, ParseError> {
- if current + 2 >= lines.len() {
- return Ok(None);
- }
- let header_line = trim_table_line(&lines[current]);
- let align_line = trim_table_line(&lines[current + 1]);
- if !header_line.text.contains('|') || !align_line.text.contains('|') {
- return Ok(None);
- }
- let header_cells = split_table_cells(&header_line);
- if header_cells.is_empty() || !cells_have_content(&header_cells) {
- return Ok(None);
- }
- let align_cells = split_table_cells(&align_line);
- let Some(alignments) = parse_alignments(&align_cells) else {
+ let Some((header_cells, alignments)) = table_header(lines, current) else {
return Ok(None);
};
- if header_cells.len() != alignments.len() {
- return Ok(None);
- }
let header_row = create_table_row(parser, &header_cells)?;
let column_count = header_cells.len();
let mut rows = Vec::new();
let mut index = current + 2;
while index < lines.len() {
let line = trim_table_line(&lines[index]);
- if !line.text.contains('|') || is_table_block_break(&line.text) {
+ if !is_table_body_line(line.text) {
break;
}
- let cells = normalize_column_count(split_table_cells(&line), column_count);
+ let cells = normalize_column_count(split_table_cells(line), column_count);
rows.push(create_table_row(parser, &cells)?);
index += 1;
}
- if rows.is_empty() {
- return Ok(None);
- }
Ok(Some(TableResult {
node: Node::Table {
header: Box::new(header_row),
@@ -1073,31 +1195,52 @@ fn parse_table(
}))
}
-fn trim_table_line(line: &Line) -> TableCellText {
- let left = trim_start(&line.text);
- let leading = line.text.len() - left.len();
- TableCellText {
- text: trim_right(left).to_owned(),
- offset: line.offset + leading,
+fn table_header(
+ lines: &[Line],
+ current: usize,
+) -> Option<(Vec<TableCellText>, Vec<TableAlignment>)> {
+ if current + 2 >= lines.len() {
+ return None;
}
+ let header_line = trim_table_line(&lines[current]);
+ let align_line = trim_table_line(&lines[current + 1]);
+ if !header_line.text.contains('|')
+ || !align_line.text.contains('|')
+ || !align_line.text.bytes().all(|byte| {
+ matches!(
+ byte,
+ b' ' | b':' | b'-' | b'|' | b'\\' | b'\t' | b'\r' | b'\n'
+ )
+ })
+ || !is_table_body_line(trim_table_line(&lines[current + 2]).text)
+ {
+ return None;
+ }
+ let header_cells = split_table_cells(header_line);
+ if header_cells.is_empty() || !cells_have_content(&header_cells) {
+ return None;
+ }
+ let alignments = parse_alignments(&split_table_cells(align_line))?;
+ (header_cells.len() == alignments.len()).then_some((header_cells, alignments))
+}
+
+fn is_table_body_line(text: &str) -> bool {
+ text.contains('|') && !is_table_block_break(text)
}
-fn trim_table_cell(cell: &TableCellText) -> TableCellText {
- let left = trim_start(&cell.text);
- let leading = cell.text.len() - left.len();
- TableCellText {
- text: trim_right(left).to_owned(),
- offset: cell.offset + leading,
+fn trim_table_line(line: &Line) -> TableLine<'_> {
+ let left = trim_start(&line.text);
+ TableLine {
+ text: trim_right(left),
+ offset: line.offset + (line.text.len() - left.len()),
}
}
-fn split_table_cells(source: &TableCellText) -> Vec<TableCellText> {
- let line = &source.text;
- let mut start = 0usize;
+fn split_table_cells(source: TableLine<'_>) -> Vec<TableCellText> {
+ let line = source.text;
+ let bytes = line.as_bytes();
+ let start = usize::from(line.starts_with('|'));
let mut end = line.len();
- if line.starts_with('|') {
- start = 1;
- }
if end > start && line.ends_with('|') {
end -= 1;
}
@@ -1107,26 +1250,30 @@ fn split_table_cells(source: &TableCellText) -> Vec<TableCellText> {
}
let mut cell = String::new();
let mut cell_offset = source.offset + start;
+ let mut segment_start = start;
let mut i = start;
while i < end {
- if byte_at(line, i) == b'\\' && i + 1 < end && byte_at(line, i + 1) == b'|' {
+ if bytes[i] == b'\\' && i + 1 < end && bytes[i + 1] == b'|' {
+ cell.push_str(&line[segment_start..i]);
cell.push('|');
i += 2;
+ segment_start = i;
continue;
}
- if byte_at(line, i) == b'|' {
+ if bytes[i] == b'|' {
+ cell.push_str(&line[segment_start..i]);
cells.push(TableCellText {
text: std::mem::take(&mut cell),
offset: cell_offset,
});
i += 1;
+ segment_start = i;
cell_offset = source.offset + i;
continue;
}
- let ch = line[i..].chars().next().unwrap_or('\0');
- cell.push(ch);
- i += ch.len_utf8();
+ i += 1;
}
+ cell.push_str(&line[segment_start..end]);
cells.push(TableCellText {
text: cell,
offset: cell_offset,
@@ -1171,11 +1318,14 @@ fn create_table_row(
) -> Result<Node, ParseError> {
let mut out = Vec::new();
for cell in cells {
- let trimmed_cell = trim_table_cell(cell);
- let value = trimmed_cell.text;
- let parsed = crate::inline::parse_inline(parser, &value, trimmed_cell.offset)?;
+ let left = trim_start(&cell.text);
+ let value = trim_right(left);
+ let offset = cell.offset + (cell.text.len() - left.len());
+ let parsed = crate::inline::parse_inline(parser, value, offset)?;
let children = if parsed.is_empty() {
- vec![Node::Text { content: value }]
+ vec![Node::Text {
+ content: value.to_owned(),
+ }]
} else {
parsed
};
@@ -1192,7 +1342,12 @@ pub(crate) fn is_block_start(line: &str, flags: u32) -> bool {
|| is_blockquote_start(line, flags)
}
-pub(crate) fn opens_code_block_midline(lines: &[Line], index: usize, flags: u32) -> bool {
+pub(crate) fn opens_code_block_midline(
+ lines: &[Line],
+ index: usize,
+ flags: u32,
+ fences: &mut ClosingFenceLines,
+) -> bool {
if !ParserFlags::has(flags, ParserFlags::ALLOW_CODE_BLOCKS) || index >= lines.len() {
return false;
}
@@ -1208,31 +1363,19 @@ pub(crate) fn opens_code_block_midline(lines: &[Line], index: usize, flags: u32)
if has_open_inline_code(prefix) {
return false;
}
- let code_lines = slice_lines_from_fence(lines, index, fence_pos);
- matches!(parse_code_block(&code_lines, 0), Ok(Some(_)))
+ scan_code_block(
+ lines,
+ index,
+ &line[fence_pos..],
+ lines[index].offset + fence_pos,
+ fences,
+ CodeBlockMode::Probe,
+ )
+ .is_some()
}
pub(crate) fn is_table_start(lines: &[Line], index: usize, flags: u32) -> bool {
- if !ParserFlags::has(flags, ParserFlags::ALLOW_TABLES) || index + 2 >= lines.len() {
- return false;
- }
- let header_line = trim_table_line(&lines[index]);
- let align_line = trim_table_line(&lines[index + 1]);
- if !header_line.text.contains('|') || !align_line.text.contains('|') {
- return false;
- }
- let header_cells = split_table_cells(&header_line);
- if header_cells.is_empty() || !cells_have_content(&header_cells) {
- return false;
- }
- let Some(alignments) = parse_alignments(&split_table_cells(&align_line)) else {
- return false;
- };
- if header_cells.len() != alignments.len() {
- return false;
- }
- let body_line = trim_table_line(&lines[index + 2]);
- body_line.text.contains('|') && !is_table_block_break(&body_line.text)
+ ParserFlags::has(flags, ParserFlags::ALLOW_TABLES) && table_header(lines, index).is_some()
}
pub(crate) fn is_heading_start(line: &str, flags: u32) -> bool {
@@ -1289,16 +1432,6 @@ fn is_escaped_fence(line: &str, fence_pos: usize) -> bool {
backslashes % 2 == 1
}
-fn slice_lines_from_fence(lines: &[Line], current: usize, fence_pos: usize) -> Vec<Line> {
- let mut out = Vec::with_capacity(lines.len() - current);
- out.push(Line {
- text: lines[current].text[fence_pos..].to_owned(),
- offset: lines[current].offset + fence_pos,
- });
- out.extend(lines[current + 1..].iter().cloned());
- out
-}
-
fn normalise_ordinal(items: &[ListItem], ordinal: Option<usize>, ordered: bool) -> Option<usize> {
if !ordered {
return None;
@@ -1334,23 +1467,20 @@ fn normalize_column_count(cells: Vec<TableCellText>, expected: usize) -> Vec<Tab
if cells.len() == expected {
return cells;
}
- let mut out = Vec::with_capacity(expected);
- for index in 0..expected {
- if let Some(cell) = cells.get(index) {
- out.push(cell.clone());
- } else {
- out.push(TableCellText {
+ let mut cells = cells.into_iter();
+ let mut out: Vec<TableCellText> = (0..expected)
+ .map(|_| {
+ cells.next().unwrap_or(TableCellText {
text: String::new(),
offset: 0,
- });
- }
- }
- if cells.len() > expected && expected > 0 {
- let mut combined = out[expected - 1].clone();
- for extra in &cells[expected..] {
- combined.text = concat3(&combined.text, "|", &extra.text);
+ })
+ })
+ .collect();
+ if let Some(last) = out.last_mut() {
+ for extra in cells {
+ last.text.push('|');
+ last.text.push_str(&extra.text);
}
- out[expected - 1] = combined;
}
out
}
diff --git a/packages/markdown_parser/rust/src/constants.rs b/packages/markdown_parser/rust/src/constants.rs
index 33b4a168c..a5fe8aad6 100644
--- a/packages/markdown_parser/rust/src/constants.rs
+++ b/packages/markdown_parser/rust/src/constants.rs
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub const CODE_FENCE_LENGTH: usize = 3;
+pub const MAX_CODE_BLOCK_SEARCH_LINES: usize = 1000;
pub const MAX_AST_NODES: usize = 10_000;
pub const MAX_INLINE_DEPTH: usize = 10;
pub const MAX_LINES: usize = 10_000;
diff --git a/packages/markdown_parser/rust/src/emoji.rs b/packages/markdown_parser/rust/src/emoji.rs
index 96c73f9b1..61fc9401b 100644
--- a/packages/markdown_parser/rust/src/emoji.rs
+++ b/packages/markdown_parser/rust/src/emoji.rs
@@ -34,6 +34,18 @@ pub struct SkinEmoji {
impl EmojiContext {
pub fn parse(bytes: &str) -> Self {
let mut context = Self::default();
+ let mut counts = [0usize; 3];
+ for line in bytes.lines() {
+ match line.as_bytes().first() {
+ Some(b'S') => counts[0] += 1,
+ Some(b'C') => counts[1] += 1,
+ Some(b'K') => counts[2] += 1,
+ _ => {}
+ }
+ }
+ context.standard.reserve_exact(counts[0]);
+ context.shortcodes.reserve_exact(counts[1]);
+ context.skins.reserve_exact(counts[2]);
for line in bytes.lines() {
if line.is_empty() {
continue;
@@ -93,20 +105,70 @@ impl EmojiContext {
_ => {}
}
}
+ context.standard.sort_by_key(|record| record.offset);
+ context
+ .shortcodes
+ .sort_by(|left, right| left.name.cmp(&right.name));
+ context
+ .skins
+ .sort_by(|left, right| (&left.name, left.tone).cmp(&(&right.name, right.tone)));
context
}
- pub fn standard_at(&self, offset: usize) -> Option<&StandardEmoji> {
- self.standard.iter().find(|record| record.offset == offset)
+ pub(crate) fn standard_cursor(&self) -> StandardCursor<'_> {
+ StandardCursor {
+ emojis: &self.standard,
+ index: 0,
+ }
}
pub fn shortcode(&self, name: &str) -> Option<&ShortcodeEmoji> {
- self.shortcodes.iter().find(|record| record.name == name)
+ let index = self
+ .shortcodes
+ .partition_point(|record| record.name.as_str() < name);
+ self.shortcodes
+ .get(index)
+ .filter(|record| record.name == name)
}
pub fn skin(&self, name: &str, tone: u8) -> Option<&SkinEmoji> {
+ let index = self
+ .skins
+ .partition_point(|record| (record.name.as_str(), record.tone) < (name, tone));
self.skins
- .iter()
- .find(|record| record.tone == tone && record.name == name)
+ .get(index)
+ .filter(|record| record.name == name && record.tone == tone)
+ }
+}
+
+pub(crate) struct StandardCursor<'a> {
+ emojis: &'a [StandardEmoji],
+ index: usize,
+}
+
+impl<'a> StandardCursor<'a> {
+ pub(crate) fn next_offset(&mut self, offset: usize) -> Option<usize> {
+ self.seek(offset);
+ self.emojis.get(self.index).map(|emoji| emoji.offset)
+ }
+
+ pub(crate) fn at(&mut self, offset: usize) -> Option<&'a StandardEmoji> {
+ self.seek(offset);
+ self.emojis
+ .get(self.index)
+ .filter(|emoji| emoji.offset == offset)
+ }
+
+ fn seek(&mut self, offset: usize) {
+ while self
+ .emojis
+ .get(self.index)
+ .is_some_and(|emoji| emoji.offset < offset)
+ {
+ self.index += 1;
+ }
+ while self.index > 0 && self.emojis[self.index - 1].offset >= offset {
+ self.index -= 1;
+ }
}
}
diff --git a/packages/markdown_parser/rust/src/inline.rs b/packages/markdown_parser/rust/src/inline.rs
index 5b0544913..979e95ce7 100644
--- a/packages/markdown_parser/rust/src/inline.rs
+++ b/packages/markdown_parser/rust/src/inline.rs
@@ -2,19 +2,20 @@
use crate::ast::{EmojiKind, Node, ParserFlags, ParserResult};
use crate::constants::{CODE_FENCE_LENGTH, MAX_INLINE_DEPTH, MAX_LINE_LENGTH};
-use crate::emoji::{EmojiContext, StandardEmoji};
+use crate::emoji::{StandardCursor, StandardEmoji};
use crate::links;
use crate::normalize::{
combine_adjacent_text, compact_empty_text_nodes, flatten_top_level_formatting,
merge_adjacent_text_simple, normalize_nodes,
};
use crate::parser::{MarkdownParser, ParseError};
+use crate::scan::{Closing, Lookahead, Pattern, Token, step};
use crate::text::{
- advance_one, append_text, byte_at, ends_with, has_visible_content, is_alpha_numeric,
- is_escapable_character, string_from_lossy,
+ append_text, byte_at, has_visible_content, is_alpha_numeric, is_escapable_character,
+ string_from_lossy,
};
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct FormattingEntry {
delimiter: u8,
is_double: bool,
@@ -22,26 +23,28 @@ struct FormattingEntry {
#[derive(Clone, Debug, Default)]
struct FormattingContext {
- entries: Vec<FormattingEntry>,
+ entries: [FormattingEntry; MAX_INLINE_DEPTH],
+ len: usize,
}
impl FormattingContext {
fn can_enter(&self, delimiter: u8, is_double: bool) -> bool {
- self.entries.len() < MAX_INLINE_DEPTH && !self.is_active(delimiter, is_double)
+ self.len < MAX_INLINE_DEPTH && !self.is_active(delimiter, is_double)
}
fn is_active(&self, delimiter: u8, is_double: bool) -> bool {
- self.entries
+ self.entries[..self.len]
.iter()
.any(|entry| entry.delimiter == delimiter && entry.is_double == is_double)
}
fn push(&mut self, delimiter: u8, is_double: bool) {
- if self.entries.len() < MAX_INLINE_DEPTH {
- self.entries.push(FormattingEntry {
+ if self.len < MAX_INLINE_DEPTH {
+ self.entries[self.len] = FormattingEntry {
delimiter,
is_double,
- });
+ };
+ self.len += 1;
}
}
}
@@ -57,15 +60,34 @@ enum FormattingNodeType {
}
#[derive(Clone, Copy, Debug)]
-struct FormattingMarkerInfo<'a> {
- marker: &'a str,
+struct FormattingMarkerInfo {
+ delimiter: u8,
node_type: FormattingNodeType,
marker_length: usize,
}
-#[derive(Clone, Copy, Debug)]
-struct FormattingEnd {
- end_position: usize,
+impl FormattingMarkerInfo {
+ fn is_double(self) -> bool {
+ self.marker_length > 1
+ }
+
+ fn is_triple_emphasis(self) -> bool {
+ self.marker_length == 3 && self.node_type == FormattingNodeType::Emphasis
+ }
+
+ fn scan_index(self) -> usize {
+ match (self.delimiter, self.marker_length) {
+ (b'*', 2) => 0,
+ (b'_', 2) => 1,
+ (b'~', _) => 2,
+ (b'|', _) => 3,
+ (b'*', 3) => 4,
+ (b'_', 3) => 5,
+ (b'*', _) => 6,
+ (b'_', _) => 7,
+ _ => 8,
+ }
+ }
}
pub fn parse_inline(
@@ -95,15 +117,39 @@ fn parse_inline_with_context(
context: &mut FormattingContext,
) -> Result<Vec<Node>, ParseError> {
let mut nodes = Vec::new();
- let mut accumulated = Vec::<u8>::new();
+ let mut accumulated = String::new();
+ let mut scan = Lookahead::new(text);
+ let emoji_context = parser.shared_emoji_context();
+ let mut emojis = emoji_context.standard_cursor();
let mut position = 0usize;
while position < text.len() {
let current = byte_at(text, position);
+ if is_plain_text_byte(current) {
+ let run_limit = emojis
+ .next_offset(base_offset + position)
+ .map_or(text.len(), |offset| (offset - base_offset).min(text.len()));
+ if run_limit > position {
+ let run_end = text.as_bytes()[position..run_limit]
+ .iter()
+ .position(|byte| !is_plain_text_byte(*byte))
+ .map_or(run_limit, |offset| position + offset);
+ let room = (MAX_LINE_LENGTH + 1)
+ .saturating_sub(accumulated.len())
+ .max(1);
+ let end = run_end.min(position + room);
+ accumulated.push_str(&text[position..end]);
+ position = end;
+ if accumulated.len() > MAX_LINE_LENGTH {
+ flush_bounded_accumulated_text(&mut nodes, &mut accumulated, MAX_LINE_LENGTH);
+ break;
+ }
+ continue;
+ }
+ }
if current == b'\\' && position + 1 < text.len() {
let next = byte_at(text, position + 1);
if next == b'_' && position > 0 && byte_at(text, position - 1) == 0xaf {
- accumulated.push(current);
- accumulated.push(next);
+ accumulated.push_str("\\_");
position += 2;
continue;
}
@@ -113,23 +159,21 @@ fn parse_inline_with_context(
kind: EmojiKind::Standard { raw, .. },
} = &result.node
{
- accumulated.extend_from_slice(raw.as_bytes());
+ accumulated.push_str(raw);
position += 1 + result.advance;
continue;
}
- if let Some(emoji) = parser
- .emoji_context()
- .standard_at(base_offset + position + 1)
+ if let Some(emoji) = emojis.at(base_offset + position + 1)
&& text[position + 1..].starts_with(&emoji.raw)
{
- accumulated.extend_from_slice(emoji.raw.as_bytes());
+ accumulated.push_str(&emoji.raw);
position += 1 + emoji.len;
continue;
}
if next == b'`' {
let run = backtick_run_length(text, position + 1);
if run >= CODE_FENCE_LENGTH {
- accumulated.extend(std::iter::repeat_n(b'`', run));
+ accumulated.extend(std::iter::repeat_n('`', run));
position += 1 + run;
continue;
}
@@ -138,18 +182,18 @@ fn parse_inline_with_context(
|| is_ordered_list_marker_dot_escape(text, position)
|| is_word_dot_escape(text, position)
{
- accumulated.push(next);
+ accumulated.push(char::from(next));
position += 2;
continue;
}
}
let remaining = &text[position..];
- if !ends_with(&accumulated, b"<\"")
- && !ends_with(&accumulated, b"<'")
+ if !accumulated.ends_with("<\"")
+ && !accumulated.ends_with("<'")
&& ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)
&& links::starts_with_url(remaining)
- && let Some(result) = links::parse_url_segment(remaining, parser.flags())
+ && let Some(result) = links::parse_url_segment(&mut scan, position, parser.flags())
{
flush_accumulated_text(&mut nodes, &mut accumulated);
nodes.push(result.node);
@@ -162,7 +206,7 @@ fn parse_inline_with_context(
&& byte_at(text, position + 1) != b'_'
&& links::is_word_underscore(text, position)
{
- accumulated.push(b'_');
+ accumulated.push('_');
position += 1;
continue;
}
@@ -171,7 +215,7 @@ fn parse_inline_with_context(
flush_accumulated_text(&mut nodes, &mut accumulated);
push_regional_indicator_pair(
&mut nodes,
- parser.emoji_context(),
+ &mut emojis,
base_offset + position,
&remaining[..result.advance],
result.node,
@@ -180,7 +224,7 @@ fn parse_inline_with_context(
continue;
}
- if let Some(emoji) = parser.emoji_context().standard_at(base_offset + position)
+ if let Some(emoji) = emojis.at(base_offset + position)
&& remaining.starts_with(&emoji.raw)
{
flush_accumulated_text(&mut nodes, &mut accumulated);
@@ -195,9 +239,8 @@ fn parse_inline_with_context(
continue;
}
- if current == b'<'
- && position + 2 < text.len()
- && let Some(result) = links::parse_custom_emoji(remaining)
+ if current == b':'
+ && let Some(result) = links::parse_emoji_shortcode(parser, remaining)
{
flush_accumulated_text(&mut nodes, &mut accumulated);
nodes.push(result.node);
@@ -206,8 +249,8 @@ fn parse_inline_with_context(
}
if current == b'<'
- && remaining.starts_with("<t:")
- && let Some(result) = links::parse_timestamp(remaining)
+ && let Some(close) = scan.next(Pattern::AngleClose, position, text.len())
+ && let Some(result) = parse_angle_bracket(parser, &mut scan, position, close)
{
flush_accumulated_text(&mut nodes, &mut accumulated);
nodes.push(result.node);
@@ -215,59 +258,6 @@ fn parse_inline_with_context(
continue;
}
- if current == b':'
- && let Some(result) = links::parse_emoji_shortcode(parser, remaining)
- {
- flush_accumulated_text(&mut nodes, &mut accumulated);
- nodes.push(result.node);
- position += result.advance;
- continue;
- }
-
- if current == b'<' && position + 1 < text.len() {
- let next = byte_at(text, position + 1);
- if next == b'+'
- && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)
- && let Some(result) = links::parse_phone_link(parser, remaining)
- {
- flush_accumulated_text(&mut nodes, &mut accumulated);
- nodes.push(result.node);
- position += result.advance;
- continue;
- }
- if remaining.starts_with("<sms:")
- && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)
- && let Some(result) = links::parse_sms_link(parser, remaining)
- {
- flush_accumulated_text(&mut nodes, &mut accumulated);
- nodes.push(result.node);
- position += result.advance;
- continue;
- }
- if matches!(next, b'@' | b'#' | b'/' | b'i')
- && let Some(result) = links::parse_mention(remaining, parser.flags())
- {
- flush_accumulated_text(&mut nodes, &mut accumulated);
- nodes.push(result.node);
- position += result.advance;
- continue;
- }
- if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS) {
- if let Some(result) = links::parse_autolink(remaining, parser.flags()) {
- flush_accumulated_text(&mut nodes, &mut accumulated);
- nodes.push(result.node);
- position += result.advance;
- continue;
- }
- if let Some(result) = links::parse_email_link(parser, remaining) {
- flush_accumulated_text(&mut nodes, &mut accumulated);
- nodes.push(result.node);
- position += result.advance;
- continue;
- }
- }
- }
-
if current == b'@' && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_EVERYONE_MENTIONS)
{
let is_escaped = position > 0 && byte_at(text, position - 1) == b'\\';
@@ -291,7 +281,7 @@ fn parse_inline_with_context(
if current >= 0x80 {
let char_len = remaining.chars().next().map_or(1, char::len_utf8);
- accumulated.extend_from_slice(&text.as_bytes()[position..position + char_len]);
+ accumulated.push_str(&text[position..position + char_len]);
position += char_len;
continue;
}
@@ -302,7 +292,7 @@ fn parse_inline_with_context(
&& (is_double_underscore
|| !(current == b'_'
&& !accumulated.is_empty()
- && is_alpha_numeric(*accumulated.last().unwrap_or(&0))))
+ && is_alpha_numeric(*accumulated.as_bytes().last().unwrap_or(&0))))
{
let previous = if position > 0 {
byte_at(text, position - 1)
@@ -311,7 +301,8 @@ fn parse_inline_with_context(
};
if let Some(result) = parse_special_sequence(
parser,
- remaining,
+ &mut scan,
+ position,
base_offset + position,
context,
previous,
@@ -323,7 +314,7 @@ fn parse_inline_with_context(
}
}
- accumulated.push(current);
+ accumulated.push(char::from(current));
position += 1;
if accumulated.len() > MAX_LINE_LENGTH {
flush_bounded_accumulated_text(&mut nodes, &mut accumulated, MAX_LINE_LENGTH);
@@ -358,17 +349,17 @@ fn parse_regional_indicator_flag(text: &str) -> Option<ParserResult> {
fn push_regional_indicator_pair(
nodes: &mut Vec<Node>,
- emoji_context: &EmojiContext,
+ emojis: &mut StandardCursor<'_>,
offset: usize,
pair: &str,
flag: Node,
) {
let (first, second) = pair.split_at(pair.len() / 2);
- match emoji_context.standard_at(offset) {
+ match emojis.at(offset) {
Some(emoji) if emoji.raw == pair => nodes.push(standard_emoji_node(emoji)),
Some(emoji) if emoji.raw == first => {
nodes.push(standard_emoji_node(emoji));
- match emoji_context.standard_at(offset + first.len()) {
+ match emojis.at(offset + first.len()) {
Some(emoji) if emoji.raw == second => nodes.push(standard_emoji_node(emoji)),
_ => nodes.extend(regional_indicator_emoji(second)),
}
@@ -429,13 +420,11 @@ fn is_ordered_list_marker_dot_escape(text: &str, backslash_position: usize) -> b
if digit_start == backslash_position {
return false;
}
- let line_start = text[..digit_start]
- .rfind('\n')
- .map_or(0, |position| position + 1);
- if text.as_bytes()[line_start..digit_start]
- .iter()
- .any(|byte| *byte != b' ')
- {
+ let mut line_start = digit_start;
+ while line_start > 0 && byte_at(text, line_start - 1) == b' ' {
+ line_start -= 1;
+ }
+ if line_start > 0 && byte_at(text, line_start - 1) != b'\n' {
return false;
}
let indent = digit_start - line_start;
@@ -450,80 +439,52 @@ fn is_word_dot_escape(text: &str, backslash_position: usize) -> bool {
previous.is_ascii_alphanumeric() || previous == b'.'
}
+fn parse_angle_bracket(
+ parser: &MarkdownParser,
+ scan: &mut Lookahead<'_>,
+ position: usize,
+ close: usize,
+) -> Option<ParserResult> {
+ let tag = &scan.text[position..=close];
+ links::parse_custom_emoji(tag)
+ .or_else(|| links::parse_timestamp(tag))
+ .or_else(|| links::parse_phone_link(parser, tag))
+ .or_else(|| links::parse_sms_link(parser, tag))
+ .or_else(|| links::parse_mention(tag, parser.flags()))
+ .or_else(|| links::parse_autolink(scan, position, close, parser.flags()))
+ .or_else(|| links::parse_email_link(parser, tag))
+}
+
fn parse_special_sequence(
parser: &mut MarkdownParser,
- text: &str,
+ scan: &mut Lookahead<'_>,
+ position: usize,
base_offset: usize,
context: &mut FormattingContext,
previous: u8,
) -> Result<Option<ParserResult>, ParseError> {
- if text.is_empty() {
- return Ok(None);
- }
- match byte_at(text, 0) {
- b'*' | b'_' | b'~' | b'|' | b'`' => {
- if let Some(result) = parse_formatting(parser, text, base_offset, context, previous)? {
- return Ok(Some(result));
- }
- }
- b'@' => {
- if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_EVERYONE_MENTIONS) {
- if text.starts_with("@everyone") {
- return Ok(Some(ParserResult {
- node: Node::Mention {
- kind: crate::ast::MentionKind::Everyone,
- },
- advance: 9,
- }));
- }
- if text.starts_with("@here") {
- return Ok(Some(ParserResult {
- node: Node::Mention {
- kind: crate::ast::MentionKind::Here,
- },
- advance: 5,
- }));
- }
- }
- }
- b'[' => {
- if let Some(result) = links::parse_timestamp(text) {
- return Ok(Some(result));
- }
- if ParserFlags::has(parser.flags(), ParserFlags::ALLOW_MASKED_LINKS)
- && let Some(result) = links::parse_masked_link(parser, text, base_offset)?
- {
- return Ok(Some(result));
- }
- }
- _ => {}
- }
- if byte_at(text, 0) != b'['
- && let Some(result) = links::parse_timestamp(text)
- {
- return Ok(Some(result));
+ if byte_at(scan.text, position) != b'[' {
+ return parse_formatting(parser, scan, position, base_offset, context, previous);
}
- if byte_at(text, 0) != b'<'
- && byte_at(text, 0) != b'['
- && ParserFlags::has(parser.flags(), ParserFlags::ALLOW_MASKED_LINKS)
- && let Some(result) = links::parse_masked_link(parser, text, base_offset)?
- {
- return Ok(Some(result));
+ if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_MASKED_LINKS) {
+ return Ok(None);
}
- Ok(None)
+ links::parse_masked_link(parser, scan, position, base_offset)
}
fn parse_formatting(
parser: &mut MarkdownParser,
- text: &str,
+ scan: &mut Lookahead<'_>,
+ position: usize,
base_offset: usize,
context: &mut FormattingContext,
previous: u8,
) -> Result<Option<ParserResult>, ParseError> {
+ let text = &scan.text[position..];
if text.len() < 2 {
return Ok(None);
}
- let Some(marker) = get_formatting_marker_info(text) else {
+ let Some(marker) = get_formatting_marker_info(scan, position) else {
return Ok(None);
};
if marker.node_type == FormattingNodeType::Spoiler
@@ -531,23 +492,23 @@ fn parse_formatting(
{
return Ok(None);
}
- if !context.can_enter(byte_at(marker.marker, 0), marker.marker.len() > 1) {
+ if !context.can_enter(marker.delimiter, marker.is_double()) {
return Ok(None);
}
if marker.marker_length == 1 && marker.node_type == FormattingNodeType::Emphasis {
let next = byte_at(text, marker.marker_length);
- if !links::can_open_single_emphasis(byte_at(marker.marker, 0), previous, next) {
+ if !links::can_open_single_emphasis(marker.delimiter, previous, next) {
return Ok(None);
}
}
- let Some(end) = find_formatting_end(text, marker) else {
+ let Some(end_position) = find_formatting_end(scan, position, marker) else {
return Ok(None);
};
- let inner = &text[marker.marker_length..end.end_position];
- if !formatting_inner_has_visible_content(marker, inner) {
+ let inner = &text[marker.marker_length..end_position];
+ if !has_visible_content(inner) {
return Ok(None);
}
- let is_block = context.is_active(byte_at(marker.marker, 0), marker.marker.len() > 1);
+ let is_block = context.is_active(marker.delimiter, marker.is_double());
let node = create_formatting_node(
parser,
marker,
@@ -557,262 +518,142 @@ fn parse_formatting(
)?;
Ok(Some(ParserResult {
node,
- advance: end.end_position + marker.marker_length,
+ advance: end_position + marker.marker_length,
}))
}
-fn formatting_inner_has_visible_content(marker: FormattingMarkerInfo<'_>, inner: &str) -> bool {
- if marker.node_type == FormattingNodeType::InlineCode {
- return has_visible_content(&unescape_inline_code(inner));
- }
- has_visible_content(inner)
-}
-
-fn get_formatting_marker_info(text: &str) -> Option<FormattingMarkerInfo<'_>> {
- if text.is_empty() || !is_formatting_char(byte_at(text, 0)) {
+fn get_formatting_marker_info(
+ scan: &mut Lookahead<'_>,
+ position: usize,
+) -> Option<FormattingMarkerInfo> {
+ let text = scan.text;
+ let first = byte_at(text, position);
+ if !is_formatting_char(first) {
return None;
}
- let first = byte_at(text, 0);
- let second = byte_at(text, 1);
- let third = byte_at(text, 2);
- if first == b'*' && second == b'*' && third == b'*' {
- return Some(marker("***", FormattingNodeType::Emphasis, 3));
- }
- if first == b'_' && second == b'_' && third == b'_' {
- return Some(marker("___", FormattingNodeType::Emphasis, 3));
- }
- if first == b'|' && second == b'|' {
- return Some(marker("||", FormattingNodeType::Spoiler, 2));
- }
- if first == b'~' && second == b'~' {
- return Some(marker("~~", FormattingNodeType::Strikethrough, 2));
- }
- if first == b'*' && second == b'*' {
- return Some(marker("**", FormattingNodeType::Strong, 2));
- }
- if first == b'_' && second == b'_' {
- return Some(marker("__", FormattingNodeType::Underline, 2));
- }
- if first == b'`' {
- let mut count = 1;
- while count < text.len() && byte_at(text, count) == b'`' {
- count += 1;
- }
- return Some(FormattingMarkerInfo {
- marker: &text[..count],
- node_type: FormattingNodeType::InlineCode,
- marker_length: count,
- });
- }
- if first == b'*' {
- return Some(marker("*", FormattingNodeType::Emphasis, 1));
- }
- if first == b'_' {
- return Some(marker("_", FormattingNodeType::Emphasis, 1));
- }
- None
-}
-
-fn marker(
- marker: &'static str,
- node_type: FormattingNodeType,
- marker_length: usize,
-) -> FormattingMarkerInfo<'static> {
- FormattingMarkerInfo {
- marker,
+ let second = byte_at(text, position + 1);
+ let third = byte_at(text, position + 2);
+ let (node_type, marker_length) = match (first, second, third) {
+ (b'*', b'*', b'*') | (b'_', b'_', b'_') => (FormattingNodeType::Emphasis, 3),
+ (b'|', b'|', _) => (FormattingNodeType::Spoiler, 2),
+ (b'~', b'~', _) => (FormattingNodeType::Strikethrough, 2),
+ (b'*', b'*', _) => (FormattingNodeType::Strong, 2),
+ (b'_', b'_', _) => (FormattingNodeType::Underline, 2),
+ (b'`', _, _) => (
+ FormattingNodeType::InlineCode,
+ scan.run_end(position, b'`') - position,
+ ),
+ (b'*' | b'_', _, _) => (FormattingNodeType::Emphasis, 1),
+ _ => return None,
+ };
+ Some(FormattingMarkerInfo {
+ delimiter: first,
node_type,
marker_length,
- }
+ })
}
-fn find_formatting_end(text: &str, marker: FormattingMarkerInfo<'_>) -> Option<FormattingEnd> {
- let mut position = marker.marker_length;
- let mut nested_level = 0usize;
- if text.len() < marker.marker_length * 2 {
+fn find_formatting_end(
+ scan: &mut Lookahead<'_>,
+ opener: usize,
+ marker: FormattingMarkerInfo,
+) -> Option<usize> {
+ let len = scan.text.len();
+ if len - opener < marker.marker_length * 2 {
return None;
}
+ let mut start = opener + marker.marker_length;
if marker.node_type == FormattingNodeType::InlineCode && marker.marker_length > 1 {
- while position < text.len() {
- let current = byte_at(text, position);
- if is_inline_code_line_break(current) {
- return None;
- }
- if current == b'`' {
- let mut count = 0usize;
- let mut check = position;
- while check < text.len() && byte_at(text, check) == b'`' {
- count += 1;
- check += 1;
- }
- if count >= marker.marker_length {
- return Some(FormattingEnd {
- end_position: position + count - marker.marker_length,
- });
- }
- position = check;
- continue;
- }
- position += advance_one(text, position);
- if position > MAX_LINE_LENGTH {
- break;
- }
- }
- return None;
+ return scan.code_span_end(opener, marker.marker_length, MAX_LINE_LENGTH);
}
- if marker.marker_length == 1 && marker.node_type == FormattingNodeType::InlineCode {
- let marker_char = byte_at(marker.marker, 0);
- while position < text.len() {
- let current = byte_at(text, position);
- if is_inline_code_line_break(current) {
- return None;
- }
- if current == b'\\' && position + 1 < text.len() {
- position += 2;
- continue;
- }
- if current == marker_char {
- if marker_char == b'`'
- && position + 1 < text.len()
- && byte_at(text, position + 1) == b'`'
- {
- while position < text.len() && byte_at(text, position) == b'`' {
- position += 1;
- }
- continue;
- }
- return Some(FormattingEnd {
- end_position: position,
- });
- }
- position += advance_one(text, position);
- if position > MAX_LINE_LENGTH {
- break;
- }
- }
- return None;
- }
- if marker.marker_length == 1 && marker.node_type == FormattingNodeType::Emphasis {
- let marker_char = byte_at(marker.marker, 0);
- while position < text.len() {
- let current = byte_at(text, position);
- if current == b'\\' && position + 1 < text.len() {
- position += 2;
- continue;
- }
- if current == marker_char {
- if marker_char == b'*'
- && let Some(nested_end) = find_nested_asterisk_run_end(text, position)
- {
- position = nested_end;
- continue;
- }
- if marker_char == b'_'
- && position + 1 < text.len()
- && byte_at(text, position + 1) == b'_'
- {
- position += 2;
- continue;
- }
- let prev = if position > 0 {
- byte_at(text, position - 1)
- } else {
- 0
- };
- let next = byte_at(text, position + 1);
- if links::can_close_single_emphasis(marker_char, prev, next) {
- return Some(FormattingEnd {
- end_position: position,
- });
- }
- }
- position += advance_one(text, position);
- if position > MAX_LINE_LENGTH {
- break;
+ if marker.node_type == FormattingNodeType::Spoiler
+ && scan.text.as_bytes()[start..].starts_with(b"||")
+ && start + 2 < len
+ {
+ start += 1;
+ }
+ let close_len = if marker.is_triple_emphasis() { 3 } else { 1 };
+ let closer = scan.closer(
+ Closing::Formatting(marker.scan_index()),
+ start,
+ close_len,
+ |scan, position| classify_formatting(scan, marker, position),
+ );
+ closer?
+ .within(opener, MAX_LINE_LENGTH)
+ .map(|end| end - opener)
+}
+
+fn classify_formatting(
+ scan: &mut Lookahead<'_>,
+ marker: FormattingMarkerInfo,
+ position: usize,
+) -> Token {
+ let text = scan.text;
+ let bytes = text.as_bytes();
+ let current = bytes[position];
+ let delimiter = marker.delimiter;
+ if marker.node_type == FormattingNodeType::InlineCode && is_inline_code_line_break(current) {
+ return Token::Fail;
+ }
+ if current == b'\\' && position + 1 < bytes.len() {
+ return Token::Skip(2);
+ }
+ let repeats = |count: usize| {
+ bytes.len() - position >= count
+ && bytes[position..position + count]
+ .iter()
+ .all(|byte| *byte == delimiter)
+ };
+ match (marker.node_type, marker.marker_length) {
+ (FormattingNodeType::InlineCode, _) if current == b'`' => {
+ if repeats(2) {
+ Token::Skip(scan.run_end(position, b'`') - position)
+ } else {
+ Token::Close
}
}
- return None;
- }
- if marker.node_type == FormattingNodeType::InlineCode {
- while position < text.len() {
- let current = byte_at(text, position);
- if is_inline_code_line_break(current) {
- return None;
- }
- if current == b'`' {
- return Some(FormattingEnd {
- end_position: position,
- });
+ (FormattingNodeType::Emphasis, 1) if current == delimiter => {
+ if delimiter == b'*'
+ && let Some(nested_end) = find_nested_asterisk_run_end(scan, position)
+ {
+ return Token::Skip(nested_end - position);
}
- position += advance_one(text, position);
- if position > MAX_LINE_LENGTH {
- break;
+ if delimiter == b'_' && repeats(2) {
+ return Token::Skip(2);
}
- }
- return None;
- }
- let first_marker_char = byte_at(marker.marker, 0);
- let is_double = marker.marker.len() > 1;
- while position < text.len() {
- if byte_at(text, position) == b'\\' && position + 1 < text.len() {
- position += 2;
- continue;
- }
- if text.as_bytes()[position..].starts_with(marker.marker.as_bytes()) {
- if nested_level == 0 {
- if marker.node_type == FormattingNodeType::Spoiler
- && position == marker.marker_length
- && position + marker.marker.len() < text.len()
- {
- position += 1;
- continue;
- }
- return Some(FormattingEnd {
- end_position: position,
- });
+ let next = byte_at(text, position + 1);
+ if links::can_close_single_emphasis(delimiter, bytes[position - 1], next) {
+ return Token::Close;
}
- nested_level -= 1;
- position += marker.marker.len();
- continue;
- }
- if is_double
- && position + 1 < text.len()
- && byte_at(text, position) == first_marker_char
- && byte_at(text, position + 1) == first_marker_char
- {
- nested_level += 1;
- }
- position += advance_one(text, position);
- if position > MAX_LINE_LENGTH {
- break;
+ step(text, position)
}
+ (FormattingNodeType::Emphasis, 3) if repeats(3) => Token::Close,
+ (FormattingNodeType::Emphasis, 3) if repeats(2) => Token::Open(1),
+ (FormattingNodeType::InlineCode | FormattingNodeType::Emphasis, _) => step(text, position),
+ _ if repeats(2) => Token::Close,
+ _ => step(text, position),
}
- None
}
fn is_inline_code_line_break(byte: u8) -> bool {
matches!(byte, b'\n' | b'\r')
}
-fn find_nested_asterisk_run_end(text: &str, position: usize) -> Option<usize> {
- let run_length = links::count_repeated_byte(text, position, b'*');
+fn find_nested_asterisk_run_end(scan: &mut Lookahead<'_>, position: usize) -> Option<usize> {
+ let text = scan.text;
+ let run_length = scan.run_end(position, b'*') - position;
if run_length < 2 {
return None;
}
- let previous = if position > 0 {
- byte_at(text, position - 1)
- } else {
- 0
- };
+ let previous = byte_at(text, position - 1);
let next = byte_at(text, position + run_length);
if !links::is_left_flanking_delimiter(previous, next) {
return None;
}
- let nested_marker = get_formatting_marker_info(&text[position..])?;
- if byte_at(nested_marker.marker, 0) != b'*' || nested_marker.marker_length < 2 {
- return None;
- }
- let nested_end = find_formatting_end(&text[position..], nested_marker)?;
- let end_position = position + nested_end.end_position + nested_marker.marker_length;
+ let nested_marker = get_formatting_marker_info(scan, position)?;
+ let nested_end = find_formatting_end(scan, position, nested_marker)?;
+ let end_position = position + nested_end + nested_marker.marker_length;
if end_position <= position || end_position > text.len() {
return None;
}
@@ -821,7 +662,7 @@ fn find_nested_asterisk_run_end(text: &str, position: usize) -> Option<usize> {
fn create_formatting_node(
parser: &mut MarkdownParser,
- marker: FormattingMarkerInfo<'_>,
+ marker: FormattingMarkerInfo,
inner: &str,
inner_offset: usize,
is_block: bool,
@@ -831,7 +672,7 @@ fn create_formatting_node(
content: unescape_inline_code(inner),
});
}
- if marker.marker == "***" || marker.marker == "___" {
+ if marker.is_triple_emphasis() {
let mut strong_context = FormattingContext::default();
strong_context.push(b'*', true);
let inner_nodes =
@@ -843,7 +684,7 @@ fn create_formatting_node(
});
}
let mut new_context = FormattingContext::default();
- new_context.push(byte_at(marker.marker, 0), marker.marker.len() > 1);
+ new_context.push(marker.delimiter, marker.is_double());
let children = parse_inline_with_context(parser, inner, inner_offset, &mut new_context)?;
Ok(match marker.node_type {
FormattingNodeType::Strong => Node::Strong { children },
@@ -887,31 +728,26 @@ fn unescape_inline_code(content: &str) -> String {
string_from_lossy(&out)
}
-fn flush_accumulated_text(nodes: &mut Vec<Node>, accumulated: &mut Vec<u8>) {
- if accumulated.is_empty() {
- return;
- }
- let content = string_from_lossy(accumulated);
- append_text(nodes, content);
+fn flush_accumulated_text(nodes: &mut Vec<Node>, accumulated: &mut String) {
+ append_text(nodes, accumulated);
accumulated.clear();
}
-fn flush_bounded_accumulated_text(
- nodes: &mut Vec<Node>,
- accumulated: &mut Vec<u8>,
- max_len: usize,
-) {
- if accumulated.is_empty() {
- return;
- }
+fn flush_bounded_accumulated_text(nodes: &mut Vec<Node>, accumulated: &mut String, max_len: usize) {
let mut prefix_len = accumulated.len().min(max_len);
- while prefix_len > 0 && std::str::from_utf8(&accumulated[..prefix_len]).is_err() {
+ while !accumulated.is_char_boundary(prefix_len) {
prefix_len -= 1;
}
- if prefix_len > 0 {
- append_text(nodes, string_from_lossy(&accumulated[..prefix_len]));
- }
- accumulated.clear();
+ accumulated.truncate(prefix_len);
+ flush_accumulated_text(nodes, accumulated);
+}
+
+fn is_plain_text_byte(byte: u8) -> bool {
+ byte.is_ascii()
+ && !matches!(
+ byte,
+ b'\\' | b'h' | b'f' | b'_' | b'<' | b':' | b'@' | b'*' | b'~' | b'|' | b'`' | b'['
+ )
}
fn is_formatting_char(char: u8) -> bool {
diff --git a/packages/markdown_parser/rust/src/json.rs b/packages/markdown_parser/rust/src/json.rs
index f4e74decc..8b6921908 100644
--- a/packages/markdown_parser/rust/src/json.rs
+++ b/packages/markdown_parser/rust/src/json.rs
@@ -1,15 +1,310 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use serde::Serialize;
+use std::fmt::Write;
-use crate::ast::Node;
-use crate::parser::ParseError;
+use crate::ast::{EmojiKind, ListItem, MentionKind, Node};
-#[derive(Serialize)]
-struct AstEnvelope<'a> {
- nodes: &'a [Node],
+macro_rules! key {
+ ($name:literal) => {
+ concat!(",\"", $name, "\":")
+ };
}
-pub fn write_ast_json(nodes: &[Node]) -> Result<String, ParseError> {
- Ok(serde_json::to_string(&AstEnvelope { nodes })?)
+pub fn write_ast_json(nodes: &[Node], capacity: usize) -> String {
+ let mut out = String::with_capacity(capacity);
+ out.push_str("{\"nodes\":");
+ write_nodes(&mut out, nodes);
+ out.push('}');
+ out
}
+
+fn write_nodes(out: &mut String, nodes: &[Node]) {
+ out.push('[');
+ for (index, node) in nodes.iter().enumerate() {
+ if index > 0 {
+ out.push(',');
+ }
+ write_node(out, node);
+ }
+ out.push(']');
+}
+
+fn write_node(out: &mut String, node: &Node) {
+ out.push_str("{\"type\":\"");
+ out.push_str(node.tag_name());
+ out.push('"');
+ match node {
+ Node::Text { content } | Node::InlineCode { content } => {
+ out.push_str(key!("content"));
+ write_str(out, content);
+ }
+ Node::Blockquote {
+ children,
+ blank_lines,
+ } => {
+ write_children(out, children);
+ if let Some(blank_lines) = blank_lines {
+ out.push_str(key!("blankLines"));
+ write_number(out, *blank_lines as u64);
+ }
+ }
+ Node::Strong { children }
+ | Node::Emphasis { children }
+ | Node::Underline { children }
+ | Node::Strikethrough { children }
+ | Node::Subtext { children }
+ | Node::Sequence { children } => write_children(out, children),
+ Node::Spoiler { children, is_block } => {
+ write_children(out, children);
+ if let Some(is_block) = is_block {
+ out.push_str(key!("isBlock"));
+ write_bool(out, *is_block);
+ }
+ }
+ Node::Heading { level, children } => {
+ out.push_str(key!("level"));
+ write_number(out, u64::from(*level));
+ write_children(out, children);
+ }
+ Node::List { ordered, items } => {
+ out.push_str(key!("ordered"));
+ write_bool(out, *ordered);
+ out.push_str(key!("items"));
+ out.push('[');
+ for (index, ListItem { children, ordinal }) in items.iter().enumerate() {
+ if index > 0 {
+ out.push(',');
+ }
+ out.push_str("{\"children\":");
+ write_nodes(out, children);
+ if let Some(ordinal) = ordinal {
+ out.push_str(key!("ordinal"));
+ write_number(out, *ordinal as u64);
+ }
+ out.push('}');
+ }
+ out.push(']');
+ }
+ Node::CodeBlock { language, content } => {
+ if let Some(language) = language {
+ out.push_str(key!("language"));
+ write_str(out, language);
+ }
+ out.push_str(key!("content"));
+ write_str(out, content);
+ }
+ Node::Link {
+ text,
+ url,
+ escaped,
+ raw_url,
+ source,
+ } => {
+ if let Some(text) = text {
+ out.push_str(key!("text"));
+ write_node(out, text);
+ }
+ out.push_str(key!("url"));
+ write_str(out, url);
+ out.push_str(key!("escaped"));
+ write_bool(out, *escaped);
+ out.push_str(key!("rawUrl"));
+ write_str(out, raw_url);
+ out.push_str(key!("source"));
+ write_str(out, source);
+ }
+ Node::Mention { kind } => {
+ out.push_str(key!("kind"));
+ write_mention_kind(out, kind);
+ }
+ Node::Timestamp { timestamp, style } => {
+ out.push_str(key!("timestamp"));
+ write_number(out, *timestamp);
+ out.push_str(key!("style"));
+ let _ = write!(out, "\"{style:?}\"");
+ }
+ Node::Emoji { kind } => {
+ out.push_str(key!("kind"));
+ write_emoji_kind(out, kind);
+ }
+ Node::Table {
+ header,
+ alignments,
+ rows,
+ } => {
+ out.push_str(key!("header"));
+ write_node(out, header);
+ out.push_str(key!("alignments"));
+ out.push('[');
+ for (index, alignment) in alignments.iter().enumerate() {
+ if index > 0 {
+ out.push(',');
+ }
+ let _ = write!(out, "\"{alignment:?}\"");
+ }
+ out.push(']');
+ out.push_str(key!("rows"));
+ write_nodes(out, rows);
+ }
+ Node::TableRow { cells } => {
+ out.push_str(key!("cells"));
+ write_nodes(out, cells);
+ }
+ Node::TableCell { children } => write_children(out, children),
+ Node::Alert {
+ alert_type,
+ children,
+ } => {
+ out.push_str(key!("alertType"));
+ let _ = write!(out, "\"{alert_type:?}\"");
+ write_children(out, children);
+ }
+ }
+ out.push('}');
+}
+
+fn write_mention_kind(out: &mut String, kind: &MentionKind) {
+ match kind {
+ MentionKind::User { id } => write_tagged_id(out, "User", id),
+ MentionKind::Channel { id } => write_tagged_id(out, "Channel", id),
+ MentionKind::Role { id } => write_tagged_id(out, "Role", id),
+ MentionKind::Command {
+ name,
+ subcommand_group,
+ subcommand,
+ id,
+ } => {
+ out.push_str("{\"kind\":\"Command\"");
+ out.push_str(key!("name"));
+ write_str(out, name);
+ if let Some(subcommand_group) = subcommand_group {
+ out.push_str(key!("subcommandGroup"));
+ write_str(out, subcommand_group);
+ }
+ if let Some(subcommand) = subcommand {
+ out.push_str(key!("subcommand"));
+ write_str(out, subcommand);
+ }
+ out.push_str(key!("id"));
+ write_str(out, id);
+ out.push('}');
+ }
+ MentionKind::GuildNavigation {
+ navigation_type,
+ id,
+ } => {
+ out.push_str("{\"kind\":\"GuildNavigation\"");
+ out.push_str(key!("navigationType"));
+ let _ = write!(out, "\"{navigation_type:?}\"");
+ if let Some(id) = id {
+ out.push_str(key!("id"));
+ write_str(out, id);
+ }
+ out.push('}');
+ }
+ MentionKind::Everyone => out.push_str("{\"kind\":\"Everyone\"}"),
+ MentionKind::Here => out.push_str("{\"kind\":\"Here\"}"),
+ }
+}
+
+fn write_tagged_id(out: &mut String, kind: &str, id: &str) {
+ out.push_str("{\"kind\":\"");
+ out.push_str(kind);
+ out.push('"');
+ out.push_str(key!("id"));
+ write_str(out, id);
+ out.push('}');
+}
+
+fn write_emoji_kind(out: &mut String, kind: &EmojiKind) {
+ match kind {
+ EmojiKind::Standard {
+ raw,
+ codepoints,
+ name,
+ } => {
+ out.push_str("{\"kind\":\"Standard\"");
+ out.push_str(key!("raw"));
+ write_str(out, raw);
+ out.push_str(key!("codepoints"));
+ write_str(out, codepoints);
+ out.push_str(key!("name"));
+ write_str(out, name);
+ }
+ EmojiKind::Custom { name, id, animated } => {
+ out.push_str("{\"kind\":\"Custom\"");
+ out.push_str(key!("name"));
+ write_str(out, name);
+ out.push_str(key!("id"));
+ write_str(out, id);
+ out.push_str(key!("animated"));
+ write_bool(out, *animated);
+ }
+ }
+ out.push('}');
+}
+
+fn write_children(out: &mut String, children: &[Node]) {
+ out.push_str(key!("children"));
+ write_nodes(out, children);
+}
+
+fn write_bool(out: &mut String, value: bool) {
+ out.push_str(if value { "true" } else { "false" });
+}
+
+fn write_number(out: &mut String, value: u64) {
+ let _ = write!(out, "{value}");
+}
+
+fn write_str(out: &mut String, value: &str) {
+ out.push('"');
+ let bytes = value.as_bytes();
+ let mut start = 0;
+ let mut index = next_escape(bytes, 0);
+ while index < bytes.len() {
+ out.push_str(&value[start..index]);
+ let byte = bytes[index];
+ match byte {
+ b'"' => out.push_str("\\\""),
+ b'\\' => out.push_str("\\\\"),
+ b'\n' => out.push_str("\\n"),
+ b'\r' => out.push_str("\\r"),
+ b'\t' => out.push_str("\\t"),
+ 0x08 => out.push_str("\\b"),
+ 0x0c => out.push_str("\\f"),
+ _ => {
+ out.push_str("\\u00");
+ out.push(char::from(HEX_DIGITS[usize::from(byte >> 4)]));
+ out.push(char::from(HEX_DIGITS[usize::from(byte & 0xf)]));
+ }
+ }
+ start = index + 1;
+ index = next_escape(bytes, start);
+ }
+ out.push_str(&value[start..]);
+ out.push('"');
+}
+
+fn next_escape(bytes: &[u8], from: usize) -> usize {
+ const LOW_BITS: u64 = u64::from_ne_bytes([0x01; 8]);
+ const HIGH_BITS: u64 = u64::from_ne_bytes([0x80; 8]);
+ let has_zero = |word: u64| word.wrapping_sub(LOW_BITS) & !word & HIGH_BITS != 0;
+ let mut index = from;
+ while let Some(chunk) = bytes.get(index..index + 8) {
+ let word = u64::from_ne_bytes(chunk.try_into().unwrap_or_default());
+ if word.wrapping_sub(LOW_BITS * 0x20) & !word & HIGH_BITS != 0
+ || has_zero(word ^ (LOW_BITS * u64::from(b'"')))
+ || has_zero(word ^ (LOW_BITS * u64::from(b'\\')))
+ {
+ break;
+ }
+ index += 8;
+ }
+ bytes[index..]
+ .iter()
+ .position(|byte| *byte < 0x20 || matches!(byte, b'"' | b'\\'))
+ .map_or(bytes.len(), |offset| index + offset)
+}
+
+const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef";
diff --git a/packages/markdown_parser/rust/src/lib.rs b/packages/markdown_parser/rust/src/lib.rs
index e7e8ccf96..b2eebb33c 100644
--- a/packages/markdown_parser/rust/src/lib.rs
+++ b/packages/markdown_parser/rust/src/lib.rs
@@ -12,6 +12,7 @@ pub mod native;
pub mod normalize;
pub mod parser;
pub mod plaintext;
+mod scan;
pub mod text;
pub mod wasm;
@@ -27,5 +28,5 @@ pub fn parse_markdown_json(
let context = EmojiContext::parse(emoji_context);
let mut parser = MarkdownParser::new(flags, context);
let nodes = parser.parse(input)?;
- json::write_ast_json(&nodes)
+ Ok(json::write_ast_json(&nodes, input.len() * 8 + 64))
}
diff --git a/packages/markdown_parser/rust/src/links.rs b/packages/markdown_parser/rust/src/links.rs
index 5fd1f9583..80001a918 100644
--- a/packages/markdown_parser/rust/src/links.rs
+++ b/packages/markdown_parser/rust/src/links.rs
@@ -5,22 +5,21 @@ use crate::ast::{
};
use crate::constants::MAX_LINK_URL_LENGTH;
use crate::parser::{MarkdownParser, ParseError};
+use crate::scan::{ANGLE_SYNTAX_PREFIXES, Lookahead, Pattern, http_scheme_len};
use crate::text::{
- advance_one, bounded_url_end, byte_at, find_from, has_visible_content, is_alpha_numeric,
+ bounded_utf8_prefix_length, byte_at, find_from, has_visible_content, is_alpha_numeric,
is_digit, is_digit_only, is_whitespace, starts_with, trim,
};
const APP_PROTOCOL_SCHEME: &str = "fluxer:";
-
-#[derive(Clone, Debug)]
-struct BracketResult<'a> {
- bracket_position: usize,
- link_text: &'a str,
-}
+const MAX_COMMAND_NAME_LENGTH: usize = 32;
+const MAX_COMMAND_SEGMENTS: usize = 3;
+const MAX_COMMAND_COLON_INDEX: usize = MAX_COMMAND_SEGMENTS * (MAX_COMMAND_NAME_LENGTH + 1);
#[derive(Clone, Debug)]
struct UrlInfo<'a> {
url: &'a str,
+ start: usize,
escaped: bool,
advance_by: usize,
}
@@ -33,11 +32,11 @@ struct ComparableUrl<'a> {
hash: &'a str,
}
-pub fn parse_custom_emoji(text: &str) -> Option<ParserResult> {
+pub(crate) fn parse_custom_emoji(text: &str) -> Option<ParserResult> {
if !(starts_with(text, "<:") || starts_with(text, "<a:")) {
return None;
}
- let end = find_from(text, 0, b'>')?;
+ let end = text.len() - 1;
if end < 4 {
return None;
}
@@ -136,11 +135,11 @@ pub fn parse_emoji_shortcode(parser: &MarkdownParser, text: &str) -> Option<Pars
})
}
-pub fn parse_timestamp(text: &str) -> Option<ParserResult> {
+pub(crate) fn parse_timestamp(text: &str) -> Option<ParserResult> {
if !starts_with(text, "<t:") {
return None;
}
- let end = find_from(text, 0, b'>')?;
+ let end = text.len() - 1;
let inner = &text[3..end];
if inner.is_empty() {
return None;
@@ -179,11 +178,11 @@ pub fn parse_timestamp(text: &str) -> Option<ParserResult> {
})
}
-pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {
+pub(crate) fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {
if text.len() < 2 || byte_at(text, 0) != b'<' {
return None;
}
- let end = find_from(text, 0, b'>')?;
+ let end = text.len() - 1;
if end < 2 {
return None;
}
@@ -219,7 +218,10 @@ pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {
if !ParserFlags::has(flags, ParserFlags::ALLOW_COMMAND_MENTIONS) {
return None;
}
- let colon = find_from(inner, 0, b':')?;
+ let colon = inner
+ .bytes()
+ .take(MAX_COMMAND_COLON_INDEX + 1)
+ .position(|byte| byte == b':')?;
let id = &inner[colon + 1..];
if !is_digit_only(id) {
return None;
@@ -227,7 +229,10 @@ pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {
let command = &inner[1..colon];
let mut segments = Vec::new();
for segment in command.split(' ') {
- if segment.is_empty() || !is_valid_command_name(segment) || segments.len() == 3 {
+ if segment.is_empty()
+ || !is_valid_command_name(segment)
+ || segments.len() == MAX_COMMAND_SEGMENTS
+ {
return None;
}
segments.push(segment);
@@ -238,7 +243,8 @@ pub fn parse_mention(text: &str, flags: u32) -> Option<ParserResult> {
return mention(
MentionKind::Command {
name: segments[0].to_owned(),
- subcommand_group: (segments.len() == 3).then(|| segments[1].to_owned()),
+ subcommand_group: (segments.len() == MAX_COMMAND_SEGMENTS)
+ .then(|| segments[1].to_owned()),
subcommand: (segments.len() >= 2)
.then(|| segments[segments.len() - 1].to_owned()),
id: id.to_owned(),
@@ -281,23 +287,28 @@ fn mention(kind: MentionKind, end: usize) -> Option<ParserResult> {
})
}
-pub fn parse_masked_link(
+pub(crate) fn parse_masked_link(
parser: &mut MarkdownParser,
- text: &str,
+ scan: &mut Lookahead<'_>,
+ position: usize,
base_offset: usize,
) -> Result<Option<ParserResult>, ParseError> {
+ let text = &scan.text[position..];
if text.is_empty() || byte_at(text, 0) != b'[' {
return Ok(None);
}
- let Some(closing) = find_closing_bracket(text) else {
+ let Some(bracket_position) = scan
+ .closing_bracket(position)
+ .map(|closing| closing - position)
+ .filter(|distance| *distance <= MAX_LINK_URL_LENGTH)
+ else {
return Ok(None);
};
- let link_text = closing.link_text;
- let trimmed_link_text = trim(link_text);
- if contains_link_syntax(link_text) {
- if closing.bracket_position + 1 < text.len()
- && byte_at(text, closing.bracket_position + 1) == b'('
- && let Some(url_info) = extract_url(text, closing.bracket_position + 2)
+ let link_text = &text[1..bracket_position];
+ if scan.has_link_syntax(position + 1, position + bracket_position) {
+ if bracket_position + 1 < text.len()
+ && byte_at(text, bracket_position + 1) == b'('
+ && let Some(url_info) = extract_url(scan, position, bracket_position + 2)
{
return Ok(Some(ParserResult {
node: Node::Text {
@@ -308,34 +319,33 @@ pub fn parse_masked_link(
}
return Ok(Some(ParserResult {
node: Node::Text {
- content: text[..closing.bracket_position + 1].to_owned(),
+ content: text[..bracket_position + 1].to_owned(),
},
- advance: closing.bracket_position + 1,
+ advance: bracket_position + 1,
}));
}
- if closing.bracket_position + 1 >= text.len()
- || byte_at(text, closing.bracket_position + 1) != b'('
- {
+ if bracket_position + 1 >= text.len() || byte_at(text, bracket_position + 1) != b'(' {
return Ok(None);
}
- let Some(url_info) = extract_url(text, closing.bracket_position + 2) else {
- if let Some(mention) = parse_mention(trimmed_link_text, parser.flags())
- && mention.advance == trimmed_link_text.len()
+ let trimmed_link_text = trim(link_text);
+ let Some(url_info) = extract_url(scan, position, bracket_position + 2) else {
+ if trimmed_link_text.find('>') == Some(trimmed_link_text.len().wrapping_sub(1))
+ && parse_mention(trimmed_link_text, parser.flags()).is_some()
{
return Ok(None);
}
return Ok(Some(ParserResult {
node: Node::Text {
- content: text[..closing.bracket_position + 1].to_owned(),
+ content: text[..bracket_position + 1].to_owned(),
},
- advance: closing.bracket_position + 1,
+ advance: bracket_position + 1,
}));
};
if !has_visible_content(link_text)
- || is_slash_command_masked_link_text(trim(link_text))
- || is_email_like(trim(link_text))
+ || is_slash_command_masked_link_text(trimmed_link_text)
+ || is_email_like(trimmed_link_text)
|| url_info.url.len() > MAX_LINK_URL_LENGTH
- || !is_valid_masked_link_url(url_info.url)
+ || !is_valid_masked_link_url(scan, position + url_info.start, url_info.url.len())
|| (starts_with_url(trimmed_link_text)
&& should_treat_as_masked_link(trimmed_link_text, url_info.url))
{
@@ -367,95 +377,42 @@ pub fn parse_masked_link(
}))
}
-fn find_closing_bracket(text: &str) -> Option<BracketResult<'_>> {
- let mut position = 1;
- let mut nested = 0usize;
- while position < text.len() {
- match byte_at(text, position) {
- b'[' => {
- nested += 1;
- position += 1;
- }
- b']' if nested > 0 => {
- nested -= 1;
- position += 1;
- }
- b']' => {
- return Some(BracketResult {
- bracket_position: position,
- link_text: &text[1..position],
- });
- }
- b'\\' => position += if position + 1 < text.len() { 2 } else { 1 },
- _ => position += advance_one(text, position),
- }
- if position > MAX_LINK_URL_LENGTH {
- break;
- }
- }
- None
-}
-
-fn extract_url(text: &str, start: usize) -> Option<UrlInfo<'_>> {
+fn extract_url<'a>(scan: &mut Lookahead<'a>, position: usize, start: usize) -> Option<UrlInfo<'a>> {
+ let text = &scan.text[position..];
if start >= text.len() {
return None;
}
if byte_at(text, start) == b'<' {
- return extract_escaped_url(text, start + 1);
- }
- extract_unescaped_url(text, start)
-}
-
-fn extract_escaped_url(text: &str, start: usize) -> Option<UrlInfo<'_>> {
- let mut pos = start;
- while pos < text.len() {
- if byte_at(text, pos) == b'>' {
- let url = &text[start..pos];
- pos += 1;
- if pos >= text.len() || byte_at(text, pos) != b')' {
- return None;
- }
- return Some(UrlInfo {
- url,
- escaped: true,
- advance_by: pos + 1,
- });
- }
- pos += advance_one(text, pos);
- }
- None
-}
-
-fn extract_unescaped_url(text: &str, start: usize) -> Option<UrlInfo<'_>> {
- let mut pos = start;
- let mut nested = 0usize;
- while pos < text.len() {
- match byte_at(text, pos) {
- b'(' => {
- nested += 1;
- pos += 1;
- }
- b')' if nested > 0 => {
- nested -= 1;
- pos += 1;
- }
- b')' => {
- return Some(UrlInfo {
- url: &text[start..pos],
- escaped: false,
- advance_by: pos + 1,
- });
- }
- _ => pos += advance_one(text, pos),
+ let close =
+ scan.next(Pattern::AngleClose, position + start + 1, scan.text.len())? - position;
+ if close + 1 >= text.len() || byte_at(text, close + 1) != b')' {
+ return None;
}
+ return Some(UrlInfo {
+ url: &text[start + 1..close],
+ start: start + 1,
+ escaped: true,
+ advance_by: close + 2,
+ });
}
- None
+ let close = scan.closing_paren(position + start)? - position;
+ Some(UrlInfo {
+ url: &text[start..close],
+ start,
+ escaped: false,
+ advance_by: close + 1,
+ })
}
-pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {
+pub(crate) fn parse_url_segment(
+ scan: &mut Lookahead<'_>,
+ position: usize,
+ flags: u32,
+) -> Option<ParserResult> {
if !ParserFlags::has(flags, ParserFlags::ALLOW_AUTOLINKS) {
return None;
}
+ let text = &scan.text[position..];
let prefix_len = if starts_with(text, "https://") {
8
} else if starts_with(text, "http://") {
@@ -465,30 +422,12 @@ pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {
} else {
return None;
};
- let mut end = prefix_len;
- let mut paren_depth = 0usize;
- while end < text.len() {
- let c = byte_at(text, end);
- if c == b'(' {
- paren_depth += 1;
- end += 1;
- } else if c == b')' {
- if paren_depth > 0 {
- paren_depth -= 1;
- end += 1;
- } else {
- break;
- }
- } else if is_url_termination_char(c) {
- break;
- } else {
- end += advance_one(text, end);
- }
- if end - prefix_len > MAX_LINK_URL_LENGTH {
- end = bounded_url_end(text, end, prefix_len);
- break;
- }
- }
+ let stop = scan.url_stop(position + prefix_len) - position;
+ let mut end = if stop - prefix_len <= MAX_LINK_URL_LENGTH {
+ stop
+ } else {
+ bounded_utf8_prefix_length(text, prefix_len + MAX_LINK_URL_LENGTH)
+ };
while end > 0
&& is_trimmed_autolink_punctuation(byte_at(text, end - 1))
&& !has_terminal_tld(&text[..end])
@@ -498,16 +437,17 @@ pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {
if end <= prefix_len {
return None;
}
- let url = &text[..end];
- if (starts_with_app_protocol_url(url) && !looks_like_valid_app_protocol_url(url))
- || contains_nested_scheme_before_path(url, prefix_len)
- || contains_formatting_marker_before_path(url, prefix_len)
- || contains_unparenthesized_nested_http_url(url)
- || contains_angle_bracket_syntax(url)
- || contains_angle_email_like(url)
+ let url_end = position + end;
+ if (starts_with_app_protocol_url(&text[..end])
+ && !is_valid_app_protocol_url(scan, position, url_end))
+ || contains_nested_scheme_before_path(scan, position + prefix_len, url_end)
+ || contains_formatting_marker_before_path(scan, position + prefix_len, url_end)
+ || contains_angle_bracket_markup(scan, position, url_end)
{
return None;
}
+ let text = &scan.text[position..];
+ let url = &text[..end];
let escaped = matches!(byte_at(text, 0), b'"' | b'\'')
|| (end < text.len() && matches!(byte_at(text, end), b'"' | b'\''));
Some(ParserResult {
@@ -522,14 +462,20 @@ pub fn parse_url_segment(text: &str, flags: u32) -> Option<ParserResult> {
})
}
-pub fn parse_autolink(text: &str, flags: u32) -> Option<ParserResult> {
+pub(crate) fn parse_autolink(
+ scan: &mut Lookahead<'_>,
+ position: usize,
+ close: usize,
+ flags: u32,
+) -> Option<ParserResult> {
+ let text = &scan.text[position..=close];
if !ParserFlags::has(flags, ParserFlags::ALLOW_AUTOLINKS) || byte_at(text, 0) != b'<' {
return None;
}
if matches!(byte_at(text, 1), b'"' | b'\'') || !starts_with_url(&text[1..]) {
return None;
}
- let end = find_from(text, 1, b'>')?;
+ let end = text.len() - 1;
let url = &text[1..end];
if url.len() > MAX_LINK_URL_LENGTH {
return None;
@@ -544,10 +490,10 @@ pub fn parse_autolink(text: &str, flags: u32) -> Option<ParserResult> {
return None;
};
if starts_with_app_protocol_url(url) {
- if !looks_like_valid_app_protocol_url(url) {
+ if !is_valid_app_protocol_url(scan, position + 1, close) {
return None;
}
- } else if !looks_like_valid_http_url_prefix(url, prefix_len) {
+ } else if !looks_like_valid_http_url_prefix(scan, position + 1, close, prefix_len, true) {
return None;
}
Some(ParserResult {
@@ -562,11 +508,11 @@ pub fn parse_autolink(text: &str, flags: u32) -> Option<ParserResult> {
})
}
-pub fn parse_email_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {
+pub(crate) fn parse_email_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {
if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS) || byte_at(text, 0) != b'<' {
return None;
}
- let end = find_from(text, 1, b'>')?;
+ let end = text.len() - 1;
let content = &text[1..end];
if starts_with(content, "http://")
|| starts_with(content, "https://")
@@ -593,11 +539,11 @@ pub fn parse_email_link(parser: &MarkdownParser, text: &str) -> Option<ParserRes
})
}
-pub fn parse_phone_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {
+pub(crate) fn parse_phone_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {
if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS) || byte_at(text, 0) != b'<' {
return None;
}
- let end = find_from(text, 1, b'>')?;
+ let end = text.len() - 1;
let content = &text[1..end];
if !is_valid_phone_number(content) {
return None;
@@ -618,13 +564,13 @@ pub fn parse_phone_link(parser: &MarkdownParser, text: &str) -> Option<ParserRes
})
}
-pub fn parse_sms_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {
+pub(crate) fn parse_sms_link(parser: &MarkdownParser, text: &str) -> Option<ParserResult> {
if !ParserFlags::has(parser.flags(), ParserFlags::ALLOW_AUTOLINKS)
|| !starts_with(text, "<sms:")
{
return None;
}
- let end = find_from(text, 1, b'>')?;
+ let end = text.len() - 1;
let content = &text[1..end];
let phone = &content[4..];
if !is_valid_phone_number(phone) {
@@ -662,30 +608,18 @@ fn starts_with_app_protocol_url(text: &str) -> bool {
)
}
-pub fn looks_like_valid_http_url_prefix(url: &str, prefix_len: usize) -> bool {
- looks_like_valid_http_url_prefix_with_userinfo(url, prefix_len, true)
-}
-
-fn looks_like_valid_http_url_prefix_with_userinfo(
- url: &str,
+fn looks_like_valid_http_url_prefix(
+ scan: &mut Lookahead<'_>,
+ start: usize,
+ end: usize,
prefix_len: usize,
allow_userinfo: bool,
) -> bool {
- if url.len() <= prefix_len
- || contains_unparenthesized_nested_http_url(url)
- || contains_angle_bracket_syntax(url)
- || contains_angle_email_like(url)
- {
+ if end - start <= prefix_len || contains_angle_bracket_markup(scan, start, end) {
return false;
}
- let mut authority_end = prefix_len;
- while authority_end < url.len() {
- let char = byte_at(url, authority_end);
- if matches!(char, b'/' | b'?' | b'#') {
- break;
- }
- authority_end += 1;
- }
+ let url = &scan.text[start..end];
+ let authority_end = path_start(scan, start + prefix_len, end) - start;
let colon_before_path_slash = authority_end < url.len()
&& byte_at(url, authority_end) == b'/'
&& authority_end > prefix_len
@@ -697,6 +631,10 @@ fn looks_like_valid_http_url_prefix_with_userinfo(
)
}
+fn path_start(scan: &mut Lookahead<'_>, from: usize, end: usize) -> usize {
+ scan.next(Pattern::PathDelimiter, from, end).unwrap_or(end)
+}
+
fn is_valid_http_authority(
authority: &str,
allow_userinfo: bool,
@@ -751,52 +689,78 @@ fn is_valid_http_authority(
true
}
-fn contains_angle_bracket_syntax(value: &str) -> bool {
- ["<:", "<a:", "<id:", "<@", "<#", "</", "<t:", "<+", "<sms:"]
- .iter()
- .any(|needle| value.contains(needle))
+fn contains_angle_bracket_markup(scan: &mut Lookahead<'_>, start: usize, end: usize) -> bool {
+ scan.next(Pattern::AngleOpen, start, end).is_some()
+ && (contains_unparenthesized_nested_http_url(scan, start, end)
+ || contains_angle_bracket_syntax(scan, start, end)
+ || scan.has_angle_email(start, end))
}
-fn contains_unparenthesized_nested_http_url(url: &str) -> bool {
- for needle in ["<http://", "<https://"] {
- if let Some(pos) = url.find(needle)
- && !url[..pos].contains('(')
+fn contains_angle_bracket_syntax(scan: &mut Lookahead<'_>, start: usize, end: usize) -> bool {
+ let mut from = start;
+ while let Some(position) = scan.next(Pattern::AngleSyntax, from, end) {
+ let window = &scan.text.as_bytes()[position..end];
+ if ANGLE_SYNTAX_PREFIXES
+ .iter()
+ .any(|prefix| window.starts_with(prefix.as_bytes()))
{
return true;
}
+ from = position + 1;
}
false
}
-fn contains_nested_scheme_before_path(url: &str, prefix_len: usize) -> bool {
- let rest = &url[prefix_len..];
- let first_path_delimiter = rest
- .bytes()
- .position(|char| matches!(char, b'/' | b'?' | b'#'))
- .unwrap_or(rest.len());
- for needle in ["http://", "https://"] {
- if let Some(pos) = rest.find(needle)
- && pos < first_path_delimiter
- && pos > 0
- && byte_at(rest, pos - 1) == b'('
- {
- return true;
+fn contains_unparenthesized_nested_http_url(
+ scan: &mut Lookahead<'_>,
+ start: usize,
+ end: usize,
+) -> bool {
+ let bytes = scan.text.as_bytes();
+ let Some(position) = scan
+ .next(Pattern::NestedHttpUrl, start, end)
+ .filter(|position| {
+ http_scheme_len(&bytes[position + 1..]).is_some_and(|len| position + 1 + len <= end)
+ })
+ else {
+ return false;
+ };
+ scan.next(Pattern::OpenParen, start, position).is_none()
+}
+
+fn contains_nested_scheme_before_path(
+ scan: &mut Lookahead<'_>,
+ rest_start: usize,
+ end: usize,
+) -> bool {
+ let bytes = scan.text.as_bytes();
+ let path = path_start(scan, rest_start, end);
+ let mut checked = [false; 2];
+ let mut from = rest_start;
+ while !(checked[0] && checked[1])
+ && let Some(position) = scan.next(Pattern::HttpScheme, from, path)
+ {
+ let len = http_scheme_len(&bytes[position..]).unwrap_or_default();
+ let kind = usize::from(len == 8);
+ if !checked[kind] {
+ checked[kind] = true;
+ if position + len <= end && position > rest_start && bytes[position - 1] == b'(' {
+ return true;
+ }
}
+ from = position + 1;
}
false
}
-fn contains_formatting_marker_before_path(url: &str, prefix_len: usize) -> bool {
- let rest = &url[prefix_len..];
- let authority_end = rest
- .bytes()
- .position(|char| matches!(char, b'/' | b'?' | b'#'))
- .unwrap_or(rest.len());
- let authority = &rest[..authority_end];
- authority.contains("__")
- || authority.contains("**")
- || authority.contains("~~")
- || authority.contains("||")
+fn contains_formatting_marker_before_path(
+ scan: &mut Lookahead<'_>,
+ rest_start: usize,
+ end: usize,
+) -> bool {
+ let path = path_start(scan, rest_start, end);
+ scan.next(Pattern::DoubledMarker, rest_start, path)
+ .is_some_and(|position| position + 2 <= path)
}
fn is_valid_bracketed_ip_literal(value: &str) -> bool {
@@ -807,15 +771,16 @@ fn is_valid_bracketed_ip_literal(value: &str) -> bool {
.all(|char| is_digit(char) || char.is_ascii_hexdigit() || matches!(char, b':' | b'.'))
}
-pub fn is_valid_masked_link_url(url: &str) -> bool {
+fn is_valid_masked_link_url(scan: &mut Lookahead<'_>, start: usize, len: usize) -> bool {
+ let url = &scan.text[start..start + len];
if url.is_empty() || url.contains('"') {
return false;
}
if starts_with(url, "https://") {
- return looks_like_valid_http_url_prefix_with_userinfo(url, 8, false);
+ return looks_like_valid_http_url_prefix(scan, start, start + len, 8, false);
}
if starts_with(url, "http://") {
- return looks_like_valid_http_url_prefix_with_userinfo(url, 7, false);
+ return looks_like_valid_http_url_prefix(scan, start, start + len, 7, false);
}
if starts_with(url, "mailto:") {
return is_valid_email(&url["mailto:".len()..]);
@@ -826,30 +791,7 @@ pub fn is_valid_masked_link_url(url: &str) -> bool {
if starts_with(url, "sms:") {
return is_valid_phone_number(&url["sms:".len()..]);
}
- if starts_with_app_protocol_url(url) {
- return looks_like_valid_app_protocol_url(url);
- }
- false
-}
-
-fn contains_angle_email_like(value: &str) -> bool {
- let mut offset = 0;
- while offset < value.len() {
- let Some(open_relative) = value[offset..].find('<') else {
- return false;
- };
- let open = offset + open_relative;
- let Some(close_relative) = value[open + 1..].find('>') else {
- return false;
- };
- let close = open + 1 + close_relative;
- let inner = &value[open + 1..close];
- if !inner.is_empty() && byte_at(inner, 0) != b'@' && inner.contains('@') {
- return true;
- }
- offset = close + 1;
- }
- false
+ starts_with_app_protocol_url(url) && is_valid_app_protocol_url(scan, start, start + len)
}
pub fn should_treat_as_masked_link(link_text: &str, url: &str) -> bool {
@@ -925,20 +867,13 @@ fn equivalent_url_path(left: &str, right: &str) -> bool {
normalized_left == normalized_right
}
-fn looks_like_valid_app_protocol_url(url: &str) -> bool {
- if !starts_with_app_protocol_url(url) {
- return false;
- }
- url[APP_PROTOCOL_SCHEME.len()..].bytes().all(|char| {
- !matches!(
- char,
- b'"' | b'\'' | b'<' | b'>' | b'\\' | b'|' | b' ' | b'\t' | b'\r' | b'\n'
- )
- })
-}
-
-pub fn is_url_termination_char(char: u8) -> bool {
- matches!(char, b' ' | b'\t' | b'\n' | b'\r' | b')' | b'"')
+fn is_valid_app_protocol_url(scan: &mut Lookahead<'_>, start: usize, end: usize) -> bool {
+ scan.next(
+ Pattern::AppProtocolInvalid,
+ start + APP_PROTOCOL_SCHEME.len(),
+ end,
+ )
+ .is_none()
}
pub fn is_word_underscore(text: &str, pos: usize) -> bool {
@@ -963,30 +898,13 @@ pub fn is_valid_emoji_name(name: &str) -> bool {
fn is_valid_command_name(name: &str) -> bool {
!name.is_empty()
- && name.len() <= 32
+ && name.len() <= MAX_COMMAND_NAME_LENGTH
&& name.bytes().all(|char| {
!char.is_ascii_uppercase()
&& (char >= 0x80 || is_alpha_numeric(char) || matches!(char, b'-' | b'_' | b'\''))
})
}
-fn contains_link_syntax(text: &str) -> bool {
- let mut search_offset = 0;
- while let Some(bracket) = text[search_offset..].find('[') {
- let bracket = search_offset + bracket;
- let after = &text[bracket + 1..];
- let Some(close) = after.find(']') else {
- return false;
- };
- let close_abs = bracket + 1 + close;
- if close_abs + 1 < text.len() && byte_at(text, close_abs + 1) == b'(' {
- return true;
- }
- search_offset = close_abs + 1;
- }
- false
-}
-
fn is_slash_command_masked_link_text(text: &str) -> bool {
starts_with(text, "</") && text.ends_with('>') && text.contains(':')
}
@@ -1008,25 +926,23 @@ fn is_email_like(text: &str) -> bool {
}
pub fn is_valid_email(text: &str) -> bool {
- if !text.contains('@') || text.contains(' ') {
- return false;
- }
- let Some(at) = text.find('@') else {
+ let Some(at) = text
+ .bytes()
+ .position(|char| {
+ !(is_alpha_numeric(char) || matches!(char, b'.' | b'_' | b'%' | b'+' | b'-'))
+ })
+ .filter(|at| byte_at(text, *at) == b'@')
+ else {
return false;
};
if at == 0 || at + 1 >= text.len() {
return false;
}
- for char in text[..at].bytes() {
- if !(is_alpha_numeric(char) || matches!(char, b'.' | b'_' | b'%' | b'+' | b'-')) {
- return false;
- }
- }
let domain = &text[at + 1..];
- let Some(dot) = domain.rfind('.') else {
- return false;
- };
- dot > 0 && dot + 2 <= domain.len() && all_ascii_domain(domain)
+ all_ascii_domain(domain)
+ && domain
+ .rfind('.')
+ .is_some_and(|dot| dot > 0 && dot + 2 <= domain.len())
}
fn all_ascii_domain(domain: &str) -> bool {
@@ -1392,11 +1308,3 @@ pub fn is_right_flanking_delimiter(previous: u8, next: u8) -> bool {
|| crate::text::is_whitespace(next)
|| crate::text::is_punctuation(next)
}
-
-pub fn count_repeated_byte(text: &str, position: usize, byte: u8) -> usize {
- let mut count = 0;
- while position + count < text.len() && byte_at(text, position + count) == byte {
- count += 1;
- }
- count
-}
diff --git a/packages/markdown_parser/rust/src/normalize.rs b/packages/markdown_parser/rust/src/normalize.rs
index 46dfcf3ef..1cc16cda5 100644
--- a/packages/markdown_parser/rust/src/normalize.rs
+++ b/packages/markdown_parser/rust/src/normalize.rs
@@ -18,7 +18,7 @@ pub fn normalize_child_nodes(nodes: &mut Vec<Node>, inside_blockquote: bool) {
pub fn normalize_node(node: &mut Node) {
match node {
- Node::Text { content } => *content = remove_text_presentation(content),
+ Node::Text { content } => remove_text_presentation(content),
Node::Strong { children }
| Node::Emphasis { children }
| Node::Underline { children }
@@ -61,7 +61,7 @@ pub fn normalize_node(node: &mut Node) {
pub fn apply_text_presentation_node(node: &mut Node) {
match node {
- Node::Text { content } => *content = remove_text_presentation(content),
+ Node::Text { content } => remove_text_presentation(content),
Node::Strong { children }
| Node::Emphasis { children }
| Node::Underline { children }
diff --git a/packages/markdown_parser/rust/src/parser.rs b/packages/markdown_parser/rust/src/parser.rs
index 98385be56..851685899 100644
--- a/packages/markdown_parser/rust/src/parser.rs
+++ b/packages/markdown_parser/rust/src/parser.rs
@@ -1,8 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::fmt;
+use std::sync::Arc;
use crate::ast::Node;
+use crate::block::ClosingFenceLines;
use crate::constants::{MAX_AST_NODES, MAX_LINE_LENGTH};
use crate::emoji::EmojiContext;
use crate::normalize::{
@@ -35,7 +37,7 @@ impl From<serde_json::Error> for ParseError {
#[derive(Clone, Debug)]
pub struct MarkdownParser {
flags: u32,
- emoji_context: EmojiContext,
+ emoji_context: Arc<EmojiContext>,
pub(crate) node_count: usize,
}
@@ -43,7 +45,7 @@ impl MarkdownParser {
pub fn new(flags: u32, emoji_context: EmojiContext) -> Self {
Self {
flags,
- emoji_context,
+ emoji_context: Arc::new(emoji_context),
node_count: 0,
}
}
@@ -56,6 +58,10 @@ impl MarkdownParser {
&self.emoji_context
}
+ pub(crate) fn shared_emoji_context(&self) -> Arc<EmojiContext> {
+ Arc::clone(&self.emoji_context)
+ }
+
pub fn parse(&mut self, input: &str) -> Result<Vec<Node>, ParseError> {
let lines = text::split_lines(input);
self.parse_lines(lines)
@@ -66,12 +72,17 @@ impl MarkdownParser {
parser: self,
lines,
current_line: 0,
+ fences: ClosingFenceLines::default(),
}
.parse()
}
pub(crate) fn child_with_flags(&self, flags: u32) -> Self {
- Self::new(flags, self.emoji_context.clone())
+ Self {
+ flags,
+ emoji_context: Arc::clone(&self.emoji_context),
+ node_count: 0,
+ }
}
}
@@ -79,9 +90,15 @@ pub(crate) struct RuntimeState<'a> {
pub(crate) parser: &'a mut MarkdownParser,
pub(crate) lines: Vec<Line>,
pub(crate) current_line: usize,
+ pub(crate) fences: ClosingFenceLines,
}
impl RuntimeState<'_> {
+ pub(crate) fn replace_line(&mut self, index: usize, text: String, offset: usize) {
+ self.lines[index] = Line { text, offset };
+ self.fences.line_replaced(index);
+ }
+
fn parse(&mut self) -> Result<Vec<Node>, ParseError> {
let mut ast = Vec::new();
if self.lines.is_empty() {
@@ -93,18 +110,12 @@ impl RuntimeState<'_> {
self.lines[self.current_line].text =
text::bounded_line_text(&self.lines[self.current_line].text).to_owned();
}
- let line = self.lines[self.current_line].text.clone();
- let trimmed = trim_start(&line);
- if trim(trimmed).is_empty() {
+ if trim(&self.lines[self.current_line].text).is_empty() {
let blank_count = self.count_blank_lines();
if !ast.is_empty() && self.current_line + blank_count < self.lines.len() {
- let next_line = self.lines[self.current_line + blank_count].clone();
- let next_trimmed = trim_start(&next_line.text);
- let next_offset =
- next_line.offset + (next_line.text.len() - next_trimmed.len());
- let is_next_heading =
- crate::block::parse_heading_node(self.parser, next_trimmed, next_offset)?
- .is_some();
+ let next_trimmed =
+ trim_start(&self.lines[self.current_line + blank_count].text);
+ let is_next_heading = crate::block::heading_content(next_trimmed).is_some();
let is_prev_heading = matches!(ast.last(), Some(Node::Heading { .. }));
if !is_next_heading && !is_prev_heading {
ast.push(Node::Text {
@@ -168,6 +179,7 @@ impl RuntimeState<'_> {
&self.lines,
self.current_line + consumed,
self.parser.flags(),
+ &mut self.fences,
)
|| trimmed_next.is_empty()
{
diff --git a/packages/markdown_parser/rust/src/scan.rs b/packages/markdown_parser/rust/src/scan.rs
new file mode 100644
index 000000000..e91659e42
--- /dev/null
+++ b/packages/markdown_parser/rust/src/scan.rs
@@ -0,0 +1,668 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+use crate::text::advance_one;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum Token {
+ Close,
+ Fail,
+ Open(usize),
+ Skip(usize),
+ Step(usize),
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) struct Closer {
+ pub(crate) at: usize,
+ last_checked: Option<usize>,
+}
+
+impl Closer {
+ fn or_checked(self, landing: Option<usize>) -> Self {
+ Self {
+ at: self.at,
+ last_checked: self.last_checked.or(landing),
+ }
+ }
+
+ pub(crate) fn within(self, opener: usize, limit: usize) -> Option<usize> {
+ self.last_checked
+ .is_none_or(|landing| landing - opener <= limit)
+ .then_some(self.at)
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+enum Frame {
+ Then { at: usize, landing: Option<usize> },
+ Inner { at: usize, inner: usize },
+ Outer { at: usize, last_checked: usize },
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct Slot {
+ state: u32,
+ last_checked: u32,
+}
+
+#[derive(Clone, Debug, Default)]
+struct ClosingMemo {
+ slots: Vec<Slot>,
+ frames: Vec<Frame>,
+}
+
+impl ClosingMemo {
+ fn get(&self, position: usize) -> Option<Option<Closer>> {
+ let slot = self.slots[position];
+ match slot.state {
+ 0 => None,
+ 1 => Some(None),
+ state => Some(Some(Closer {
+ at: state as usize - 2,
+ last_checked: slot.last_checked.checked_sub(1).map(|value| value as usize),
+ })),
+ }
+ }
+
+ fn set(&mut self, position: usize, value: Option<Closer>) {
+ let slot = match value {
+ None => Slot {
+ state: 1,
+ last_checked: 0,
+ },
+ Some(closer) => Slot {
+ state: closer.at as u32 + 2,
+ last_checked: closer.last_checked.map_or(0, |value| value as u32 + 1),
+ },
+ };
+ self.slots[position] = slot;
+ }
+
+ fn resolve(
+ &mut self,
+ len: usize,
+ start: usize,
+ close_len: usize,
+ mut classify: impl FnMut(usize) -> Token,
+ ) -> Option<Closer> {
+ if self.slots.is_empty() {
+ self.slots = vec![Slot::default(); len + 1];
+ }
+ let mut frames = std::mem::take(&mut self.frames);
+ let mut position = start;
+ let mut result;
+ 'descend: loop {
+ result = loop {
+ if position >= len {
+ break None;
+ }
+ if let Some(known) = self.get(position) {
+ break known;
+ }
+ match classify(position) {
+ Token::Close => {
+ let closer = Some(Closer {
+ at: position,
+ last_checked: None,
+ });
+ self.set(position, closer);
+ break closer;
+ }
+ Token::Fail => {
+ self.set(position, None);
+ break None;
+ }
+ Token::Skip(advance) => {
+ frames.push(Frame::Then {
+ at: position,
+ landing: None,
+ });
+ position += advance;
+ }
+ Token::Step(advance) => {
+ frames.push(Frame::Then {
+ at: position,
+ landing: Some(position + advance),
+ });
+ position += advance;
+ }
+ Token::Open(advance) => {
+ frames.push(Frame::Inner {
+ at: position,
+ inner: position + advance,
+ });
+ position += advance;
+ }
+ }
+ };
+ while let Some(frame) = frames.pop() {
+ match frame {
+ Frame::Then { at, landing } => {
+ result = result.map(|closer| closer.or_checked(landing));
+ self.set(at, result);
+ }
+ Frame::Inner { at, inner } => match result {
+ None => self.set(at, None),
+ Some(closer) => {
+ frames.push(Frame::Outer {
+ at,
+ last_checked: closer.last_checked.unwrap_or(inner),
+ });
+ position = closer.at + close_len;
+ continue 'descend;
+ }
+ },
+ Frame::Outer { at, last_checked } => {
+ result = result.map(|closer| closer.or_checked(Some(last_checked)));
+ self.set(at, result);
+ }
+ }
+ }
+ break;
+ }
+ self.frames = frames;
+ result
+ }
+}
+
+const MIN_UNINDEXED_STEPS: usize = 256;
+
+fn scan_briefly(
+ len: usize,
+ start: usize,
+ close_len: usize,
+ steps: &mut usize,
+ classify: &mut impl FnMut(usize) -> Token,
+) -> Option<Option<Closer>> {
+ let mut position = start;
+ let mut depth = 0usize;
+ let mut last_checked = None;
+ while *steps > 0 {
+ *steps -= 1;
+ if position >= len {
+ return Some(None);
+ }
+ match classify(position) {
+ Token::Close if depth == 0 => {
+ return Some(Some(Closer {
+ at: position,
+ last_checked,
+ }));
+ }
+ Token::Close => {
+ depth -= 1;
+ position += close_len;
+ }
+ Token::Fail => return Some(None),
+ Token::Skip(advance) => position += advance,
+ Token::Open(advance) => {
+ depth += 1;
+ position += advance;
+ last_checked = Some(position);
+ }
+ Token::Step(advance) => {
+ position += advance;
+ last_checked = Some(position);
+ }
+ }
+ }
+ None
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum Pattern {
+ AngleOpen,
+ AngleClose,
+ At,
+ OpenBracket,
+ CloseBracket,
+ OpenParen,
+ Paren,
+ UrlTerminator,
+ PathDelimiter,
+ AppProtocolInvalid,
+ HttpScheme,
+ NestedHttpUrl,
+ AngleSyntax,
+ DoubledMarker,
+}
+
+const PATTERNS: usize = 14;
+
+pub(crate) const ANGLE_SYNTAX_PREFIXES: [&str; 9] =
+ ["<:", "<a:", "<id:", "<@", "<#", "</", "<t:", "<+", "<sms:"];
+
+impl Pattern {
+ fn find(self, bytes: &[u8], from: usize, end: usize) -> Option<usize> {
+ let mut window = bytes[from..end].iter();
+ let offset = match self {
+ Self::AngleOpen => window.position(|byte| *byte == b'<'),
+ Self::AngleClose => window.position(|byte| *byte == b'>'),
+ Self::At => window.position(|byte| *byte == b'@'),
+ Self::OpenBracket => window.position(|byte| *byte == b'['),
+ Self::CloseBracket => window.position(|byte| *byte == b']'),
+ Self::OpenParen => window.position(|byte| *byte == b'('),
+ Self::Paren => window.position(|byte| matches!(byte, b'(' | b')')),
+ Self::UrlTerminator => window.position(|byte| is_url_terminator(*byte)),
+ Self::PathDelimiter => window.position(|byte| matches!(byte, b'/' | b'?' | b'#')),
+ _ => (from..end).position(|position| self.matches(bytes, position)),
+ };
+ offset.map(|offset| from + offset)
+ }
+
+ fn matches(self, bytes: &[u8], position: usize) -> bool {
+ let byte = bytes[position];
+ let rest = &bytes[position..];
+ match self {
+ Self::AngleOpen => byte == b'<',
+ Self::AngleClose => byte == b'>',
+ Self::At => byte == b'@',
+ Self::OpenBracket => byte == b'[',
+ Self::CloseBracket => byte == b']',
+ Self::OpenParen => byte == b'(',
+ Self::Paren => matches!(byte, b'(' | b')'),
+ Self::UrlTerminator => is_url_terminator(byte),
+ Self::PathDelimiter => matches!(byte, b'/' | b'?' | b'#'),
+ Self::AppProtocolInvalid => matches!(
+ byte,
+ b'"' | b'\'' | b'<' | b'>' | b'\\' | b'|' | b' ' | b'\t' | b'\r' | b'\n'
+ ),
+ Self::HttpScheme => byte == b'h' && http_scheme_len(rest).is_some(),
+ Self::NestedHttpUrl => byte == b'<' && http_scheme_len(&rest[1..]).is_some(),
+ Self::AngleSyntax => {
+ byte == b'<'
+ && ANGLE_SYNTAX_PREFIXES
+ .iter()
+ .any(|prefix| rest.starts_with(prefix.as_bytes()))
+ }
+ Self::DoubledMarker => {
+ matches!(byte, b'_' | b'*' | b'~' | b'|') && rest.get(1) == Some(&byte)
+ }
+ }
+ }
+}
+
+pub(crate) fn http_scheme_len(bytes: &[u8]) -> Option<usize> {
+ if bytes.starts_with(b"http://") {
+ Some(7)
+ } else if bytes.starts_with(b"https://") {
+ Some(8)
+ } else {
+ None
+ }
+}
+
+#[derive(Clone, Debug, Default)]
+struct NextMatch {
+ next: Vec<u32>,
+}
+
+impl NextMatch {
+ fn find(&mut self, bytes: &[u8], from: usize, limit: usize, pattern: Pattern) -> Option<usize> {
+ let len = bytes.len();
+ if self.next.is_empty() {
+ self.next = vec![0; len];
+ }
+ let mut position = from;
+ let found = loop {
+ if position == len {
+ break len;
+ }
+ let known = self.next[position];
+ if known != 0 {
+ break known as usize - 1;
+ }
+ if pattern.matches(bytes, position) {
+ break position;
+ }
+ position += 1;
+ };
+ let stored = found as u32 + 1;
+ self.next[from..position].fill(stored);
+ if found == position && found < len {
+ self.next[found] = stored;
+ }
+ (found < limit).then_some(found)
+ }
+}
+
+enum ChainLink {
+ Hit(usize),
+ Next(usize),
+ End,
+}
+
+#[derive(Clone, Debug, Default)]
+struct ChainMemo {
+ hits: Vec<u32>,
+ chain: Vec<usize>,
+}
+
+impl ChainMemo {
+ fn resolve(
+ &mut self,
+ len: usize,
+ first: usize,
+ mut link: impl FnMut(usize) -> ChainLink,
+ ) -> Option<usize> {
+ if self.hits.is_empty() {
+ self.hits = vec![0; len];
+ }
+ self.chain.clear();
+ let mut node = first;
+ let found = loop {
+ match self.hits[node] {
+ 0 => {}
+ 1 => break None,
+ known => break Some(known as usize - 2),
+ }
+ self.chain.push(node);
+ match link(node) {
+ ChainLink::Hit(at) => break Some(at),
+ ChainLink::Next(next) => node = next,
+ ChainLink::End => break None,
+ }
+ };
+ let stored = found.map_or(1, |at| at as u32 + 2);
+ for &node in &self.chain {
+ self.hits[node] = stored;
+ }
+ found
+ }
+}
+
+fn is_url_terminator(byte: u8) -> bool {
+ matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'"')
+}
+
+pub(crate) fn step(text: &str, position: usize) -> Token {
+ Token::Step(advance_one(text, position))
+}
+
+pub(crate) const FORMATTING_SCANS: usize = 9;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum Closing {
+ Formatting(usize),
+ Bracket,
+ Paren,
+ UrlParen,
+}
+
+impl Closing {
+ fn index(self) -> usize {
+ match self {
+ Self::Formatting(index) => index,
+ Self::Bracket => FORMATTING_SCANS,
+ Self::Paren => FORMATTING_SCANS + 1,
+ Self::UrlParen => FORMATTING_SCANS + 2,
+ }
+ }
+}
+
+const CLOSINGS: usize = FORMATTING_SCANS + 3;
+
+#[derive(Clone, Debug, Default)]
+struct Memos {
+ formatting: [ClosingMemo; FORMATTING_SCANS],
+ brackets: ClosingMemo,
+ parens: ClosingMemo,
+ url_parens: ClosingMemo,
+ next: [NextMatch; PATTERNS],
+ link_syntax: ChainMemo,
+ angle_email: ChainMemo,
+}
+
+impl Memos {
+ fn closing(&mut self, closing: Closing) -> &mut ClosingMemo {
+ match closing {
+ Closing::Formatting(index) => &mut self.formatting[index],
+ Closing::Bracket => &mut self.brackets,
+ Closing::Paren => &mut self.parens,
+ Closing::UrlParen => &mut self.url_parens,
+ }
+ }
+}
+
+#[derive(Clone, Debug, Default)]
+struct CodeSpanScan {
+ start: usize,
+ frontier: usize,
+ ended: bool,
+ longest_runs: Vec<(usize, usize)>,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct Lookahead<'a> {
+ pub(crate) text: &'a str,
+ backtick_run: (usize, usize),
+ asterisk_run: (usize, usize),
+ code_span: CodeSpanScan,
+ closing_steps: [usize; CLOSINGS],
+ pattern_steps: [usize; PATTERNS],
+ memos: Option<Box<Memos>>,
+}
+
+impl<'a> Lookahead<'a> {
+ pub(crate) fn new(text: &'a str) -> Self {
+ Self {
+ text,
+ backtick_run: (0, 0),
+ asterisk_run: (0, 0),
+ code_span: CodeSpanScan::default(),
+ closing_steps: [text.len() + MIN_UNINDEXED_STEPS; CLOSINGS],
+ pattern_steps: [text.len() + MIN_UNINDEXED_STEPS; PATTERNS],
+ memos: None,
+ }
+ }
+
+ fn memos(&mut self) -> &mut Memos {
+ self.memos.get_or_insert_with(Box::default)
+ }
+
+ pub(crate) fn next(&mut self, pattern: Pattern, from: usize, limit: usize) -> Option<usize> {
+ if from >= limit {
+ return None;
+ }
+ let bytes = self.text.as_bytes();
+ let steps = &mut self.pattern_steps[pattern as usize];
+ if *steps > 0 {
+ let brief_end = limit.min(from + *steps);
+ let found = pattern.find(bytes, from, brief_end);
+ *steps -= found.map_or(brief_end, |found| found + 1) - from;
+ if found.is_some() || brief_end == limit {
+ return found;
+ }
+ }
+ self.memos().next[pattern as usize].find(bytes, from, limit, pattern)
+ }
+
+ pub(crate) fn closer(
+ &mut self,
+ closing: Closing,
+ start: usize,
+ close_len: usize,
+ mut classify: impl FnMut(&mut Self, usize) -> Token,
+ ) -> Option<Closer> {
+ let len = self.text.len();
+ let mut steps = self.closing_steps[closing.index()];
+ if steps > 0 {
+ let result = scan_briefly(len, start, close_len, &mut steps, &mut |position| {
+ classify(self, position)
+ });
+ self.closing_steps[closing.index()] = steps;
+ if let Some(result) = result {
+ return result;
+ }
+ }
+ let mut memo = std::mem::take(self.memos().closing(closing));
+ let result = memo.resolve(len, start, close_len, |position| classify(self, position));
+ *self.memos().closing(closing) = memo;
+ result
+ }
+
+ pub(crate) fn run_end(&mut self, position: usize, byte: u8) -> usize {
+ let cache = if byte == b'`' {
+ &mut self.backtick_run
+ } else {
+ &mut self.asterisk_run
+ };
+ let (start, end) = *cache;
+ if start <= position && position < end {
+ return end;
+ }
+ let bytes = self.text.as_bytes();
+ let mut end = position;
+ while end < bytes.len() && bytes[end] == byte {
+ end += 1;
+ }
+ *cache = (position, end);
+ end
+ }
+
+ pub(crate) fn code_span_end(
+ &mut self,
+ opener: usize,
+ marker_length: usize,
+ limit: usize,
+ ) -> Option<usize> {
+ let start = opener + marker_length;
+ if self.code_span.start != start || self.code_span.frontier < start {
+ self.code_span = CodeSpanScan {
+ start,
+ frontier: start,
+ ended: false,
+ longest_runs: Vec::new(),
+ };
+ }
+ let bytes = self.text.as_bytes();
+ loop {
+ let scan = &mut self.code_span;
+ let index = scan
+ .longest_runs
+ .partition_point(|(count, _)| *count < marker_length);
+ if let Some(&(count, run_start)) = scan.longest_runs.get(index) {
+ return (run_start - opener <= limit)
+ .then(|| run_start + count - marker_length - opener);
+ }
+ if scan.ended || scan.frontier - opener > limit {
+ return None;
+ }
+ let Some(offset) = bytes[scan.frontier..]
+ .iter()
+ .position(|byte| matches!(byte, b'`' | b'\n' | b'\r'))
+ else {
+ scan.ended = true;
+ continue;
+ };
+ let run_start = scan.frontier + offset;
+ if bytes[run_start] != b'`' {
+ scan.ended = true;
+ continue;
+ }
+ let mut run_end = run_start;
+ while run_end < bytes.len() && bytes[run_end] == b'`' {
+ run_end += 1;
+ }
+ let count = run_end - run_start;
+ if scan
+ .longest_runs
+ .last()
+ .is_none_or(|(longest, _)| count > *longest)
+ {
+ scan.longest_runs.push((count, run_start));
+ }
+ scan.frontier = run_end;
+ }
+ }
+
+ pub(crate) fn closing_bracket(&mut self, opener: usize) -> Option<usize> {
+ let text = self.text;
+ let bytes = text.as_bytes();
+ self.closer(Closing::Bracket, opener + 1, 1, |_, position| {
+ match bytes[position] {
+ b'[' => Token::Open(1),
+ b']' => Token::Close,
+ b'\\' if position + 1 < bytes.len() => Token::Skip(2),
+ _ => step(text, position),
+ }
+ })
+ .map(|closer| closer.at)
+ }
+
+ pub(crate) fn closing_paren(&mut self, start: usize) -> Option<usize> {
+ let bytes = self.text.as_bytes();
+ self.closer(Closing::Paren, start, 1, |_, position| {
+ match bytes[position] {
+ b'(' => Token::Open(1),
+ b')' => Token::Close,
+ _ => Token::Step(1),
+ }
+ })
+ .map(|closer| closer.at)
+ }
+
+ pub(crate) fn url_stop(&mut self, start: usize) -> usize {
+ let bytes = self.text.as_bytes();
+ let terminator = self
+ .next(Pattern::UrlTerminator, start, bytes.len())
+ .unwrap_or(bytes.len());
+ if self.next(Pattern::Paren, start, terminator).is_none() {
+ return terminator;
+ }
+ self.closer(Closing::UrlParen, start, 1, |_, position| {
+ match bytes[position] {
+ b'(' => Token::Open(1),
+ b')' => Token::Close,
+ byte if is_url_terminator(byte) => Token::Fail,
+ _ => Token::Step(1),
+ }
+ })
+ .map_or(terminator, |closer| closer.at)
+ }
+
+ pub(crate) fn has_link_syntax(&mut self, start: usize, end: usize) -> bool {
+ let Some(first) = self.next(Pattern::OpenBracket, start, end) else {
+ return false;
+ };
+ let bytes = self.text.as_bytes();
+ let len = bytes.len();
+ let mut link_syntax = std::mem::take(&mut self.memos().link_syntax);
+ let found = link_syntax.resolve(len, first, |open| {
+ let Some(close) = self.next(Pattern::CloseBracket, open + 1, len) else {
+ return ChainLink::End;
+ };
+ if bytes.get(close + 1) == Some(&b'(') {
+ return ChainLink::Hit(close);
+ }
+ self.next(Pattern::OpenBracket, close + 1, len)
+ .map_or(ChainLink::End, ChainLink::Next)
+ });
+ self.memos().link_syntax = link_syntax;
+ found.is_some_and(|close| close + 1 < end)
+ }
+
+ pub(crate) fn has_angle_email(&mut self, start: usize, end: usize) -> bool {
+ let Some(first) = self.next(Pattern::AngleOpen, start, end) else {
+ return false;
+ };
+ let bytes = self.text.as_bytes();
+ let len = bytes.len();
+ let mut angle_email = std::mem::take(&mut self.memos().angle_email);
+ let found = angle_email.resolve(len, first, |open| {
+ let Some(close) = self.next(Pattern::AngleClose, open + 1, len) else {
+ return ChainLink::End;
+ };
+ if close > open + 1
+ && bytes[open + 1] != b'@'
+ && self.next(Pattern::At, open + 1, close).is_some()
+ {
+ return ChainLink::Hit(close);
+ }
+ self.next(Pattern::AngleOpen, close + 1, len)
+ .map_or(ChainLink::End, ChainLink::Next)
+ });
+ self.memos().angle_email = angle_email;
+ found.is_some_and(|close| close < end)
+ }
+}
diff --git a/packages/markdown_parser/rust/src/text.rs b/packages/markdown_parser/rust/src/text.rs
index 5e9b3fc6b..55c08abc2 100644
--- a/packages/markdown_parser/rust/src/text.rs
+++ b/packages/markdown_parser/rust/src/text.rs
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::ast::Node;
-use crate::constants::{MAX_LINE_LENGTH, MAX_LINK_URL_LENGTH};
+use crate::constants::MAX_LINE_LENGTH;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Line {
@@ -75,24 +75,17 @@ pub fn concat(left: &str, right: &str) -> String {
out
}
-pub fn concat3(a: &str, b: &str, c: &str) -> String {
- let mut out = String::with_capacity(a.len() + b.len() + c.len());
- out.push_str(a);
- out.push_str(b);
- out.push_str(c);
- out
-}
-
-pub fn append_text(nodes: &mut Vec<Node>, content: impl Into<String>) {
- let content = content.into();
+pub fn append_text(nodes: &mut Vec<Node>, content: &str) {
if content.is_empty() {
return;
}
if let Some(Node::Text { content: previous }) = nodes.last_mut() {
- previous.push_str(&content);
+ previous.push_str(content);
return;
}
- nodes.push(Node::Text { content });
+ nodes.push(Node::Text {
+ content: content.to_owned(),
+ });
}
pub fn line_as_text(lines: &[Line], current: usize) -> String {
@@ -141,11 +134,6 @@ pub fn starts_with(value: &str, prefix: &str) -> bool {
value.as_bytes().starts_with(prefix.as_bytes())
}
-#[inline]
-pub fn ends_with(value: &[u8], suffix: &[u8]) -> bool {
- value.ends_with(suffix)
-}
-
#[inline]
pub fn is_digit(byte: u8) -> bool {
byte.is_ascii_digit()
@@ -241,17 +229,13 @@ pub fn find_from(value: &str, start: usize, needle: u8) -> Option<usize> {
.map(|pos| start + pos)
}
-pub fn bounded_url_end(value: &str, end: usize, prefix_len: usize) -> usize {
- bounded_utf8_prefix_length(&value[..end], prefix_len + MAX_LINK_URL_LENGTH)
-}
-
-pub fn remove_text_presentation(text: &str) -> String {
- if !text.contains('\u{fe0f}') {
- return text.to_owned();
+pub fn remove_text_presentation(text: &mut String) {
+ if !text.is_ascii() && text.contains('\u{fe0f}') {
+ *text = text
+ .replace("™️", "™")
+ .replace("©️", "©")
+ .replace("®️", "®");
}
- text.replace("™️", "™")
- .replace("©️", "©")
- .replace("®️", "®")
}
pub fn trim_start_newline_whitespace(value: &str) -> &str {
diff --git a/packages/markdown_parser/rust/tests/fixtures.rs b/packages/markdown_parser/rust/tests/fixtures.rs
index 5ab3d637e..ea9a6c0dd 100644
--- a/packages/markdown_parser/rust/tests/fixtures.rs
+++ b/packages/markdown_parser/rust/tests/fixtures.rs
@@ -6,7 +6,14 @@ use serde_json::json;
fn parse(input: &str, flags: u32, emoji_context: &str) -> serde_json::Value {
let mut parser = MarkdownParser::new(flags, EmojiContext::parse(emoji_context));
let nodes = parser.parse(input).expect("parse succeeds");
- serde_json::to_value(json!({"nodes": nodes})).expect("serialize succeeds")
+ let value = serde_json::to_value(json!({"nodes": nodes})).expect("serialize succeeds");
+ let written = fluxer_markdown_parser::parse_markdown_json(input, flags, emoji_context)
+ .expect("parse succeeds");
+ assert_eq!(
+ serde_json::from_str::<serde_json::Value>(&written).expect("written json parses"),
+ value
+ );
+ value
}
#[test]
diff --git a/packages/markdown_parser/rust/tests/linear_time.rs b/packages/markdown_parser/rust/tests/linear_time.rs
new file mode 100644
index 000000000..582542390
--- /dev/null
+++ b/packages/markdown_parser/rust/tests/linear_time.rs
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+use std::time::{Duration, Instant};
+
+use fluxer_markdown_parser::{EmojiContext, MarkdownParser, ParserFlags};
+
+#[test]
+fn unclosed_masked_links_parse_in_linear_time() {
+ let line = "[a](".repeat(1024);
+ let input = format!("{line}\n").repeat(400);
+ let mut parser = MarkdownParser::new(ParserFlags::ALL, EmojiContext::default());
+ let started = Instant::now();
+ let nodes = parser.parse(&input).expect("parse succeeds");
+ let elapsed = started.elapsed();
+ assert_eq!(nodes.len(), 1);
+ assert!(
+ elapsed < Duration::from_secs(20),
+ "parsing {} bytes took {elapsed:?}",
+ input.len()
+ );
+}