A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(app): keep composer markdown highlighting aligned on long messages (#1915)
Hampusb1fb2c14a135
diff --git a/fluxer_app/src/features/lexical/composer/markdownSpans.ts b/fluxer_app/src/features/lexical/composer/markdownSpans.tsindex e1dfce829..9fac5ed42 100644--- a/fluxer_app/src/features/lexical/composer/markdownSpans.ts+++ b/fluxer_app/src/features/lexical/composer/markdownSpans.ts@@ -1,8 +1,10 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport {ParserFlags} from '@app/features/messaging/utils/markdown/parser/Enums';+import {MAX_LINE_LENGTH} from '@app/features/messaging/utils/markdown/parser/MarkdownConstants';import {parseMarkdownAstWithWasm} from '@app/features/messaging/utils/markdown/parser/MarkdownParserWasm';import type {Node} from '@app/features/messaging/utils/markdown/parser/Nodes';+import {normalizeUrl} from '@app/features/messaging/utils/markdown/parser/UrlUtils';import {findUrlEnd} from '@app/features/messaging/utils/markdown/UrlSpanUtils';export const MarkdownHl = {@@ -48,54 +50,148 @@ export const DEFAULT_COMPOSER_MARKDOWN_FLAGS =ParserFlags.ALLOW_TABLES |ParserFlags.ALLOW_ALERTS;+export interface MarkdownRecoveredRange {+ start: number;+ end: number;+}++export interface MarkdownHighlightResult {+ spans: Array<MarkdownSpan>;+ recovered: Array<MarkdownRecoveredRange>;+}+interface AlignContext {source: string;pos: number;spans: Array<MarkdownSpan>;failed: boolean;listIndentLevel: number | null;+ closers: Array<string>;}+type AlignAttempt = {type: 'aligned'; spans: Array<MarkdownSpan>} | {type: 'desynced'; pos: number};++const MAX_ALIGN_ATTEMPTS = 64;+const MAX_URL_REALIGN_STEPS = 256;+export function computeMarkdownHighlightSpans(source: string,parserFlags = DEFAULT_COMPOSER_MARKDOWN_FLAGS,): Array<MarkdownSpan> {+ return computeMarkdownHighlightResult(source, parserFlags).spans;+}++export function computeMarkdownHighlightResult(+ source: string,+ parserFlags = DEFAULT_COMPOSER_MARKDOWN_FLAGS,+): MarkdownHighlightResult {if (source.length === 0) {- return [];+ return {spans: [], recovered: []};+ }+ const spans: Array<MarkdownSpan> = [];+ const recovered: Array<MarkdownRecoveredRange> = [];+ let offset = 0;+ let attempts = 0;+ let segmentLimit = source.length;+ while (offset < source.length) {+ attempts += 1;+ if (attempts > MAX_ALIGN_ATTEMPTS) {+ recoverRange(spans, recovered, offset, source.length);+ break;+ }+ const segmentEnd = Math.min(segmentEndFrom(source, offset), segmentLimit);+ const attempt = alignSegment(source.slice(offset, segmentEnd), parserFlags);+ if (attempt.type === 'aligned') {+ for (const span of attempt.spans) {+ spans.push({start: span.start + offset, end: span.end + offset, role: span.role, format: span.format});+ }+ offset = segmentEnd;+ segmentLimit = source.length;+ continue;+ }+ const safe = resyncStart(source, offset, offset + attempt.pos);+ if (safe > offset) {+ segmentLimit = safe;+ continue;+ }+ const newline = source.indexOf('\n', offset);+ const resume = newline < 0 || newline >= segmentEnd ? segmentEnd : newline + 1;+ recoverRange(spans, recovered, offset, resume);+ offset = resume;+ segmentLimit = source.length;}- let nodes: Array<Node>;- try {- nodes = parseMarkdownAstWithWasm(source, parserFlags & DEFAULT_COMPOSER_MARKDOWN_FLAGS).nodes;- } catch {- return [{start: 0, end: source.length, role: 'content', format: MarkdownHl.none}];+ return {spans: coalesce(spans), recovered};+}++function recoverRange(+ spans: Array<MarkdownSpan>,+ recovered: Array<MarkdownRecoveredRange>,+ start: number,+ end: number,+): void {+ if (end <= start) {+ return;}- const ctx: AlignContext = {source, pos: 0, spans: [], failed: false, listIndentLevel: null};- while (ctx.pos < source.length) {- const blank = /^[ \t\r]*\n/.exec(source.slice(ctx.pos));- if (blank == null) {- break;+ spans.push({start, end, role: 'content', format: MarkdownHl.none});+ recovered.push({start, end});+}++function resyncStart(source: string, lowerBound: number, failurePos: number): number {+ if (failurePos <= lowerBound) {+ return lowerBound;+ }+ return Math.max(lowerBound, source.lastIndexOf('\n', failurePos - 1) + 1);+}++function segmentEndFrom(source: string, offset: number): number {+ let lineStart = offset;+ while (lineStart < source.length) {+ const newline = source.indexOf('\n', lineStart);+ const lineEnd = newline < 0 ? source.length : newline;+ if (lineEnd - lineStart > MAX_LINE_LENGTH) {+ return lineStart > offset ? lineStart : lineStart + MAX_LINE_LENGTH;}- pushContent(ctx, ctx.pos + blank[0].length, MarkdownHl.none);+ if (newline < 0) {+ return source.length;+ }+ lineStart = newline + 1;}- for (let index = 0; index < nodes.length; index += 1) {- if (ctx.failed) {- break;+ return source.length;+}++function alignSegment(source: string, parserFlags: number): AlignAttempt {+ const ctx: AlignContext = {source, pos: 0, spans: [], failed: false, listIndentLevel: null, closers: []};+ try {+ const nodes = parseMarkdownAstWithWasm(source, parserFlags & DEFAULT_COMPOSER_MARKDOWN_FLAGS).nodes;+ while (ctx.pos < source.length) {+ const blank = /^[ \t\r]*\n/.exec(source.slice(ctx.pos));+ if (blank == null) {+ break;+ }+ pushContent(ctx, ctx.pos + blank[0].length, MarkdownHl.none);}- if (index > 0) {- advanceSequenceGap(nodes, index, MarkdownHl.none, ctx);+ for (let index = 0; index < nodes.length; index += 1) {+ if (ctx.failed) {+ break;+ }+ if (index > 0) {+ advanceSequenceGap(nodes, index, MarkdownHl.none, ctx);+ }+ alignNode(nodes[index]!, MarkdownHl.none, ctx);}- alignNode(nodes[index]!, MarkdownHl.none, ctx);+ } catch {+ return {type: 'desynced', pos: ctx.pos};}if (ctx.failed) {- return [{start: 0, end: source.length, role: 'content', format: MarkdownHl.none}];+ return {type: 'desynced', pos: ctx.pos};}if (ctx.pos < source.length) {if (!/^[ \t\r\n]*$/.test(source.slice(ctx.pos))) {- return [{start: 0, end: source.length, role: 'content', format: MarkdownHl.none}];+ return {type: 'desynced', pos: ctx.pos};}pushContent(ctx, source.length, MarkdownHl.none);}- return coalesce(ctx.spans);+ return {type: 'aligned', spans: ctx.spans};}function pushMarker(ctx: AlignContext, length: number, format: MarkdownHlFormat): void {@@ -136,6 +232,12 @@ const WRAPPERS: Partial<Record<Node['type'], {marker: string; bit: number}>> = {Strikethrough: {marker: '~~', bit: MarkdownHl.strike},};+function alignEnclosed(children: Array<Node>, closer: string, format: MarkdownHlFormat, ctx: AlignContext): void {+ ctx.closers.push(closer);+ alignChildren(children, format, ctx);+ ctx.closers.pop();+}+function alignNode(node: Node, format: MarkdownHlFormat, ctx: AlignContext): void {const wrapper = WRAPPERS[node.type];if (wrapper != null && 'children' in node) {@@ -143,18 +245,21 @@ function alignNode(node: Node, format: MarkdownHlFormat, ctx: AlignContext): voiif (!expectMarker(ctx, wrapper.marker, next)) {return;}- alignChildren(node.children, next, ctx);+ alignEnclosed(node.children, wrapper.marker, next, ctx);expectMarker(ctx, wrapper.marker, next);return;}switch (node.type) {case 'Text': {- const end = advancePastText(ctx.source, ctx.pos, node.content, ctx.listIndentLevel);- if (end == null) {+ const advance = advancePastText(ctx.source, ctx.pos, node.content, ctx.listIndentLevel, format);+ for (const marker of advance.markers) {+ pushContent(ctx, marker.start, format);+ pushMarker(ctx, marker.end - marker.start, format);+ }+ pushContent(ctx, advance.end, format);+ if (!advance.complete) {ctx.failed = true;- return;}- pushContent(ctx, end, format);return;}case 'Strong': {@@ -169,7 +274,7 @@ function alignNode(node: Node, format: MarkdownHlFormat, ctx: AlignContext): voi}const next = format | MarkdownHl.bold;pushMarker(ctx, delimiter.length, next);- alignChildren(node.children, next, ctx);+ alignEnclosed(node.children, delimiter, next, ctx);expectMarker(ctx, delimiter, next);return;}@@ -181,7 +286,7 @@ function alignNode(node: Node, format: MarkdownHlFormat, ctx: AlignContext): voi}const next = format | MarkdownHl.italic;pushMarker(ctx, 1, next);- alignChildren(node.children, next, ctx);+ alignEnclosed(node.children, delimiter, next, ctx);if (ctx.source[ctx.pos] === delimiter) {pushMarker(ctx, 1, next);} else {@@ -198,7 +303,7 @@ function alignNode(node: Node, format: MarkdownHlFormat, ctx: AlignContext): voiif (!expectMarker(ctx, '||', next)) {return;}- alignChildren(node.children, next, ctx);+ alignEnclosed(node.children, '||', next, ctx);expectMarker(ctx, '||', next);return;}@@ -298,7 +403,7 @@ function alignLink(node: Extract<Node, {type: 'Link'}>, format: MarkdownHlFormatpushContent(ctx, close + 1, format);return;}- const end = findUrlEnd(ctx.source, ctx.pos);+ const end = urlExtentEnd(ctx, node.url);if (end <= ctx.pos) {ctx.failed = true;return;@@ -309,7 +414,9 @@ function alignLink(node: Extract<Node, {type: 'Link'}>, format: MarkdownHlFormatif (!expectMarker(ctx, '[', format)) {return;}+ ctx.closers.push(']');alignNode(node.text, format | MarkdownHl.link, ctx);+ ctx.closers.pop();if (ctx.failed) {return;}@@ -325,6 +432,25 @@ function alignLink(node: Extract<Node, {type: 'Link'}>, format: MarkdownHlFormatexpectMarker(ctx, ')', format);}+function urlExtentEnd(ctx: AlignContext, url: string): number {+ const scanned = findUrlEnd(ctx.source, ctx.pos);+ const lowest = Math.max(ctx.pos + 1, scanned - MAX_URL_REALIGN_STEPS);+ for (let end = scanned; end >= lowest; end -= 1) {+ if (normalizeUrl(foldUnpairedSurrogates(ctx.source.slice(ctx.pos, end))) !== url) {+ continue;+ }+ return end >= scanned ? scanned : findUrlEnd(ctx.source.slice(0, end), ctx.pos);+ }+ let limit = ctx.source.length;+ for (const closer of ctx.closers) {+ const found = ctx.source.indexOf(closer, ctx.pos + 1);+ if (found > ctx.pos && found < limit) {+ limit = found;+ }+ }+ return limit >= scanned ? scanned : findUrlEnd(ctx.source.slice(0, limit), ctx.pos);+}+function scanMaskedUrlEnd(source: string, start: number): number {if (source[start] === '<') {const close = source.indexOf('>', start + 1);@@ -445,7 +571,7 @@ function alignCodeBlock(node: Extract<Node, {type: 'CodeBlock'}>, format: Markdoconst next = format | MarkdownHl.code | MarkdownHl.codeBlock;if (inlineClosing >= 0) {const contentEnd = languagePartStart + inlineClosing;- if (node.language != null || ctx.source.slice(languagePartStart, contentEnd) !== node.content) {+ if (node.language != null || !equalsParserText(ctx.source.slice(languagePartStart, contentEnd), node.content)) {ctx.failed = true;return;}@@ -459,7 +585,7 @@ function alignCodeBlock(node: Extract<Node, {type: 'CodeBlock'}>, format: Markdoreturn;}const language = node.language;- if (language != null && trimCodeFenceInfo(languagePart) !== language) {+ if (language != null && !equalsParserText(trimCodeFenceInfo(languagePart), language)) {ctx.failed = true;return;}@@ -494,7 +620,7 @@ function alignCodeBlock(node: Extract<Node, {type: 'CodeBlock'}>, format: Markdo}currentLineStart = newline + 1;}- if (closingStart < 0 || expectedContent !== node.content) {+ if (closingStart < 0 || !equalsParserText(expectedContent, node.content)) {ctx.failed = true;return;}@@ -554,7 +680,11 @@ function alignInlineCode(content: string, format: MarkdownHlFormat, ctx: AlignCo}const contentStart = openingStart + ticks;const closingStart = findInlineCodeClosing(ctx.source, contentStart, ticks);- if (closingStart < 0 || unescapeInlineCode(ctx.source.slice(contentStart, closingStart)) !== content) {+ if (closingStart < 0) {+ ctx.failed = true;+ return;+ }+ if (!equalsParserText(unescapeInlineCode(ctx.source.slice(contentStart, closingStart)), content)) {ctx.failed = true;return;}@@ -705,6 +835,7 @@ function alignBlockquote(children: Array<Node>, format: MarkdownHlFormat, ctx: Aspans: [],failed: false,listIndentLevel: null,+ closers: [],};if (children.length === 0) {pushContent(virtualContext, virtualSource.length, format);@@ -895,10 +1026,17 @@ function alignNodeSequence(nodes: Array<Node>, format: MarkdownHlFormat, ctx: Al}function advanceSequenceGap(nodes: Array<Node>, index: number, format: MarkdownHlFormat, ctx: AlignContext): void {- let sourceLineBreaks = 0;- while (ctx.source[ctx.pos + sourceLineBreaks] === '\n') {- sourceLineBreaks += 1;+ const breakEnds: Array<number> = [];+ let scan = ctx.pos;+ while (ctx.source[scan] != null) {+ const blank = (breakEnds.length === 0 ? LINE_BREAK_RE : BLANK_LINE_RE).exec(ctx.source.slice(scan));+ if (blank == null) {+ break;+ }+ scan += blank[0].length;+ breakEnds.push(scan);}+ const sourceLineBreaks = breakEnds.length;let contentLineBreaks = 0;for (let i = index; i < nodes.length; i += 1) {const next = nodes[i]!;@@ -915,7 +1053,52 @@ function advanceSequenceGap(nodes: Array<Node>, index: number, format: MarkdownH}}const structuralLineBreaks = Math.max(0, sourceLineBreaks - contentLineBreaks);- pushContent(ctx, ctx.pos + structuralLineBreaks, format);+ if (structuralLineBreaks > 0) {+ pushContent(ctx, breakEnds[structuralLineBreaks - 1]!, format);+ }+}++interface TextAdvance {+ end: number;+ markers: Array<MarkdownRecoveredRange>;+ complete: boolean;+}++function runLength(value: string, start: number, char: string): number {+ let length = 0;+ while (value[start + length] === char) {+ length += 1;+ }+ return length;+}++const LINE_BREAK_RE = /^\n/;+const BLANK_LINE_RE = /^[ \t\r]*\n/;+const DROPPED_WHITESPACE_RE = /^[ \t\r]+\n/;+const LINE_WHITESPACE_RE = /^[ \t\r]+/;++const REPLACEMENT_CHARACTER = '\uFFFD';+const UNPAIRED_SURROGATE_RE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;++function isUnpairedSurrogateAt(text: string, index: number): boolean {+ const code = text.charCodeAt(index);+ if (code >= 0xd800 && code <= 0xdbff) {+ const next = text.charCodeAt(index + 1);+ return !(next >= 0xdc00 && next <= 0xdfff);+ }+ if (code >= 0xdc00 && code <= 0xdfff) {+ const previous = text.charCodeAt(index - 1);+ return !(previous >= 0xd800 && previous <= 0xdbff);+ }+ return false;+}++export function foldUnpairedSurrogates(text: string): string {+ return text.replace(UNPAIRED_SURROGATE_RE, REPLACEMENT_CHARACTER);+}++export function equalsParserText(sourceText: string, parserText: string): boolean {+ return sourceText === parserText || foldUnpairedSurrogates(sourceText) === parserText;}function advancePastText(@@ -923,25 +1106,70 @@ function advancePastText(start: number,content: string,listIndentLevel: number | null,-): number | null {+ format: MarkdownHlFormat,+): TextAdvance {+ const markers: Array<MarkdownRecoveredRange> = [];let pos = start;let ci = 0;while (ci < content.length && pos < source.length) {- if (source[pos] === '\\' && pos + 1 < source.length && source[pos + 1] === content[ci]) {- pos += 2;+ if (source[pos] === '\\') {+ const sourceRun = runLength(source, pos, '\\');+ const contentRun = runLength(content, ci, '\\');+ if (contentRun > 0 && contentRun === sourceRun) {+ pos += sourceRun;+ ci += sourceRun;+ continue;+ }+ if (contentRun > 0 && contentRun === Math.floor(sourceRun / 2)) {+ for (let pair = 0; pair < contentRun; pair += 1) {+ markers.push({start: pos + pair * 2, end: pos + pair * 2 + 1});+ }+ pos += contentRun * 2;+ ci += contentRun;+ continue;+ }+ if (contentRun === 0 && pos + 1 < source.length && source[pos + 1] === content[ci]) {+ markers.push({start: pos, end: pos + 1});+ pos += 2;+ ci += 1;+ continue;+ }+ }+ if (source[pos] === content[ci]) {+ pos += 1;ci += 1;continue;}- if (source[pos] === content[ci]) {+ if (content[ci] === REPLACEMENT_CHARACTER && isUnpairedSurrogateAt(source, pos)) {pos += 1;ci += 1;continue;}+ if (content[ci] === '\n') {+ const dropped = DROPPED_WHITESPACE_RE.exec(source.slice(pos));+ if (dropped != null) {+ pos += dropped[0].length;+ ci += 1;+ continue;+ }+ }if (/\s/.test(source[pos]!) && /\s/.test(content[ci]!)) {pos += 1;ci += 1;continue;}+ if (pos === 0 || source[pos - 1] === '\n') {+ const indent = LINE_WHITESPACE_RE.exec(source.slice(pos));+ if (indent != null) {+ pos += indent[0].length;+ continue;+ }+ }+ if ((format & MarkdownHl.italic) !== 0 && (source[pos] === '*' || source[pos] === '_')) {+ markers.push({start: pos, end: pos + 1});+ pos += 1;+ continue;+ }if (listIndentLevel != null) {const continuationEnd = listContinuationEnd(source, pos, listIndentLevel);if (continuationEnd > pos) {@@ -951,7 +1179,7 @@ function advancePastText(}break;}- return ci === content.length ? pos : null;+ return {end: pos, markers, complete: ci === content.length};}function coalesce(spans: Array<MarkdownSpan>): Array<MarkdownSpan> {