Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): block-split long backslash lines without inline lex (#11376) (#11501)

Sourav Rajvic5f1f7946bf4
diff --git a/studio/frontend/package-lock.json b/studio/frontend/package-lock.json
index 5b70943e693..2fe0b95a72f 100644
--- a/studio/frontend/package-lock.json
+++ b/studio/frontend/package-lock.json
@@ -53,6 +53,7 @@
"katex": "^0.16.28",
"lucide-react": "^1.7.0",
"mammoth": "^1.11.0",
+ "marked": "17.0.6",
"mdast-util-from-markdown": "2.0.3",
"mdast-util-gfm": "3.1.0",
"mdast-util-math": "3.0.0",
@@ -11396,9 +11397,9 @@
}
},
"node_modules/marked": {
- "version": "16.4.2",
- "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz",
- "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==",
+ "version": "17.0.6",
+ "resolved": "https://registry.npmjs.org/marked/-/marked-17.0.6.tgz",
+ "integrity": "sha512-gB0gkNafnonOw0obSTEGZTT86IuhILt2Wfx0mWH/1Au83kybTayroZ/V6nS25mN7u8ASy+5fMhgB3XPNrOZdmA==",
"license": "MIT",
"bin": {
"marked": "bin/marked.js"
@@ -11799,6 +11800,18 @@
"uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0"
}
},
+ "node_modules/mermaid/node_modules/marked": {
+ "version": "16.4.2",
+ "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz",
+ "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==",
+ "license": "MIT",
+ "bin": {
+ "marked": "bin/marked.js"
+ },
+ "engines": {
+ "node": ">= 20"
+ }
+ },
"node_modules/micromark": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz",
@@ -14650,18 +14663,6 @@
"react-dom": "^18.0.0 || ^19.0.0"
}
},
- "node_modules/streamdown/node_modules/marked": {
- "version": "17.0.6",
- "resolved": "https://registry.npmjs.org/marked/-/marked-17.0.6.tgz",
- "integrity": "sha512-gB0gkNafnonOw0obSTEGZTT86IuhILt2Wfx0mWH/1Au83kybTayroZ/V6nS25mN7u8ASy+5fMhgB3XPNrOZdmA==",
- "license": "MIT",
- "bin": {
- "marked": "bin/marked.js"
- },
- "engines": {
- "node": ">= 20"
- }
- },
"node_modules/strict-event-emitter": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz",
diff --git a/studio/frontend/package.json b/studio/frontend/package.json
index 71a76104ab3..8ecb3690278 100644
--- a/studio/frontend/package.json
+++ b/studio/frontend/package.json
@@ -68,6 +68,7 @@
"katex": "^0.16.28",
"lucide-react": "^1.7.0",
"mammoth": "^1.11.0",
+ "marked": "17.0.6",
"mdast-util-from-markdown": "2.0.3",
"mdast-util-gfm": "3.1.0",
"mdast-util-math": "3.0.0",
diff --git a/studio/frontend/src/components/assistant-ui/streaming-markdown.ts b/studio/frontend/src/components/assistant-ui/streaming-markdown.ts
index a657f6aa7b7..3f3393ea72a 100644
--- a/studio/frontend/src/components/assistant-ui/streaming-markdown.ts
+++ b/studio/frontend/src/components/assistant-ui/streaming-markdown.ts
@@ -7,7 +7,7 @@ import { mathFromMarkdown } from "mdast-util-math";
import { gfm } from "micromark-extension-gfm";
import { math } from "micromark-extension-math";
import remend from "remend";
-import { parseMarkdownIntoBlocks } from "streamdown";
+import { parseMarkdownIntoBlocks } from "../../lib/parse-markdown-blocks.ts";
// Cheap gate: a bullet marker then only thematic-break punctuation. Not
// asterisk specific, `- --verbose` and `* ___under___` flash too.
diff --git a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts
index ae191413407..07b9f34f828 100644
--- a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts
+++ b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts
@@ -2,7 +2,8 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import remend from "remend";
-import { type BlockProps, parseMarkdownIntoBlocks } from "streamdown";
+import { type BlockProps } from "streamdown";
+import { parseMarkdownIntoBlocks } from "../../lib/parse-markdown-blocks.ts";
// How far behind the live edge a block has to be before it can be retained.
// The block list interleaves "\n\n" separators, so this is about four
diff --git a/studio/frontend/src/lib/latex.ts b/studio/frontend/src/lib/latex.ts
index adfbdd385c2..c5095f0db3e 100644
--- a/studio/frontend/src/lib/latex.ts
+++ b/studio/frontend/src/lib/latex.ts
@@ -9,7 +9,7 @@
// 2. Escape currency dollar signs so they are not misinterpreted as LaTeX
// math delimiters when singleDollarTextMath is enabled.
-import { parseMarkdownIntoBlocks } from "streamdown";
+import { parseMarkdownIntoBlocks } from "./parse-markdown-blocks.ts";
import {
codeSpans,
crossesCodeSpanBlockBoundary,
diff --git a/studio/frontend/src/lib/parse-markdown-blocks.ts b/studio/frontend/src/lib/parse-markdown-blocks.ts
new file mode 100644
index 00000000000..5d06c9b8567
--- /dev/null
+++ b/studio/frontend/src/lib/parse-markdown-blocks.ts
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+// streamdown 2.5 parseMarkdownIntoBlocks with blockTokens instead of Lexer.lex (vercel/streamdown#608):
+// inline lexing of one long backslash line costs seconds (#11376). Drop when streamdown is bumped past #608.
+
+import { Lexer, type Token } from "marked";
+
+const footnoteReferencePattern = /\[\^[\w-]{1,200}\](?!:)/;
+const footnoteDefinitionPattern = /\[\^[\w-]{1,200}\]:/;
+const openingTagPattern = /<(\w+)[\s>]/;
+
+const voidElements = new Set([
+ "area",
+ "base",
+ "br",
+ "col",
+ "embed",
+ "hr",
+ "img",
+ "input",
+ "link",
+ "meta",
+ "param",
+ "source",
+ "track",
+ "wbr",
+]);
+
+const openTagPatternCache = new Map<string, RegExp>();
+const closeTagPatternCache = new Map<string, RegExp>();
+
+const getOpenTagPattern = (tagName: string): RegExp => {
+ const normalizedTag = tagName.toLowerCase();
+ const cached = openTagPatternCache.get(normalizedTag);
+ if (cached) {
+ return cached;
+ }
+ const pattern = new RegExp(`<${normalizedTag}(?=[\\s>/])[^>]*>`, "gi");
+ openTagPatternCache.set(normalizedTag, pattern);
+ return pattern;
+};
+
+const getCloseTagPattern = (tagName: string): RegExp => {
+ const normalizedTag = tagName.toLowerCase();
+ const cached = closeTagPatternCache.get(normalizedTag);
+ if (cached) {
+ return cached;
+ }
+ const pattern = new RegExp(`</${normalizedTag}(?=[\\s>])[^>]*>`, "gi");
+ closeTagPatternCache.set(normalizedTag, pattern);
+ return pattern;
+};
+
+const countNonSelfClosingOpenTags = (
+ block: string,
+ tagName: string,
+): number => {
+ if (voidElements.has(tagName.toLowerCase())) {
+ return 0;
+ }
+ const matches = block.match(getOpenTagPattern(tagName));
+ if (!matches) {
+ return 0;
+ }
+ let count = 0;
+ for (const match of matches) {
+ if (!match.trimEnd().endsWith("/>")) {
+ count += 1;
+ }
+ }
+ return count;
+};
+
+const countClosingTags = (block: string, tagName: string): number => {
+ const matches = block.match(getCloseTagPattern(tagName));
+ return matches ? matches.length : 0;
+};
+
+const countDoubleDollars = (str: string): number => {
+ let count = 0;
+ for (let i = 0; i < str.length - 1; i += 1) {
+ if (str[i] === "$" && str[i + 1] === "$") {
+ count += 1;
+ i += 1;
+ }
+ }
+ return count;
+};
+
+const lineEndingPattern = /\r\n|\r/g;
+
+const lexBlocks = (markdown: string): Token[] =>
+ new Lexer({ gfm: true }).blockTokens(markdown);
+
+const mergeTokensIntoBlocks = (tokens: Token[]): string[] => {
+ const mergedBlocks: string[] = [];
+ const htmlStack: string[] = [];
+ let previousTokenWasCode = false;
+
+ for (const token of tokens) {
+ const currentBlock = token.raw;
+ const mergedBlocksLen = mergedBlocks.length;
+
+ if (htmlStack.length > 0) {
+ mergedBlocks[mergedBlocksLen - 1] += currentBlock;
+
+ const trackedTag = htmlStack.at(-1) as string;
+ const newOpenTags = countNonSelfClosingOpenTags(currentBlock, trackedTag);
+ const newCloseTags = countClosingTags(currentBlock, trackedTag);
+
+ for (let i = 0; i < newOpenTags; i += 1) {
+ htmlStack.push(trackedTag);
+ }
+ for (let i = 0; i < newCloseTags; i += 1) {
+ if (htmlStack.length > 0 && htmlStack.at(-1) === trackedTag) {
+ htmlStack.pop();
+ }
+ }
+ continue;
+ }
+
+ if (token.type === "html" && token.block) {
+ const openingTagMatch = currentBlock.match(openingTagPattern);
+ if (openingTagMatch) {
+ const tagName = openingTagMatch[1];
+ const openTags = countNonSelfClosingOpenTags(currentBlock, tagName);
+ const closeTags = countClosingTags(currentBlock, tagName);
+ if (openTags > closeTags) {
+ htmlStack.push(tagName);
+ }
+ }
+ }
+
+ if (mergedBlocksLen > 0 && !previousTokenWasCode) {
+ const previousBlock = mergedBlocks[mergedBlocksLen - 1];
+ const prevDollarCount = countDoubleDollars(previousBlock);
+
+ if (prevDollarCount % 2 === 1) {
+ mergedBlocks[mergedBlocksLen - 1] = previousBlock + currentBlock;
+ continue;
+ }
+ }
+
+ mergedBlocks.push(currentBlock);
+
+ if (token.type !== "space") {
+ previousTokenWasCode = token.type === "code";
+ }
+ }
+
+ return mergedBlocks;
+};
+
+export function parseMarkdownIntoBlocks(markdown: string): string[] {
+ const hasFootnoteReference = footnoteReferencePattern.test(markdown);
+ const hasFootnoteDefinition = footnoteDefinitionPattern.test(markdown);
+
+ if (hasFootnoteReference || hasFootnoteDefinition) {
+ return [markdown];
+ }
+
+ const input = markdown.includes("\r")
+ ? markdown.replace(lineEndingPattern, "\n")
+ : markdown;
+
+ return mergeTokensIntoBlocks(lexBlocks(input));
+}
diff --git a/studio/frontend/tests/parse-markdown-blocks-perf.test.ts b/studio/frontend/tests/parse-markdown-blocks-perf.test.ts
new file mode 100644
index 00000000000..4f7fdd2ab74
--- /dev/null
+++ b/studio/frontend/tests/parse-markdown-blocks-perf.test.ts
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parseMarkdownIntoBlocks as streamdownSplit } from "streamdown";
+
+import { parseMarkdownIntoBlocks } from "../src/lib/parse-markdown-blocks.ts";
+
+const longBackslashLine = (chars: number): string =>
+ `[${"\\".repeat(1500)}]: `.repeat(Math.ceil(chars / 1504)).slice(0, chars);
+
+const SAMPLES = [
+ "# Title\n\npara with *em* and `code`\n\n```py\nprint('$$')\n```\n",
+ "$$\na=b\n\n$$\n\nafter\n",
+ "<div>\n\nhi\n\n</div>\n\nafter\n",
+ "<details><summary>s</summary>\n\nbody\n\n</details>\n",
+ "<br/>\n<div>\n\ninside\n\n</div>",
+ "<my-tag>\n\nx\n\n</my-tag>\n",
+ "| a | b |\n|---|---|\n| 1 | 2 |\n\n- x\n- y\n",
+ "line\r\nwith crlf\r\n\r\nnext\r\n",
+ "text[^1]\n\n[^1]: note\n",
+ "C:\\Users\\a\\b \\* \\_ \\(x\\) \\[y\\]\n\n$$ unclosed\n",
+ longBackslashLine(5_000),
+];
+
+test("block split matches streamdown", () => {
+ for (const markdown of SAMPLES) {
+ assert.deepEqual(
+ parseMarkdownIntoBlocks(markdown),
+ streamdownSplit(markdown),
+ );
+ }
+});
+
+test("block split stays fast on a long backslash line (#11376)", () => {
+ const markdown = longBackslashLine(200_000);
+ const start = performance.now();
+ parseMarkdownIntoBlocks(markdown);
+ const elapsed = performance.now() - start;
+ assert.ok(elapsed < 500, `block split took ${elapsed.toFixed(1)}ms`);
+});