Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): block-split long backslash lines without inline lex (#11376) (#11501)
Sourav Rajvic5f1f7946bf4
diff --git a/studio/frontend/package-lock.json b/studio/frontend/package-lock.jsonindex 5b70943e693..2fe0b95a72f 100644--- a/studio/frontend/package-lock.json+++ b/studio/frontend/package-lock.json@@ -53,6 +53,7 @@"katex": "^0.16.28","lucide-react": "^1.7.0","mammoth": "^1.11.0",+ "marked": "17.0.6","mdast-util-from-markdown": "2.0.3","mdast-util-gfm": "3.1.0","mdast-util-math": "3.0.0",@@ -11396,9 +11397,9 @@}},"node_modules/marked": {- "version": "16.4.2",- "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz",- "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==",+ "version": "17.0.6",+ "resolved": "https://registry.npmjs.org/marked/-/marked-17.0.6.tgz",+ "integrity": "sha512-gB0gkNafnonOw0obSTEGZTT86IuhILt2Wfx0mWH/1Au83kybTayroZ/V6nS25mN7u8ASy+5fMhgB3XPNrOZdmA==","license": "MIT","bin": {"marked": "bin/marked.js"@@ -11799,6 +11800,18 @@"uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0"}},+ "node_modules/mermaid/node_modules/marked": {+ "version": "16.4.2",+ "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz",+ "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==",+ "license": "MIT",+ "bin": {+ "marked": "bin/marked.js"+ },+ "engines": {+ "node": ">= 20"+ }+ },"node_modules/micromark": {"version": "4.0.2","resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz",@@ -14650,18 +14663,6 @@"react-dom": "^18.0.0 || ^19.0.0"}},- "node_modules/streamdown/node_modules/marked": {- "version": "17.0.6",- "resolved": "https://registry.npmjs.org/marked/-/marked-17.0.6.tgz",- "integrity": "sha512-gB0gkNafnonOw0obSTEGZTT86IuhILt2Wfx0mWH/1Au83kybTayroZ/V6nS25mN7u8ASy+5fMhgB3XPNrOZdmA==",- "license": "MIT",- "bin": {- "marked": "bin/marked.js"- },- "engines": {- "node": ">= 20"- }- },"node_modules/strict-event-emitter": {"version": "0.5.1","resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz",diff --git a/studio/frontend/package.json b/studio/frontend/package.jsonindex 71a76104ab3..8ecb3690278 100644--- a/studio/frontend/package.json+++ b/studio/frontend/package.json@@ -68,6 +68,7 @@"katex": "^0.16.28","lucide-react": "^1.7.0","mammoth": "^1.11.0",+ "marked": "17.0.6","mdast-util-from-markdown": "2.0.3","mdast-util-gfm": "3.1.0","mdast-util-math": "3.0.0",diff --git a/studio/frontend/src/components/assistant-ui/streaming-markdown.ts b/studio/frontend/src/components/assistant-ui/streaming-markdown.tsindex a657f6aa7b7..3f3393ea72a 100644--- a/studio/frontend/src/components/assistant-ui/streaming-markdown.ts+++ b/studio/frontend/src/components/assistant-ui/streaming-markdown.ts@@ -7,7 +7,7 @@ import { mathFromMarkdown } from "mdast-util-math";import { gfm } from "micromark-extension-gfm";import { math } from "micromark-extension-math";import remend from "remend";-import { parseMarkdownIntoBlocks } from "streamdown";+import { parseMarkdownIntoBlocks } from "../../lib/parse-markdown-blocks.ts";// Cheap gate: a bullet marker then only thematic-break punctuation. Not// asterisk specific, `- --verbose` and `* ___under___` flash too.diff --git a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.tsindex ae191413407..07b9f34f828 100644--- a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts+++ b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts@@ -2,7 +2,8 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import remend from "remend";-import { type BlockProps, parseMarkdownIntoBlocks } from "streamdown";+import { type BlockProps } from "streamdown";+import { parseMarkdownIntoBlocks } from "../../lib/parse-markdown-blocks.ts";// How far behind the live edge a block has to be before it can be retained.// The block list interleaves "\n\n" separators, so this is about fourdiff --git a/studio/frontend/src/lib/latex.ts b/studio/frontend/src/lib/latex.tsindex adfbdd385c2..c5095f0db3e 100644--- a/studio/frontend/src/lib/latex.ts+++ b/studio/frontend/src/lib/latex.ts@@ -9,7 +9,7 @@// 2. Escape currency dollar signs so they are not misinterpreted as LaTeX// math delimiters when singleDollarTextMath is enabled.-import { parseMarkdownIntoBlocks } from "streamdown";+import { parseMarkdownIntoBlocks } from "./parse-markdown-blocks.ts";import {codeSpans,crossesCodeSpanBlockBoundary,diff --git a/studio/frontend/src/lib/parse-markdown-blocks.ts b/studio/frontend/src/lib/parse-markdown-blocks.tsnew file mode 100644index 00000000000..5d06c9b8567--- /dev/null+++ b/studio/frontend/src/lib/parse-markdown-blocks.ts@@ -0,0 +1,168 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++// streamdown 2.5 parseMarkdownIntoBlocks with blockTokens instead of Lexer.lex (vercel/streamdown#608):+// inline lexing of one long backslash line costs seconds (#11376). Drop when streamdown is bumped past #608.++import { Lexer, type Token } from "marked";++const footnoteReferencePattern = /\[\^[\w-]{1,200}\](?!:)/;+const footnoteDefinitionPattern = /\[\^[\w-]{1,200}\]:/;+const openingTagPattern = /<(\w+)[\s>]/;++const voidElements = new Set([+ "area",+ "base",+ "br",+ "col",+ "embed",+ "hr",+ "img",+ "input",+ "link",+ "meta",+ "param",+ "source",+ "track",+ "wbr",+]);++const openTagPatternCache = new Map<string, RegExp>();+const closeTagPatternCache = new Map<string, RegExp>();++const getOpenTagPattern = (tagName: string): RegExp => {+ const normalizedTag = tagName.toLowerCase();+ const cached = openTagPatternCache.get(normalizedTag);+ if (cached) {+ return cached;+ }+ const pattern = new RegExp(`<${normalizedTag}(?=[\\s>/])[^>]*>`, "gi");+ openTagPatternCache.set(normalizedTag, pattern);+ return pattern;+};++const getCloseTagPattern = (tagName: string): RegExp => {+ const normalizedTag = tagName.toLowerCase();+ const cached = closeTagPatternCache.get(normalizedTag);+ if (cached) {+ return cached;+ }+ const pattern = new RegExp(`</${normalizedTag}(?=[\\s>])[^>]*>`, "gi");+ closeTagPatternCache.set(normalizedTag, pattern);+ return pattern;+};++const countNonSelfClosingOpenTags = (+ block: string,+ tagName: string,+): number => {+ if (voidElements.has(tagName.toLowerCase())) {+ return 0;+ }+ const matches = block.match(getOpenTagPattern(tagName));+ if (!matches) {+ return 0;+ }+ let count = 0;+ for (const match of matches) {+ if (!match.trimEnd().endsWith("/>")) {+ count += 1;+ }+ }+ return count;+};++const countClosingTags = (block: string, tagName: string): number => {+ const matches = block.match(getCloseTagPattern(tagName));+ return matches ? matches.length : 0;+};++const countDoubleDollars = (str: string): number => {+ let count = 0;+ for (let i = 0; i < str.length - 1; i += 1) {+ if (str[i] === "$" && str[i + 1] === "$") {+ count += 1;+ i += 1;+ }+ }+ return count;+};++const lineEndingPattern = /\r\n|\r/g;++const lexBlocks = (markdown: string): Token[] =>+ new Lexer({ gfm: true }).blockTokens(markdown);++const mergeTokensIntoBlocks = (tokens: Token[]): string[] => {+ const mergedBlocks: string[] = [];+ const htmlStack: string[] = [];+ let previousTokenWasCode = false;++ for (const token of tokens) {+ const currentBlock = token.raw;+ const mergedBlocksLen = mergedBlocks.length;++ if (htmlStack.length > 0) {+ mergedBlocks[mergedBlocksLen - 1] += currentBlock;++ const trackedTag = htmlStack.at(-1) as string;+ const newOpenTags = countNonSelfClosingOpenTags(currentBlock, trackedTag);+ const newCloseTags = countClosingTags(currentBlock, trackedTag);++ for (let i = 0; i < newOpenTags; i += 1) {+ htmlStack.push(trackedTag);+ }+ for (let i = 0; i < newCloseTags; i += 1) {+ if (htmlStack.length > 0 && htmlStack.at(-1) === trackedTag) {+ htmlStack.pop();+ }+ }+ continue;+ }++ if (token.type === "html" && token.block) {+ const openingTagMatch = currentBlock.match(openingTagPattern);+ if (openingTagMatch) {+ const tagName = openingTagMatch[1];+ const openTags = countNonSelfClosingOpenTags(currentBlock, tagName);+ const closeTags = countClosingTags(currentBlock, tagName);+ if (openTags > closeTags) {+ htmlStack.push(tagName);+ }+ }+ }++ if (mergedBlocksLen > 0 && !previousTokenWasCode) {+ const previousBlock = mergedBlocks[mergedBlocksLen - 1];+ const prevDollarCount = countDoubleDollars(previousBlock);++ if (prevDollarCount % 2 === 1) {+ mergedBlocks[mergedBlocksLen - 1] = previousBlock + currentBlock;+ continue;+ }+ }++ mergedBlocks.push(currentBlock);++ if (token.type !== "space") {+ previousTokenWasCode = token.type === "code";+ }+ }++ return mergedBlocks;+};++export function parseMarkdownIntoBlocks(markdown: string): string[] {+ const hasFootnoteReference = footnoteReferencePattern.test(markdown);+ const hasFootnoteDefinition = footnoteDefinitionPattern.test(markdown);++ if (hasFootnoteReference || hasFootnoteDefinition) {+ return [markdown];+ }++ const input = markdown.includes("\r")+ ? markdown.replace(lineEndingPattern, "\n")+ : markdown;++ return mergeTokensIntoBlocks(lexBlocks(input));+}diff --git a/studio/frontend/tests/parse-markdown-blocks-perf.test.ts b/studio/frontend/tests/parse-markdown-blocks-perf.test.tsnew file mode 100644index 00000000000..4f7fdd2ab74--- /dev/null+++ b/studio/frontend/tests/parse-markdown-blocks-perf.test.ts@@ -0,0 +1,43 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import assert from "node:assert/strict";+import test from "node:test";++import { parseMarkdownIntoBlocks as streamdownSplit } from "streamdown";++import { parseMarkdownIntoBlocks } from "../src/lib/parse-markdown-blocks.ts";++const longBackslashLine = (chars: number): string =>+ `[${"\\".repeat(1500)}]: `.repeat(Math.ceil(chars / 1504)).slice(0, chars);++const SAMPLES = [+ "# Title\n\npara with *em* and `code`\n\n```py\nprint('$$')\n```\n",+ "$$\na=b\n\n$$\n\nafter\n",+ "<div>\n\nhi\n\n</div>\n\nafter\n",+ "<details><summary>s</summary>\n\nbody\n\n</details>\n",+ "<br/>\n<div>\n\ninside\n\n</div>",+ "<my-tag>\n\nx\n\n</my-tag>\n",+ "| a | b |\n|---|---|\n| 1 | 2 |\n\n- x\n- y\n",+ "line\r\nwith crlf\r\n\r\nnext\r\n",+ "text[^1]\n\n[^1]: note\n",+ "C:\\Users\\a\\b \\* \\_ \\(x\\) \\[y\\]\n\n$$ unclosed\n",+ longBackslashLine(5_000),+];++test("block split matches streamdown", () => {+ for (const markdown of SAMPLES) {+ assert.deepEqual(+ parseMarkdownIntoBlocks(markdown),+ streamdownSplit(markdown),+ );+ }+});++test("block split stays fast on a long backslash line (#11376)", () => {+ const markdown = longBackslashLine(200_000);+ const start = performance.now();+ parseMarkdownIntoBlocks(markdown);+ const elapsed = performance.now() - start;+ assert.ok(elapsed < 500, `block split took ${elapsed.toFixed(1)}ms`);+});