Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep $PATH, $HOME and other shell variables as text, not maths (#12911)
Nilaya54bf4f7672f
diff --git a/studio/frontend/src/components/assistant-ui/markdown-text.tsx b/studio/frontend/src/components/assistant-ui/markdown-text.tsxindex 391065384ec..3a618656bdb 100644--- a/studio/frontend/src/components/assistant-ui/markdown-text.tsx+++ b/studio/frontend/src/components/assistant-ui/markdown-text.tsx@@ -1214,6 +1214,7 @@ function MarkdownTextRenderer({searchImages,),),+ isStreaming,),isStreaming,),diff --git a/studio/frontend/src/lib/latex.ts b/studio/frontend/src/lib/latex.tsindex c5095f0db3e..8c26975f67c 100644--- a/studio/frontend/src/lib/latex.ts+++ b/studio/frontend/src/lib/latex.ts@@ -25,21 +25,35 @@ import {quoteState,} from "./markdown-list-columns.ts";-/**- * Matches a single $ followed by a number pattern (currency), e.g.:- * $5, $1,000, $5.99, $100K, $3.5M- *- * Does NOT match:- * $$ (display math), \$ (already escaped), $\alpha (LaTeX command)- */-const CURRENCY_REGEX =- /(?<![\\$])\$(?!\$)(?=\d+(?:,\d{3})*(?:\.\d+)?[KMBkmb]?(?:\s|$|[^a-zA-Z\d]))/g;+/** matches currency bodies after `$`, including separators, decimals, and K/M/B suffixes. */+const CURRENCY_REGEX = /\d+(?:,\d{3})*(?:\.\d+)?[KMBkmb]?(?:\s|$|[^a-zA-Z\d])/y;++const HEADING_LINE_RE = / {0,3}#{1,6}(?=[ \t\r\n]|$)/y;+const TABLE_ROW_RE = /[ \t]*\|/y;+const BLOCK_BREAK_RE =+ /\n[ \t\r]*(?:\n|#{1,6}(?=[ \t\r\n])|[>|]|[-*+][ \t]|1[.)][ \t]|```|~~~|(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|=+[ \t]*)(?=\r?(?:\n|$)))/;++/** matches prose-like `$NAME ... $word` spans without math symbols. */+// non-ASCII letters and CJK punctuation are prose (KaTeX rejects them in math mode).+const VARIABLE_PROSE_RE =+ /^(?!\w+\s+$)(?:[A-Za-z]{2,}\w*|_\w+|\{[A-Za-z_]\w*\})[\w\s.,;:!?'"()/`|&<>=*\-\p{L}\p{M}\u3000-\u303f\uff00-\uff65]*(?:[\s/:,.;|<>=\-\u3000-\u303f\uff00-\uff65]|[^\P{L}\p{ASCII}]|[\s(]["'(`])$/u;+const NEW_TOKEN_RE = /[\w{\\]/;+const isAsciiLetter = (c: number) => (c | 32) >= 97 && (c | 32) <= 122;+const isWordChar = (c: number) =>+ isAsciiLetter(c) || (c >= 48 && c <= 57) || c === 95;++/** necessary prefix of VARIABLE_PROSE_RE, checked before slicing the span. */+function startsLikeVariable(text: string, at: number): boolean {+ const a = text.charCodeAt(at);+ const b = text.charCodeAt(at + 1);+ if (isAsciiLetter(a)) return isAsciiLetter(b);+ if (a === 95) return isWordChar(b);+ return a === 123 && (isAsciiLetter(b) || b === 95);+}+// an entity stays literal in Markdown without showing an escape slash in raw HTML.+const VARIABLE_DOLLAR = "$";-/**- * Union of two span lists, each ascending by start (overlap within a list is- * fine, non-ascending input silently drops spans). The sorted, non-overlapping- * result is the shape `isInRegion`'s binary search needs.- */+/** merges ascending spans; overlaps merge and non-ascending spans drop. */function mergeRegions(left: ReadonlyArray<readonly [number, number]>,right: ReadonlyArray<readonly [number, number]>,@@ -357,43 +371,63 @@ export function findCodeBlockRegions(content: string): Array<[number, number]> {const blocks = mergeRegions(fenced, indented);const inline = findInlineCodeRegions(content, blocks);- // An inline span can CONTAIN a fence (`` `~~~a~~~ $5` ``); that overlap made- // the binary search land on the inner span and miss the outer one.+ // merge overlaps so binary search does not miss a containing inline span.return mergeRegions(blocks, inline);}-/**- * Match an inline link/image `[text](DEST)`, capturing the destination as group 1- * with the `d` flag so its span is read straight from `match.indices` (the text- * can contain an escaped `\](`, so a string search for the separator is unsafe).- * The text disallows unescaped `]`; the destination allows escapes and one level- * of balanced parens.- */+/** destinations allow escapes and one level of balanced parentheses. */const LINK_DEST_RE =/!?\[(?:\\.|[^\]\\])*?\]\(((?:\\.|[^()\\]|\([^()]*\))*)\)/dg;+const AUTOLINK_RE =+ /<(?:[a-z][a-z0-9+.-]*:[^\s<>]*|[^\s<>@]+@[^\s<>@]+)>|\b(?:https?:\/\/|www\.)[^\s<]+/gi;-/**- * Find the destination spans of inline links/images, so a `\(...\)` written with- * escaped parens inside a URL isn't rewritten as math (which would break the- * link). Only the destination is returned, not the link text, so math in the- * visible text still converts. Sorted, non-overlapping (matches are disjoint).- */+/** shields link destinations from math rewrites; link text stays convertible. */function findLinkDestinationRegions(content: string): Array<[number, number]> {if (!content.includes("](")) return [];const regions: Array<[number, number]> = [];let match: RegExpExecArray | null;LINK_DEST_RE.lastIndex = 0;while ((match = LINK_DEST_RE.exec(content)) !== null) {- // `indices` is present (the `d` flag); group 1 spans the destination.+ // escaped separators make searches unsafe; use the `d` flag's bounds.regions.push(match.indices![1]);}return regions;}-/**- * Binary search to check if a position falls inside any region. Regions must be- * sorted by start and non-overlapping.- */+// raw-text element bodies do not decode character references.+const RAW_TEXT_ELEMENT_RE =+ /<(script|style|xmp|iframe|noembed|noframes)\b[^>]*>[\s\S]*?(?:<\/\1\s*>|$)/gi;++function findRawTextRegions(content: string): Array<[number, number]> {+ if (!content.includes("<")) return [];+ const regions: Array<[number, number]> = [];+ for (const match of content.matchAll(RAW_TEXT_ELEMENT_RE)) {+ regions.push([match.index, match.index + match[0].length]);+ }+ return regions;+}++function findAutolinkRegions(content: string): Array<[number, number]> {+ const regions: Array<[number, number]> = [];+ AUTOLINK_RE.lastIndex = 0;+ for (const match of content.matchAll(AUTOLINK_RE)) {+ regions.push([match.index, match.index + match[0].length]);+ }+ return regions;+}++/** `isInRegion` for ascending positions: amortised O(1) per query. */+function regionCursor(+ regions: Array<[number, number]>,+): (position: number) => boolean {+ let k = 0;+ return (position) => {+ while (k < regions.length && regions[k][1] <= position) k++;+ return k < regions.length && regions[k][0] <= position;+ };+}++/** regions must be sorted by start and non-overlapping. */export function isInRegion(position: number,regions: Array<[number, number]>,@@ -521,10 +555,41 @@ function hasInlineMathCloser(return false;}-/**- * Matches a `\[...\]` (display) or `\(...\)` (inline) LaTeX span. The body- * is capped so repeated incomplete openers stay linear during streaming.- */+// remark-math closes an open span at the next single `$`, escaped or not, within the block.+function findInlineMathCloser(+ content: string,+ offset: number,+ lineStart: number,+ lineEnd: number,+ tableRow: boolean,+): number {+ let i = content.indexOf("$", offset + 1);+ while (i !== -1 && content[i + 1] === "$") {+ while (content[i + 1] === "$") i++;+ i = content.indexOf("$", i + 1);+ }+ if (i === -1) return -1;+ // hot path: only multi-line spans can cross a block break or leave a heading.+ const multiline = lineEnd !== -1 && lineEnd < i;+ if (multiline) {+ if (BLOCK_BREAK_RE.test(content.slice(offset + 1, i))) return -1;+ HEADING_LINE_RE.lastIndex = lineStart;+ if (HEADING_LINE_RE.test(content)) return -1;+ }+ if (tableRow) {+ if (multiline) return -1;+ for (+ let p = content.indexOf("|", offset + 1);+ p !== -1 && p < i;+ p = content.indexOf("|", p + 1)+ ) {+ if (content[p - 1] !== "\\") return -1;+ }+ }+ return i;+}++/** caps LaTeX delimiter bodies at 4,096 characters so repeated incomplete openers stay linear while streaming. */const CONVERT_LATEX_DELIM_RE =/(?<!\\)\\\[([\s\S]{0,4096}?)\\\]|(?<!\\)\\\(([\s\S]{0,4096}?)\\\)/g;@@ -627,40 +692,101 @@ function convertLatexDelimiters(content: string): {return { text: parts.join(""), mathRegions };}-/**- * Preprocess a markdown string so LaTeX renders: convert bracket delimiters to- * dollar forms, then escape currency dollar signs so they are not parsed as- * math delimiters.- *- * - `\[E = mc^2\]` becomes a `$$` display block on its own lines (display math)- * - `\(\alpha\)` becomes `$\alpha$` (inline math)- * - `\(x\)` in a code span is untouched- * - `$5` alone becomes `\$5` (currency, not math)- * - `$\alpha$` is untouched (real LaTeX)- * - `$30^\circ$` is untouched (LaTeX whose body starts with a digit)- * - `**$30^\circ$**` is untouched (LaTeX wrapped in bold)- * - `$$E = mc^2$$` is untouched (display math)- * - Currency inside code blocks/spans is untouched- */-export function preprocessLaTeX(content: string): string {+/** converts bracketed LaTeX and protects currency or shell variables from single-dollar math. */+export function preprocessLaTeX(content: string, isStreaming = false): string {const { text, mathRegions } = convertLatexDelimiters(content);if (!text.includes("$")) return text;const codeRegions = findCodeBlockRegions(text);-- return text.replace(CURRENCY_REGEX, (match, offset) => {- if (isInRegion(offset, codeRegions)) {- return match;+ const linkRegions = mergeRegions(+ findLinkDestinationRegions(text),+ text.includes("<") || text.includes("://") || text.includes("www.")+ ? findAutolinkRegions(text)+ : [],+ );+ const rawTextRegions = findRawTextRegions(text);+ const inCode = regionCursor(codeRegions);+ const inLink = regionCursor(linkRegions);+ const inMath = regionCursor(mathRegions);+ const inRawText = regionCursor(rawTextRegions);+ let closer = -1;+ let lineStart = 0;+ let nextNewline = text.indexOf("\n");+ let tableRow: boolean | null = null;++ const rewrite = (offset: number): string => {+ // evaluate all cursors so each advances past `offset`.+ const code = inCode(offset);+ const link = inLink(offset);+ const math = inMath(offset);+ if (code || link) {+ return "$";+ }+ // preserve converted numeric math to avoid currency re-escaping+ if (math) {+ return "$";+ }+ const digit = text.charCodeAt(offset + 1) - 48;+ CURRENCY_REGEX.lastIndex = offset + 1;+ const currency = digit >= 0 && digit <= 9 && CURRENCY_REGEX.test(text);+ if (currency && !hasInlineMathCloser(text, offset, mathRegions)) {+ return "\\$";+ }+ if (offset === closer) {+ return "$";}- // Skip the spans we just created from `\(...\)` so a numeric body like- // `$5$` isn't re-escaped back to literal `\$5$`.- if (isInRegion(offset, mathRegions)) {- return match;+ while (nextNewline !== -1 && nextNewline < offset) {+ lineStart = nextNewline + 1;+ nextNewline = text.indexOf("\n", lineStart);+ tableRow = null;+ }+ if (tableRow === null) {+ TABLE_ROW_RE.lastIndex = lineStart;+ tableRow = TABLE_ROW_RE.test(text);+ }+ const next = findInlineMathCloser(+ text,+ offset,+ lineStart,+ nextNewline,+ tableRow,+ );+ if (+ !currency &&+ next !== -1 &&+ startsLikeVariable(text, offset + 1) &&+ // a closer at the end of a streaming reply may still be followed by a name.+ (next + 1 === text.length+ ? isStreaming+ : NEW_TOKEN_RE.test(text[next + 1])) &&+ VARIABLE_PROSE_RE.test(text.slice(offset + 1, next)) &&+ !inRawText(offset)+ ) {+ return VARIABLE_DOLLAR;}- if (hasInlineMathCloser(text, offset, mathRegions)) {- return match;+ closer = next;+ return "$";+ };++ // a single `$`: not escaped, not part of `$$`. A plain scan, since a regex replace+ // callback per dollar dominated the streaming re-render cost.+ const parts: string[] = [];+ let last = 0;+ for (+ let offset = text.indexOf("$");+ offset !== -1;+ offset = text.indexOf("$", offset + 1)+ ) {+ const prev = text[offset - 1];+ if (prev === "\\" || prev === "$" || text[offset + 1] === "$") continue;+ const out = rewrite(offset);+ if (out !== "$") {+ parts.push(text.slice(last, offset), out);+ last = offset + 1;}- return "\\" + match;- });+ }+ if (last === 0) return text;+ parts.push(text.slice(last));+ return parts.join("");}diff --git a/studio/frontend/tests/dollar-variables-render-as-text.test.ts b/studio/frontend/tests/dollar-variables-render-as-text.test.tsnew file mode 100644index 00000000000..3fe4f7f7c89--- /dev/null+++ b/studio/frontend/tests/dollar-variables-render-as-text.test.ts@@ -0,0 +1,180 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";+import { createMathPlugin } from "@streamdown/math";+import React from "react";+import { renderToStaticMarkup } from "react-dom/server";+import { Streamdown } from "streamdown";+import { stabilizeStreamingMarkdown } from "../src/components/assistant-ui/streaming-markdown.ts";+import { normalizeEscapedInlineMath } from "../src/lib/escaped-inline-math.ts";+import { preprocessLaTeX } from "../src/lib/latex.ts";++const math = createMathPlugin({ singleDollarTextMath: true });++const HTML_ENTITY_TEXT: Record<string, string> = {+ "&": "&",+ "<": "<",+ ">": ">",+ """: '"',+ "'": "'",+};++function render(source: string, isStreaming = false): string {+ return renderToStaticMarkup(+ React.createElement(+ Streamdown,+ {+ mode: "streaming",+ isAnimating: isStreaming,+ plugins: { math },+ },+ stabilizeStreamingMarkdown(+ preprocessLaTeX(normalizeEscapedInlineMath(source), isStreaming),+ isStreaming,+ ),+ ),+ );+}++function renderedMath(html: string): string[] {+ return [+ ...html.matchAll(+ /<annotation encoding="application\/x-tex">([\s\S]*?)<\/annotation>/g,+ ),+ ].map((match) =>+ match[1].replace(+ /&(?:amp|lt|gt|quot|#x27);/g,+ (entity) => HTML_ENTITY_TEXT[entity] ?? entity,+ ),+ );+}++const SHELL_VARIABLE_REPLIES: Array<[string, string[]]> = [+ [+ "Add the bin folder to $PATH, then check $HOME/.bashrc.",+ ["$PATH, then check $HOME/.bashrc."],+ ],+ [+ "Export $CUDA_HOME and $LD_LIBRARY_PATH before building.",+ ["$CUDA_HOME and $LD_LIBRARY_PATH"],+ ],+ ["Run echo $PATH\nthen echo $HOME", ["$PATH", "$HOME"]],+ ["Check $HOME, $PATH, and $USER first.", ["$HOME, $PATH, and $USER"]],+ ['Quote them: "$HOME" and "$PATH".', ["$HOME", "$PATH"]],+ ["Files live under $HOME/$USER/data.", ["$HOME/$USER/data"]],+ ["Use ${HOME} and ${PATH} in scripts.", ["${HOME} and ${PATH}"]],+ ["Set it to $PATH:$HOME/bin now.", ["$PATH:$HOME/bin"]],+ ["#define HOME $HOME\nthen $PATH", ["$HOME", "$PATH"]],+ ["Use $PATH\n***\nthen $HOME and $USER", ["$PATH", "$HOME and $USER"]],+ [+ "Try $PATH || $HOME, $HOME > $LOG or $PATH=$HOME/bin.",+ ["$PATH || $HOME", "$PATH=$HOME/bin"],+ ],+ ["PHP reads $_GET and $_POST.", ["$_GET and $"]],+ ["Use $PATH **and then** $HOME.", ["$PATH", "and then", "$HOME"]],+ ["\u4f7f\u7528 $PATH\uff0c\u7136\u540e\u68c0\u67e5 $HOME", ["$PATH\uff0c\u7136\u540e\u68c0\u67e5 $HOME"]],+ ["\u4f7f\u7528$PATH\uff0c\u7136\u540e\u68c0\u67e5$HOME", ["$PATH\uff0c\u7136\u540e\u68c0\u67e5$HOME"]],+ ["F\u00fcge $PATH hinzu, pr\u00fcfe dann $HOME", ["$PATH hinzu, pr\u00fcfe dann $HOME"]],+ [+ "Add it to $PATH, then run `echo $HOME`.",+ ["$PATH, then run", "echo $HOME"],+ ],+];++test("shell variables in a reply render as text, not maths", () => {+ for (const [source, texts] of SHELL_VARIABLE_REPLIES) {+ const html = render(source);+ assert.deepEqual(renderedMath(html), [], source);+ for (const text of texts) {+ assert.ok(html.includes(text), `${source} lost ${text}`);+ }+ }+});++test("shell variables never flash as maths while a reply streams", () => {+ for (const [source] of SHELL_VARIABLE_REPLIES) {+ for (let end = 1; end <= source.length; end += 1) {+ const prefix = source.slice(0, end);+ assert.deepEqual(renderedMath(render(prefix, true)), [], prefix);+ }+ }+});++const MATH_REPLIES: Array<[string, string[]]> = [+ ["$x$", ["x"]],+ ["In triangle $ABC$, the side $AB = 5$.", ["ABC", "AB = 5"]],+ ["The slope is $dy/dx$ and the line is $ax + b$.", ["dy/dx", "ax + b"]],+ ["Use $\\alpha + 1$ and $\\alpha$.", ["\\alpha + 1", "\\alpha"]],+ ["Lines $AB$ and $CD$ meet at $P$.", ["AB", "CD", "P"]],+ ["Segments $AB and CD$ are equal.", ["AB and CD"]],+ ["The $n$th element and the $k$th one.", ["n", "k"]],+ [+ "We have $sin x$ and $n log n$ and $sin theta$.",+ ["sin x", "n log n", "sin theta"],+ ],+ ["Points $x_1, x_2$ and $v_s$ and $a_{ij}$.", ["x_1, x_2", "v_s", "a_{ij}"]],+ ["So $E = mc^2$ and $f(x)$ and $O(n)$.", ["E = mc^2", "f(x)", "O(n)"]],+ ["Then $P(A and B)$ holds.", ["P(A and B)"]],+ ["Pick ${n \\choose k}$ ways.", ["{n \\choose k}"]],+ ["Write $ab/cd$ or $AB/CD$.", ["ab/cd", "AB/CD"]],+ ["Then $\\text{if } x > 0$ and $|x|$.", ["\\text{if } x > 0", "|x|"]],+ ["An angle of $30^\\circ$ here.", ["30^\\circ"]],+ ["**$90 - x$** is the rest.", ["90 - x"]],+ ["Let $x \\in A$ and $AB $ hold.", ["x \\in A", "AB "]],+ ["The value \\(\\beta\\) and $\\gamma$.", ["\\beta", "\\gamma"]],+ ["Sum $a +\nb$ over lines.", ["a +\nb"]],+ ["Segment $AB'$ and the derivative $uv'$.", ["AB'", "uv'"]],+ [+ "About $\\sim$2x faster: the $k$th token and the $n$th layer.",+ ["\\sim", "k", "n"],+ ],+ [+ "| Item | Price ($) |\n|---|---|\n| the $n$th row and the $m$th column | 5 |",+ ["n", "m"],+ ],+ ["# Cost in $\nThe $n$th row and the $m$th column.", ["n", "m"]],+ ["- Ends with $\n- The $n$th row and the $m$th column", ["n", "m"]],+ ["We have $sin theta $ and $AB / CD $ here.", ["sin theta ", "AB / CD "]],+ ["Use $HOME/$USER and $\\alpha$ here.", ["\\alpha"]],+ ["\u8bbe $ABC$ \u4e3a\u4e09\u89d2\u5f62\uff0c$AB = 5$\u3002", ["ABC", "AB = 5"]],+ ["\u4e09\u89d2\u5f62$ABC$\u4e2d\uff0c$AB$\u8fb9", ["ABC", "AB"]],+ ["Die Gr\u00f6\u00dfe $AB$ und $CD$", ["AB", "CD"]],+ ["Write $a * b$ and $AB * CD$.", ["a * b", "AB * CD"]],+ ["The result is $sin theta.$", ["sin theta."]],+ ["So $ab = c.$", ["ab = c."]],+];++test("real maths still renders", () => {+ for (const [source, expected] of MATH_REPLIES) {+ assert.deepEqual(renderedMath(render(source)), expected, source);+ }+});++test("maths and currency next to shell variables keep rendering", () => {+ const withMath = render("Set $x$ from $PATH, then $HOME.");+ assert.deepEqual(renderedMath(withMath), ["x"]);+ assert.ok(withMath.includes("$PATH, then $HOME."));++ const withCurrency = render("It costs $5 to set $PATH, then $HOME.");+ assert.deepEqual(renderedMath(withCurrency), []);+ assert.ok(withCurrency.includes("$5 to set $PATH, then $HOME."));+});++test("shell variables keep literal dollars in raw HTML and URLs", () => {+ for (const source of [+ "<pre>$HOME and $USER</pre>",+ "https://example.com/$HOME/$USER",+ "<https://example.com/$HOME/$USER>",+ ]) {+ const html = render(source);+ assert.deepEqual(renderedMath(html), [], source);+ assert.ok(html.includes("$HOME"), `${source} lost its first dollar`);+ assert.ok(!html.includes("\\$HOME"), `${source} showed an escape slash`);+ }+ // raw-text elements never decode `$`, so their dollars stay as written.+ const style = render("<style>$primary: red; $secondary: blue;</style>");+ assert.ok(style.includes("$primary: red; $secondary: blue;"), style);+ assert.ok(!style.includes("#36;"), style);+});diff --git a/studio/frontend/tests/streaming-latex-corpus.test.ts b/studio/frontend/tests/streaming-latex-corpus.test.tsindex 3a1e8790033..9cc3bfe5b88 100644--- a/studio/frontend/tests/streaming-latex-corpus.test.ts+++ b/studio/frontend/tests/streaming-latex-corpus.test.ts@@ -58,6 +58,10 @@ const NAMED_CASES: Array<[string, string]> = [["escaped-dollar", "A literal \\$5 stays literal.\n\n"],["escaped-backslash", "A path C:\\\\temp and \\\\(not math\\\\) here.\n\n"],["currency-suffix", "Around $5K then $5Ki then $3.5M.\n\n"],+ [+ "shell-variables",+ "Add $PATH, then check $HOME/.bashrc\nand run `echo $CUDA_HOME` with $x$.\n\n",+ ],["cjk", "\u4fa1\u683c\u306f $5 \u3067\u3059\u3002\u6570\u5f0f \\(x^2\\) \u3082\u3042\u308a\u307e\u3059\u3002\n\n"],["rtl-arabic", "\u0627\u0644\u0633\u0639\u0631 $5 \u0648\u0627\u0644\u0645\u0639\u0627\u062f\u0644\u0629 \\(x^2\\) \u0647\u0646\u0627.\n\n"],["rtl-hebrew", "\u05d4\u05de\u05d7\u05d9\u05e8 $10 \u05d5\u05d4\u05e0\u05d5\u05e1\u05d7\u05d4 \\(y\\).\n\n"],