Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: turn [1]-style citations in replies into links (#12912)
Nilaydbb2fa6b4474
diff --git a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.tsindex 8ba8ed48ba7..56b6a2bbc3c 100644--- a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts+++ b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts@@ -3,7 +3,10 @@import remend from "remend";import { type BlockProps } from "streamdown";-import { parseMarkdownIntoBlocks } from "../../lib/parse-markdown-blocks.ts";+import {+ parseMarkdownBlockDetails,+ parseMarkdownIntoBlocks,+} from "../../lib/parse-markdown-blocks.ts";// How far behind the live edge a block has to be before it can be retained.// The block list interleaves "\n\n" separators, so this is about four@@ -212,6 +215,52 @@ function hasLinkReference(text: string): boolean {}return false;}+// A shortcut `[label]` or collapsed `[label][]` resolves against a definition too. Code spans,+// inline links and the like are deliberately not excluded: that only adds false positives.+const SHORTCUT_REFERENCE_RE = /\[((?:\\[\s\S]|[^[\]\\]){1,999})\]/gu;+const DEFINITION_LABEL_RE = /\[((?:\\[\s\S]|[^\]\\]){1,999})\]:/u;++// micromark's `normalizeIdentifier`, so `[SS]` finds `[\u1E9E]:` as the renderer does.+function normalizeLabel(label: string): string {+ return label+ .replace(/[\t\n\r ]+/g, " ")+ .replace(/^ | $/g, "")+ .toLowerCase()+ .toUpperCase();+}++function hasShortcutReference(+ prose: string,+ references: string,+ definitions: readonly string[],+): boolean {+ const labels = new Set<string>();+ // Marked's tokens as well: an unmatched `[` line before a definition widens the regex's label.+ for (const definition of [+ ...definitions,+ ...(prose.match(LINK_DEFINITION_KEY_RE) ?? []),+ ]) {+ const label = DEFINITION_LABEL_RE.exec(definition)?.[1];+ if (label !== undefined) {+ labels.add(normalizeLabel(label));+ }+ }+ labels.delete("");+ if (labels.size === 0) {+ return false;+ }+ // Marked's definitions, not a regex: `[1]: <broken` is prose whose `[1]` is a reference.+ const uses = normalizeLineEndings(references);+ for (const match of uses.matchAll(SHORTCUT_REFERENCE_RE)) {+ if (+ !isEscaped(uses, match.index) &&+ labels.has(normalizeLabel(match[1]))+ ) {+ return true;+ }+ }+ return false;+}const WORD_CHARACTER_RE = /[\p{L}\p{N}_]/u;const HTML_TAG_START_RE = /[a-zA-Z/]/;@@ -221,15 +270,30 @@ const HTML_TAG_START_RE = /[a-zA-Z/]/;// paying for exactly the one split it already paid for before any of this existed.let splitMarkdown: string | null = null;let splitBlocks: readonly string[] = [];+let splitReferenceProse = "";+let splitDefinitions: readonly string[] = [];function blocksOf(markdown: string): readonly string[] {if (splitMarkdown !== markdown) {splitMarkdown = markdown;- splitBlocks = parseMarkdownIntoBlocks(markdown);+ const details = parseMarkdownBlockDetails(markdown);+ splitBlocks = details.blocks;+ splitReferenceProse = details.referenceProse.join("\n\n");+ splitDefinitions = details.definitions;}return splitBlocks;}+function referenceProseOf(markdown: string): string {+ blocksOf(markdown);+ return splitReferenceProse;+}++function definitionsOf(markdown: string): readonly string[] {+ blocksOf(markdown);+ return splitDefinitions;+}+// Which replies have to be lexed in one piece.//// marked keeps link reference definitions in one document-wide map and emits no token for a@@ -252,12 +316,9 @@ function blocksOf(markdown: string): readonly string[] {// not, so the scope would otherwise follow the reply's line ending. NOT for// `blocksOf`, whose one memo slot is shared with `parseMarkdownIntoRenderableBlocks`:// a normalised copy misses it and costs a CRLF reply two splits per render.-// Definition first is a cost decision: both are pure so the conjunction is unchanged, but only-// the one asked SECOND is skipped, and the reference scan is the dearer. `][` without a `]:` is-// the shape that separates them.+// A shortcut reference can be any `[label]`, so a definition alone is enough to pay for the split.function documentProse(markdown: string): string | null {- const normalized = normalizeLineEndings(markdown);- if (!hasLinkDefinition(normalized) || !hasLinkReference(normalized)) {+ if (!hasLinkDefinition(normalizeLineEndings(markdown))) {return null;}const prose = normalizeLineEndings(@@ -265,7 +326,13 @@ function documentProse(markdown: string): string | null {.filter((block) => !isCodeBlock(block)).join("\n"),);- return LINK_DEFINITION_LINE_RE.test(prose) && hasLinkReference(prose)+ return LINK_DEFINITION_LINE_RE.test(prose) &&+ (hasLinkReference(prose) ||+ hasShortcutReference(+ prose,+ referenceProseOf(markdown),+ definitionsOf(markdown),+ ))? prose: null;}diff --git a/studio/frontend/src/lib/parse-markdown-blocks.ts b/studio/frontend/src/lib/parse-markdown-blocks.tsindex 5d06c9b8567..dc1434515e0 100644--- a/studio/frontend/src/lib/parse-markdown-blocks.ts+++ b/studio/frontend/src/lib/parse-markdown-blocks.ts@@ -152,6 +152,57 @@ const mergeTokensIntoBlocks = (tokens: Token[]): string[] => {return mergedBlocks;};+// Prose that can still hold a link reference (code and definitions removed), and Marked's+// own definitions.+const collectReferenceProse = (+ tokens: Token[],+ out: { referenceProse: string[]; definitions: string[] },+): { referenceProse: string[]; definitions: string[] } => {+ for (const token of tokens) {+ // Indented code stays: Marked and remark disagree on some list continuations.+ if (+ (token.type === "code" && token.codeBlockStyle !== "indented") ||+ token.type === "space"+ ) {+ continue;+ }+ if (token.type === "def") {+ out.definitions.push(token.raw);+ } else if (token.type === "blockquote") {+ collectReferenceProse(token.tokens ?? [], out);+ } else if (token.type === "list") {+ for (const item of token.items) {+ collectReferenceProse(item.tokens, out);+ }+ } else {+ out.referenceProse.push(token.raw);+ }+ }+ return out;+};++// One lex for both the split and the reference prose.+export function parseMarkdownBlockDetails(markdown: string): {+ blocks: string[];+ referenceProse: string[];+ definitions: string[];+} {+ if (+ footnoteReferencePattern.test(markdown) ||+ footnoteDefinitionPattern.test(markdown)+ ) {+ return { blocks: [markdown], referenceProse: [markdown], definitions: [] };+ }+ const input = markdown.includes("\r")+ ? markdown.replace(lineEndingPattern, "\n")+ : markdown;+ const tokens = lexBlocks(input);+ return {+ blocks: mergeTokensIntoBlocks(tokens),+ ...collectReferenceProse(tokens, { referenceProse: [], definitions: [] }),+ };+}+export function parseMarkdownIntoBlocks(markdown: string): string[] {const hasFootnoteReference = footnoteReferencePattern.test(markdown);const hasFootnoteDefinition = footnoteDefinitionPattern.test(markdown);diff --git a/studio/frontend/tests/link-definition-oracle.test.ts b/studio/frontend/tests/link-definition-oracle.test.tsindex 86791e6f8de..11d5694555f 100644--- a/studio/frontend/tests/link-definition-oracle.test.ts+++ b/studio/frontend/tests/link-definition-oracle.test.ts@@ -270,3 +270,130 @@ test("ordinary code is still rendered per block", () => {);}});++test("a shortcut or collapsed reference is never split from its definition", () => {+ const failures: string[] = [];+ for (const definition of DEFINITION_CONTEXTS) {+ for (const neutral of NEUTRAL_BLOCKS) {+ for (const reference of ["[g]", "[g][]", "![g]", "[G]", "[ g ]"]) {+ for (const reply of [+ `See ${reference}.\n\n${neutral}\n\n${definition}\n`,+ `${definition}\n\n${neutral}\n\nSee ${reference}.\n`,+ ]) {+ if (asOneDocument(reply) <= asBlocks(reply)) {+ continue;+ }+ if (markdownRenderScope(reply) !== "document") {+ failures.push(JSON.stringify(reply));+ }+ }+ }+ }+ }+ assert.deepEqual(failures, [], failures.join("\n"));+});++test("a shortcut reference matches its definition the way CommonMark matches labels", () => {+ for (const [reference, definition] of [+ ["Paris is the capital [1].", "[1]: https://en.wikipedia.org/wiki/Paris"],+ ["Read [Unsloth docs] first.", "[unsloth DOCS]: https://docs.unsloth.ai"],+ ["Read [Unsloth\ndocs] first.", "[unsloth docs]: https://docs.unsloth.ai"],+ ["Read [unsloth docs] first.", "[Unsloth\tDocs]: https://docs.unsloth.ai"],+ ["Read [a\\]b] first.", "[a\\]b]: https://x.test/ab"],+ ["Read [`code`] first.", "[`code`]: https://x.test/code"],+ ["Read [1](not a link) first.", "[1]: https://x.test/one"],+ ["Read [1](foo(and(bar)) first.", "[1]: https://x.test/one"],+ ["Read [1](<foo\nbar>) first.", "[1]: https://x.test/one"],+ ['Read [1](<https://x.test>"title") first.', "[1]: https://x.test/one"],+ [+ "Read [site](https://x.test/`tag) [1] ` first.",+ "[1]: https://x.test/one",+ ],+ ["Read <https://x.test/`> [1] ` first.", "[1]: https://x.test/one"],+ ["Read <foo`[email protected]> [1] ` first.", "[1]: https://x.test/one"],+ ["Read https://x.test/a`b [1] ` first.", "[1]: https://x.test/one"],+ ["Read www.x.test/a`b [1] ` first.", "[1]: https://x.test/one"],+ [+ "Read [a [b]](https://x.test/`tag) [1] ` first.",+ "[1]: https://x.test/one",+ ],+ [+ "Read [a `[`](https://x.test/`tag) [1] ` first.",+ "[1]: https://x.test/one",+ ],+ [+ "Read [site `]`](https://x.test/`tag) [1] ` first.",+ "[1]: https://x.test/one",+ ],+ ["Read [site](foo`bar\\ ) [1] ` first.", "[1]: https://x.test/one"],+ ["[site](foo`bar\\ ) [1] `", "[1]: https://x.test/one"],+ ['[site](foo`bar "title\\\ncontinued") [1] `', "[1]: https://x.test/one"],+ ["`[x](foo`[site](url`tag)) [1] `", "[1]: https://x.test/one"],+ ["[x]: <broken [1]", "[1]: /one"],+ ['Read <span title="`"> [1] ` first.', "[1]: https://x.test/one"],+ ['Read <span hidden title="`"> [1] ` first.', "[1]: https://x.test/one"],+ ['Read <span title=">`"> [1] ` first.', "[1]: https://x.test/one"],+ ["Read <!-- ` --> [1] ` first.", "[1]: https://x.test/one"],+ ["Read <!A`> [1] ` first.", "[1]: https://x.test/one"],+ ['Read <a_b title="`"> ` [1] ` first.', "[1]: https://x.test/one"],+ ["> `open\n>\n> [1]\n> `", "[1]: https://x.test/one"],+ ["> `open\n> # heading\n> [1]\n> `", "[1]: https://x.test/one"],+ [+ '> [1](/inline "title\n> # heading\n> continuation")',+ "[1]: https://x.test/one",+ ],+ [+ `Read [1](${"(".repeat(33)}x${")".repeat(33)}) first.`,+ "[1]: https://x.test/one",+ ],+ ["See [1].\n\n[broken", "[1]: /one"],+ ["See [1].\n\n> [broken", "- [1]: /one"],+ ["See [^source note].", "[^source note]: https://x.test/source"],+ ["> [1]: /url\n2. item\n\n See [1].", "[2]: /two"],+ ["Read [SS] first.", "[\u1E9E]: https://x.test/ss"],+ ["Read [Stra\u00DFe] first.", "[STRASSE]: https://x.test/strasse"],+ ]) {+ const reply = `${reference}\n\n${definition}\n`;+ assert.ok(asOneDocument(reply) > asBlocks(reply), JSON.stringify(reply));+ assert.equal(markdownRenderScope(reply), "document", JSON.stringify(reply));+ }+});++test("inline math does not lend its backticks to a later code span", () => {+ assert.equal(+ markdownRenderScope("Math $a ` b$ [1] `\n\n[1]: /one\n"),+ "document",+ );+ assert.equal(+ markdownRenderScope("Cost $\n\nMath $a ` b$ [1] `\n\n[1]: /one\n"),+ "document",+ );+});++test("escaped backticks do not close code spans", () => {+ assert.equal(+ markdownRenderScope("Read ` [1] \\`\n\n[1]: /one\n"),+ "document",+ );+});++test("table cells do not share code span delimiters", () => {+ const reply =+ "| left | right |\n| --- | --- |\n| `open | [1] |\n| x | close` |\n\n[1]: /one\n";+ assert.ok(asOneDocument(reply) > asBlocks(reply));+ assert.equal(markdownRenderScope(reply), "document");+});++// Only labels that cannot resolve. Code spans and inline links may still pick the document+// path: that costs Copy and Download controls, never a link.+test("a bracketed label that is not a shortcut reference keeps block rendering", () => {+ for (const reply of [+ "Not a link \\[1] here.\n\n[1]: https://x.test\n",+ "Cites [2].\n\n[1]: https://x.test/1\n",+ "No uses.\n\n[1]: https://x.test/a\n[1]: https://x.test/b\n",+ "```py\nx = a[1]\n```\n\n[1]: https://x.test\n",+ ]) {+ assert.equal(asOneDocument(reply), asBlocks(reply), JSON.stringify(reply));+ assert.equal(markdownRenderScope(reply), "blocks", JSON.stringify(reply));+ }+});diff --git a/studio/frontend/tests/streaming-render-schedule.test.ts b/studio/frontend/tests/streaming-render-schedule.test.tsindex a3bb9101724..b5e04981bd4 100644--- a/studio/frontend/tests/streaming-render-schedule.test.ts+++ b/studio/frontend/tests/streaming-render-schedule.test.ts@@ -612,6 +612,28 @@ test("a bracket-dense reply does not stall the scan", () => {assert.ok(median < 150, `scope took ${median.toFixed(1)}ms on a 100k bracket-dense reply`);});+test("malformed inline-link titles do not rescan the remaining reply", () => {+ const reply = `${"[x](/url (".repeat(20_000)}\` [1] \`\n\n[1]: /one\n`;+ const started = performance.now();+ markdownRenderScope(reply);+ const elapsed = performance.now() - started;+ assert.ok(+ elapsed < 500,+ `20k malformed inline links took ${elapsed.toFixed(1)}ms`,+ );+});++test("unterminated HTML delimiters do not rescan the remaining reply", () => {+ const reply = `${"<!--".repeat(20_000)}\` [1]\n\n[1]: /one\n`;+ const started = performance.now();+ assert.equal(markdownRenderScope(reply), "document");+ const elapsed = performance.now() - started;+ assert.ok(+ elapsed < 500,+ `20k unterminated HTML comments took ${elapsed.toFixed(1)}ms`,+ );+});+// Scope decides what is committed, so it cannot follow the reply's line ending.// This label is 999 normalised, 1000 raw under CRLF.test("the render scope does not depend on the reply's line ending", () => {@@ -980,6 +1002,24 @@ test("a reply dense with `]:` and no definition does not pay per occurrence", ()`500k of \`[]:\` cost ${invalidMedian.toFixed(1)}ms; invalid candidates are being rescanned`);});+test("unclosed backtick runs of many widths do not rescan the paragraph", () => {+ // widths 1..800 expose lazy-regex tail rescans; 100ms is well below the ~900ms regression.+ let runs = "";+ for (let width = 1; width <= 800; width += 1) {+ runs += `${"`".repeat(width)}a`;+ }+ const reply = `See [1]. ${runs}\n\n[1]: https://x.test\n`;+ for (let i = 0; i < 3; i += 1) markdownRenderScope(reply + " ");+ const samples: number[] = [];+ for (let i = 0; i < 5; i += 1) {+ const t0 = performance.now();+ assert.equal(markdownRenderScope(reply + " ".repeat(i)), "document");+ samples.push(performance.now() - t0);+ }+ const median = samples.sort((a, b) => a - b)[2]!;+ assert.ok(median < 100, `321k of backtick runs cost ${median.toFixed(1)}ms`);+});+test("a reference label past the old cap still resolves against its definition", () => {// A label resolves only when BOTH probes admit it, and the definition one moved to 999 while// this one stayed at 200, so 201..999 stayed on the blocks path (unslothai/unsloth#9540).