Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: turn [1]-style citations in replies into links (#12912)

Nilaydbb2fa6b4474
diff --git a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts
index 8ba8ed48ba7..56b6a2bbc3c 100644
--- a/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts
+++ b/studio/frontend/src/components/assistant-ui/streaming-render-schedule.ts
@@ -3,7 +3,10 @@
import remend from "remend";
import { type BlockProps } from "streamdown";
-import { parseMarkdownIntoBlocks } from "../../lib/parse-markdown-blocks.ts";
+import {
+ parseMarkdownBlockDetails,
+ parseMarkdownIntoBlocks,
+} from "../../lib/parse-markdown-blocks.ts";
// How far behind the live edge a block has to be before it can be retained.
// The block list interleaves "\n\n" separators, so this is about four
@@ -212,6 +215,52 @@ function hasLinkReference(text: string): boolean {
}
return false;
}
+// A shortcut `[label]` or collapsed `[label][]` resolves against a definition too. Code spans,
+// inline links and the like are deliberately not excluded: that only adds false positives.
+const SHORTCUT_REFERENCE_RE = /\[((?:\\[\s\S]|[^[\]\\]){1,999})\]/gu;
+const DEFINITION_LABEL_RE = /\[((?:\\[\s\S]|[^\]\\]){1,999})\]:/u;
+
+// micromark's `normalizeIdentifier`, so `[SS]` finds `[\u1E9E]:` as the renderer does.
+function normalizeLabel(label: string): string {
+ return label
+ .replace(/[\t\n\r ]+/g, " ")
+ .replace(/^ | $/g, "")
+ .toLowerCase()
+ .toUpperCase();
+}
+
+function hasShortcutReference(
+ prose: string,
+ references: string,
+ definitions: readonly string[],
+): boolean {
+ const labels = new Set<string>();
+ // Marked's tokens as well: an unmatched `[` line before a definition widens the regex's label.
+ for (const definition of [
+ ...definitions,
+ ...(prose.match(LINK_DEFINITION_KEY_RE) ?? []),
+ ]) {
+ const label = DEFINITION_LABEL_RE.exec(definition)?.[1];
+ if (label !== undefined) {
+ labels.add(normalizeLabel(label));
+ }
+ }
+ labels.delete("");
+ if (labels.size === 0) {
+ return false;
+ }
+ // Marked's definitions, not a regex: `[1]: <broken` is prose whose `[1]` is a reference.
+ const uses = normalizeLineEndings(references);
+ for (const match of uses.matchAll(SHORTCUT_REFERENCE_RE)) {
+ if (
+ !isEscaped(uses, match.index) &&
+ labels.has(normalizeLabel(match[1]))
+ ) {
+ return true;
+ }
+ }
+ return false;
+}
const WORD_CHARACTER_RE = /[\p{L}\p{N}_]/u;
const HTML_TAG_START_RE = /[a-zA-Z/]/;
@@ -221,15 +270,30 @@ const HTML_TAG_START_RE = /[a-zA-Z/]/;
// paying for exactly the one split it already paid for before any of this existed.
let splitMarkdown: string | null = null;
let splitBlocks: readonly string[] = [];
+let splitReferenceProse = "";
+let splitDefinitions: readonly string[] = [];
function blocksOf(markdown: string): readonly string[] {
if (splitMarkdown !== markdown) {
splitMarkdown = markdown;
- splitBlocks = parseMarkdownIntoBlocks(markdown);
+ const details = parseMarkdownBlockDetails(markdown);
+ splitBlocks = details.blocks;
+ splitReferenceProse = details.referenceProse.join("\n\n");
+ splitDefinitions = details.definitions;
}
return splitBlocks;
}
+function referenceProseOf(markdown: string): string {
+ blocksOf(markdown);
+ return splitReferenceProse;
+}
+
+function definitionsOf(markdown: string): readonly string[] {
+ blocksOf(markdown);
+ return splitDefinitions;
+}
+
// Which replies have to be lexed in one piece.
//
// marked keeps link reference definitions in one document-wide map and emits no token for a
@@ -252,12 +316,9 @@ function blocksOf(markdown: string): readonly string[] {
// not, so the scope would otherwise follow the reply's line ending. NOT for
// `blocksOf`, whose one memo slot is shared with `parseMarkdownIntoRenderableBlocks`:
// a normalised copy misses it and costs a CRLF reply two splits per render.
-// Definition first is a cost decision: both are pure so the conjunction is unchanged, but only
-// the one asked SECOND is skipped, and the reference scan is the dearer. `][` without a `]:` is
-// the shape that separates them.
+// A shortcut reference can be any `[label]`, so a definition alone is enough to pay for the split.
function documentProse(markdown: string): string | null {
- const normalized = normalizeLineEndings(markdown);
- if (!hasLinkDefinition(normalized) || !hasLinkReference(normalized)) {
+ if (!hasLinkDefinition(normalizeLineEndings(markdown))) {
return null;
}
const prose = normalizeLineEndings(
@@ -265,7 +326,13 @@ function documentProse(markdown: string): string | null {
.filter((block) => !isCodeBlock(block))
.join("\n"),
);
- return LINK_DEFINITION_LINE_RE.test(prose) && hasLinkReference(prose)
+ return LINK_DEFINITION_LINE_RE.test(prose) &&
+ (hasLinkReference(prose) ||
+ hasShortcutReference(
+ prose,
+ referenceProseOf(markdown),
+ definitionsOf(markdown),
+ ))
? prose
: null;
}
diff --git a/studio/frontend/src/lib/parse-markdown-blocks.ts b/studio/frontend/src/lib/parse-markdown-blocks.ts
index 5d06c9b8567..dc1434515e0 100644
--- a/studio/frontend/src/lib/parse-markdown-blocks.ts
+++ b/studio/frontend/src/lib/parse-markdown-blocks.ts
@@ -152,6 +152,57 @@ const mergeTokensIntoBlocks = (tokens: Token[]): string[] => {
return mergedBlocks;
};
+// Prose that can still hold a link reference (code and definitions removed), and Marked's
+// own definitions.
+const collectReferenceProse = (
+ tokens: Token[],
+ out: { referenceProse: string[]; definitions: string[] },
+): { referenceProse: string[]; definitions: string[] } => {
+ for (const token of tokens) {
+ // Indented code stays: Marked and remark disagree on some list continuations.
+ if (
+ (token.type === "code" && token.codeBlockStyle !== "indented") ||
+ token.type === "space"
+ ) {
+ continue;
+ }
+ if (token.type === "def") {
+ out.definitions.push(token.raw);
+ } else if (token.type === "blockquote") {
+ collectReferenceProse(token.tokens ?? [], out);
+ } else if (token.type === "list") {
+ for (const item of token.items) {
+ collectReferenceProse(item.tokens, out);
+ }
+ } else {
+ out.referenceProse.push(token.raw);
+ }
+ }
+ return out;
+};
+
+// One lex for both the split and the reference prose.
+export function parseMarkdownBlockDetails(markdown: string): {
+ blocks: string[];
+ referenceProse: string[];
+ definitions: string[];
+} {
+ if (
+ footnoteReferencePattern.test(markdown) ||
+ footnoteDefinitionPattern.test(markdown)
+ ) {
+ return { blocks: [markdown], referenceProse: [markdown], definitions: [] };
+ }
+ const input = markdown.includes("\r")
+ ? markdown.replace(lineEndingPattern, "\n")
+ : markdown;
+ const tokens = lexBlocks(input);
+ return {
+ blocks: mergeTokensIntoBlocks(tokens),
+ ...collectReferenceProse(tokens, { referenceProse: [], definitions: [] }),
+ };
+}
+
export function parseMarkdownIntoBlocks(markdown: string): string[] {
const hasFootnoteReference = footnoteReferencePattern.test(markdown);
const hasFootnoteDefinition = footnoteDefinitionPattern.test(markdown);
diff --git a/studio/frontend/tests/link-definition-oracle.test.ts b/studio/frontend/tests/link-definition-oracle.test.ts
index 86791e6f8de..11d5694555f 100644
--- a/studio/frontend/tests/link-definition-oracle.test.ts
+++ b/studio/frontend/tests/link-definition-oracle.test.ts
@@ -270,3 +270,130 @@ test("ordinary code is still rendered per block", () => {
);
}
});
+
+test("a shortcut or collapsed reference is never split from its definition", () => {
+ const failures: string[] = [];
+ for (const definition of DEFINITION_CONTEXTS) {
+ for (const neutral of NEUTRAL_BLOCKS) {
+ for (const reference of ["[g]", "[g][]", "![g]", "[G]", "[ g ]"]) {
+ for (const reply of [
+ `See ${reference}.\n\n${neutral}\n\n${definition}\n`,
+ `${definition}\n\n${neutral}\n\nSee ${reference}.\n`,
+ ]) {
+ if (asOneDocument(reply) <= asBlocks(reply)) {
+ continue;
+ }
+ if (markdownRenderScope(reply) !== "document") {
+ failures.push(JSON.stringify(reply));
+ }
+ }
+ }
+ }
+ }
+ assert.deepEqual(failures, [], failures.join("\n"));
+});
+
+test("a shortcut reference matches its definition the way CommonMark matches labels", () => {
+ for (const [reference, definition] of [
+ ["Paris is the capital [1].", "[1]: https://en.wikipedia.org/wiki/Paris"],
+ ["Read [Unsloth docs] first.", "[unsloth DOCS]: https://docs.unsloth.ai"],
+ ["Read [Unsloth\ndocs] first.", "[unsloth docs]: https://docs.unsloth.ai"],
+ ["Read [unsloth docs] first.", "[Unsloth\tDocs]: https://docs.unsloth.ai"],
+ ["Read [a\\]b] first.", "[a\\]b]: https://x.test/ab"],
+ ["Read [`code`] first.", "[`code`]: https://x.test/code"],
+ ["Read [1](not a link) first.", "[1]: https://x.test/one"],
+ ["Read [1](foo(and(bar)) first.", "[1]: https://x.test/one"],
+ ["Read [1](<foo\nbar>) first.", "[1]: https://x.test/one"],
+ ['Read [1](<https://x.test>"title") first.', "[1]: https://x.test/one"],
+ [
+ "Read [site](https://x.test/`tag) [1] ` first.",
+ "[1]: https://x.test/one",
+ ],
+ ["Read <https://x.test/`> [1] ` first.", "[1]: https://x.test/one"],
+ ["Read <foo`[email protected]> [1] ` first.", "[1]: https://x.test/one"],
+ ["Read https://x.test/a`b [1] ` first.", "[1]: https://x.test/one"],
+ ["Read www.x.test/a`b [1] ` first.", "[1]: https://x.test/one"],
+ [
+ "Read [a [b]](https://x.test/`tag) [1] ` first.",
+ "[1]: https://x.test/one",
+ ],
+ [
+ "Read [a `[`](https://x.test/`tag) [1] ` first.",
+ "[1]: https://x.test/one",
+ ],
+ [
+ "Read [site `]`](https://x.test/`tag) [1] ` first.",
+ "[1]: https://x.test/one",
+ ],
+ ["Read [site](foo`bar\\ ) [1] ` first.", "[1]: https://x.test/one"],
+ ["[site](foo`bar\\ ) [1] `", "[1]: https://x.test/one"],
+ ['[site](foo`bar "title\\\ncontinued") [1] `', "[1]: https://x.test/one"],
+ ["`[x](foo`[site](url`tag)) [1] `", "[1]: https://x.test/one"],
+ ["[x]: <broken [1]", "[1]: /one"],
+ ['Read <span title="`"> [1] ` first.', "[1]: https://x.test/one"],
+ ['Read <span hidden title="`"> [1] ` first.', "[1]: https://x.test/one"],
+ ['Read <span title=">`"> [1] ` first.', "[1]: https://x.test/one"],
+ ["Read <!-- ` --> [1] ` first.", "[1]: https://x.test/one"],
+ ["Read <!A`> [1] ` first.", "[1]: https://x.test/one"],
+ ['Read <a_b title="`"> ` [1] ` first.', "[1]: https://x.test/one"],
+ ["> `open\n>\n> [1]\n> `", "[1]: https://x.test/one"],
+ ["> `open\n> # heading\n> [1]\n> `", "[1]: https://x.test/one"],
+ [
+ '> [1](/inline "title\n> # heading\n> continuation")',
+ "[1]: https://x.test/one",
+ ],
+ [
+ `Read [1](${"(".repeat(33)}x${")".repeat(33)}) first.`,
+ "[1]: https://x.test/one",
+ ],
+ ["See [1].\n\n[broken", "[1]: /one"],
+ ["See [1].\n\n> [broken", "- [1]: /one"],
+ ["See [^source note].", "[^source note]: https://x.test/source"],
+ ["> [1]: /url\n2. item\n\n See [1].", "[2]: /two"],
+ ["Read [SS] first.", "[\u1E9E]: https://x.test/ss"],
+ ["Read [Stra\u00DFe] first.", "[STRASSE]: https://x.test/strasse"],
+ ]) {
+ const reply = `${reference}\n\n${definition}\n`;
+ assert.ok(asOneDocument(reply) > asBlocks(reply), JSON.stringify(reply));
+ assert.equal(markdownRenderScope(reply), "document", JSON.stringify(reply));
+ }
+});
+
+test("inline math does not lend its backticks to a later code span", () => {
+ assert.equal(
+ markdownRenderScope("Math $a ` b$ [1] `\n\n[1]: /one\n"),
+ "document",
+ );
+ assert.equal(
+ markdownRenderScope("Cost $\n\nMath $a ` b$ [1] `\n\n[1]: /one\n"),
+ "document",
+ );
+});
+
+test("escaped backticks do not close code spans", () => {
+ assert.equal(
+ markdownRenderScope("Read ` [1] \\`\n\n[1]: /one\n"),
+ "document",
+ );
+});
+
+test("table cells do not share code span delimiters", () => {
+ const reply =
+ "| left | right |\n| --- | --- |\n| `open | [1] |\n| x | close` |\n\n[1]: /one\n";
+ assert.ok(asOneDocument(reply) > asBlocks(reply));
+ assert.equal(markdownRenderScope(reply), "document");
+});
+
+// Only labels that cannot resolve. Code spans and inline links may still pick the document
+// path: that costs Copy and Download controls, never a link.
+test("a bracketed label that is not a shortcut reference keeps block rendering", () => {
+ for (const reply of [
+ "Not a link \\[1] here.\n\n[1]: https://x.test\n",
+ "Cites [2].\n\n[1]: https://x.test/1\n",
+ "No uses.\n\n[1]: https://x.test/a\n[1]: https://x.test/b\n",
+ "```py\nx = a[1]\n```\n\n[1]: https://x.test\n",
+ ]) {
+ assert.equal(asOneDocument(reply), asBlocks(reply), JSON.stringify(reply));
+ assert.equal(markdownRenderScope(reply), "blocks", JSON.stringify(reply));
+ }
+});
diff --git a/studio/frontend/tests/streaming-render-schedule.test.ts b/studio/frontend/tests/streaming-render-schedule.test.ts
index a3bb9101724..b5e04981bd4 100644
--- a/studio/frontend/tests/streaming-render-schedule.test.ts
+++ b/studio/frontend/tests/streaming-render-schedule.test.ts
@@ -612,6 +612,28 @@ test("a bracket-dense reply does not stall the scan", () => {
assert.ok(median < 150, `scope took ${median.toFixed(1)}ms on a 100k bracket-dense reply`);
});
+test("malformed inline-link titles do not rescan the remaining reply", () => {
+ const reply = `${"[x](/url (".repeat(20_000)}\` [1] \`\n\n[1]: /one\n`;
+ const started = performance.now();
+ markdownRenderScope(reply);
+ const elapsed = performance.now() - started;
+ assert.ok(
+ elapsed < 500,
+ `20k malformed inline links took ${elapsed.toFixed(1)}ms`,
+ );
+});
+
+test("unterminated HTML delimiters do not rescan the remaining reply", () => {
+ const reply = `${"<!--".repeat(20_000)}\` [1]\n\n[1]: /one\n`;
+ const started = performance.now();
+ assert.equal(markdownRenderScope(reply), "document");
+ const elapsed = performance.now() - started;
+ assert.ok(
+ elapsed < 500,
+ `20k unterminated HTML comments took ${elapsed.toFixed(1)}ms`,
+ );
+});
+
// Scope decides what is committed, so it cannot follow the reply's line ending.
// This label is 999 normalised, 1000 raw under CRLF.
test("the render scope does not depend on the reply's line ending", () => {
@@ -980,6 +1002,24 @@ test("a reply dense with `]:` and no definition does not pay per occurrence", ()
`500k of \`[]:\` cost ${invalidMedian.toFixed(1)}ms; invalid candidates are being rescanned`);
});
+test("unclosed backtick runs of many widths do not rescan the paragraph", () => {
+ // widths 1..800 expose lazy-regex tail rescans; 100ms is well below the ~900ms regression.
+ let runs = "";
+ for (let width = 1; width <= 800; width += 1) {
+ runs += `${"`".repeat(width)}a`;
+ }
+ const reply = `See [1]. ${runs}\n\n[1]: https://x.test\n`;
+ for (let i = 0; i < 3; i += 1) markdownRenderScope(reply + " ");
+ const samples: number[] = [];
+ for (let i = 0; i < 5; i += 1) {
+ const t0 = performance.now();
+ assert.equal(markdownRenderScope(reply + " ".repeat(i)), "document");
+ samples.push(performance.now() - t0);
+ }
+ const median = samples.sort((a, b) => a - b)[2]!;
+ assert.ok(median < 100, `321k of backtick runs cost ${median.toFixed(1)}ms`);
+});
+
test("a reference label past the old cap still resolves against its definition", () => {
// A label resolves only when BOTH probes admit it, and the definition one moved to 999 while
// this one stayed at 200, so 201..999 stayed on the blocks path (unslothai/unsloth#9540).