LLM inference in C/C++
C++ ggml
ui: rendering performance follow-up (#26097)
Aleksander Grygier6e2bc65fb2a8
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelteindex acf2de12ac..92652a0a86 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte@@ -11,8 +11,7 @@classifyToolResult,formatJsonPretty,parseToolResultWithImages,- type AgenticSection,- type ToolResultLine+ type AgenticSection} from '$lib/utils';import { getBuiltinToolUi } from '$lib/constants/built-in-tools';import type { DatabaseMessageExtra } from '$lib/types';@@ -29,11 +28,10 @@let { section, open, isStreaming, attachments, onToggle }: Props = $props();const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');-- const parsedLines: ToolResultLine[] = $derived(+ const outputKind = $derived(classifyToolResult(section.toolResult));+ const parsedLines = $derived(section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []);- const outputKind = $derived(classifyToolResult(section.toolResult));</script><ToolCallBlock {section} {open} {isStreaming} meta={null} {title} {onToggle}>diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelteindex 6f30060f54..b990c3898b 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte@@ -15,7 +15,6 @@let { section, open, isStreaming, onToggle }: Props = $props();const editFileMeta = $derived(parseEditFileMeta(section));-const editDiffs = $derived((editFileMeta?.edits ?? []).map((edit) => computeLineDiff(edit.oldText, edit.newText)));diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelteindex e4b4adf151..60862dd063 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte@@ -27,7 +27,7 @@const isStreamingCall = $derived(section.type === AgenticSectionType.TOOL_CALL_STREAMING);const showSpinner = $derived(isPending || (isStreamingCall && isStreaming));- const results: SearchResult[] = $derived(extractSearchResults(section.toolResult));+ const results = $derived(extractSearchResults(section.toolResult));const query = $derived(extractSearchQuery(section.toolArgs));// Same icon-resolution chain as ChatMessageToolCallBlockDefault sodiff --git a/tools/ui/src/lib/constants/latex-protection.ts b/tools/ui/src/lib/constants/latex-protection.tsindex da27c00837..c42aec41ac 100644--- a/tools/ui/src/lib/constants/latex-protection.ts+++ b/tools/ui/src/lib/constants/latex-protection.ts@@ -28,6 +28,18 @@ export const LATEX_MATH_AND_CODE_PATTERN =/** Regex to capture the content of a $$...\\\\...$$ block (display-formula with line-break) */export const LATEX_LINEBREAK_REGEXP = /\$\$([\s\S]*?\\\\[\s\S]*?)\$\$/;+/**+ * Matches the unescaped `\[...\]` display-math delimiter and surrounding+ * context so callers can insert line-breaks around the placeholder or convert+ * to inline when the formula has a non-empty trailing context (e.g. a table+ * cell that opens with `\[` and closes with content after `\]`).+ *+ * group 1: prefix before `\[`+ * group 2: formula body+ * group 3: trailing context after `\]`+ */+export const LATEX_DISPLAY_BLOCK_REGEXP = /([\S].*?)\\\[([\s\S]*?)\\\](.*)/g;+/*** Cheap gate for `preprocessLaTeX`. Every transformation it performs is triggered* by a `$` (inline/display math, currency escaping) or a backslash escape@@ -36,6 +48,76 @@ export const LATEX_LINEBREAK_REGEXP = /\$\$([\s\S]*?\\\\[\s\S]*?)\$\$/;*/export const LATEX_TRIGGER_REGEXP = /[$\\]/;+/** Inline LaTeX math delimiter (the dollar sign). */+export const LATEX_INLINE_DELIMITER = '$';++/** Display LaTeX math delimiter (paired dollar signs). */+export const LATEX_DISPLAY_DELIMITER = '$$';++/** Matches a single non-whitespace character. */+export const LATEX_NON_WHITESPACE_REGEXP = /\S/;++/** Matches a character that may appear adjacent to `$`, indicating a non-TeX+ * context such as an identifier (`var$`, `$var`), currency ($5), or code. */+export const LATEX_NEIGHBOR_CHAR_REGEXP = /[A-Za-z0-9_$-]/;++/** Matches a single digit (used to detect currency-like `$5`). */+export const LATEX_DIGIT_REGEXP = /[0-9]/;++/** Matches the leading blockquote prefix (`> ` or `>`) on a markdown line. */+export const LATEX_BLOCKQUOTE_PREFIX_REGEXP = /^(>\s*)/;++/** Matches the placeholder inserted by the protect/restore pipeline for a+ * protected LaTeX expression. Group 1 is the index into `latexExpressions`. */+export const LATEX_PLACEHOLDER_REGEXP = /<<LATEX_(\d+)>>/g;++/** Matches the placeholder inserted by the protect/restore pipeline for a+ * protected code block. Group 1 is the index into `codeBlocks`. */+export const CODE_BLOCK_PLACEHOLDER_REGEXP = /<<CODE_BLOCK_(\d+)>>/g;++/** Matches a `$` immediately followed by a digit, which is treated as a+ * currency amount (e.g. `$5`) and escaped to `\$5` so it isn't parsed as math. */+export const LATEX_CURRENCY_DOLLAR_REGEXP = /\$(?=\d)/g;++/** Captures remaining `$$...$$`, `\[...\]`, `\(...\)` (only unescaped via+ * `(?<!\\)`) after the display-block pass has run. Group 1 holds the+ * matched formula. */+export const LATEX_PROTECT_REGEXP =+ /(\$\$[\s\S]*?\$\$|(?<!\\)\\\[[\s\S]*?\\\]|(?<!\\)\\\(.*?\\\))/g;++/** Matches unescaped inline `\(...\)` (at least one char inside) used to+ * convert `\(` → `$` after the protect pass. */+export const LATEX_INLINE_CONVERT_REGEXP = /(?<!\\)\\\((.+?)\\\)/g;++/** Matches unescaped display `\[...\]` used to convert `\[` → `$$`+ * after the protect pass. */+export const LATEX_DISPLAY_CONVERT_REGEXP = /(?<!\\)\\\[([\s\S]*?)\\\]/g;++/** `\(` — opens an inline LaTeX math block. */+export const LATEX_INLINE_OPEN = '\\(';++/** `\)` — closes an inline LaTeX math block. */+export const LATEX_INLINE_CLOSE = '\\)';++/** `\[` — opens a display LaTeX math block. */+export const LATEX_DISPLAY_OPEN = '\\[';++/** `\]` — closes a display LaTeX math block. */+export const LATEX_DISPLAY_CLOSE = '\\]';++/** `\` — the LaTeX escape character. */+export const LATEX_BACKSLASH = '\\';++/** `\$` — dollar sign escaped so it isn't parsed as math (used to disambiguate+ * currency amounts like `$5`). */+export const LATEX_CURRENCY_ESCAPE = '\\$';++/** `\ce{` — mhchem chemistry command prefix. */+export const LATEX_MHCHEM_CE = '\\ce{';++/** `\pu{` — mhchem physics-unit command prefix. */+export const LATEX_MHCHEM_PU = '\\pu{';+/** map from mchem-regexp to replacement */export const MHCHEM_PATTERN_MAP: readonly [RegExp, string][] = [[/(\s)\$\\ce{/g, '$1$\\\\ce{'],diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.tsindex aa8c287c3a..e670367512 100644--- a/tools/ui/src/lib/utils/agentic.ts+++ b/tools/ui/src/lib/utils/agentic.ts@@ -92,8 +92,17 @@ function deriveSingleTurnSections(// 3. Persisted tool calls (from message.toolCalls field)const toolCalls = parseToolCalls(message.toolCalls);++ // Index tool messages by toolCallId for O(1) lookup instead of O(n) find()+ const toolMsgById = new Map<string, DatabaseMessage>();+ for (const tm of toolMessages) {+ if (tm.toolCallId && !toolMsgById.has(tm.toolCallId)) {+ toolMsgById.set(tm.toolCallId, tm);+ }+ }+for (const tc of toolCalls) {- const resultMsg = toolMessages.find((m) => m.toolCallId === tc.id);+ const resultMsg = tc.id ? toolMsgById.get(tc.id) : undefined;// Only show as pending/loading if we're actively streaming; otherwise it's just a tool call without resultconst type = resultMsg? AgenticSectionType.TOOL_CALL@@ -112,9 +121,10 @@ function deriveSingleTurnSections(}// 4. Streaming tool calls (not yet persisted - currently being received)+ const persistedIds = new Set(toolCalls.map((t) => t.id).filter(Boolean));for (const tc of streamingToolCalls) {// Skip if already in persisted tool calls- if (tc.id && toolCalls.find((t) => t.id === tc.id)) continue;+ if (tc.id && persistedIds.has(tc.id)) continue;sections.push({type: AgenticSectionType.TOOL_CALL_STREAMING,content: '',@@ -281,15 +291,31 @@ export function splitSearchSummaryList(return { lines };}+/** Bounded cache for parseToolResultWithImages results. */+const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32;+const toolResultLinesCache = new Map<string, ToolResultLine[]>();+/*** Parse tool result text into lines, matching image attachments by name.+ * Memoized: called per render during streaming on unchanged tool result+ * strings with unchanged extras.*/export function parseToolResultWithImages(toolResult: string,extras?: DatabaseMessageExtra[]): ToolResultLine[] {+ // Cache key includes image attachment names so we recompute when+ // attachments change, even if the count stays the same.+ const imageNames = (extras ?? [])+ .filter((e): e is DatabaseMessageExtraImageFile => e.type === AttachmentType.IMAGE)+ .map((e) => e.name)+ .join(NEWLINE);+ const cacheKey = `${imageNames}:${toolResult}`;+ const cached = toolResultLinesCache.get(cacheKey);+ if (cached !== undefined) return cached;+const lines = toolResult.split(NEWLINE);- return lines.map((line) => {+ const result = lines.map((line) => {const match = line.match(ATTACHMENT_SAVED_REGEX);if (!match || !extras) return { text: line };@@ -301,8 +327,19 @@ export function parseToolResultWithImages(return { text: line, image };});++ if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) {+ toolResultLinesCache.delete(toolResultLinesCache.keys().next().value!);+ }+ toolResultLinesCache.set(cacheKey, result);++ return result;}+/** Bounded cache for classifyToolResult results. */+const CLASSIFY_CACHE_MAX_SIZE = 32;+const classifyCache = new Map<string, ToolResultKind>();+/*** Pick a renderer tier for a tool's result content.*@@ -312,25 +349,39 @@ export function parseToolResultWithImages(* through MarkdownContent for proper formatting.* text - everything else, rendered as plain text lines (with image* attachment resolution as a side effect).+ * Memoized: called per render during streaming on unchanged content.*/export function classifyToolResult(content: string | undefined): ToolResultKind {if (!content) return ToolResultKind.TEXT;++ const cached = classifyCache.get(content);+ if (cached !== undefined) return cached;+const trimmed = content.trim();if (!trimmed) return ToolResultKind.TEXT;+ let result: ToolResultKind = ToolResultKind.TEXT;+// Strongest signal: JSON object/array round-trips through JSON.parse.if (TOOL_RESULT_JSON_OPEN_REGEX.test(trimmed)) {try {JSON.parse(trimmed);- return ToolResultKind.JSON;+ result = ToolResultKind.JSON;} catch (error) {console.error('[agentic] tool result looked like JSON but failed to parse:', error);}}- if (looksLikeMarkdown(trimmed)) return ToolResultKind.MARKDOWN;+ if (result === ToolResultKind.TEXT && looksLikeMarkdown(trimmed)) {+ result = ToolResultKind.MARKDOWN;+ }++ if (classifyCache.size >= CLASSIFY_CACHE_MAX_SIZE) {+ classifyCache.delete(classifyCache.keys().next().value!);+ }+ classifyCache.set(content, result);- return ToolResultKind.TEXT;+ return result;}/**@@ -370,19 +421,35 @@ function looksLikeMarkdown(content: string): boolean {return false;}+/** Bounded cache for parsed tool-call JSON blobs. */+const TOOL_CALLS_CACHE_MAX_SIZE = 64;+const toolCallsParseCache = new Map<string, ApiChatCompletionToolCall[]>();+/*** Safely parse the toolCalls JSON string from a DatabaseMessage.+ * Memoized: the same JSON string is re-parsed on every render during+ * streaming, which is wasted CPU since tool calls don't change mid-stream.*/function parseToolCalls(toolCallsJson?: string): ApiChatCompletionToolCall[] {if (!toolCallsJson) return [];+ const cached = toolCallsParseCache.get(toolCallsJson);+ if (cached) return cached;++ let result: ApiChatCompletionToolCall[];try {const parsed = JSON.parse(toolCallsJson);-- return Array.isArray(parsed) ? parsed : [];+ result = Array.isArray(parsed) ? parsed : [];} catch {- return [];+ result = [];+ }++ if (toolCallsParseCache.size >= TOOL_CALLS_CACHE_MAX_SIZE) {+ toolCallsParseCache.delete(toolCallsParseCache.keys().next().value!);}+ toolCallsParseCache.set(toolCallsJson, result);++ return result;}/**diff --git a/tools/ui/src/lib/utils/code.ts b/tools/ui/src/lib/utils/code.tsindex 44b9b48415..35f3877f4a 100644--- a/tools/ui/src/lib/utils/code.ts+++ b/tools/ui/src/lib/utils/code.ts@@ -34,6 +34,10 @@ function escapeCode(code: string): string {return code.replace(AMPERSAND_REGEX, '&').replace(LT_REGEX, '<').replace(GT_REGEX, '>');}+/** Bounded cache for highlightCode results. */+const HIGHLIGHT_CACHE_MAX_SIZE = 64;+const highlightCache = new Map<string, string>();+/*** Highlights code using highlight.js* @param code - The code to highlight@@ -47,23 +51,37 @@ function escapeCode(code: string): string {export function highlightCode(code: string, language: string, autoDetect = true): string {if (!code) return '';+ // Cache key includes language and autoDetect flag since results differ.+ // During streaming, the same code string may be highlighted repeatedly+ // (e.g., when text after a code block changes but the code itself doesn't).+ const cacheKey = `${language}:${autoDetect}:${code}`;+ const cached = highlightCache.get(cacheKey);+ if (cached) return cached;+const trimmed = trimCodePadding(code);+ let result: string;try {const lang = language.toLowerCase();const isSupported = hljs.getLanguage(lang);if (isSupported) {- return hljs.highlight(trimmed, { language: lang }).value;+ result = hljs.highlight(trimmed, { language: lang }).value;} else if (autoDetect) {- return hljs.highlightAuto(trimmed).value;+ result = hljs.highlightAuto(trimmed).value;} else {- return escapeCode(trimmed);+ result = escapeCode(trimmed);}} catch {- // Fallback to escaped plain text- return escapeCode(trimmed);+ result = escapeCode(trimmed);}++ if (highlightCache.size >= HIGHLIGHT_CACHE_MAX_SIZE) {+ highlightCache.delete(highlightCache.keys().next().value!);+ }+ highlightCache.set(cacheKey, result);++ return result;}export { trimCodePadding };diff --git a/tools/ui/src/lib/utils/latex-protection.ts b/tools/ui/src/lib/utils/latex-protection.tsindex 573eb92978..bbeed82500 100644--- a/tools/ui/src/lib/utils/latex-protection.ts+++ b/tools/ui/src/lib/utils/latex-protection.ts@@ -1,9 +1,31 @@import {+ CODE_BLOCK_PLACEHOLDER_REGEXP,CODE_BLOCK_REGEXP,+ LATEX_BACKSLASH,+ LATEX_BLOCKQUOTE_PREFIX_REGEXP,+ LATEX_CURRENCY_DOLLAR_REGEXP,+ LATEX_CURRENCY_ESCAPE,+ LATEX_DIGIT_REGEXP,+ LATEX_DISPLAY_BLOCK_REGEXP,+ LATEX_DISPLAY_CLOSE,+ LATEX_DISPLAY_CONVERT_REGEXP,+ LATEX_DISPLAY_DELIMITER,+ LATEX_DISPLAY_OPEN,+ LATEX_INLINE_CLOSE,+ LATEX_INLINE_CONVERT_REGEXP,+ LATEX_INLINE_DELIMITER,+ LATEX_INLINE_OPEN,LATEX_MATH_AND_CODE_PATTERN,+ LATEX_MHCHEM_CE,+ LATEX_MHCHEM_PU,LATEX_LINEBREAK_REGEXP,+ LATEX_NEIGHBOR_CHAR_REGEXP,+ LATEX_NON_WHITESPACE_REGEXP,+ LATEX_PLACEHOLDER_REGEXP,+ LATEX_PROTECT_REGEXP,LATEX_TRIGGER_REGEXP,- MHCHEM_PATTERN_MAP+ MHCHEM_PATTERN_MAP,+ NEWLINE} from '$lib/constants';/**@@ -20,13 +42,13 @@ import {* @returns The processed string with LaTeX replaced by placeholders.*/export function maskInlineLaTeX(content: string, latexExpressions: string[]): string {- if (!content.includes('$')) {+ if (!content.includes(LATEX_INLINE_DELIMITER)) {return content;}return content- .split('\n')+ .split(NEWLINE).map((line) => {- if (line.indexOf('$') == -1) {+ if (line.indexOf(LATEX_INLINE_DELIMITER) == -1) {return line;}@@ -34,7 +56,7 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): stlet currentPosition = 0;while (currentPosition < line.length) {- const openDollarIndex = line.indexOf('$', currentPosition);+ const openDollarIndex = line.indexOf(LATEX_INLINE_DELIMITER, currentPosition);if (openDollarIndex == -1) {processedLine += line.slice(currentPosition);@@ -42,7 +64,7 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): st}// Is there a next $-sign?- const closeDollarIndex = line.indexOf('$', openDollarIndex + 1);+ const closeDollarIndex = line.indexOf(LATEX_INLINE_DELIMITER, openDollarIndex + 1);if (closeDollarIndex == -1) {processedLine += line.slice(currentPosition);@@ -62,14 +84,14 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): stshouldSkipAsNonLatex = true;}- if (/[A-Za-z0-9_$-]/.test(charBeforeOpen)) {+ if (LATEX_NEIGHBOR_CHAR_REGEXP.test(charBeforeOpen)) {// Character, digit, $, _ or - before first '$', no TeX.shouldSkipAsNonLatex = true;}if (- /[0-9]/.test(charAfterOpen) &&- (/[A-Za-z0-9_$-]/.test(charAfterClose) || ' ' == charBeforeClose)+ LATEX_DIGIT_REGEXP.test(charAfterOpen) &&+ (LATEX_NEIGHBOR_CHAR_REGEXP.test(charAfterClose) || ' ' == charBeforeClose)) {// First $ seems to belong to an amount.shouldSkipAsNonLatex = true;@@ -92,7 +114,7 @@ export function maskInlineLaTeX(content: string, latexExpressions: string[]): streturn processedLine;})- .join('\n');+ .join(NEWLINE);}function escapeBrackets(text: string): string {@@ -107,9 +129,9 @@ function escapeBrackets(text: string): string {if (codeBlock != null) {return codeBlock;} else if (squareBracket != null) {- return `$$${squareBracket}$$`;+ return `${LATEX_DISPLAY_DELIMITER}${squareBracket}${LATEX_DISPLAY_DELIMITER}`;} else if (roundBracket != null) {- return `$${roundBracket}$`;+ return `${LATEX_INLINE_DELIMITER}${roundBracket}${LATEX_INLINE_DELIMITER}`;}return match;@@ -145,32 +167,49 @@ const doEscapeMhchem = false;* preprocessLaTeX("Price: $10. The equation is \\(x^2\\).")* // → "Price: $10. The equation is $x^2$."*/+/** Bounded cache for preprocessLaTeX results. */+const LATEX_CACHE_MAX_SIZE = 64;+const latexCache = new Map<string, string>();+export function preprocessLaTeX(content: string): string {// See also:// https://github.com/danny-avila/LibreChat/blob/main/client/src/utils/latex.ts+ // Memoize on the input string. During streaming the prefix before an+ // incomplete code block stays the same across multiple tokens, so the+ // full protect/restore pipeline would re-run unnecessarily.+ const cached = latexCache.get(content);+ if (cached !== undefined) return cached;++ // Save original before the function mutates `content` through steps 0-8+ const originalContent = content;+// Every step below keys off a `$` or a backslash escape (\[ \] \( \) \ce{ \pu{).// With neither present the protect/restore passes round-trip the input// unchanged, so skip them: the step 2 scan is O(n^2) in line length and costs// ~90ms on a 26KB single-line message that contains no math at all. This// matters during streaming, where the whole message is reprocessed per frame.if (!LATEX_TRIGGER_REGEXP.test(content)) {+ if (latexCache.size >= LATEX_CACHE_MAX_SIZE) {+ latexCache.delete(latexCache.keys().next().value!);+ }+ latexCache.set(originalContent, content);return content;}// Step 0: Temporarily remove blockquote markers (>) to process LaTeX correctly// Store the structure so we can restore it laterconst blockquoteMarkers: Map<number, string> = new Map();- const lines = content.split('\n');+ const lines = content.split(NEWLINE);const processedLines = lines.map((line, index) => {- const match = line.match(/^(>\s*)/);+ const match = line.match(LATEX_BLOCKQUOTE_PREFIX_REGEXP);if (match) {blockquoteMarkers.set(index, match[1]);return line.slice(match[1].length);}return line;});- content = processedLines.join('\n');+ content = processedLines.join(NEWLINE);// Step 1: Protect code blocksconst codeBlocks: string[] = [];@@ -187,58 +226,52 @@ export function preprocessLaTeX(content: string): string {// Match \S...\[...\] and protect them and insert a line-break.// Guarded: with no `\[` present this pattern still probes every start offset,// expanding `.*?` to the end of each line before failing - O(n^2) for nothing.- if (content.includes('\\[')) {- content = content.replace(- /([\S].*?)\\\[([\s\S]*?)\\\](.*)/g,- (match, group1, group2, group3) => {- // Check if there are characters following the formula (display-formula in a table-cell?)- if (group1.endsWith('\\')) {- return match; // Backslash before \[, do nothing.- }- const hasSuffix = /\S/.test(group3);- let optBreak;-- if (hasSuffix) {- latexExpressions.push(`\\(${group2.trim()}\\)`); // Convert into inline.- optBreak = '';- } else {- latexExpressions.push(`\\[${group2}\\]`);- optBreak = '\n';- }-- return `${group1}${optBreak}<<LATEX_${latexExpressions.length - 1}>>${optBreak}${group3}`;+ if (content.includes(LATEX_DISPLAY_OPEN)) {+ content = content.replace(LATEX_DISPLAY_BLOCK_REGEXP, (match, group1, group2, group3) => {+ // Check if there are characters following the formula (display-formula in a table-cell?)+ if (group1.endsWith(LATEX_BACKSLASH)) {+ return match; // Backslash before \[, do nothing.}- );+ const hasSuffix = LATEX_NON_WHITESPACE_REGEXP.test(group3);+ let optBreak;++ if (hasSuffix) {+ latexExpressions.push(`${LATEX_INLINE_OPEN}${group2.trim()}${LATEX_INLINE_CLOSE}`); // Convert into inline.+ optBreak = '';+ } else {+ latexExpressions.push(`${LATEX_DISPLAY_OPEN}${group2}${LATEX_DISPLAY_CLOSE}`);+ optBreak = NEWLINE;+ }++ return `${group1}${optBreak}<<LATEX_${latexExpressions.length - 1}>>${optBreak}${group3}`;+ });}// Match \(...\), \[...\], $$...$$ and protect them- content = content.replace(- /(\$\$[\s\S]*?\$\$|(?<!\\)\\\[[\s\S]*?\\\]|(?<!\\)\\\(.*?\\\))/g,- (match) => {- latexExpressions.push(match);+ content = content.replace(LATEX_PROTECT_REGEXP, (match) => {+ latexExpressions.push(match);- return `<<LATEX_${latexExpressions.length - 1}>>`;- }- );+ return `<<LATEX_${latexExpressions.length - 1}>>`;+ });// Protect inline $...$ but NOT if it looks like money (e.g., $10, $3.99)content = maskInlineLaTeX(content, latexExpressions);// Step 3: Escape standalone $ before digits (currency like $5 → \$5)// (Now that inline math is protected, this will only escape dollars not already protected)- content = content.replace(/\$(?=\d)/g, '\\$');+ content = content.replace(LATEX_CURRENCY_DOLLAR_REGEXP, LATEX_CURRENCY_ESCAPE);// Step 4: Restore protected LaTeX expressions (they are valid)- content = content.replace(/<<LATEX_(\d+)>>/g, (_, index) => {+ content = content.replace(LATEX_PLACEHOLDER_REGEXP, (_, index) => {let expr = latexExpressions[parseInt(index)];const match = expr.match(LATEX_LINEBREAK_REGEXP);if (match) {// Katex: The $$-delimiters should be in their own line// if there are \\-line-breaks.const formula = match[1];- const prefix = formula.startsWith('\n') ? '' : '\n';- const suffix = formula.endsWith('\n') ? '' : '\n';- expr = '$$' + prefix + formula + suffix + '$$';+ const prefix = formula.startsWith(NEWLINE) ? '' : NEWLINE;+ const suffix = formula.endsWith(NEWLINE) ? '' : NEWLINE;+ expr = LATEX_DISPLAY_DELIMITER + prefix + formula + suffix + LATEX_DISPLAY_DELIMITER;}return expr;});@@ -247,7 +280,7 @@ export function preprocessLaTeX(content: string): string {// This must happen BEFORE restoring code blocks to avoid affecting code contentcontent = escapeBrackets(content);- if (doEscapeMhchem && (content.includes('\\ce{') || content.includes('\\pu{'))) {+ if (doEscapeMhchem && (content.includes(LATEX_MHCHEM_CE) || content.includes(LATEX_MHCHEM_PU))) {content = escapeMhchem(content);}@@ -257,31 +290,38 @@ export function preprocessLaTeX(content: string): string {// Using the look‑behind pattern `(?<!\\)` we skip matches// that are preceded by a backslash, e.g.// `Definitions\\(also called macros)` (title of chapter 20 in The TeXbook).- .replace(/(?<!\\)\\\((.+?)\\\)/g, '$$$1$') // inline+ .replace(LATEX_INLINE_CONVERT_REGEXP, (_, formula: string) => {+ return `${LATEX_INLINE_DELIMITER}${formula}${LATEX_INLINE_DELIMITER}`;+ }) // inline.replace(// Using the look‑behind pattern `(?<!\\)` we skip matches// that are preceded by a backslash, e.g. `\\[4pt]`.- /(?<!\\)\\\[([\s\S]*?)\\\]/g, // display, see also PR #16599- (_, content: string) => {- return `$$${content}$$`;+ LATEX_DISPLAY_CONVERT_REGEXP, // display, see also PR #16599+ (_, formula: string) => {+ return `${LATEX_DISPLAY_DELIMITER}${formula}${LATEX_DISPLAY_DELIMITER}`;});// Step 7: Restore code blocks// This happens AFTER all LaTeX conversions to preserve code content- content = content.replace(/<<CODE_BLOCK_(\d+)>>/g, (_, index) => {+ content = content.replace(CODE_BLOCK_PLACEHOLDER_REGEXP, (_, index) => {return codeBlocks[parseInt(index)];});// Step 8: Restore blockquote markersif (blockquoteMarkers.size > 0) {- const finalLines = content.split('\n');+ const finalLines = content.split(NEWLINE);const restoredLines = finalLines.map((line, index) => {const marker = blockquoteMarkers.get(index);return marker ? marker + line : line;});- content = restoredLines.join('\n');+ content = restoredLines.join(NEWLINE);+ }++ if (latexCache.size >= LATEX_CACHE_MAX_SIZE) {+ latexCache.delete(latexCache.keys().next().value!);}+ latexCache.set(originalContent, content);return content;}diff --git a/tools/ui/src/lib/utils/parse-partial-json-args.ts b/tools/ui/src/lib/utils/parse-partial-json-args.tsindex 09c1122890..58439bd6e3 100644--- a/tools/ui/src/lib/utils/parse-partial-json-args.ts+++ b/tools/ui/src/lib/utils/parse-partial-json-args.ts@@ -14,70 +14,94 @@ const JSON_ARRAY_CLOSE = ']';// comma when the model cut off mid-key.const TRAILING_JSON_PUNCTUATION_REGEX = /,?\s*$/;+/** Bounded cache for parsePartialJsonArgs results. */+const PARTIAL_JSON_CACHE_MAX_SIZE = 32;+const partialJsonCache = new Map<string, Record<string, unknown> | null>();++function cacheResult(input: string, result: Record<string, unknown> | null): void {+ if (partialJsonCache.size >= PARTIAL_JSON_CACHE_MAX_SIZE) {+ partialJsonCache.delete(partialJsonCache.keys().next().value!);+ }+ partialJsonCache.set(input, result);+}+// Parse partial tool-arg JSON streamed token-by-token. Closes any// unterminated string and dangling open containers (in reverse order),// so parsers can still surface keys already received while the call-// is still in flight.+// is still in flight. Memoized: the char-by-char scanner runs on every+// render during streaming even when toolArgs hasn't changed.export function parsePartialJsonArgs(toolArgsString: string): Record<string, unknown> | null {+ const cached = partialJsonCache.get(toolArgsString);+ if (cached !== undefined) return cached;++ let result: Record<string, unknown> | null;+try {const parsed: unknown = JSON.parse(toolArgsString);- if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {- return parsed as Record<string, unknown>;- }- return null;+ result =+ parsed && typeof parsed === 'object' && !Array.isArray(parsed)+ ? (parsed as Record<string, unknown>)+ : null;} catch {- let inString = false;- let escape = false;- const stack: ('{' | '[')[] = [];+ result = scanPartialJson(toolArgsString);+ }- for (let i = 0; i < toolArgsString.length; i++) {- const ch = toolArgsString[i];- if (escape) {- escape = false;- continue;- }- if (ch === JSON_BACKSLASH && inString) {- escape = true;- continue;- }- if (ch === JSON_QUOTE) {- inString = !inString;- continue;- }- if (inString) continue;- if (ch === JSON_OBJECT_OPEN) stack.push(JSON_OBJECT_OPEN);- else if (ch === JSON_OBJECT_CLOSE) {- if (stack.length === 0 || stack[stack.length - 1] !== JSON_OBJECT_OPEN) return null;- stack.pop();- } else if (ch === JSON_ARRAY_OPEN) stack.push(JSON_ARRAY_OPEN);- else if (ch === JSON_ARRAY_CLOSE) {- if (stack.length === 0 || stack[stack.length - 1] !== JSON_ARRAY_OPEN) return null;- stack.pop();- }- }+ cacheResult(toolArgsString, result);+ return result;+}++/** Char-by-char scanner for unterminated partial JSON. */+function scanPartialJson(toolArgsString: string): Record<string, unknown> | null {+ let inString = false;+ let escape = false;+ const stack: ('{' | '[')[] = [];- let completed = toolArgsString;+ for (let i = 0; i < toolArgsString.length; i++) {+ const ch = toolArgsString[i];if (escape) {- // Dangling escape at end of partial JSON: escape the trailing- // backslash as a literal so we can close the string cleanly.- completed += JSON_BACKSLASH;+ escape = false;+ continue;}- if (inString) completed += JSON_QUOTE;- if (!inString) completed = completed.replace(TRAILING_JSON_PUNCTUATION_REGEX, '');-- // Close in reverse nesting order: innermost container first.- for (let i = stack.length - 1; i >= 0; i--) {- completed += stack[i] === JSON_OBJECT_OPEN ? JSON_OBJECT_CLOSE : JSON_ARRAY_CLOSE;+ if (ch === JSON_BACKSLASH && inString) {+ escape = true;+ continue;}-- try {- const parsed: unknown = JSON.parse(completed);- if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {- return parsed as Record<string, unknown>;- }- return null;- } catch {- return null;+ if (ch === JSON_QUOTE) {+ inString = !inString;+ continue;+ }+ if (inString) continue;+ if (ch === JSON_OBJECT_OPEN) stack.push(JSON_OBJECT_OPEN);+ else if (ch === JSON_OBJECT_CLOSE) {+ if (stack.length === 0 || stack[stack.length - 1] !== JSON_OBJECT_OPEN) return null;+ stack.pop();+ } else if (ch === JSON_ARRAY_OPEN) stack.push(JSON_ARRAY_OPEN);+ else if (ch === JSON_ARRAY_CLOSE) {+ if (stack.length === 0 || stack[stack.length - 1] !== JSON_ARRAY_OPEN) return null;+ stack.pop();}}++ let completed = toolArgsString;+ if (escape) {+ // Dangling escape at end of partial JSON: escape the trailing+ // backslash as a literal so we can close the string cleanly.+ completed += JSON_BACKSLASH;+ }+ if (inString) completed += JSON_QUOTE;+ if (!inString) completed = completed.replace(TRAILING_JSON_PUNCTUATION_REGEX, '');++ // Close in reverse nesting order: innermost container first.+ for (let i = stack.length - 1; i >= 0; i--) {+ completed += stack[i] === JSON_OBJECT_OPEN ? JSON_OBJECT_CLOSE : JSON_ARRAY_CLOSE;+ }++ try {+ const parsed: unknown = JSON.parse(completed);+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed)+ ? (parsed as Record<string, unknown>)+ : null;+ } catch {+ return null;+ }}diff --git a/tools/ui/src/lib/utils/search-results.ts b/tools/ui/src/lib/utils/search-results.tsindex 861b0d2d32..d090e6dc4b 100644--- a/tools/ui/src/lib/utils/search-results.ts+++ b/tools/ui/src/lib/utils/search-results.ts@@ -155,41 +155,71 @@ function parseChunk(chunk: string): SearchResult | null {return result;}+/** Bounded cache for extractSearchResults results. */+const SEARCH_RESULTS_CACHE_MAX_SIZE = 32;+const searchResultsCache = new Map<string, SearchResult[]>();+/*** Extract a SearchResult[] from a tool-result string. Returns `[]` when* the input does not match the expected shape — useful for branching* between dedicated search-results rendering and the generic tool-call- * block.+ * block. Memoized: called per render during streaming on unchanged+ * tool result strings.*/export function extractSearchResults(text: string | undefined | null): SearchResult[] {if (!text) return [];+ const cached = searchResultsCache.get(text);+ if (cached) return cached;+const results: SearchResult[] = [];for (const chunk of splitChunks(text)) {const parsed = parseChunk(chunk);if (parsed) results.push(parsed);}++ if (searchResultsCache.size >= SEARCH_RESULTS_CACHE_MAX_SIZE) {+ searchResultsCache.delete(searchResultsCache.keys().next().value!);+ }+ searchResultsCache.set(text, results);+return results;}+/** Bounded cache for extractSearchQuery results. */+const SEARCH_QUERY_CACHE_MAX_SIZE = 32;+const searchQueryCache = new Map<string, string>();+/*** Best-effort extraction of the search query out of a tool call's JSON* argument blob. Currently looks for a `query` field (the convention* used by Exa and most web-search MCP servers); returns an empty string- * if it cannot be located.+ * if it cannot be located. Memoized: called per render during streaming+ * on unchanged tool args strings.*/export function extractSearchQuery(toolArgs: string | undefined | null): string {if (!toolArgs) return '';++ const cached = searchQueryCache.get(toolArgs);+ if (cached !== undefined) return cached;++ let result = '';try {const parsed: unknown = JSON.parse(toolArgs);if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {const candidate = (parsed as Record<string, unknown>)[SEARCH_TOOL_QUERY_FIELD];- if (typeof candidate === 'string') return candidate.trim();+ if (typeof candidate === 'string') result = candidate.trim();}} catch {- return '';+ result = '';+ }++ if (searchQueryCache.size >= SEARCH_QUERY_CACHE_MAX_SIZE) {+ searchQueryCache.delete(searchQueryCache.keys().next().value!);}- return '';+ searchQueryCache.set(toolArgs, result);++ return result;}/**diff --git a/tools/ui/tests/unit/parse-toolcalls-memo.test.ts b/tools/ui/tests/unit/parse-toolcalls-memo.test.tsnew file mode 100644index 0000000000..85187febcb--- /dev/null+++ b/tools/ui/tests/unit/parse-toolcalls-memo.test.ts@@ -0,0 +1,146 @@+// Tests for the memoized parseToolCalls and O(1) tool message lookup in+// deriveAgenticSections. These were added to prevent regressions where+// streaming text tokens trigger redundant JSON.parse calls on unchanged+// tool call data.++import { describe, it, expect, vi } from 'vitest';+import { deriveAgenticSections } from '$lib/utils/agentic';+import type { ApiChatCompletionToolCall } from '$lib/types/api';+import type { DatabaseMessage } from '$lib/types/database';+import { MessageRole, AgenticSectionType } from '$lib/enums';++function makeMessage(overrides: Partial<DatabaseMessage>): DatabaseMessage {+ return {+ id: 'm1',+ convId: 'c1',+ type: 'text',+ timestamp: 0,+ role: MessageRole.ASSISTANT,+ content: '',+ parent: null,+ children: [],+ ...overrides+ } as DatabaseMessage;+}++describe('parseToolCalls memoization', () => {+ it('returns the same array reference for the same JSON string', () => {+ // parseToolCalls is not exported, but deriveAgenticSections uses it+ // internally. We verify memoization through behavior: calling+ // deriveAgenticSections twice with the same toolCalls should not+ // re-parse (which we verify by checking the returned sections+ // are equivalent).+ const toolCallsJson = JSON.stringify([+ { id: 'call_1', type: 'function', function: { name: 'test', arguments: '{}' } }+ ]);++ const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });+ const sections1 = deriveAgenticSections(msg, [], [], false);+ const sections2 = deriveAgenticSections(msg, [], [], false);++ expect(sections1).toHaveLength(sections2.length);+ expect(sections1[0].type).toBe(sections2[0].type);+ });++ it('does not re-parse JSON on cache hit', () => {+ const toolCallsJson = JSON.stringify([+ { id: 'call_1', type: 'function', function: { name: 'test', arguments: '{}' } }+ ]);++ const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });+ const spy = vi.spyOn(JSON, 'parse');++ deriveAgenticSections(msg, [], [], false);+ const callsAfterFirst = spy.mock.calls.length;++ deriveAgenticSections(msg, [], [], false);+ expect(spy.mock.calls.length).toBe(callsAfterFirst);++ spy.mockRestore();+ });++ it('handles empty/undefined toolCalls without error', () => {+ const msg = makeMessage({ content: 'hello' });+ const sections = deriveAgenticSections(msg, [], [], false);++ expect(sections).toHaveLength(1);+ expect(sections[0].type).toBe(AgenticSectionType.TEXT);+ });++ it('handles invalid JSON gracefully', () => {+ const msg = makeMessage({ content: 'hello', toolCalls: '{invalid' });+ const sections = deriveAgenticSections(msg, [], [], false);++ // Should return just the text section, no tool call sections+ expect(sections).toHaveLength(1);+ expect(sections[0].type).toBe(AgenticSectionType.TEXT);+ });+});++describe('deriveAgenticSections O(1) tool message lookup', () => {+ it('matches tool messages to tool calls by toolCallId', () => {+ const toolCallsJson = JSON.stringify([+ { id: 'call_1', type: 'function', function: { name: 'test_1', arguments: '{}' } },+ { id: 'call_2', type: 'function', function: { name: 'test_2', arguments: '{}' } }+ ]);++ const toolMessages = [+ makeMessage({ role: MessageRole.TOOL, toolCallId: 'call_1', content: 'result_1' }),+ makeMessage({ role: MessageRole.TOOL, toolCallId: 'call_2', content: 'result_2' })+ ];++ const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });+ const sections = deriveAgenticSections(msg, toolMessages, [], false);++ // Expect: TEXT + 2 TOOL_CALL sections+ const toolCallSections = sections.filter((s) => s.type === AgenticSectionType.TOOL_CALL);+ expect(toolCallSections).toHaveLength(2);+ expect(toolCallSections[0].toolResult).toBe('result_1');+ expect(toolCallSections[1].toolResult).toBe('result_2');+ });++ it('handles missing tool messages (pending calls during streaming)', () => {+ const toolCallsJson = JSON.stringify([+ { id: 'call_1', type: 'function', function: { name: 'test', arguments: '{}' } }+ ]);++ const msg = makeMessage({ content: '', toolCalls: toolCallsJson });+ const sections = deriveAgenticSections(msg, [], [], true);++ const toolCallSection = sections.find((s) => s.type === AgenticSectionType.TOOL_CALL_PENDING);+ expect(toolCallSection).toBeDefined();+ expect(toolCallSection?.content).toBe('');+ });++ it('scales with many tool calls (no O(n^2) blowup)', () => {+ const N = 100;+ const toolCalls = Array.from(+ { length: N },+ (_, i): ApiChatCompletionToolCall => ({+ id: `call_${i}`,+ type: 'function',+ function: { name: `tool_${i}`, arguments: '{}' }+ })+ );+ const toolCallsJson = JSON.stringify(toolCalls);++ const toolMessages = Array.from({ length: N }, (_, i) =>+ makeMessage({+ role: MessageRole.TOOL,+ toolCallId: `call_${i}`,+ content: `result_${i}`+ })+ );++ const msg = makeMessage({ content: 'hello', toolCalls: toolCallsJson });++ // If the lookup were still O(n^2), this would be noticeably slow+ const start = Date.now();+ const sections = deriveAgenticSections(msg, toolMessages, [], false);+ const elapsed = Date.now() - start;++ const toolCallSections = sections.filter((s) => s.type === AgenticSectionType.TOOL_CALL);+ expect(toolCallSections).toHaveLength(N);+ expect(elapsed).toBeLessThan(100); // Should be fast with O(1) lookup+ });+});