LLM inference in C/C++
C++ ggml
ui: reduce per-token render cost when streaming (#26053)
Zach Winter555881ebc8b0
diff --git a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelteindex ea7fe7b952..ad70322619 100644--- a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte+++ b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte@@ -89,10 +89,15 @@</Collapsible.Trigger><Collapsible.Content>- <div class="pl-1.5 grid min-w-0" style="min-height: var(--min-message-height);">- <div class="min-w-0 border-l border-muted-foreground/20 pl-4 pb-2 my-2">- {@render children()}+ <!-- Collapsible.Content renders its children unconditionally and only sets+ `hidden`, so a closed block would keep re-rendering its whole body on+ every streamed token. Gate on `open` so collapsed content costs nothing. -->+ {#if open}+ <div class="pl-1.5 grid min-w-0" style="min-height: var(--min-message-height);">+ <div class="min-w-0 border-l border-muted-foreground/20 pl-4 pb-2 my-2">+ {@render children()}+ </div></div>- </div>+ {/if}</Collapsible.Content></Collapsible.Root>diff --git a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelteindex 4370aea42a..5cbe003bd7 100644--- a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte+++ b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte@@ -90,8 +90,12 @@</Collapsible.Trigger><Collapsible.Content>- <div class="p-3 pt-1">- {@render children()}- </div>+ <!-- See CollapsibleContentBlock: bits-ui keeps closed content mounted, which+ makes a collapsed tool result re-render on every streamed token. -->+ {#if open}+ <div class="p-3 pt-1">+ {@render children()}+ </div>+ {/if}</Collapsible.Content></Collapsible.Root>diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelteindex 2c17f42345..fc7e314122 100644--- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte+++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte@@ -107,6 +107,15 @@return null;});const liveSvgHtml = $derived(streamingSvgCode !== null ? sanitizeSvg(streamingSvgCode) : '');++ // Derived rather than called inline in the template so it only recomputes when+ // the block actually changes. Auto-detection is disabled while streaming: it+ // costs ~38ms a call and re-guesses the language on every chunk.+ const streamingCodeHtml = $derived(+ incompleteCodeBlock+ ? highlightCode(incompleteCodeBlock.code, incompleteCodeBlock.language || 'text', false)+ : ''+ );let previewDialogOpen = $state(false);let previewCode = $state('');let previewLanguage = $state('text');@@ -903,10 +912,7 @@><pre class="streaming-code-pre"><codeclass="hljs language-{incompleteCodeBlock.language || 'text'}"- >{@html highlightCode(- incompleteCodeBlock.code,- incompleteCodeBlock.language || 'text'- )}</code+ >{@html streamingCodeHtml}</code></pre></div></div>diff --git a/tools/ui/src/lib/constants/latex-protection.ts b/tools/ui/src/lib/constants/latex-protection.tsindex 27c88e7252..da27c00837 100644--- a/tools/ui/src/lib/constants/latex-protection.ts+++ b/tools/ui/src/lib/constants/latex-protection.ts@@ -28,6 +28,14 @@ export const LATEX_MATH_AND_CODE_PATTERN =/** Regex to capture the content of a $$...\\\\...$$ block (display-formula with line-break) */export const LATEX_LINEBREAK_REGEXP = /\$\$([\s\S]*?\\\\[\s\S]*?)\$\$/;+/**+ * Cheap gate for `preprocessLaTeX`. Every transformation it performs is triggered+ * by a `$` (inline/display math, currency escaping) or a backslash escape+ * (`\(`, `\[`, `\ce{`, `\pu{`). Text containing neither is returned untouched, so+ * this lets the caller skip the whole protect/restore pipeline.+ */+export const LATEX_TRIGGER_REGEXP = /[$\\]/;+/** map from mchem-regexp to replacement */export const MHCHEM_PATTERN_MAP: readonly [RegExp, string][] = [[/(\s)\$\\ce{/g, '$1$\\\\ce{'],diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.tsindex 9649cfe03d..bc2feefd3d 100644--- a/tools/ui/src/lib/stores/conversations.svelte.ts+++ b/tools/ui/src/lib/stores/conversations.svelte.ts@@ -169,8 +169,21 @@ class ConversationsStore {* Updates a message at a specific index in active messages*/updateMessageAtIndex(index: number, updates: Partial<DatabaseMessage>): void {- if (index !== -1 && this.activeMessages[index]) {- this.activeMessages[index] = { ...this.activeMessages[index], ...updates };+ const message = index === -1 ? undefined : this.activeMessages[index];++ if (!message) return;++ // Assign field by field rather than replacing the object. Replacing it+ // changes the array slot, which invalidates every consumer that merely+ // walks the list - notably ChatMessages.displayMessages, which rebuilds+ // entries for every message in the conversation. Deep $state proxies make+ // per-field writes fine-grained, so only readers of the changed field wake.+ const target = message as unknown as Record<string, unknown>;++ for (const [key, value] of Object.entries(updates)) {+ if (target[key] !== value) {+ target[key] = value;+ }}}diff --git a/tools/ui/src/lib/utils/code.ts b/tools/ui/src/lib/utils/code.tsindex a29028c741..44b9b48415 100644--- a/tools/ui/src/lib/utils/code.ts+++ b/tools/ui/src/lib/utils/code.ts@@ -30,13 +30,21 @@ function trimCodePadding(code: string): string {return code.replace(TRIM_LEADING_PADDING_REGEX, '').replace(TRIM_TRAILING_PADDING_REGEX, '');}+function escapeCode(code: string): string {+ return code.replace(AMPERSAND_REGEX, '&').replace(LT_REGEX, '<').replace(GT_REGEX, '>');+}+/*** Highlights code using highlight.js* @param code - The code to highlight* @param language - The programming language+ * @param autoDetect - Fall back to `highlightAuto` when `language` is unknown.+ * Callers rendering a still-streaming block should pass false: auto-detection+ * costs ~38ms per call and re-guesses on every chunk, so the language (and+ * therefore the whole highlight) flickers as the block grows.* @returns HTML string with syntax highlighting*/-export function highlightCode(code: string, language: string): string {+export function highlightCode(code: string, language: string, autoDetect = true): string {if (!code) return '';const trimmed = trimCodePadding(code);@@ -47,15 +55,14 @@ export function highlightCode(code: string, language: string): string {if (isSupported) {return hljs.highlight(trimmed, { language: lang }).value;- } else {+ } else if (autoDetect) {return hljs.highlightAuto(trimmed).value;+ } else {+ return escapeCode(trimmed);}} catch {// Fallback to escaped plain text- return trimmed- .replace(AMPERSAND_REGEX, '&')- .replace(LT_REGEX, '<')- .replace(GT_REGEX, '>');+ return escapeCode(trimmed);}}diff --git a/tools/ui/src/lib/utils/latex-protection.ts b/tools/ui/src/lib/utils/latex-protection.tsindex 839306978f..573eb92978 100644--- a/tools/ui/src/lib/utils/latex-protection.ts+++ b/tools/ui/src/lib/utils/latex-protection.ts@@ -2,6 +2,7 @@ import {CODE_BLOCK_REGEXP,LATEX_MATH_AND_CODE_PATTERN,LATEX_LINEBREAK_REGEXP,+ LATEX_TRIGGER_REGEXP,MHCHEM_PATTERN_MAP} from '$lib/constants';@@ -148,6 +149,15 @@ export function preprocessLaTeX(content: string): string {// See also:// https://github.com/danny-avila/LibreChat/blob/main/client/src/utils/latex.ts+ // Every step below keys off a `$` or a backslash escape (\[ \] \( \) \ce{ \pu{).+ // With neither present the protect/restore passes round-trip the input+ // unchanged, so skip them: the step 2 scan is O(n^2) in line length and costs+ // ~90ms on a 26KB single-line message that contains no math at all. This+ // matters during streaming, where the whole message is reprocessed per frame.+ if (!LATEX_TRIGGER_REGEXP.test(content)) {+ return content;+ }+// Step 0: Temporarily remove blockquote markers (>) to process LaTeX correctly// Store the structure so we can restore it laterconst blockquoteMarkers: Map<number, string> = new Map();@@ -175,24 +185,31 @@ export function preprocessLaTeX(content: string): string {const latexExpressions: string[] = [];// Match \S...\[...\] and protect them and insert a line-break.- content = content.replace(/([\S].*?)\\\[([\s\S]*?)\\\](.*)/g, (match, group1, group2, group3) => {- // Check if there are characters following the formula (display-formula in a table-cell?)- if (group1.endsWith('\\')) {- return match; // Backslash before \[, do nothing.- }- const hasSuffix = /\S/.test(group3);- let optBreak;-- if (hasSuffix) {- latexExpressions.push(`\\(${group2.trim()}\\)`); // Convert into inline.- optBreak = '';- } else {- latexExpressions.push(`\\[${group2}\\]`);- optBreak = '\n';- }+ // Guarded: with no `\[` present this pattern still probes every start offset,+ // expanding `.*?` to the end of each line before failing - O(n^2) for nothing.+ if (content.includes('\\[')) {+ content = content.replace(+ /([\S].*?)\\\[([\s\S]*?)\\\](.*)/g,+ (match, group1, group2, group3) => {+ // Check if there are characters following the formula (display-formula in a table-cell?)+ if (group1.endsWith('\\')) {+ return match; // Backslash before \[, do nothing.+ }+ const hasSuffix = /\S/.test(group3);+ let optBreak;++ if (hasSuffix) {+ latexExpressions.push(`\\(${group2.trim()}\\)`); // Convert into inline.+ optBreak = '';+ } else {+ latexExpressions.push(`\\[${group2}\\]`);+ optBreak = '\n';+ }- return `${group1}${optBreak}<<LATEX_${latexExpressions.length - 1}>>${optBreak}${group3}`;- });+ return `${group1}${optBreak}<<LATEX_${latexExpressions.length - 1}>>${optBreak}${group3}`;+ }+ );+ }// Match \(...\), \[...\], $$...$$ and protect themcontent = content.replace(diff --git a/tools/ui/tests/client/README-perf.md b/tools/ui/tests/client/README-perf.mdnew file mode 100644index 0000000000..a4e3e4c644--- /dev/null+++ b/tools/ui/tests/client/README-perf.md@@ -0,0 +1,58 @@+# Agentic thread perf harness++Two tiers, both reusing the existing vitest projects (see `vite.config.ts`).++## Tier 1 - `agentic-stream.perf.svelte.test.ts` (project: `client`, real Chromium)++Mounts `ChatMessageAgenticContent` and replays a stream, replacing the message+object on each chunk exactly as the real pipeline does:++- `chat.svelte.ts` `updateStreamingUI()` runs per SSE chunk+- `conversations.svelte.ts` `updateMessageAtIndex` does `{ ...old, ...updates }`++That new object identity is the thing under test: it cascades through+`deriveAgenticSections` (which returns fresh `AgenticSection` objects) into every+tool-call block in the message, including completed ones.++```+npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.ts+```++### Reading the output++- `mean` / `p95` / `max` - the synchronous window per token: prop write,+ `await tick()`, then a forced `offsetHeight` read so style and layout are+ included rather than deferred.+- `sync` - sum of those windows. This is the number to optimize.+- `wall` - the whole run including work `MarkdownContent` defers into its own+ `requestAnimationFrame`. It carries a ~16.7ms/token idle floor because the+ harness yields a frame each iteration, so compare `wall` **across fixtures**,+ never against `sync`.++### The knobs, and what each one discriminates++The point of the harness is the _scaling curve_, not any single number.++| Knob | Reads on |+| --------------------------- | ---------------------------------------------------------------------------------------------------- |+| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. |+| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). |+| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. |+| `openCodeFence` | `hljs.highlightAuto` on partial code. |++Deliberately no hard assertions: CI timing is noisy and the value here is the+before/after delta, not a gate.++### Caveat++This measures one message's subtree. In the real app `ChatMessages.svelte`+rebuilds its whole `displayMessages` list per token, so multiply by the number+of rendered messages to get the conversation-level cost.++## Tier 2 - `../unit/agentic-hotpath.bench.ts` (project: `unit`, node)++Per-call costs for the pure functions the curve implicates.++```+npx vitest bench --project=unit --run tests/unit/agentic-hotpath.bench.ts+```diff --git a/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts b/tools/ui/tests/client/agentic-stream.perf.svelte.test.tsnew file mode 100644index 0000000000..c13f98cfc3--- /dev/null+++ b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts@@ -0,0 +1,410 @@+// Tier 1 perf harness for the agentic thread.+//+// Drives ChatMessageAgenticContent the way the real streaming pipeline does:+// chat.svelte.ts -> conversations.svelte.ts:184 replaces the message object per+// SSE chunk (`{ ...old, ...updates }`), which changes prop identity and cascades+// through deriveAgenticSections into every tool-call block in the message.+//+// The fixture is parameterized so the *scaling curve* identifies the culprit -+// a single number would not. See tests/client/README-perf.md.+//+// Run: npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.ts++import { describe, it } from 'vitest';+import { render } from 'vitest-browser-svelte';+import { tick } from 'svelte';+import AgenticPerfWrapper from './components/AgenticPerfWrapper.svelte';+import ChatMessagesPerfWrapper from './components/ChatMessagesPerfWrapper.svelte';+import { perfState } from './components/agentic-perf-state.svelte';+import { conversationsStore } from '$lib/stores/conversations.svelte';+import type { DatabaseMessage } from '$lib/types';+import { MessageRole } from '$lib/enums';++// --- fixture construction -------------------------------------------------++interface FixtureOpts {+ /** Completed tool-call sections preceding the streaming text. */+ priorToolCalls: number;+ /** Size of each tool result blob. */+ toolResultBytes: number;+ /** Number of edit_file calls (each a 400x400-line diff). */+ editFileEdits: number;+ /** Leave an unclosed ``` fence at the end of the streamed content. */+ openCodeFence: boolean;+ /**+ * Emit a blank line every N chunks so the content forms real markdown+ * blocks. 0 = one unbroken paragraph, which defeats MarkdownContent's+ * stable-block cache entirely (worst case). Typical prose has breaks.+ */+ paragraphEvery: number;+}++const DEFAULTS: FixtureOpts = {+ priorToolCalls: 0,+ toolResultBytes: 1024,+ editFileEdits: 0,+ openCodeFence: false,+ paragraphEvery: 0+};++function blob(bytes: number, seed: string): string {+ const line = `${seed} output line with some representative width to it`;+ const n = Math.max(1, Math.ceil(bytes / (line.length + 1)));+ const out: string[] = [];+ for (let i = 0; i < n; i++) out.push(`${line} ${i}`);+ return out.join('\n');+}++function diffLines(n: number, seed: string): string {+ const out: string[] = [];+ for (let i = 0; i < n; i++) out.push(`${seed} line ${i} const value_${i} = compute(${i});`);+ return out.join('\n');+}++let msgSeq = 0;++function baseMessage(overrides: Partial<DatabaseMessage>): DatabaseMessage {+ return {+ id: `m${msgSeq++}`,+ convId: 'perf-conv',+ type: 'text',+ timestamp: 0,+ role: MessageRole.ASSISTANT,+ content: '',+ parent: null,+ children: [],+ ...overrides+ } as DatabaseMessage;+}++function buildFixture(opts: FixtureOpts): {+ message: DatabaseMessage;+ toolMessages: DatabaseMessage[];+} {+ const toolCalls: unknown[] = [];+ const toolMessages: DatabaseMessage[] = [];++ for (let i = 0; i < opts.priorToolCalls; i++) {+ const id = `call_${i}`;+ toolCalls.push({+ id,+ type: 'function',+ function: {+ name: 'exec_shell_command',+ arguments: JSON.stringify({ command: `grep -rn "thing_${i}" src/` })+ }+ });+ toolMessages.push(+ baseMessage({+ role: MessageRole.TOOL,+ toolCallId: id,+ content: `${blob(opts.toolResultBytes, `t${i}`)}\n[exit code: 0]`+ })+ );+ }++ for (let i = 0; i < opts.editFileEdits; i++) {+ const id = `edit_${i}`;+ toolCalls.push({+ id,+ type: 'function',+ function: {+ name: 'edit_file',+ arguments: JSON.stringify({+ path: `/src/file_${i}.ts`,+ edits: [{ old_text: diffLines(400, 'old'), new_text: diffLines(400, 'new') }]+ })+ }+ });+ toolMessages.push(+ baseMessage({+ role: MessageRole.TOOL,+ toolCallId: id,+ content: JSON.stringify({ result: 'ok', edits_applied: 1 })+ })+ );+ }++ const message = baseMessage({+ toolCalls: toolCalls.length > 0 ? JSON.stringify(toolCalls) : undefined,+ content: ''+ });++ return { message, toolMessages };+}++// --- the driver -----------------------------------------------------------++interface Sample {+ label: string;+ tokens: number;+ mean: number;+ p95: number;+ max: number;+ /** Sum of the synchronous per-token windows. */+ total: number;+ /** Wall-clock for the whole run incl. deferred rAF work, then settle. */+ wall: number;+}++function nextFrame(): Promise<void> {+ return new Promise((resolve) => requestAnimationFrame(() => resolve()));+}++const results: Sample[] = [];++/**+ * Replays `tokens` streamed chunks, replacing the message object each time+ * exactly as conversations.svelte.ts:184 does, flushing Svelte and forcing+ * layout so the measurement includes render + style/layout, not just script.+ */+async function measure(label: string, partial: Partial<FixtureOpts>, tokens = 60) {+ const opts = { ...DEFAULTS, ...partial };+ const { message, toolMessages } = buildFixture(opts);++ perfState.message = message;+ perfState.toolMessages = toolMessages;+ perfState.isStreaming = true;++ // Every wrapper reads the same module state, so a leftover mount from a+ // previous fixture would re-render on each mutation and fold its cost into+ // this measurement. Tear down explicitly between fixtures.+ const { unmount } = render(AgenticPerfWrapper);++ await tick();++ const CHUNK = 'The quick brown fox jumps over the lazy dog. ';+ let accumulated = opts.openCodeFence ? '```notalanguage\n' : '';+ const durations: number[] = [];++ const wallStart = performance.now();++ for (let i = 0; i < tokens; i++) {+ accumulated += CHUNK;+ if (opts.paragraphEvery > 0 && (i + 1) % opts.paragraphEvery === 0) {+ accumulated += '\n\n';+ }++ const t0 = performance.now();+ // Mirrors updateMessageAtIndex: a brand-new object identity per chunk.+ perfState.message = { ...perfState.message!, content: accumulated };+ await tick();+ void document.body.offsetHeight; // force style + layout+ durations.push(performance.now() - t0);++ // MarkdownContent coalesces its parse into a rAF, so that work lands+ // outside the window above. Yield a frame each iteration so it is+ // captured in `wall` - the gap between `wall` and `total` is the+ // deferred cost.+ await nextFrame();+ }++ // Let any trailing coalesced work drain before stopping the clock.+ await nextFrame();+ await nextFrame();+ const wall = performance.now() - wallStart;++ await unmount();++ durations.sort((a, b) => a - b);+ const total = durations.reduce((a, b) => a + b, 0);++ results.push({+ label,+ tokens,+ mean: total / durations.length,+ p95: durations[Math.floor(durations.length * 0.95)],+ max: durations[durations.length - 1],+ total,+ wall+ });+}++function report() {+ const pad = (s: string, n: number) => s.padEnd(n);+ const num = (n: number) => n.toFixed(2).padStart(8);++ const header = `${pad('fixture', 40)}${pad('tok', 5)}${'mean'.padStart(8)}${'p95'.padStart(8)}${'max'.padStart(8)}${'sync'.padStart(9)}${'wall'.padStart(9)}`;+ const lines = [+ '',+ '=== Tier 1: ms per streamed token (agentic content subtree) ===',+ 'mean/p95/max = synchronous window per token (script + style + layout).',+ 'sync = sum of those windows.',+ 'wall = whole run, incl. work MarkdownContent defers into its own rAF.',+ ' NOTE: wall carries a ~16.7ms/token idle floor from the harness',+ ' yielding a frame each iteration (60 tokens => ~1000ms floor).',+ ' Compare wall ACROSS fixtures / against the baseline row,',+ ' never against sync.',+ '',+ header,+ '-'.repeat(header.length)+ ];++ for (const r of results) {+ lines.push(+ `${pad(r.label, 40)}${pad(String(r.tokens), 5)}${num(r.mean)}${num(r.p95)}${num(r.max)}${num(r.total)}${num(r.wall)}`+ );+ }++ lines.push('');+ console.log(lines.join('\n'));+}++// --- conversation-level driver --------------------------------------------+// The per-message driver above cannot see the fan-out in ChatMessages: a single+// token mutation invalidates `displayMessages`, which rebuilds a fresh+// toolMessages array for EVERY message in the conversation. Drive the real+// store through the real list component to measure that.++async function measureConversation(+ label: string,+ priorMessages: number,+ tokens = 60,+ /**+ * Give each prior assistant turn a resolved tool call, so the fixture pays+ * `hasAgenticContent`'s JSON.parse and the tool-message grouping walk that a+ * real agent thread would - plain prose messages skip both.+ */+ agentic = false+) {+ const history: DatabaseMessage[] = [];+ for (let i = 0; i < priorMessages; i++) {+ const isAssistant = i % 2 !== 0;++ if (isAssistant && agentic) {+ const id = `prior_call_${i}`;+ history.push(+ baseMessage({+ role: MessageRole.ASSISTANT,+ content: `Message ${i}`,+ toolCalls: JSON.stringify([+ {+ id,+ type: 'function',+ function: {+ name: 'exec_shell_command',+ arguments: JSON.stringify({ command: `grep -rn "thing_${i}" src/` })+ }+ }+ ])+ })+ );+ history.push(+ baseMessage({+ role: MessageRole.TOOL,+ toolCallId: id,+ content: `${blob(1024, `r${i}`)}\n[exit code: 0]`+ })+ );+ continue;+ }++ history.push(+ baseMessage({+ role: isAssistant ? MessageRole.ASSISTANT : MessageRole.USER,+ content: `Message ${i}: ${blob(512, `m${i}`)}`+ })+ );+ }++ const streaming = baseMessage({ role: MessageRole.ASSISTANT, content: '' });+ history.push(streaming);++ conversationsStore.activeMessages = history;++ const { unmount } = render(ChatMessagesPerfWrapper);+ await tick();++ const idx = conversationsStore.findMessageIndex(streaming.id);+ const CHUNK = 'The quick brown fox jumps over the lazy dog. ';+ let accumulated = '';+ const durations: number[] = [];+ const wallStart = performance.now();++ for (let i = 0; i < tokens; i++) {+ accumulated += CHUNK;++ const t0 = performance.now();+ // The real path: chat.svelte.ts -> conversations.svelte.ts.+ conversationsStore.updateMessageAtIndex(idx, { content: accumulated });+ await tick();+ void document.body.offsetHeight;+ durations.push(performance.now() - t0);++ await nextFrame();+ }++ await nextFrame();+ await nextFrame();+ const wall = performance.now() - wallStart;++ await unmount();+ conversationsStore.activeMessages = [];++ durations.sort((a, b) => a - b);+ const total = durations.reduce((a, b) => a + b, 0);++ results.push({+ label,+ tokens,+ mean: total / durations.length,+ p95: durations[Math.floor(durations.length * 0.95)],+ max: durations[durations.length - 1],+ total,+ wall+ });+}++// --- the matrix -----------------------------------------------------------+// Sequential, in one test, so the table prints together and the samples do not+// interleave with other suites competing for the main thread.++describe('agentic streaming perf', () => {+ it('scales', { timeout: 600_000 }, async () => {+ // Baseline: plain text streaming, nothing agentic.+ await measure('baseline: no tool calls', {});++ // Knob: priorToolCalls. Flat => no fan-out. Linear => fan-out confirmed.+ await measure('priorToolCalls=1 (1KB results)', { priorToolCalls: 1 });+ await measure('priorToolCalls=5 (1KB results)', { priorToolCalls: 5 });+ await measure('priorToolCalls=20 (1KB results)', { priorToolCalls: 20 });++ // Knob: toolResultBytes, at fixed section count.+ await measure('5 calls x 200KB results', {+ priorToolCalls: 5,+ toolResultBytes: 200 * 1024+ });++ // Knob: editFileEdits (computeLineDiff, 400x400 LCS).+ await measure('3 edit_file calls (400x400 diff)', { editFileEdits: 3 });++ // Knob: openCodeFence (hljs highlightAuto on partial code).+ await measure('open code fence, unknown language', { openCodeFence: true });++ // Conversation level: does streaming one message cost more as the+ // conversation grows? Flat => no fan-out. Linear => confirmed.+ await measureConversation('convo: 1 prior message', 1);+ await measureConversation('convo: 10 prior messages', 10);+ await measureConversation('convo: 40 prior messages', 40);++ // Same, but each prior assistant turn carries a resolved tool call.+ await measureConversation('convo: 10 prior, agentic', 10, 60, true);+ await measureConversation('convo: 40 prior, agentic', 40, 60, true);++ // Message length: MarkdownContent re-parses the whole accumulated string+ // each frame, so per-token cost should climb as the response grows.+ // A rising mean across these three rows means O(n^2) over the stream.+ // One unbroken paragraph: worst case, stable-block cache never applies.+ await measure('len 60tok 1-para (worst case)', {}, 60);+ await measure('len 250tok 1-para (worst case)', {}, 250);+ await measure('len 600tok 1-para (worst case)', {}, 600);++ // Same lengths, broken into paragraphs every 8 chunks: the typical shape,+ // where only the trailing paragraph should be unstable.+ await measure('len 60tok paras (typical)', { paragraphEvery: 8 }, 60);+ await measure('len 250tok paras (typical)', { paragraphEvery: 8 }, 250);+ await measure('len 600tok paras (typical)', { paragraphEvery: 8 }, 600);++ report();+ });+});diff --git a/tools/ui/tests/client/collapsible-lazy-body.svelte.test.ts b/tools/ui/tests/client/collapsible-lazy-body.svelte.test.tsnew file mode 100644index 0000000000..9b29bea736--- /dev/null+++ b/tools/ui/tests/client/collapsible-lazy-body.svelte.test.ts@@ -0,0 +1,37 @@+// Guards the lazy-body behaviour of the shared collapsible wrappers.+//+// bits-ui's Collapsible.Content renders its children unconditionally and only+// sets `hidden`, so before this was gated a collapsed tool result kept its whole+// body in the DOM and re-rendered it on every streamed token (measured at+// ~41ms/section/token for a 200KB result). These tests pin the fix: closed means+// not rendered, and opening still mounts the body.++import { describe, it, expect } from 'vitest';+import { render } from 'vitest-browser-svelte';+import { tick } from 'svelte';+import CollapsibleLazyBodyHarness from './components/CollapsibleLazyBodyHarness.svelte';++const MARKER = 'collapsible-body-marker';++describe('collapsible wrappers render their body lazily', () => {+ for (const variant of ['content', 'terminal'] as const) {+ it(`${variant}: body is absent while closed and present once open`, async () => {+ const screen = render(CollapsibleLazyBodyHarness, { variant, open: false });+ await tick();++ expect(document.body.textContent).not.toContain(MARKER);++ await screen.rerender({ variant, open: true });+ await tick();++ expect(document.body.textContent).toContain(MARKER);++ // And it unmounts again on close, so a collapsed block stops costing+ // anything during streaming.+ await screen.rerender({ variant, open: false });+ await tick();++ expect(document.body.textContent).not.toContain(MARKER);+ });+ }+});diff --git a/tools/ui/tests/client/components/AgenticPerfWrapper.svelte b/tools/ui/tests/client/components/AgenticPerfWrapper.sveltenew file mode 100644index 0000000000..8f95860b72--- /dev/null+++ b/tools/ui/tests/client/components/AgenticPerfWrapper.svelte@@ -0,0 +1,16 @@+<script lang="ts">+ import * as Tooltip from '$lib/components/ui/tooltip';+ import ChatMessageAgenticContent from '$lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte';+ import { perfState } from './agentic-perf-state.svelte';+</script>++<Tooltip.Provider>+ {#if perfState.message}+ <ChatMessageAgenticContent+ message={perfState.message}+ toolMessages={perfState.toolMessages}+ isStreaming={perfState.isStreaming}+ isLastAssistantMessage+ />+ {/if}+</Tooltip.Provider>diff --git a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.sveltenew file mode 100644index 0000000000..14d9389787--- /dev/null+++ b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte@@ -0,0 +1,12 @@+<script lang="ts">+ // Mounts the real ChatMessages list against the real conversations store, so+ // the harness exercises `displayMessages` (which rebuilds every message's+ // toolMessages array) rather than a single message subtree.+ import * as Tooltip from '$lib/components/ui/tooltip';+ import ChatMessages from '$lib/components/app/chat/ChatMessages/ChatMessages.svelte';+ import { conversationsStore } from '$lib/stores/conversations.svelte';+</script>++<Tooltip.Provider>+ <ChatMessages messages={conversationsStore.activeMessages} />+</Tooltip.Provider>diff --git a/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.sveltenew file mode 100644index 0000000000..53d3b9cfd1--- /dev/null+++ b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte@@ -0,0 +1,21 @@+<script lang="ts">+ import CollapsibleContentBlock from '$lib/components/app/content/CollapsibleContentBlock.svelte';+ import CollapsibleTerminalBlock from '$lib/components/app/content/CollapsibleTerminalBlock.svelte';++ interface Props {+ variant: 'content' | 'terminal';+ open: boolean;+ }++ let { variant, open }: Props = $props();+</script>++{#if variant === 'content'}+ <CollapsibleContentBlock {open} title="Block">+ <span>collapsible-body-marker</span>+ </CollapsibleContentBlock>+{:else}+ <CollapsibleTerminalBlock {open} title="Block">+ <span>collapsible-body-marker</span>+ </CollapsibleTerminalBlock>+{/if}diff --git a/tools/ui/tests/client/components/agentic-perf-state.svelte.ts b/tools/ui/tests/client/components/agentic-perf-state.svelte.tsnew file mode 100644index 0000000000..f0cc8c3b88--- /dev/null+++ b/tools/ui/tests/client/components/agentic-perf-state.svelte.ts@@ -0,0 +1,17 @@+// Shared reactive fixture state for the Tier 1 perf harness.+//+// The wrapper component reads this module directly rather than taking props,+// so the driver can mutate it without depending on how the test renderer+// forwards props.++import type { DatabaseMessage } from '$lib/types';++export const perfState = $state<{+ message: DatabaseMessage | null;+ toolMessages: DatabaseMessage[];+ isStreaming: boolean;+}>({+ message: null,+ toolMessages: [],+ isStreaming: true+});diff --git a/tools/ui/tests/client/update-message-in-place.svelte.test.ts b/tools/ui/tests/client/update-message-in-place.svelte.test.tsnew file mode 100644index 0000000000..54c215fef5--- /dev/null+++ b/tools/ui/tests/client/update-message-in-place.svelte.test.ts@@ -0,0 +1,64 @@+// Pins the contract that makes streaming cheap: updateMessageAtIndex mutates the+// existing message object instead of replacing it.+//+// Replacing it (`{ ...old, ...updates }`) changes the array slot, which+// invalidates every consumer that merely walks the list - ChatMessages'+// `displayMessages` rebuilds entries for EVERY message in the conversation. That+// made per-token cost scale with conversation length (1.26ms at 1 prior message+// -> 3.07ms at 40). Mutating in place keeps it flat.++import { describe, it, expect } from 'vitest';+import { conversationsStore } from '$lib/stores/conversations.svelte';+import type { DatabaseMessage } from '$lib/types';+import { MessageRole } from '$lib/enums';++function makeMessage(id: string): DatabaseMessage {+ return {+ id,+ convId: 'c1',+ type: 'text',+ timestamp: 0,+ role: MessageRole.ASSISTANT,+ content: '',+ parent: null,+ children: []+ } as DatabaseMessage;+}++describe('conversationsStore.updateMessageAtIndex', () => {+ it('mutates in place, preserving object identity', () => {+ conversationsStore.activeMessages = [makeMessage('a'), makeMessage('b')];+ const before = conversationsStore.activeMessages[1];++ conversationsStore.updateMessageAtIndex(1, { content: 'hello' });++ expect(conversationsStore.activeMessages[1].content).toBe('hello');+ expect(conversationsStore.activeMessages[1]).toBe(before);++ conversationsStore.activeMessages = [];+ });++ it('leaves other messages and unrelated fields untouched', () => {+ conversationsStore.activeMessages = [makeMessage('a'), makeMessage('b')];+ const untouched = conversationsStore.activeMessages[0];++ conversationsStore.updateMessageAtIndex(1, { content: 'x', model: 'm1' });++ expect(conversationsStore.activeMessages[0]).toBe(untouched);+ expect(conversationsStore.activeMessages[0].content).toBe('');+ expect(conversationsStore.activeMessages[1].model).toBe('m1');+ expect(conversationsStore.activeMessages[1].id).toBe('b');++ conversationsStore.activeMessages = [];+ });++ it('is a no-op for an index of -1 or out of range', () => {+ conversationsStore.activeMessages = [makeMessage('a')];++ expect(() => conversationsStore.updateMessageAtIndex(-1, { content: 'x' })).not.toThrow();+ expect(() => conversationsStore.updateMessageAtIndex(9, { content: 'x' })).not.toThrow();+ expect(conversationsStore.activeMessages[0].content).toBe('');++ conversationsStore.activeMessages = [];+ });+});diff --git a/tools/ui/tests/unit/agentic-hotpath.bench.ts b/tools/ui/tests/unit/agentic-hotpath.bench.tsnew file mode 100644index 0000000000..51b99c31f6--- /dev/null+++ b/tools/ui/tests/unit/agentic-hotpath.bench.ts@@ -0,0 +1,248 @@+// Microbenchmarks for the functions on the agentic-thread streaming hot path.+//+// Every function here is invoked from a `$derived` that is invalidated on each+// streamed token, for every tool-call section in the message. These numbers give+// the per-call cost; multiply by (tokens x sections) for the real damage.+//+// Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts++import { bench, describe } from 'vitest';++import { remark } from 'remark';+import remarkBreaks from 'remark-breaks';+import remarkGfm from 'remark-gfm';+import remarkMath from 'remark-math';+import remarkRehype from 'remark-rehype';+import rehypeKatex from 'rehype-katex';+import rehypeHighlight from 'rehype-highlight';+import rehypeStringify from 'rehype-stringify';+import { all as lowlightAll } from 'lowlight';++import { computeLineDiff } from '$lib/utils/compute-line-diff';+import { extractSearchResults, extractSearchQuery } from '$lib/utils/search-results';+import { parsePartialJsonArgs } from '$lib/utils/parse-partial-json-args';+import { highlightCode, detectIncompleteCodeBlock } from '$lib/utils/code';+import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic';+import { preprocessLaTeX } from '$lib/utils/latex-protection';++// --- fixtures -------------------------------------------------------------++function lines(n: number, seed: string): string {+ const out: string[] = [];+ for (let i = 0; i < n; i++) out.push(`${seed} line ${i} const value_${i} = compute(${i});`);+ return out.join('\n');+}++const SHELL_OUTPUT_1KB = lines(12, 'out');+const SHELL_OUTPUT_200KB = lines(2600, 'out');+const SHELL_OUTPUT_2MB = lines(26000, 'out');++// Realistic exec_shell_command result: the exit-code marker is the final line,+// which is what the un-anchored EXIT_CODE regex has to scan the whole blob for.+const SHELL_2MB_WITH_EXIT = `${SHELL_OUTPUT_2MB}\n[exit code: 0]`;++const EDIT_OLD_400 = lines(400, 'old');+const EDIT_NEW_400 = lines(400, 'new');+const EDIT_OLD_50 = lines(50, 'old');+const EDIT_NEW_50 = lines(50, 'new');++const WRITE_FILE_ARGS = JSON.stringify({+ path: '/src/lib/thing.ts',+ content: lines(1500, 'src')+});++const MARKDOWN_50KB = Array.from(+ { length: 400 },+ (_, i) =>+ `## Section ${i}\n\nSome **bold** prose with a [link](https://example.com) and \`inline\` code.\n\n- bullet one\n- bullet two\n\n\`\`\`ts\nconst x${i} = ${i};\n\`\`\`\n`+).join('\n');++const CODE_BLOCK_5KB = lines(60, 'code');++// --- A: computeLineDiff (O(m*n) LCS, allocates a full matrix) --------------++describe('computeLineDiff', () => {+ bench('50 x 50 lines', () => {+ computeLineDiff(EDIT_OLD_50, EDIT_NEW_50);+ });++ bench('400 x 400 lines', () => {+ computeLineDiff(EDIT_OLD_400, EDIT_NEW_400);+ });+});++// --- B: the unified processor, rebuilt per call ---------------------------+// Mirrors MarkdownContent.svelte:144-176. The local rehype/remark plugins are+// omitted (they pull in browser-only modules); rehypeHighlight + lowlightAll is+// the dominant term, so this is a lower bound on the real cost.++function buildProcessor() {+ // eslint-disable-next-line @typescript-eslint/no-explicit-any+ let proc: any = remark().use(remarkGfm);+ proc = proc.use(remarkMath).use(remarkBreaks).use(remarkRehype).use(rehypeKatex);+ return proc.use(rehypeHighlight, { languages: lowlightAll }).use(rehypeStringify, {+ allowDangerousHtml: true+ });+}++describe('markdown processor', () => {+ bench('build processor (per processMarkdown call, x2)', () => {+ buildProcessor();+ });++ const prebuilt = buildProcessor();++ bench('parse 50KB markdown with a prebuilt processor', () => {+ prebuilt.parse(MARKDOWN_50KB);+ });+});++// Isolates the O(n^2) term measured in Tier 1: processMarkdown re-parses the+// WHOLE accumulated string every frame, while only the last block can have+// changed. If parse() dominates at these sizes, incremental parsing (re-parsing+// only the tail after the last stable block) is the fix; if not, the cost is+// downstream in transform/stringify/DOM.+describe('markdown parse scaling (whole-string reparse per frame)', () => {+ const proc = buildProcessor();+ const prose = (kb: number) =>+ Array.from(+ { length: Math.ceil((kb * 1024) / 64) },+ (_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.`+ ).join(' ');++ const MD_3KB = prose(3);+ const MD_11KB = prose(11);+ const MD_26KB = prose(26);++ bench('parse 3KB', () => {+ proc.parse(MD_3KB);+ });++ bench('parse 11KB', () => {+ proc.parse(MD_11KB);+ });++ bench('parse 26KB', () => {+ proc.parse(MD_26KB);+ });++ bench('parse only a 200-char tail (proposed incremental)', () => {+ proc.parse(MD_26KB.slice(-200));+ });+});++// processMarkdown runs these over the WHOLE accumulated string every frame too,+// before parse() even starts. Measure them before assuming parse is the target.+describe('other whole-string passes per frame', () => {+ const prose = (kb: number) =>+ Array.from(+ { length: Math.ceil((kb * 1024) / 64) },+ (_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.`+ ).join(' ');++ const MD_3KB = prose(3);+ const MD_11KB = prose(11);+ const MD_26KB = prose(26);+ const MD_26KB_LATEX = `${MD_26KB} and some math $x^2 + y^2 = z^2$ inline.`;++ bench('preprocessLaTeX 3KB (no latex present)', () => {+ preprocessLaTeX(MD_3KB);+ });++ bench('preprocessLaTeX 11KB (no latex present)', () => {+ preprocessLaTeX(MD_11KB);+ });++ bench('preprocessLaTeX 26KB (no latex present)', () => {+ preprocessLaTeX(MD_26KB);+ });++ bench('preprocessLaTeX 26KB (latex present)', () => {+ preprocessLaTeX(MD_26KB_LATEX);+ });++ bench('detectIncompleteCodeBlock 26KB', () => {+ detectIncompleteCodeBlock(MD_26KB);+ });+});++// --- C: extractSearchResults, run for EVERY tool of every type ------------++describe('extractSearchResults (only .length > 0 is consumed)', () => {+ bench('1KB non-search tool result', () => {+ extractSearchResults(SHELL_OUTPUT_1KB);+ });++ bench('200KB non-search tool result', () => {+ extractSearchResults(SHELL_OUTPUT_200KB);+ });++ bench('2MB non-search tool result', () => {+ extractSearchResults(SHELL_OUTPUT_2MB);+ });+});++describe('extractSearchQuery', () => {+ bench('write_file args (throws on the JSON.parse path)', () => {+ extractSearchQuery(WRITE_FILE_ARGS);+ });+});++// --- E: the un-anchored exit-code regex -----------------------------------+// Reproduced inline so the bench is independent of the current call site.++const EXIT_CODE_TAIL = /\[exit code: (-?\d+)\]\s*$/;++describe('exit-code regex', () => {+ bench('whole 2MB blob (current behaviour)', () => {+ SHELL_2MB_WITH_EXIT.match(EXIT_CODE_TAIL);+ });++ bench('last 64 chars only (proposed)', () => {+ SHELL_2MB_WITH_EXIT.slice(-64).match(EXIT_CODE_TAIL);+ });+});++// --- per-line result parsers ----------------------------------------------++describe('parseToolResultWithImages', () => {+ bench('1KB', () => {+ parseToolResultWithImages(SHELL_OUTPUT_1KB, []);+ });++ bench('200KB', () => {+ parseToolResultWithImages(SHELL_OUTPUT_200KB, []);+ });++ bench('2MB', () => {+ parseToolResultWithImages(SHELL_OUTPUT_2MB, []);+ });+});++describe('classifyToolResult', () => {+ bench('200KB plain text (falls through to looksLikeMarkdown)', () => {+ classifyToolResult(SHELL_OUTPUT_200KB);+ });+});++describe('parsePartialJsonArgs', () => {+ bench('write_file args, ~60KB (char-by-char scan)', () => {+ parsePartialJsonArgs(WRITE_FILE_ARGS);+ });+});++// --- F / I: highlight.js ---------------------------------------------------++describe('highlightCode', () => {+ bench('short bash command (title row, per token)', () => {+ highlightCode('grep -rn "foo" src/ | head -50', 'bash');+ });++ bench('5KB known language', () => {+ highlightCode(CODE_BLOCK_5KB, 'typescript');+ });++ bench('5KB unknown language -> highlightAuto', () => {+ highlightCode(CODE_BLOCK_5KB, 'not-a-language');+ });+});diff --git a/tools/ui/tests/unit/code.test.ts b/tools/ui/tests/unit/code.test.tsindex 944cf8b735..7820c98072 100644--- a/tools/ui/tests/unit/code.test.ts+++ b/tools/ui/tests/unit/code.test.ts@@ -79,4 +79,25 @@ describe('highlightCode', () => {const framed = highlightCode('\nfunction foo() {}\n', 'javascript');expect(framed).toBe(trimmed);});++ it('auto-detects an unknown language by default', () => {+ const html = highlightCode('const answer = 42;', 'not-a-language');+ expect(html).toContain('hljs-');+ });++ it('escapes instead of auto-detecting when autoDetect is false', () => {+ const html = highlightCode('const answer = 42;', 'not-a-language', false);+ expect(html).not.toContain('hljs-');+ expect(html).toBe('const answer = 42;');+ });++ it('still highlights a known language when autoDetect is false', () => {+ const html = highlightCode('const answer = 42;', 'javascript', false);+ expect(html).toContain('hljs-');+ });++ it('escapes html metacharacters when falling back to plain text', () => {+ const html = highlightCode('<script>a && b</script>', 'not-a-language', false);+ expect(html).toBe('<script>a && b</script>');+ });});diff --git a/tools/ui/tests/unit/latex-protection.test.ts b/tools/ui/tests/unit/latex-protection.test.tsindex 84328dbc17..011b9c0af7 100644--- a/tools/ui/tests/unit/latex-protection.test.ts+++ b/tools/ui/tests/unit/latex-protection.test.ts@@ -374,3 +374,41 @@ $$\n\\pi_n(\\mathbb{S}^3) = \\begin{cases}expect(output).toBe('Regular text with $x^2$.\n\n> Quote with $y^2$.\n\nMore text with $z^2$.');});});++// The fast path skips the whole protect/restore pipeline when the content has+// neither `$` nor a backslash. These pin the assumption it relies on: such+// content is returned byte-for-byte unchanged, and anything carrying a trigger+// character still goes through the full pipeline.+describe('preprocessLaTeX fast path', () => {+ const untouched = [+ ['plain prose', 'The quick brown fox jumps over the lazy dog.'],+ ['headings and lists', '# Title\n\n- one\n- two\n\n## Sub\n\n1. first\n2. second'],+ ['fenced code without escapes', '```ts\nconst x = 1;\n```'],+ ['inline code and emphasis', 'Use `npm run build` with **bold** and _italic_.'],+ ['blockquotes', '> quoted line\n> another line'],+ ['tables', '| a | b |\n| --- | --- |\n| 1 | 2 |'],+ ['brackets and parens without escapes', 'array[0] and call(arg) and [link](/url)'],+ ['multi-paragraph prose', 'First para.\n\nSecond para.\n\nThird para.']+ ];++ for (const [label, input] of untouched) {+ test(`returns ${label} unchanged`, () => {+ expect(preprocessLaTeX(input)).toBe(input);+ });+ }++ test('still processes content once a trigger character is present', () => {+ expect(preprocessLaTeX('inline \\(x^2\\) here')).toBe('inline $x^2$ here');+ expect(preprocessLaTeX('Price: $5')).toBe('Price: \\$5');+ });++ test('fast path result matches the full pipeline for trigger-free content', () => {+ // Appending a trigger char forces the slow path; stripping it back off+ // must agree with what the fast path returned for the same prefix.+ const body = 'Plain paragraph with no math.\n\nAnother paragraph here.';+ const viaFastPath = preprocessLaTeX(body);+ const viaFullPipeline = preprocessLaTeX(`${body}\n\n\\(z\\)`);++ expect(viaFullPipeline.startsWith(viaFastPath)).toBe(true);+ });+});