Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(ai): avoid cubic reprocessing of tool-call-heavy chat streams (#5234)
Eric Hayes67712c26d52c
diff --git a/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx b/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsxindex 8e1276e4b7..f3b60c3d1a 100644--- a/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx+++ b/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx@@ -79,7 +79,13 @@ export function ChatMessages(props: ChatMessagesProps) {let messagesRef: HTMLDivElement | undefined;- const generatingMessage = () => {+ /*+ Memoized: asChatMessage reprocesses the whole stream from scratch, and+ this accessor is read from four places below (lastPair,+ allButLastMessagePair, isEmptyChat, and the streaming Show) — without a+ memo each of those reruns it independently on every stream update.+ */+ const generatingMessage = createMemo(() => {const s = stream();if (!s) return;if (s.isDone()) return;@@ -88,7 +94,7 @@ export function ChatMessages(props: ChatMessagesProps) {if (!message) return;if (messageContentIsEmpty(message)) return;return message;- };+ });const isStream = () => {const s = stream();diff --git a/apps/web/src/lib/core/component/AI/util/message.test.ts b/apps/web/src/lib/core/component/AI/util/message.test.tsnew file mode 100644index 0000000000..e1e00c776b--- /dev/null+++ b/apps/web/src/lib/core/component/AI/util/message.test.ts@@ -0,0 +1,103 @@+import type { ChatStream } from '@service-cognition/generated/schemas';+import { describe, expect, it } from 'vitest';+import { asChatMessage } from './message';++function textItem(text: string): ChatStream {+ return {+ type: 'chat_message_response',+ chat_id: 'chat',+ message_id: 'message',+ stream_id: 'stream',+ content: { type: 'text', text },+ };+}++function toolCallItem(i: number): ChatStream {+ return {+ type: 'chat_message_response',+ chat_id: 'chat',+ message_id: 'message',+ stream_id: 'stream',+ content: { type: 'toolCall', name: `tool${i}`, json: {}, id: `call${i}` },+ };+}++function toolResponseItem(i: number): ChatStream {+ return {+ type: 'chat_message_response',+ chat_id: 'chat',+ message_id: 'message',+ stream_id: 'stream',+ content: {+ type: 'toolCallResponseJson',+ name: `tool${i}`,+ json: { ok: true },+ id: `call${i}`,+ },+ };+}++/* Each tool call/response pair is its own unmergeable part, so a message with+ `count` tool calls ends up with roughly 2*count distinct top-level parts —+ exactly the shape a "did a bunch of tool calls" turn produces. */+function buildToolCallHeavyItems(count: number): ChatStream[] {+ const items: ChatStream[] = [];+ for (let i = 0; i < count; i++) {+ items.push(toolCallItem(i));+ items.push(toolResponseItem(i));+ }+ items.push(textItem('done'));+ return items;+}++/* Mirrors how ChatMessages.tsx's generatingMessage() actually calls this:+ once per stream update, on the whole array received so far. */+function simulateTicks(items: ChatStream[]): number {+ const start = performance.now();+ for (let i = 1; i <= items.length; i++) {+ asChatMessage(items.slice(0, i));+ }+ return performance.now() - start;+}++describe('asChatMessage', () => {+ it('merges consecutive text parts, keeps tool calls as separate parts', () => {+ const items = [+ textItem('Hello '),+ textItem('world'),+ toolCallItem(1),+ toolResponseItem(1),+ textItem('done'),+ ];+ const message = asChatMessage(items);+ expect(message?.content).toEqual([+ { type: 'text', text: 'Hello world' },+ { type: 'toolCall', name: 'tool1', json: {}, id: 'call1' },+ {+ type: 'toolCallResponseJson',+ name: 'tool1',+ json: { ok: true },+ id: 'call1',+ },+ { type: 'text', text: 'done' },+ ]);+ });++ it('does not blow up cubically over a stream of many tool calls', () => {+ /* Every item in the reduce used to spread-copy the whole accumulated+ parts array so far (`[...acc.slice(0, -1), x]` / `[...acc, x]`),+ regardless of whether it merges into the last part. Tool calls never+ merge, so each one grows the accumulator — one call already costs+ O(n^2) for a tool-call-heavy message. Since the caller+ (ChatMessages.tsx's generatingMessage) re-invokes asChatMessage on the+ whole stream once per incoming chunk, that O(n^2) per call becomes+ O(n^3) over the life of a "did a bunch of tool calls" turn — this is+ what made those responses stay slow even after the bufferedStream fix. */+ const smallTime = simulateTicks(buildToolCallHeavyItems(300));+ const largeTime = simulateTicks(buildToolCallHeavyItems(600));++ /* Doubling input: linear is ~2x, quadratic ~4x, cubic (the bug) ~8x.+ 5x sits clearly between quadratic and cubic. */+ expect(largeTime).toBeLessThan(smallTime * 5 + 20);+ });+});diff --git a/apps/web/src/lib/core/component/AI/util/message.ts b/apps/web/src/lib/core/component/AI/util/message.tsindex 03f2aa8c80..5fbc66d299 100644--- a/apps/web/src/lib/core/component/AI/util/message.ts+++ b/apps/web/src/lib/core/component/AI/util/message.ts@@ -13,34 +13,31 @@ export function asChatMessage(): ChatMessageWithAttachments | undefined {if (items.length === 0) return;- const newMessageParts: AssistantMessagePart[] = items.reduce((acc, item) => {+ /*+ Build in place instead of `[...acc.slice(0, -1), x]` / `[...acc, x]`: those+ copied every part accumulated so far on every single item, regardless of+ whether it merged into the last part. Plain text stays cheap either way+ (it all merges into one entry), but a message with many tool calls never+ merges — each one grows the accumulator — so the old approach cost+ O(n^2) per call, compounding into O(n^3) since the caller re-invokes this+ over the whole stream on every incoming chunk.+ */+ const newMessageParts: AssistantMessagePart[] = [];+ for (const item of items) {// ignore other message types- if (item.type !== 'chat_message_response') return acc;- if (acc.length === 0) {- return [item.content];- }- const last = acc[acc.length - 1];+ if (item.type !== 'chat_message_response') continue;+ const last = newMessageParts[newMessageParts.length - 1];- if (last.type === 'text' && item.content.type === 'text') {- return [- ...acc.slice(0, -1),- {- type: 'text',- text: last.text + item.content.text,- },- ];- } else if (last.type === 'thinking' && item.content.type === 'thinking') {- return [- ...acc.slice(0, -1),- {- type: 'thinking',- thinking: last.thinking + item.content.thinking,- },- ];+ if (last?.type === 'text' && item.content.type === 'text') {+ last.text += item.content.text;+ } else if (last?.type === 'thinking' && item.content.type === 'thinking') {+ last.thinking += item.content.thinking;} else {- return [...acc, item.content];+ // shallow copy: a later merge mutates `last` in place, and must never+ // touch the stream's own (otherwise-immutable) part objects+ newMessageParts.push({ ...item.content });}- }, [] as AssistantMessagePart[]);+ }const message = items.find((msg) => msg.type === 'chat_message_response');if (!message) return;