Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(ai): avoid cubic reprocessing of tool-call-heavy chat streams (#5234)

Eric Hayes67712c26d52c
diff --git a/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx b/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx
index 8e1276e4b7..f3b60c3d1a 100644
--- a/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx
+++ b/apps/web/src/lib/core/component/AI/component/message/ChatMessages.tsx
@@ -79,7 +79,13 @@ export function ChatMessages(props: ChatMessagesProps) {
let messagesRef: HTMLDivElement | undefined;
- const generatingMessage = () => {
+ /*
+ Memoized: asChatMessage reprocesses the whole stream from scratch, and
+ this accessor is read from four places below (lastPair,
+ allButLastMessagePair, isEmptyChat, and the streaming Show) — without a
+ memo each of those reruns it independently on every stream update.
+ */
+ const generatingMessage = createMemo(() => {
const s = stream();
if (!s) return;
if (s.isDone()) return;
@@ -88,7 +94,7 @@ export function ChatMessages(props: ChatMessagesProps) {
if (!message) return;
if (messageContentIsEmpty(message)) return;
return message;
- };
+ });
const isStream = () => {
const s = stream();
diff --git a/apps/web/src/lib/core/component/AI/util/message.test.ts b/apps/web/src/lib/core/component/AI/util/message.test.ts
new file mode 100644
index 0000000000..e1e00c776b
--- /dev/null
+++ b/apps/web/src/lib/core/component/AI/util/message.test.ts
@@ -0,0 +1,103 @@
+import type { ChatStream } from '@service-cognition/generated/schemas';
+import { describe, expect, it } from 'vitest';
+import { asChatMessage } from './message';
+
+function textItem(text: string): ChatStream {
+ return {
+ type: 'chat_message_response',
+ chat_id: 'chat',
+ message_id: 'message',
+ stream_id: 'stream',
+ content: { type: 'text', text },
+ };
+}
+
+function toolCallItem(i: number): ChatStream {
+ return {
+ type: 'chat_message_response',
+ chat_id: 'chat',
+ message_id: 'message',
+ stream_id: 'stream',
+ content: { type: 'toolCall', name: `tool${i}`, json: {}, id: `call${i}` },
+ };
+}
+
+function toolResponseItem(i: number): ChatStream {
+ return {
+ type: 'chat_message_response',
+ chat_id: 'chat',
+ message_id: 'message',
+ stream_id: 'stream',
+ content: {
+ type: 'toolCallResponseJson',
+ name: `tool${i}`,
+ json: { ok: true },
+ id: `call${i}`,
+ },
+ };
+}
+
+/* Each tool call/response pair is its own unmergeable part, so a message with
+ `count` tool calls ends up with roughly 2*count distinct top-level parts —
+ exactly the shape a "did a bunch of tool calls" turn produces. */
+function buildToolCallHeavyItems(count: number): ChatStream[] {
+ const items: ChatStream[] = [];
+ for (let i = 0; i < count; i++) {
+ items.push(toolCallItem(i));
+ items.push(toolResponseItem(i));
+ }
+ items.push(textItem('done'));
+ return items;
+}
+
+/* Mirrors how ChatMessages.tsx's generatingMessage() actually calls this:
+ once per stream update, on the whole array received so far. */
+function simulateTicks(items: ChatStream[]): number {
+ const start = performance.now();
+ for (let i = 1; i <= items.length; i++) {
+ asChatMessage(items.slice(0, i));
+ }
+ return performance.now() - start;
+}
+
+describe('asChatMessage', () => {
+ it('merges consecutive text parts, keeps tool calls as separate parts', () => {
+ const items = [
+ textItem('Hello '),
+ textItem('world'),
+ toolCallItem(1),
+ toolResponseItem(1),
+ textItem('done'),
+ ];
+ const message = asChatMessage(items);
+ expect(message?.content).toEqual([
+ { type: 'text', text: 'Hello world' },
+ { type: 'toolCall', name: 'tool1', json: {}, id: 'call1' },
+ {
+ type: 'toolCallResponseJson',
+ name: 'tool1',
+ json: { ok: true },
+ id: 'call1',
+ },
+ { type: 'text', text: 'done' },
+ ]);
+ });
+
+ it('does not blow up cubically over a stream of many tool calls', () => {
+ /* Every item in the reduce used to spread-copy the whole accumulated
+ parts array so far (`[...acc.slice(0, -1), x]` / `[...acc, x]`),
+ regardless of whether it merges into the last part. Tool calls never
+ merge, so each one grows the accumulator — one call already costs
+ O(n^2) for a tool-call-heavy message. Since the caller
+ (ChatMessages.tsx's generatingMessage) re-invokes asChatMessage on the
+ whole stream once per incoming chunk, that O(n^2) per call becomes
+ O(n^3) over the life of a "did a bunch of tool calls" turn — this is
+ what made those responses stay slow even after the bufferedStream fix. */
+ const smallTime = simulateTicks(buildToolCallHeavyItems(300));
+ const largeTime = simulateTicks(buildToolCallHeavyItems(600));
+
+ /* Doubling input: linear is ~2x, quadratic ~4x, cubic (the bug) ~8x.
+ 5x sits clearly between quadratic and cubic. */
+ expect(largeTime).toBeLessThan(smallTime * 5 + 20);
+ });
+});
diff --git a/apps/web/src/lib/core/component/AI/util/message.ts b/apps/web/src/lib/core/component/AI/util/message.ts
index 03f2aa8c80..5fbc66d299 100644
--- a/apps/web/src/lib/core/component/AI/util/message.ts
+++ b/apps/web/src/lib/core/component/AI/util/message.ts
@@ -13,34 +13,31 @@ export function asChatMessage(
): ChatMessageWithAttachments | undefined {
if (items.length === 0) return;
- const newMessageParts: AssistantMessagePart[] = items.reduce((acc, item) => {
+ /*
+ Build in place instead of `[...acc.slice(0, -1), x]` / `[...acc, x]`: those
+ copied every part accumulated so far on every single item, regardless of
+ whether it merged into the last part. Plain text stays cheap either way
+ (it all merges into one entry), but a message with many tool calls never
+ merges — each one grows the accumulator — so the old approach cost
+ O(n^2) per call, compounding into O(n^3) since the caller re-invokes this
+ over the whole stream on every incoming chunk.
+ */
+ const newMessageParts: AssistantMessagePart[] = [];
+ for (const item of items) {
// ignore other message types
- if (item.type !== 'chat_message_response') return acc;
- if (acc.length === 0) {
- return [item.content];
- }
- const last = acc[acc.length - 1];
+ if (item.type !== 'chat_message_response') continue;
+ const last = newMessageParts[newMessageParts.length - 1];
- if (last.type === 'text' && item.content.type === 'text') {
- return [
- ...acc.slice(0, -1),
- {
- type: 'text',
- text: last.text + item.content.text,
- },
- ];
- } else if (last.type === 'thinking' && item.content.type === 'thinking') {
- return [
- ...acc.slice(0, -1),
- {
- type: 'thinking',
- thinking: last.thinking + item.content.thinking,
- },
- ];
+ if (last?.type === 'text' && item.content.type === 'text') {
+ last.text += item.content.text;
+ } else if (last?.type === 'thinking' && item.content.type === 'thinking') {
+ last.thinking += item.content.thinking;
} else {
- return [...acc, item.content];
+ // shallow copy: a later merge mutates `last` in place, and must never
+ // touch the stream's own (otherwise-immutable) part objects
+ newMessageParts.push({ ...item.content });
}
- }, [] as AssistantMessagePart[]);
+ }
const message = items.find((msg) => msg.type === 'chat_message_response');
if (!message) return;