Skip to content

Open source CapCut alternative.

TypeScript

fix(site): keep tool frames out of the chat journal [rebuild]

David Lecebadcb061c7
diff --git a/site/src/cut/server/ai/bridge.ts b/site/src/cut/server/ai/bridge.ts
index 79d07f83..ac23e927 100644
--- a/site/src/cut/server/ai/bridge.ts
+++ b/site/src/cut/server/ai/bridge.ts
@@ -75,6 +75,20 @@ export function unregisterSession(key: string) {
sessionProjects.delete(key);
}
+/** The chat stream carries a tool's output without its frames: the model
+ * gets them through the returned result, the page never draws them, and one
+ * watch_video's contact sheets would otherwise fill the turn journal. */
+function streamedOutput(output: unknown): unknown {
+ if (!output || typeof output !== "object" || Array.isArray(output)) {
+ return output;
+ }
+ const { image, images, ...rest } = output as Record<string, unknown>;
+ if (image === undefined && images === undefined) {
+ return output;
+ }
+ return { ...rest, imagesOmitted: true };
+}
+
const TOOL_TIMEOUT_MS = 120_000; // subtitles generation can take a while
function detachedTool(projectId: string, toolName: string, input: unknown): Promise<{ output?: unknown; errorText?: string }> {
@@ -105,7 +119,7 @@ export function callBrowserTool(
return detachedTool(projectId, toolName, input).then((result) => {
session?.writer.write(result.errorText !== undefined
? { type: "tool-output-error", toolCallId, errorText: result.errorText }
- : { type: "tool-output-available", toolCallId, output: result.output ?? null });
+ : { type: "tool-output-available", toolCallId, output: streamedOutput(result.output ?? null) });
return result;
});
}
@@ -133,7 +147,7 @@ export function callBrowserTool(
if (r.errorText !== undefined) {
session.writer.write({ type: "tool-output-error", toolCallId, errorText: r.errorText });
} else {
- session.writer.write({ type: "tool-output-available", toolCallId, output: r.output ?? null });
+ session.writer.write({ type: "tool-output-available", toolCallId, output: streamedOutput(r.output ?? null) });
}
resolve(r);
},
diff --git a/site/src/cut/server/ai/turnStreams.test.ts b/site/src/cut/server/ai/turnStreams.test.ts
index 07f8368e..d95868b8 100644
--- a/site/src/cut/server/ai/turnStreams.test.ts
+++ b/site/src/cut/server/ai/turnStreams.test.ts
@@ -4,7 +4,7 @@ import os from "node:os";
import path from "node:path";
import type { UIMessageChunk } from "ai";
import { cancelTurnStream, followTurnStream, startTurnStream } from "./turnStreams";
-import { callBrowserTool, claimBrowserTool, detachSession, registerSession, unregisterSession } from "./bridge";
+import { callBrowserTool, claimBrowserTool, detachSession, registerSession, resolveBrowserTool, unregisterSession } from "./bridge";
import { createProject, readProject } from "../projects";
const root = await mkdtemp(path.join(os.tmpdir(), "donkey-chat-test-"));
@@ -93,3 +93,19 @@ test("an unclaimed tool transfers to the engine when the tab disconnects", async
expect(chunks.filter((chunk) => chunk.toolCallId === id).map((chunk) => chunk.type)).toEqual(["tool-input-available", "tool-output-available"]);
unregisterSession("unclaimed");
});
+
+test("frames reach the model and stay out of the turn journal", async () => {
+ const chunks: Record<string, unknown>[] = [];
+ registerSession("frames", { write: (chunk) => chunks.push(chunk) });
+ const pending = callBrowserTool("frames", "watch_video", { assetId: "a" });
+ const id = String(chunks[0].toolCallId);
+ const frame = `data:image/jpeg;base64,${"A".repeat(2_000_000)}`;
+ resolveBrowserTool("frames", id, { output: { duration: 10, image: frame, images: [frame, frame] } });
+ const result = (await pending).output as { image?: string; images?: string[] };
+ expect(result.image).toBe(frame);
+ expect(result.images).toEqual([frame, frame]);
+ const journaled = chunks.find((chunk) => chunk.type === "tool-output-available");
+ expect(journaled?.output).toEqual({ duration: 10, imagesOmitted: true });
+ expect(JSON.stringify(journaled).length).toBeLessThan(1000);
+ unregisterSession("frames");
+});