Open source CapCut alternative.
TypeScript
fix(site): keep tool frames out of the chat journal [rebuild]
David Lecebadcb061c7
diff --git a/site/src/cut/server/ai/bridge.ts b/site/src/cut/server/ai/bridge.tsindex 79d07f83..ac23e927 100644--- a/site/src/cut/server/ai/bridge.ts+++ b/site/src/cut/server/ai/bridge.ts@@ -75,6 +75,20 @@ export function unregisterSession(key: string) {sessionProjects.delete(key);}+/** The chat stream carries a tool's output without its frames: the model+ * gets them through the returned result, the page never draws them, and one+ * watch_video's contact sheets would otherwise fill the turn journal. */+function streamedOutput(output: unknown): unknown {+ if (!output || typeof output !== "object" || Array.isArray(output)) {+ return output;+ }+ const { image, images, ...rest } = output as Record<string, unknown>;+ if (image === undefined && images === undefined) {+ return output;+ }+ return { ...rest, imagesOmitted: true };+}+const TOOL_TIMEOUT_MS = 120_000; // subtitles generation can take a whilefunction detachedTool(projectId: string, toolName: string, input: unknown): Promise<{ output?: unknown; errorText?: string }> {@@ -105,7 +119,7 @@ export function callBrowserTool(return detachedTool(projectId, toolName, input).then((result) => {session?.writer.write(result.errorText !== undefined? { type: "tool-output-error", toolCallId, errorText: result.errorText }- : { type: "tool-output-available", toolCallId, output: result.output ?? null });+ : { type: "tool-output-available", toolCallId, output: streamedOutput(result.output ?? null) });return result;});}@@ -133,7 +147,7 @@ export function callBrowserTool(if (r.errorText !== undefined) {session.writer.write({ type: "tool-output-error", toolCallId, errorText: r.errorText });} else {- session.writer.write({ type: "tool-output-available", toolCallId, output: r.output ?? null });+ session.writer.write({ type: "tool-output-available", toolCallId, output: streamedOutput(r.output ?? null) });}resolve(r);},diff --git a/site/src/cut/server/ai/turnStreams.test.ts b/site/src/cut/server/ai/turnStreams.test.tsindex 07f8368e..d95868b8 100644--- a/site/src/cut/server/ai/turnStreams.test.ts+++ b/site/src/cut/server/ai/turnStreams.test.ts@@ -4,7 +4,7 @@ import os from "node:os";import path from "node:path";import type { UIMessageChunk } from "ai";import { cancelTurnStream, followTurnStream, startTurnStream } from "./turnStreams";-import { callBrowserTool, claimBrowserTool, detachSession, registerSession, unregisterSession } from "./bridge";+import { callBrowserTool, claimBrowserTool, detachSession, registerSession, resolveBrowserTool, unregisterSession } from "./bridge";import { createProject, readProject } from "../projects";const root = await mkdtemp(path.join(os.tmpdir(), "donkey-chat-test-"));@@ -93,3 +93,19 @@ test("an unclaimed tool transfers to the engine when the tab disconnects", asyncexpect(chunks.filter((chunk) => chunk.toolCallId === id).map((chunk) => chunk.type)).toEqual(["tool-input-available", "tool-output-available"]);unregisterSession("unclaimed");});++test("frames reach the model and stay out of the turn journal", async () => {+ const chunks: Record<string, unknown>[] = [];+ registerSession("frames", { write: (chunk) => chunks.push(chunk) });+ const pending = callBrowserTool("frames", "watch_video", { assetId: "a" });+ const id = String(chunks[0].toolCallId);+ const frame = `data:image/jpeg;base64,${"A".repeat(2_000_000)}`;+ resolveBrowserTool("frames", id, { output: { duration: 10, image: frame, images: [frame, frame] } });+ const result = (await pending).output as { image?: string; images?: string[] };+ expect(result.image).toBe(frame);+ expect(result.images).toEqual([frame, frame]);+ const journaled = chunks.find((chunk) => chunk.type === "tool-output-available");+ expect(journaled?.output).toEqual({ duration: 10, imagesOmitted: true });+ expect(JSON.stringify(journaled).length).toBeLessThan(1000);+ unregisterSession("frames");+});