Open source CapCut alternative.
TypeScript
fix(site): trim chat request media to budget so long turns never overflow
David Leae779688d436
diff --git a/site/src/cut/lib/geminiChat.ts b/site/src/cut/lib/geminiChat.tsindex a30dd5a4..14b89108 100644--- a/site/src/cut/lib/geminiChat.ts+++ b/site/src/cut/lib/geminiChat.ts@@ -8,6 +8,7 @@ import { runAiTool } from "./aiTools";import { normalizeRef } from "./assetRef";import { NO_CREDITS_MESSAGE, useGenerate } from "./generate";import { hostedPost } from "./hosted";+import { enforceMediaBudget } from "./mediaBudget";import { refsToParts } from "./refMedia";import { parseTurnIntent, TURN_INTENT_PROMPT, turnIntentInput, type TurnIntent } from "./turnIntent";@@ -146,6 +147,8 @@ async function emitStepLimitSummary({}): Promise<void> {let text = STEP_LIMIT_FALLBACK;try {+ // The last round may have appended fresh tool media after its own trim.+ enforceMediaBudget(input);const summaryInput: Item[] = [...input,{@@ -411,6 +414,9 @@ export function streamGeminiChat({let scenePlannedThisTurn = false;for (let round = 0; round < MAX_TOOL_ROUNDS && !settled; round++) {+ // Media accumulated over past rounds is trimmed to budget first, so+ // this round's request fits the hosted route whatever came before.+ enforceMediaBudget(input);// The round streams: text lands in the reply bubble as Gemini writes// it, instead of popping in whole seconds later when the round-trip// settles. The block opens on the first real (non-whitespace) delta,diff --git a/site/src/cut/lib/mediaBudget.test.ts b/site/src/cut/lib/mediaBudget.test.tsnew file mode 100644index 00000000..9a8932af--- /dev/null+++ b/site/src/cut/lib/mediaBudget.test.ts@@ -0,0 +1,82 @@+import { describe, expect, test } from "bun:test";+import { enforceMediaBudget } from "./mediaBudget";++type Item = Record<string, unknown>;++/** A media part whose payload decodes to `bytes`. */+const media = (bytes: number): Item => ({+ type: "input_audio",+ dataBase64: "A".repeat(Math.ceil((bytes * 4) / 3)),+ mimeType: "audio/aac",+});++const mediaTotal = (input: Item[]): number =>+ input.reduce((n, item) => {+ const content = item.content;+ if (!Array.isArray(content)) return n;+ return (+ n ++ content.reduce(+ (m: number, p: Item) =>+ m + (typeof p.dataBase64 === "string" ? Math.floor(p.dataBase64.length * 0.75) : 0),+ 0+ )+ );+ }, 0);++describe("enforceMediaBudget", () => {+ test("leaves an input under budget untouched", () => {+ const turn: Item = { role: "user", content: [{ text: "hear this:" }, media(100)] };+ const input = [turn];+ enforceMediaBudget(input, 1000);+ expect(input[0]).toBe(turn);+ expect((input[0].content as Item[]).length).toBe(2);+ });++ test("drops the oldest media first and keeps the newest", () => {+ const input: Item[] = [+ { role: "user", content: [{ text: "old audio:" }, media(600)] },+ { role: "user", content: [{ text: "new audio:" }, media(600)] },+ ];+ enforceMediaBudget(input, 1000);+ const oldParts = input[0].content as Item[];+ const newParts = input[1].content as Item[];+ expect(oldParts.some((p) => typeof p.dataBase64 === "string")).toBe(false);+ expect(oldParts[0]).toEqual({ text: "old audio:" });+ expect(String(oldParts.at(-1)?.text)).toContain("dropped");+ expect(newParts.some((p) => typeof p.dataBase64 === "string")).toBe(true);+ expect(mediaTotal(input)).toBeLessThanOrEqual(1000);+ });++ test("trims within a step and keeps the parts that still fit", () => {+ const input: Item[] = [+ { role: "user", content: [media(600), media(600), { text: "label" }] },+ ];+ enforceMediaBudget(input, 700);+ const parts = input[0].content as Item[];+ expect(parts.filter((p) => typeof p.dataBase64 === "string").length).toBe(1);+ expect(parts.some((p) => p.text === "label")).toBe(true);+ expect(String(parts.at(-1)?.text)).toContain("dropped");+ expect(mediaTotal(input)).toBeLessThanOrEqual(700);+ });++ test("always lands under budget, even when the newest step alone exceeds it", () => {+ const input: Item[] = [+ { role: "user", content: [media(500)] },+ { role: "user", content: [media(900), media(900)] },+ ];+ enforceMediaBudget(input, 1000);+ expect(mediaTotal(input)).toBeLessThanOrEqual(1000);+ });++ test("keeps assistant function calls and thought signatures intact", () => {+ const call: Item = {+ role: "assistant",+ content: [{ functionCall: { id: "x", name: "seek", args: {} }, thoughtSignature: "sig" }],+ };+ const input: Item[] = [call, { role: "user", content: [media(600), media(600)] }];+ enforceMediaBudget(input, 700);+ expect(input[0]).toBe(call);+ expect((input[0].content as Item[])[0].thoughtSignature).toBe("sig");+ });+});diff --git a/site/src/cut/lib/mediaBudget.ts b/site/src/cut/lib/mediaBudget.tsnew file mode 100644index 00000000..59ef1dd7--- /dev/null+++ b/site/src/cut/lib/mediaBudget.ts@@ -0,0 +1,65 @@+// A chat turn replays its whole conversation on every tool round, and media —+// listen_audio sound, watch_video contact sheets, attachment payloads — rides+// that input inline. The hosted route refuses any request whose media tops its+// per-request cap (lib/inference/blobs.ts), so on a large project a long+// editing pass would grow round over round until every remaining round is+// refused. Trimming the input to this budget before each round makes that+// impossible by construction: the oldest media leaves first, replaced by a note+// saying how to fetch it again, and the request always fits no matter how many+// rounds the turn runs or how large the project is. The model keeps everything+// it concluded — text, tool calls, and tool results all survive; only the raw+// pixels and sound it already took in are released.++type Item = Record<string, unknown>;++// Sized under the route's 20MB resolved-media cap, with headroom for the+// counting difference: this counts every media part, the route counts only the+// offloaded ones, so a request trimmed here always clears the route.+export const MEDIA_BUDGET_BYTES = 16 * 1024 * 1024;++/** Raw bytes behind one content part's inline media. Chat media parts carry+ * base64 under `dataBase64` (refsToParts, functionResponseParts). */+const mediaBytes = (part: Item): number =>+ typeof part.dataBase64 === "string" ? Math.floor(part.dataBase64.length * 0.75) : 0;++const droppedNote = (count: number): Item => ({+ text: `[${count === 1 ? "A media attachment" : `${count} media attachments`} in this step ${+ count === 1 ? "was" : "were"+ } dropped to keep this request under the media size limit. Fetch what you still need with the media tools (listen_audio, watch_video, capture_frame), a narrower range at a time.]`,+});++/**+ * Trim `input` in place until its inline media fits `budget`. Media parts are+ * dropped oldest-first — input order is arrival order, so what leaves is what+ * the model has had longest — and each trimmed step gains one note naming what+ * left and how to get it back. Text parts, function calls and responses, and+ * thought signatures all stay.+ */+export function enforceMediaBudget(input: Item[], budget = MEDIA_BUDGET_BYTES): void {+ let total = 0;+ for (const item of input) {+ const content = item.content;+ if (!Array.isArray(content)) continue;+ for (const part of content) total += mediaBytes(part as Item);+ }+ for (const item of input) {+ if (total <= budget) return;+ const content = item.content;+ if (!Array.isArray(content)) continue;+ let dropped = 0;+ const kept: Item[] = [];+ for (const part of content as Item[]) {+ const bytes = mediaBytes(part);+ if (bytes > 0 && total > budget) {+ total -= bytes;+ dropped += 1;+ continue;+ }+ kept.push(part);+ }+ if (dropped > 0) {+ kept.push(droppedNote(dropped));+ item.content = kept;+ }+ }+}