Skip to content

Open source CapCut alternative.

TypeScript

fix(site): trim chat request media to budget so long turns never overflow

David Leae779688d436
diff --git a/site/src/cut/lib/geminiChat.ts b/site/src/cut/lib/geminiChat.ts
index a30dd5a4..14b89108 100644
--- a/site/src/cut/lib/geminiChat.ts
+++ b/site/src/cut/lib/geminiChat.ts
@@ -8,6 +8,7 @@ import { runAiTool } from "./aiTools";
import { normalizeRef } from "./assetRef";
import { NO_CREDITS_MESSAGE, useGenerate } from "./generate";
import { hostedPost } from "./hosted";
+import { enforceMediaBudget } from "./mediaBudget";
import { refsToParts } from "./refMedia";
import { parseTurnIntent, TURN_INTENT_PROMPT, turnIntentInput, type TurnIntent } from "./turnIntent";
@@ -146,6 +147,8 @@ async function emitStepLimitSummary({
}): Promise<void> {
let text = STEP_LIMIT_FALLBACK;
try {
+ // The last round may have appended fresh tool media after its own trim.
+ enforceMediaBudget(input);
const summaryInput: Item[] = [
...input,
{
@@ -411,6 +414,9 @@ export function streamGeminiChat({
let scenePlannedThisTurn = false;
for (let round = 0; round < MAX_TOOL_ROUNDS && !settled; round++) {
+ // Media accumulated over past rounds is trimmed to budget first, so
+ // this round's request fits the hosted route whatever came before.
+ enforceMediaBudget(input);
// The round streams: text lands in the reply bubble as Gemini writes
// it, instead of popping in whole seconds later when the round-trip
// settles. The block opens on the first real (non-whitespace) delta,
diff --git a/site/src/cut/lib/mediaBudget.test.ts b/site/src/cut/lib/mediaBudget.test.ts
new file mode 100644
index 00000000..9a8932af
--- /dev/null
+++ b/site/src/cut/lib/mediaBudget.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect, test } from "bun:test";
+import { enforceMediaBudget } from "./mediaBudget";
+
+type Item = Record<string, unknown>;
+
+/** A media part whose payload decodes to `bytes`. */
+const media = (bytes: number): Item => ({
+ type: "input_audio",
+ dataBase64: "A".repeat(Math.ceil((bytes * 4) / 3)),
+ mimeType: "audio/aac",
+});
+
+const mediaTotal = (input: Item[]): number =>
+ input.reduce((n, item) => {
+ const content = item.content;
+ if (!Array.isArray(content)) return n;
+ return (
+ n +
+ content.reduce(
+ (m: number, p: Item) =>
+ m + (typeof p.dataBase64 === "string" ? Math.floor(p.dataBase64.length * 0.75) : 0),
+ 0
+ )
+ );
+ }, 0);
+
+describe("enforceMediaBudget", () => {
+ test("leaves an input under budget untouched", () => {
+ const turn: Item = { role: "user", content: [{ text: "hear this:" }, media(100)] };
+ const input = [turn];
+ enforceMediaBudget(input, 1000);
+ expect(input[0]).toBe(turn);
+ expect((input[0].content as Item[]).length).toBe(2);
+ });
+
+ test("drops the oldest media first and keeps the newest", () => {
+ const input: Item[] = [
+ { role: "user", content: [{ text: "old audio:" }, media(600)] },
+ { role: "user", content: [{ text: "new audio:" }, media(600)] },
+ ];
+ enforceMediaBudget(input, 1000);
+ const oldParts = input[0].content as Item[];
+ const newParts = input[1].content as Item[];
+ expect(oldParts.some((p) => typeof p.dataBase64 === "string")).toBe(false);
+ expect(oldParts[0]).toEqual({ text: "old audio:" });
+ expect(String(oldParts.at(-1)?.text)).toContain("dropped");
+ expect(newParts.some((p) => typeof p.dataBase64 === "string")).toBe(true);
+ expect(mediaTotal(input)).toBeLessThanOrEqual(1000);
+ });
+
+ test("trims within a step and keeps the parts that still fit", () => {
+ const input: Item[] = [
+ { role: "user", content: [media(600), media(600), { text: "label" }] },
+ ];
+ enforceMediaBudget(input, 700);
+ const parts = input[0].content as Item[];
+ expect(parts.filter((p) => typeof p.dataBase64 === "string").length).toBe(1);
+ expect(parts.some((p) => p.text === "label")).toBe(true);
+ expect(String(parts.at(-1)?.text)).toContain("dropped");
+ expect(mediaTotal(input)).toBeLessThanOrEqual(700);
+ });
+
+ test("always lands under budget, even when the newest step alone exceeds it", () => {
+ const input: Item[] = [
+ { role: "user", content: [media(500)] },
+ { role: "user", content: [media(900), media(900)] },
+ ];
+ enforceMediaBudget(input, 1000);
+ expect(mediaTotal(input)).toBeLessThanOrEqual(1000);
+ });
+
+ test("keeps assistant function calls and thought signatures intact", () => {
+ const call: Item = {
+ role: "assistant",
+ content: [{ functionCall: { id: "x", name: "seek", args: {} }, thoughtSignature: "sig" }],
+ };
+ const input: Item[] = [call, { role: "user", content: [media(600), media(600)] }];
+ enforceMediaBudget(input, 700);
+ expect(input[0]).toBe(call);
+ expect((input[0].content as Item[])[0].thoughtSignature).toBe("sig");
+ });
+});
diff --git a/site/src/cut/lib/mediaBudget.ts b/site/src/cut/lib/mediaBudget.ts
new file mode 100644
index 00000000..59ef1dd7
--- /dev/null
+++ b/site/src/cut/lib/mediaBudget.ts
@@ -0,0 +1,65 @@
+// A chat turn replays its whole conversation on every tool round, and media —
+// listen_audio sound, watch_video contact sheets, attachment payloads — rides
+// that input inline. The hosted route refuses any request whose media tops its
+// per-request cap (lib/inference/blobs.ts), so on a large project a long
+// editing pass would grow round over round until every remaining round is
+// refused. Trimming the input to this budget before each round makes that
+// impossible by construction: the oldest media leaves first, replaced by a note
+// saying how to fetch it again, and the request always fits no matter how many
+// rounds the turn runs or how large the project is. The model keeps everything
+// it concluded — text, tool calls, and tool results all survive; only the raw
+// pixels and sound it already took in are released.
+
+type Item = Record<string, unknown>;
+
+// Sized under the route's 20MB resolved-media cap, with headroom for the
+// counting difference: this counts every media part, the route counts only the
+// offloaded ones, so a request trimmed here always clears the route.
+export const MEDIA_BUDGET_BYTES = 16 * 1024 * 1024;
+
+/** Raw bytes behind one content part's inline media. Chat media parts carry
+ * base64 under `dataBase64` (refsToParts, functionResponseParts). */
+const mediaBytes = (part: Item): number =>
+ typeof part.dataBase64 === "string" ? Math.floor(part.dataBase64.length * 0.75) : 0;
+
+const droppedNote = (count: number): Item => ({
+ text: `[${count === 1 ? "A media attachment" : `${count} media attachments`} in this step ${
+ count === 1 ? "was" : "were"
+ } dropped to keep this request under the media size limit. Fetch what you still need with the media tools (listen_audio, watch_video, capture_frame), a narrower range at a time.]`,
+});
+
+/**
+ * Trim `input` in place until its inline media fits `budget`. Media parts are
+ * dropped oldest-first — input order is arrival order, so what leaves is what
+ * the model has had longest — and each trimmed step gains one note naming what
+ * left and how to get it back. Text parts, function calls and responses, and
+ * thought signatures all stay.
+ */
+export function enforceMediaBudget(input: Item[], budget = MEDIA_BUDGET_BYTES): void {
+ let total = 0;
+ for (const item of input) {
+ const content = item.content;
+ if (!Array.isArray(content)) continue;
+ for (const part of content) total += mediaBytes(part as Item);
+ }
+ for (const item of input) {
+ if (total <= budget) return;
+ const content = item.content;
+ if (!Array.isArray(content)) continue;
+ let dropped = 0;
+ const kept: Item[] = [];
+ for (const part of content as Item[]) {
+ const bytes = mediaBytes(part);
+ if (bytes > 0 && total > budget) {
+ total -= bytes;
+ dropped += 1;
+ continue;
+ }
+ kept.push(part);
+ }
+ if (dropped > 0) {
+ kept.push(droppedNote(dropped));
+ item.content = kept;
+ }
+ }
+}