Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): compact a long chat's cloud copy to fit and say when a cloud save fails

David Le8042dfe04e8b
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsx
index 537b68cc..9db352bd 100644
--- a/site/src/cut/components/AiPanel.tsx
+++ b/site/src/cut/components/AiPanel.tsx
@@ -287,9 +287,9 @@ function readThreads(projectId: string): ChatThread[] {
return readRawThreads(projectId).filter((t) => isStoredThread(t) && !t.deleted) as ChatThread[];
}
-/** Persisted copies drop frame payloads (data URLs) from tool outputs — one
- * watch_video result carries a few MB of frames and localStorage holds a few
- * MB per origin in total. The live thread keeps its images; replayed turns only
+/** Persisted copies drop frame and sound payloads (data URLs) from tool
+ * outputs — one watch_video result carries a few MB of frames, and every
+ * project's history shares this browser's storage. The live thread keeps its images; replayed turns only
* ever reuse text parts, so nothing downstream misses them. */
function slimForStorage(list: ChatThread[]): ChatThread[] {
const bulky = (v: unknown) =>
@@ -343,18 +343,16 @@ function slimForStorage(list: ChatThread[]): ChatThread[] {
const out = (p as { output?: unknown }).output;
if (!out || typeof out !== "object") return p;
const o = out as Record<string, unknown>;
- if (
- !bulky(o.image) &&
- !(Array.isArray(o.images) && o.images.some(bulky))
- )
- return p;
+ const frames = bulky(o.image) || (Array.isArray(o.images) && o.images.some(bulky));
+ // listen_audio's clip, as the server's stream drops it too.
+ const sound = typeof o.audio === "string" && o.audio.startsWith("data:");
+ if (!frames && !sound) return p;
return {
...p,
output: {
...o,
- image: undefined,
- images: undefined,
- imagesOmitted: true,
+ ...(frames ? { image: undefined, images: undefined, imagesOmitted: true } : {}),
+ ...(sound ? { audio: undefined, audioOmitted: true } : {}),
},
} as typeof p;
}),
diff --git a/site/src/cut/lib/chatCloud.test.ts b/site/src/cut/lib/chatCloud.test.ts
index 47f489a8..d909b0c5 100644
--- a/site/src/cut/lib/chatCloud.test.ts
+++ b/site/src/cut/lib/chatCloud.test.ts
@@ -1,8 +1,9 @@
-import { afterEach, beforeEach, expect, test } from "bun:test";
+import { afterEach, beforeEach, expect, spyOn, test } from "bun:test";
import { bindCutMode, releaseCutMode } from "./backend";
import { cloudBackend, knownDocVersion } from "./backend/cloud";
-import { ensureCloudThreads } from "./chatCloud";
-import { chatThreadDeleted, deleteStoredThread, readProjectThreads, writeProjectThreads } from "./chatThreads";
+import { ensureCloudThreads, flushCloudThreadSaves, putCloudThread, queueCloudThreadSave } from "./chatCloud";
+import { CLOUD_THREAD_BUDGET } from "./chatCompact";
+import { chatThreadDeleted, deleteStoredThread, readProjectThreads, threadSaveFailure, writeProjectThreads } from "./chatThreads";
const originalFetch = globalThis.fetch;
const storage = new Map<string, string>();
@@ -86,3 +87,33 @@ test("observing a newer scene document preserves the loaded cloud save base", as
await cloudBackend.fetch(path, { method: "PUT", body: "{}" });
expect(putBase).toBe("1");
});
+
+test("a long thread goes up compacted, and a refused cloud save is told until one lands", async () => {
+ const parts = Array.from({ length: 295 }, (_, i) => ({
+ type: "tool-listen_audio", toolCallId: `c${i}`, state: "output-available", input: {},
+ output: { name: "voice", audio: `data:audio/wav;base64,${"x".repeat(20_000)}` },
+ }));
+ const thread = { id: "long", updatedAt: 1, messages: [{ id: "a", role: "assistant", parts }] };
+ const bodies: string[] = [];
+ let status = 500;
+ stubFetch((_url, init) => {
+ if (init?.method === "PUT") { bodies.push(String(init.body)); return Response.json({}, { status }); }
+ return Response.json([]);
+ });
+ const warn = spyOn(console, "warn").mockImplementation(() => {});
+ try {
+ queueCloudThreadSave("compact", thread);
+ await flushCloudThreadSaves();
+ expect(bodies[0].length).toBeLessThanOrEqual(CLOUD_THREAD_BUDGET);
+ expect(JSON.parse(bodies[0]).messages[0].parts).toHaveLength(295);
+ expect(threadSaveFailure()).toContain("cloud");
+ await expect(putCloudThread(cloudBackend, "compact", thread)).rejects.toThrow();
+ expect(warn.mock.calls.filter((c) => String(c[0]).includes("cloud"))).toHaveLength(1);
+ status = 200;
+ queueCloudThreadSave("compact", { ...thread, updatedAt: 2 });
+ await flushCloudThreadSaves();
+ expect(threadSaveFailure()).toBeNull();
+ } finally {
+ warn.mockRestore();
+ }
+});
diff --git a/site/src/cut/lib/chatCloud.ts b/site/src/cut/lib/chatCloud.ts
index 2b6c9076..06740fb5 100644
--- a/site/src/cut/lib/chatCloud.ts
+++ b/site/src/cut/lib/chatCloud.ts
@@ -5,8 +5,11 @@
// to merge, and ships the same slimmed payload the panel writes to storage.
import { cutMode, getBackend, type CutBackend } from "./backend";
import { cloudBackend } from "./backend/cloud";
+import { compactThreadForCloud } from "./chatCompact";
import {
chatThreadDeleted,
+ cloudThreadSaved,
+ cloudThreadSaveFailed,
deleteStoredThread,
isStoredThread,
loadProjectThreads,
@@ -51,13 +54,37 @@ export async function putCloudThread(
thread: StoredThread
): Promise<void> {
if (backend.kind !== "cloud") return;
- const res = await backend.fetch(`/api/cut/projects/${projectId}/chats/${thread.id}`, {
- method: "PUT",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify(thread),
- });
- if (res.status === 410) { deleteStoredThread(projectId, thread.id); return; }
- if (!res.ok) throw new Error("Could not save a chat thread.");
+ await sendCloudThread(backend, projectId, thread);
+}
+
+/** PUT a thread's cloud copy — compacted to fit the server's limit — and say
+ * when it doesn't land. A 410 means the thread was deleted elsewhere; a 409 is
+ * another tab's save winning a race, which the next sync merges. */
+async function sendCloudThread(backend: CutBackend, projectId: string, thread: unknown, keepalive = false): Promise<void> {
+ const id = (thread as StoredThread).id;
+ const body = JSON.stringify(compactThreadForCloud(thread));
+ let res: Response;
+ try {
+ res = await backend.fetch(`/api/cut/projects/${projectId}/chats/${id}`, {
+ method: "PUT",
+ headers: { "Content-Type": "application/json" },
+ body,
+ // A keepalive request carries at most 64 KB; a bigger one would be
+ // refused outright, so it goes as a plain request and may not finish.
+ keepalive: keepalive && body.length < 60_000,
+ });
+ } catch (err) {
+ cloudThreadSaveFailed(err);
+ throw err;
+ }
+ if (res.status === 410) { deleteStoredThread(projectId, id); return; }
+ if (res.status === 409) throw new Error("The conversation changed on the server.");
+ if (!res.ok) {
+ const err = new Error(`Could not save a chat thread (${res.status}).`);
+ cloudThreadSaveFailed(err);
+ throw err;
+ }
+ cloudThreadSaved();
}
/** Merge changed conversations and retry local saves, including offline deletions. */
@@ -164,14 +191,7 @@ async function flushOne(key: string, keepalive = false): Promise<void> {
// a cloud project, and by the time the debounce fires the user may have
// closed it — with the Mac app running, the ambient backend is then the
// engine, which has no chats route and no copy of this project.
- const response = await cloudBackend.fetch(`/api/cut/projects/${entry.projectId}/chats/${entry.threadId}`, {
- method: "PUT",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify(entry.data),
- keepalive,
- });
- if (response.status === 410) deleteStoredThread(entry.projectId, entry.threadId);
- else if (!response.ok) throw new Error("Could not save the conversation.");
+ await sendCloudThread(cloudBackend, entry.projectId, entry.data, keepalive);
} catch {
if (syncState?.projectId === entry.projectId) {
syncState.versions.delete(entry.threadId);
@@ -182,8 +202,8 @@ async function flushOne(key: string, keepalive = false): Promise<void> {
}
/** Push every queued save now instead of waiting out the debounce. */
-export function flushCloudThreadSaves(keepalive = false): void {
- for (const key of [...pending.keys()]) void flushOne(key, keepalive);
+export function flushCloudThreadSaves(keepalive = false): Promise<void> {
+ return Promise.all([...pending.keys()].map((key) => flushOne(key, keepalive))).then(() => undefined);
}
/** Delete the server copy of a thread (deleted or pruned locally). */
diff --git a/site/src/cut/lib/chatCompact.test.ts b/site/src/cut/lib/chatCompact.test.ts
new file mode 100644
index 00000000..c662c72d
--- /dev/null
+++ b/site/src/cut/lib/chatCompact.test.ts
@@ -0,0 +1,68 @@
+import { expect, test } from "bun:test";
+import { cloudThreadBytes, compactThreadForCloud, CLOUD_THREAD_BUDGET } from "./chatCompact";
+
+type Part = { type: string; toolCallId?: string; state?: string; input?: unknown; output?: unknown; text?: string };
+type Message = { id: string; role: string; parts: Part[] };
+type Thread = { id: string; updatedAt: number; messages: Message[]; pi?: unknown[] };
+
+const blob = (n: number) => "x".repeat(n);
+// One agent turn the size of the incident's: 295 calls, each with its input
+// and a JSON output, a few carrying a listen_audio clip the slim kept.
+function turn(index: number, outputChars: number, audioEvery = 0): Message[] {
+ const parts: Part[] = [{ type: "step-start" }, { type: "text", text: `turn ${index} reasoning` }];
+ for (let call = 0; call < 295; call++) {
+ const output: Record<string, unknown> = { ok: true, detail: blob(outputChars) };
+ if (audioEvery && call % audioEvery === 0) output.audio = `data:audio/wav;base64,${blob(400_000)}`;
+ parts.push({ type: "tool-add_title", toolCallId: `t${index}-${call}`, state: "output-available", input: { text: `title ${call}` }, output });
+ }
+ parts.push({ type: "text", text: `turn ${index} report` });
+ return [
+ { id: `u${index}`, role: "user", parts: [{ type: "text", text: `ask ${index}` }] },
+ { id: `a${index}`, role: "assistant", parts },
+ ];
+}
+const thread = (messages: Message[], pi?: unknown[]): Thread => ({ id: "th", updatedAt: 5, messages, ...(pi ? { pi } : {}) });
+const toolParts = (t: Thread) => t.messages.flatMap((m) => m.parts.filter((p) => p.type.startsWith("tool-")));
+
+test("a thread under the limit goes up as it is", () => {
+ const small = thread(turn(0, 100));
+ expect(compactThreadForCloud(small)).toBe(small);
+});
+
+test("a long chat's history fits the cloud copy with every turn, call and report kept", () => {
+ // Fifteen incident-sized turns with listen_audio clips: well past the
+ // server's 4,000,000-character limit.
+ const big = thread(Array.from({ length: 15 }, (_, i) => turn(i, 1000, 60)).flat());
+ expect(JSON.stringify(big).length).toBeGreaterThan(4_000_000);
+ const fit = compactThreadForCloud(big) as Thread;
+ expect(cloudThreadBytes(fit)).toBeLessThanOrEqual(CLOUD_THREAD_BUDGET);
+ expect(fit.messages.map((m) => m.id)).toEqual(big.messages.map((m) => m.id));
+ expect(toolParts(fit).map((p) => [p.toolCallId, p.state, p.input])).toEqual(toolParts(big).map((p) => [p.toolCallId, p.state, p.input]));
+ const texts = (t: Thread) => t.messages.flatMap((m) => m.parts.filter((p) => p.type === "text").map((p) => p.text));
+ expect(texts(fit)).toEqual(texts(big));
+ // The media went first; the original is untouched.
+ expect(JSON.stringify(fit)).not.toContain("data:audio");
+ expect(JSON.stringify(big)).toContain("data:audio");
+});
+
+test("when media alone isn't enough, the oldest turns' outputs are trimmed and the newest stay whole", () => {
+ const big = thread(Array.from({ length: 6 }, (_, i) => turn(i, 3000)).flat());
+ const fit = compactThreadForCloud(big, 3_000_000) as Thread;
+ expect(cloudThreadBytes(fit)).toBeLessThanOrEqual(3_000_000);
+ const outputOf = (t: Thread, id: string) => t.messages.find((m) => m.id === id)!.parts.find((p) => p.type.startsWith("tool-"))!.output as Record<string, unknown>;
+ expect(outputOf(fit, "a0").trimmed).toBe(true);
+ expect(outputOf(fit, "a5")).toEqual(outputOf(big, "a5"));
+});
+
+test("the pi context's oldest tool results are trimmed with the transcript's", () => {
+ const pi = Array.from({ length: 600 }, (_, i) => i % 2
+ ? { role: "toolResult", toolCallId: `c${i}`, content: [{ type: "text", text: blob(4000) }], details: { response: { detail: blob(4000) } } }
+ : { role: "assistant", content: [{ type: "toolCall", id: `c${i + 1}`, name: "add_title", arguments: {} }] });
+ const big = thread(turn(0, 100), pi);
+ const fit = compactThreadForCloud(big, 1_500_000) as Thread;
+ expect(cloudThreadBytes(fit)).toBeLessThanOrEqual(1_500_000);
+ const results = (fit.pi as { role: string; toolCallId?: string; details?: { trimmed?: boolean } }[]).filter((m) => m.role === "toolResult");
+ expect(results).toHaveLength(300);
+ expect(results[0].details?.trimmed).toBe(true);
+ expect(results.at(-1)).toEqual((big.pi as unknown[]).at(-1) as never);
+});
diff --git a/site/src/cut/lib/chatCompact.ts b/site/src/cut/lib/chatCompact.ts
new file mode 100644
index 00000000..58b2254a
--- /dev/null
+++ b/site/src/cut/lib/chatCompact.ts
@@ -0,0 +1,142 @@
+// The cloud copy of a chat thread has to fit one request: the server refuses a
+// thread past 4,000,000 characters (and Vercel any body past 4.5 MB). The
+// browser keeps the whole thread; what goes up is this compacted copy, so a
+// long chat's history always reaches the account instead of failing the save.
+//
+// In order, until it fits: inline media still in the thread (a listen_audio
+// clip, any data URL the storage slim didn't know) gives way to a note; then
+// the bulkiest tool results, oldest first, across the transcript and the pi
+// context, keep only a preview; then the pi context goes (the browser still
+// holds it); last, the oldest messages. Every call's id, state and input stay
+// until that last step, so the transcript reads the same and a resumed turn
+// still finds its tool parts.
+
+/** What a compacted thread may weigh, in UTF-8 bytes: under the server's
+ * character limit and the body limit, with room for the request around it. */
+export const CLOUD_THREAD_BUDGET = 3_500_000;
+
+const MEDIA_NOTE = "[media omitted from the cloud copy]";
+const TRIM_NOTE = "Trimmed from the cloud copy; run the tool again if needed.";
+/** Results smaller than this aren't worth a trim: a trimmed one weighs ~250. */
+const TRIM_MIN_BYTES = 400;
+const PREVIEW_CHARS = 160;
+
+const encoder = new TextEncoder();
+const bytesOf = (value: unknown) => encoder.encode(JSON.stringify(value) ?? "").length;
+
+/** A thread's size as the cloud save sends it. */
+export function cloudThreadBytes(thread: unknown): number {
+ return bytesOf(thread);
+}
+
+type Part = { type?: unknown; output?: unknown; [key: string]: unknown };
+type Message = { parts?: unknown; [key: string]: unknown };
+type PiMessage = { role?: unknown; content?: unknown; details?: unknown; [key: string]: unknown };
+type Thread = { messages?: unknown; pi?: unknown; [key: string]: unknown };
+
+const isRecord = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v);
+const isToolPart = (p: Part) => typeof p.type === "string" && (p.type.startsWith("tool-") || p.type === "dynamic-tool");
+
+/** The value with every inline data URL replaced by a note; the same object
+ * when there was none. */
+function withoutMedia(value: unknown): unknown {
+ if (typeof value === "string") return value.length > 256 && /^data:[^;,]+;base64,/.test(value) ? MEDIA_NOTE : value;
+ if (Array.isArray(value)) {
+ let changed = false;
+ const out = value.map((v) => {
+ const next = withoutMedia(v);
+ if (next !== v) changed = true;
+ return next;
+ });
+ return changed ? out : value;
+ }
+ if (isRecord(value)) {
+ let changed = false;
+ const out: Record<string, unknown> = {};
+ for (const [key, v] of Object.entries(value)) {
+ const next = withoutMedia(v);
+ if (next !== v) changed = true;
+ out[key] = next;
+ }
+ return changed ? out : value;
+ }
+ return value;
+}
+
+const preview = (value: unknown) => (JSON.stringify(value) ?? "").slice(0, PREVIEW_CHARS);
+
+/** One trimmable result: where it sits in history (0 oldest, 1 newest), what
+ * trimming it saves, and how. */
+type Candidate = { age: number; saves: number; trim: () => void };
+
+export function compactThreadForCloud<T>(thread: T, budget = CLOUD_THREAD_BUDGET): T {
+ let size = bytesOf(thread);
+ if (size <= budget || !isRecord(thread)) return thread;
+
+ // Media first: nothing of it is readable as text anyway.
+ const source = thread as Thread;
+ let messages: Message[] = Array.isArray(source.messages) ? (source.messages as Message[]) : [];
+ let pi: PiMessage[] | undefined = Array.isArray(source.pi) ? (source.pi as PiMessage[]) : undefined;
+ messages = withoutMedia(messages) as Message[];
+ if (pi) pi = withoutMedia(pi) as PiMessage[];
+ // Copies from here on, so trims never reach the browser's own thread.
+ messages = messages.map((m) => (Array.isArray(m.parts) ? { ...m, parts: [...(m.parts as Part[])] } : { ...m }));
+ if (pi) pi = [...pi];
+ const assemble = (): Thread => {
+ const out: Thread = { ...source, messages };
+ if (pi) out.pi = pi;
+ else delete out.pi;
+ return out;
+ };
+ size = bytesOf(assemble());
+ if (size <= budget) return assemble() as T;
+
+ // Then the bulkiest results, oldest first, transcript and pi alike.
+ const candidates: Candidate[] = [];
+ messages.forEach((message, m) => {
+ const parts = message.parts as Part[] | undefined;
+ parts?.forEach((part, p) => {
+ if (!isRecord(part) || !isToolPart(part) || part.output === undefined) return;
+ const before = bytesOf(part.output);
+ if (before < TRIM_MIN_BYTES) return;
+ const output = { trimmed: true, note: TRIM_NOTE, preview: preview(part.output) };
+ candidates.push({
+ age: messages.length > 1 ? m / (messages.length - 1) : 0,
+ saves: before - bytesOf(output),
+ trim: () => { parts[p] = { ...part, output }; },
+ });
+ });
+ });
+ pi?.forEach((message, i) => {
+ if (!isRecord(message) || message.role !== "toolResult") return;
+ const before = bytesOf(message.content) + bytesOf(message.details);
+ if (before < TRIM_MIN_BYTES) return;
+ const content = [{ type: "text", text: `${TRIM_NOTE} ${preview(message.content)}` }];
+ const details = { trimmed: true };
+ candidates.push({
+ age: pi!.length > 1 ? i / (pi!.length - 1) : 0,
+ saves: before - bytesOf(content) - bytesOf(details),
+ trim: () => { pi![i] = { ...message, content, details }; },
+ });
+ });
+ candidates.sort((a, b) => a.age - b.age);
+ for (const candidate of candidates) {
+ if (size <= budget) break;
+ if (candidate.saves <= 0) continue;
+ candidate.trim();
+ size -= candidate.saves;
+ }
+ // The running total is exact for the parts it replaced; confirm it whole.
+ size = bytesOf(assemble());
+ if (size <= budget) return assemble() as T;
+
+ // Then the pi context, which the browser still holds for its own resumes.
+ pi = undefined;
+ size = bytesOf(assemble());
+ // Last, the oldest messages — the newest always goes up.
+ while (size > budget && messages.length > 1) {
+ size -= bytesOf(messages[0]) + 1;
+ messages = messages.slice(1);
+ }
+ return assemble() as T;
+}
diff --git a/site/src/cut/lib/chatThreads.ts b/site/src/cut/lib/chatThreads.ts
index 5afd569b..c41adea9 100644
--- a/site/src/cut/lib/chatThreads.ts
+++ b/site/src/cut/lib/chatThreads.ts
@@ -122,6 +122,7 @@ export function useThreadStore(store: ThreadStore | null): void {
queue = Promise.resolve();
flushScheduled = false;
goneChecked.clear();
+ warned.local = false;
setSaveFailure(null);
}
@@ -358,18 +359,22 @@ export async function flushThreadSaves(): Promise<void> {
}
// A failed save is told once in the console and shown in the panel until a
-// save lands again.
-let saveFailure: string | null = null;
-let warned = false;
+// save lands again — for this browser's copy and for a cloud project's server
+// copy alike.
+const failures: Record<"local" | "cloud", string | null> = { local: null, cloud: null };
+const warned = { local: false, cloud: false };
const failureChanges = new Set<() => void>();
-function setSaveFailure(message: string | null): void {
- if (message === saveFailure) return;
- saveFailure = message;
+function setFailure(where: "local" | "cloud", message: string | null): void {
+ if (message === failures[where]) return;
+ failures[where] = message;
for (const onChange of failureChanges) onChange();
}
+function setSaveFailure(message: string | null): void {
+ setFailure("local", message);
+}
function saveFailed(err: unknown): void {
- if (!warned) {
- warned = true;
+ if (!warned.local) {
+ warned.local = true;
console.warn("[chat] couldn't save chat history in this browser", err);
}
setSaveFailure(
@@ -378,8 +383,20 @@ function saveFailed(err: unknown): void {
: "Chat history couldn't be saved in this browser.",
);
}
+/** A cloud project's thread didn't reach the server. It is still in this
+ * browser and the next sync retries it. */
+export function cloudThreadSaveFailed(err: unknown): void {
+ if (!warned.cloud) {
+ warned.cloud = true;
+ console.warn("[chat] couldn't save chat history to the cloud", err);
+ }
+ setFailure("cloud", "Chat history couldn't be saved to the cloud — it's kept in this browser and will retry.");
+}
+export function cloudThreadSaved(): void {
+ setFailure("cloud", null);
+}
export function threadSaveFailure(): string | null {
- return saveFailure;
+ return failures.local ?? failures.cloud;
}
export function subscribeThreadSaveFailure(onChange: () => void): () => void {
failureChanges.add(onChange);
diff --git a/site/src/cut/server/cloud/chats.ts b/site/src/cut/server/cloud/chats.ts
index a2447a75..305fe0d5 100644
--- a/site/src/cut/server/cloud/chats.ts
+++ b/site/src/cut/server/cloud/chats.ts
@@ -7,8 +7,8 @@ import type { Prisma } from "@/generated/prisma/client";
import { caught, err } from "./util";
-// Well above a slimmed thread (data-URL frames are stripped client-side), well
-// below anything that could hurt the database.
+// Well below anything that could hurt the database. The client compacts its
+// cloud copy under this (chatCompact.ts), trimming the oldest bulky results.
const MAX_THREAD_BYTES = 4_000_000;
/** Polling reads row versions; transcript bodies are fetched only after a change. */