Open source CapCut alternative.
TypeScript
fix(cut): compact a long chat's cloud copy to fit and say when a cloud save fails
David Le8042dfe04e8b
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsxindex 537b68cc..9db352bd 100644--- a/site/src/cut/components/AiPanel.tsx+++ b/site/src/cut/components/AiPanel.tsx@@ -287,9 +287,9 @@ function readThreads(projectId: string): ChatThread[] {return readRawThreads(projectId).filter((t) => isStoredThread(t) && !t.deleted) as ChatThread[];}-/** Persisted copies drop frame payloads (data URLs) from tool outputs — one- * watch_video result carries a few MB of frames and localStorage holds a few- * MB per origin in total. The live thread keeps its images; replayed turns only+/** Persisted copies drop frame and sound payloads (data URLs) from tool+ * outputs — one watch_video result carries a few MB of frames, and every+ * project's history shares this browser's storage. The live thread keeps its images; replayed turns only* ever reuse text parts, so nothing downstream misses them. */function slimForStorage(list: ChatThread[]): ChatThread[] {const bulky = (v: unknown) =>@@ -343,18 +343,16 @@ function slimForStorage(list: ChatThread[]): ChatThread[] {const out = (p as { output?: unknown }).output;if (!out || typeof out !== "object") return p;const o = out as Record<string, unknown>;- if (- !bulky(o.image) &&- !(Array.isArray(o.images) && o.images.some(bulky))- )- return p;+ const frames = bulky(o.image) || (Array.isArray(o.images) && o.images.some(bulky));+ // listen_audio's clip, as the server's stream drops it too.+ const sound = typeof o.audio === "string" && o.audio.startsWith("data:");+ if (!frames && !sound) return p;return {...p,output: {...o,- image: undefined,- images: undefined,- imagesOmitted: true,+ ...(frames ? { image: undefined, images: undefined, imagesOmitted: true } : {}),+ ...(sound ? { audio: undefined, audioOmitted: true } : {}),},} as typeof p;}),diff --git a/site/src/cut/lib/chatCloud.test.ts b/site/src/cut/lib/chatCloud.test.tsindex 47f489a8..d909b0c5 100644--- a/site/src/cut/lib/chatCloud.test.ts+++ b/site/src/cut/lib/chatCloud.test.ts@@ -1,8 +1,9 @@-import { afterEach, beforeEach, expect, test } from "bun:test";+import { afterEach, beforeEach, expect, spyOn, test } from "bun:test";import { bindCutMode, releaseCutMode } from "./backend";import { cloudBackend, knownDocVersion } from "./backend/cloud";-import { ensureCloudThreads } from "./chatCloud";-import { chatThreadDeleted, deleteStoredThread, readProjectThreads, writeProjectThreads } from "./chatThreads";+import { ensureCloudThreads, flushCloudThreadSaves, putCloudThread, queueCloudThreadSave } from "./chatCloud";+import { CLOUD_THREAD_BUDGET } from "./chatCompact";+import { chatThreadDeleted, deleteStoredThread, readProjectThreads, threadSaveFailure, writeProjectThreads } from "./chatThreads";const originalFetch = globalThis.fetch;const storage = new Map<string, string>();@@ -86,3 +87,33 @@ test("observing a newer scene document preserves the loaded cloud save base", asawait cloudBackend.fetch(path, { method: "PUT", body: "{}" });expect(putBase).toBe("1");});++test("a long thread goes up compacted, and a refused cloud save is told until one lands", async () => {+ const parts = Array.from({ length: 295 }, (_, i) => ({+ type: "tool-listen_audio", toolCallId: `c${i}`, state: "output-available", input: {},+ output: { name: "voice", audio: `data:audio/wav;base64,${"x".repeat(20_000)}` },+ }));+ const thread = { id: "long", updatedAt: 1, messages: [{ id: "a", role: "assistant", parts }] };+ const bodies: string[] = [];+ let status = 500;+ stubFetch((_url, init) => {+ if (init?.method === "PUT") { bodies.push(String(init.body)); return Response.json({}, { status }); }+ return Response.json([]);+ });+ const warn = spyOn(console, "warn").mockImplementation(() => {});+ try {+ queueCloudThreadSave("compact", thread);+ await flushCloudThreadSaves();+ expect(bodies[0].length).toBeLessThanOrEqual(CLOUD_THREAD_BUDGET);+ expect(JSON.parse(bodies[0]).messages[0].parts).toHaveLength(295);+ expect(threadSaveFailure()).toContain("cloud");+ await expect(putCloudThread(cloudBackend, "compact", thread)).rejects.toThrow();+ expect(warn.mock.calls.filter((c) => String(c[0]).includes("cloud"))).toHaveLength(1);+ status = 200;+ queueCloudThreadSave("compact", { ...thread, updatedAt: 2 });+ await flushCloudThreadSaves();+ expect(threadSaveFailure()).toBeNull();+ } finally {+ warn.mockRestore();+ }+});diff --git a/site/src/cut/lib/chatCloud.ts b/site/src/cut/lib/chatCloud.tsindex 2b6c9076..06740fb5 100644--- a/site/src/cut/lib/chatCloud.ts+++ b/site/src/cut/lib/chatCloud.ts@@ -5,8 +5,11 @@// to merge, and ships the same slimmed payload the panel writes to storage.import { cutMode, getBackend, type CutBackend } from "./backend";import { cloudBackend } from "./backend/cloud";+import { compactThreadForCloud } from "./chatCompact";import {chatThreadDeleted,+ cloudThreadSaved,+ cloudThreadSaveFailed,deleteStoredThread,isStoredThread,loadProjectThreads,@@ -51,13 +54,37 @@ export async function putCloudThread(thread: StoredThread): Promise<void> {if (backend.kind !== "cloud") return;- const res = await backend.fetch(`/api/cut/projects/${projectId}/chats/${thread.id}`, {- method: "PUT",- headers: { "Content-Type": "application/json" },- body: JSON.stringify(thread),- });- if (res.status === 410) { deleteStoredThread(projectId, thread.id); return; }- if (!res.ok) throw new Error("Could not save a chat thread.");+ await sendCloudThread(backend, projectId, thread);+}++/** PUT a thread's cloud copy — compacted to fit the server's limit — and say+ * when it doesn't land. A 410 means the thread was deleted elsewhere; a 409 is+ * another tab's save winning a race, which the next sync merges. */+async function sendCloudThread(backend: CutBackend, projectId: string, thread: unknown, keepalive = false): Promise<void> {+ const id = (thread as StoredThread).id;+ const body = JSON.stringify(compactThreadForCloud(thread));+ let res: Response;+ try {+ res = await backend.fetch(`/api/cut/projects/${projectId}/chats/${id}`, {+ method: "PUT",+ headers: { "Content-Type": "application/json" },+ body,+ // A keepalive request carries at most 64 KB; a bigger one would be+ // refused outright, so it goes as a plain request and may not finish.+ keepalive: keepalive && body.length < 60_000,+ });+ } catch (err) {+ cloudThreadSaveFailed(err);+ throw err;+ }+ if (res.status === 410) { deleteStoredThread(projectId, id); return; }+ if (res.status === 409) throw new Error("The conversation changed on the server.");+ if (!res.ok) {+ const err = new Error(`Could not save a chat thread (${res.status}).`);+ cloudThreadSaveFailed(err);+ throw err;+ }+ cloudThreadSaved();}/** Merge changed conversations and retry local saves, including offline deletions. */@@ -164,14 +191,7 @@ async function flushOne(key: string, keepalive = false): Promise<void> {// a cloud project, and by the time the debounce fires the user may have// closed it — with the Mac app running, the ambient backend is then the// engine, which has no chats route and no copy of this project.- const response = await cloudBackend.fetch(`/api/cut/projects/${entry.projectId}/chats/${entry.threadId}`, {- method: "PUT",- headers: { "Content-Type": "application/json" },- body: JSON.stringify(entry.data),- keepalive,- });- if (response.status === 410) deleteStoredThread(entry.projectId, entry.threadId);- else if (!response.ok) throw new Error("Could not save the conversation.");+ await sendCloudThread(cloudBackend, entry.projectId, entry.data, keepalive);} catch {if (syncState?.projectId === entry.projectId) {syncState.versions.delete(entry.threadId);@@ -182,8 +202,8 @@ async function flushOne(key: string, keepalive = false): Promise<void> {}/** Push every queued save now instead of waiting out the debounce. */-export function flushCloudThreadSaves(keepalive = false): void {- for (const key of [...pending.keys()]) void flushOne(key, keepalive);+export function flushCloudThreadSaves(keepalive = false): Promise<void> {+ return Promise.all([...pending.keys()].map((key) => flushOne(key, keepalive))).then(() => undefined);}/** Delete the server copy of a thread (deleted or pruned locally). */diff --git a/site/src/cut/lib/chatCompact.test.ts b/site/src/cut/lib/chatCompact.test.tsnew file mode 100644index 00000000..c662c72d--- /dev/null+++ b/site/src/cut/lib/chatCompact.test.ts@@ -0,0 +1,68 @@+import { expect, test } from "bun:test";+import { cloudThreadBytes, compactThreadForCloud, CLOUD_THREAD_BUDGET } from "./chatCompact";++type Part = { type: string; toolCallId?: string; state?: string; input?: unknown; output?: unknown; text?: string };+type Message = { id: string; role: string; parts: Part[] };+type Thread = { id: string; updatedAt: number; messages: Message[]; pi?: unknown[] };++const blob = (n: number) => "x".repeat(n);+// One agent turn the size of the incident's: 295 calls, each with its input+// and a JSON output, a few carrying a listen_audio clip the slim kept.+function turn(index: number, outputChars: number, audioEvery = 0): Message[] {+ const parts: Part[] = [{ type: "step-start" }, { type: "text", text: `turn ${index} reasoning` }];+ for (let call = 0; call < 295; call++) {+ const output: Record<string, unknown> = { ok: true, detail: blob(outputChars) };+ if (audioEvery && call % audioEvery === 0) output.audio = `data:audio/wav;base64,${blob(400_000)}`;+ parts.push({ type: "tool-add_title", toolCallId: `t${index}-${call}`, state: "output-available", input: { text: `title ${call}` }, output });+ }+ parts.push({ type: "text", text: `turn ${index} report` });+ return [+ { id: `u${index}`, role: "user", parts: [{ type: "text", text: `ask ${index}` }] },+ { id: `a${index}`, role: "assistant", parts },+ ];+}+const thread = (messages: Message[], pi?: unknown[]): Thread => ({ id: "th", updatedAt: 5, messages, ...(pi ? { pi } : {}) });+const toolParts = (t: Thread) => t.messages.flatMap((m) => m.parts.filter((p) => p.type.startsWith("tool-")));++test("a thread under the limit goes up as it is", () => {+ const small = thread(turn(0, 100));+ expect(compactThreadForCloud(small)).toBe(small);+});++test("a long chat's history fits the cloud copy with every turn, call and report kept", () => {+ // Fifteen incident-sized turns with listen_audio clips: well past the+ // server's 4,000,000-character limit.+ const big = thread(Array.from({ length: 15 }, (_, i) => turn(i, 1000, 60)).flat());+ expect(JSON.stringify(big).length).toBeGreaterThan(4_000_000);+ const fit = compactThreadForCloud(big) as Thread;+ expect(cloudThreadBytes(fit)).toBeLessThanOrEqual(CLOUD_THREAD_BUDGET);+ expect(fit.messages.map((m) => m.id)).toEqual(big.messages.map((m) => m.id));+ expect(toolParts(fit).map((p) => [p.toolCallId, p.state, p.input])).toEqual(toolParts(big).map((p) => [p.toolCallId, p.state, p.input]));+ const texts = (t: Thread) => t.messages.flatMap((m) => m.parts.filter((p) => p.type === "text").map((p) => p.text));+ expect(texts(fit)).toEqual(texts(big));+ // The media went first; the original is untouched.+ expect(JSON.stringify(fit)).not.toContain("data:audio");+ expect(JSON.stringify(big)).toContain("data:audio");+});++test("when media alone isn't enough, the oldest turns' outputs are trimmed and the newest stay whole", () => {+ const big = thread(Array.from({ length: 6 }, (_, i) => turn(i, 3000)).flat());+ const fit = compactThreadForCloud(big, 3_000_000) as Thread;+ expect(cloudThreadBytes(fit)).toBeLessThanOrEqual(3_000_000);+ const outputOf = (t: Thread, id: string) => t.messages.find((m) => m.id === id)!.parts.find((p) => p.type.startsWith("tool-"))!.output as Record<string, unknown>;+ expect(outputOf(fit, "a0").trimmed).toBe(true);+ expect(outputOf(fit, "a5")).toEqual(outputOf(big, "a5"));+});++test("the pi context's oldest tool results are trimmed with the transcript's", () => {+ const pi = Array.from({ length: 600 }, (_, i) => i % 2+ ? { role: "toolResult", toolCallId: `c${i}`, content: [{ type: "text", text: blob(4000) }], details: { response: { detail: blob(4000) } } }+ : { role: "assistant", content: [{ type: "toolCall", id: `c${i + 1}`, name: "add_title", arguments: {} }] });+ const big = thread(turn(0, 100), pi);+ const fit = compactThreadForCloud(big, 1_500_000) as Thread;+ expect(cloudThreadBytes(fit)).toBeLessThanOrEqual(1_500_000);+ const results = (fit.pi as { role: string; toolCallId?: string; details?: { trimmed?: boolean } }[]).filter((m) => m.role === "toolResult");+ expect(results).toHaveLength(300);+ expect(results[0].details?.trimmed).toBe(true);+ expect(results.at(-1)).toEqual((big.pi as unknown[]).at(-1) as never);+});diff --git a/site/src/cut/lib/chatCompact.ts b/site/src/cut/lib/chatCompact.tsnew file mode 100644index 00000000..58b2254a--- /dev/null+++ b/site/src/cut/lib/chatCompact.ts@@ -0,0 +1,142 @@+// The cloud copy of a chat thread has to fit one request: the server refuses a+// thread past 4,000,000 characters (and Vercel any body past 4.5 MB). The+// browser keeps the whole thread; what goes up is this compacted copy, so a+// long chat's history always reaches the account instead of failing the save.+//+// In order, until it fits: inline media still in the thread (a listen_audio+// clip, any data URL the storage slim didn't know) gives way to a note; then+// the bulkiest tool results, oldest first, across the transcript and the pi+// context, keep only a preview; then the pi context goes (the browser still+// holds it); last, the oldest messages. Every call's id, state and input stay+// until that last step, so the transcript reads the same and a resumed turn+// still finds its tool parts.++/** What a compacted thread may weigh, in UTF-8 bytes: under the server's+ * character limit and the body limit, with room for the request around it. */+export const CLOUD_THREAD_BUDGET = 3_500_000;++const MEDIA_NOTE = "[media omitted from the cloud copy]";+const TRIM_NOTE = "Trimmed from the cloud copy; run the tool again if needed.";+/** Results smaller than this aren't worth a trim: a trimmed one weighs ~250. */+const TRIM_MIN_BYTES = 400;+const PREVIEW_CHARS = 160;++const encoder = new TextEncoder();+const bytesOf = (value: unknown) => encoder.encode(JSON.stringify(value) ?? "").length;++/** A thread's size as the cloud save sends it. */+export function cloudThreadBytes(thread: unknown): number {+ return bytesOf(thread);+}++type Part = { type?: unknown; output?: unknown; [key: string]: unknown };+type Message = { parts?: unknown; [key: string]: unknown };+type PiMessage = { role?: unknown; content?: unknown; details?: unknown; [key: string]: unknown };+type Thread = { messages?: unknown; pi?: unknown; [key: string]: unknown };++const isRecord = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v);+const isToolPart = (p: Part) => typeof p.type === "string" && (p.type.startsWith("tool-") || p.type === "dynamic-tool");++/** The value with every inline data URL replaced by a note; the same object+ * when there was none. */+function withoutMedia(value: unknown): unknown {+ if (typeof value === "string") return value.length > 256 && /^data:[^;,]+;base64,/.test(value) ? MEDIA_NOTE : value;+ if (Array.isArray(value)) {+ let changed = false;+ const out = value.map((v) => {+ const next = withoutMedia(v);+ if (next !== v) changed = true;+ return next;+ });+ return changed ? out : value;+ }+ if (isRecord(value)) {+ let changed = false;+ const out: Record<string, unknown> = {};+ for (const [key, v] of Object.entries(value)) {+ const next = withoutMedia(v);+ if (next !== v) changed = true;+ out[key] = next;+ }+ return changed ? out : value;+ }+ return value;+}++const preview = (value: unknown) => (JSON.stringify(value) ?? "").slice(0, PREVIEW_CHARS);++/** One trimmable result: where it sits in history (0 oldest, 1 newest), what+ * trimming it saves, and how. */+type Candidate = { age: number; saves: number; trim: () => void };++export function compactThreadForCloud<T>(thread: T, budget = CLOUD_THREAD_BUDGET): T {+ let size = bytesOf(thread);+ if (size <= budget || !isRecord(thread)) return thread;++ // Media first: nothing of it is readable as text anyway.+ const source = thread as Thread;+ let messages: Message[] = Array.isArray(source.messages) ? (source.messages as Message[]) : [];+ let pi: PiMessage[] | undefined = Array.isArray(source.pi) ? (source.pi as PiMessage[]) : undefined;+ messages = withoutMedia(messages) as Message[];+ if (pi) pi = withoutMedia(pi) as PiMessage[];+ // Copies from here on, so trims never reach the browser's own thread.+ messages = messages.map((m) => (Array.isArray(m.parts) ? { ...m, parts: [...(m.parts as Part[])] } : { ...m }));+ if (pi) pi = [...pi];+ const assemble = (): Thread => {+ const out: Thread = { ...source, messages };+ if (pi) out.pi = pi;+ else delete out.pi;+ return out;+ };+ size = bytesOf(assemble());+ if (size <= budget) return assemble() as T;++ // Then the bulkiest results, oldest first, transcript and pi alike.+ const candidates: Candidate[] = [];+ messages.forEach((message, m) => {+ const parts = message.parts as Part[] | undefined;+ parts?.forEach((part, p) => {+ if (!isRecord(part) || !isToolPart(part) || part.output === undefined) return;+ const before = bytesOf(part.output);+ if (before < TRIM_MIN_BYTES) return;+ const output = { trimmed: true, note: TRIM_NOTE, preview: preview(part.output) };+ candidates.push({+ age: messages.length > 1 ? m / (messages.length - 1) : 0,+ saves: before - bytesOf(output),+ trim: () => { parts[p] = { ...part, output }; },+ });+ });+ });+ pi?.forEach((message, i) => {+ if (!isRecord(message) || message.role !== "toolResult") return;+ const before = bytesOf(message.content) + bytesOf(message.details);+ if (before < TRIM_MIN_BYTES) return;+ const content = [{ type: "text", text: `${TRIM_NOTE} ${preview(message.content)}` }];+ const details = { trimmed: true };+ candidates.push({+ age: pi!.length > 1 ? i / (pi!.length - 1) : 0,+ saves: before - bytesOf(content) - bytesOf(details),+ trim: () => { pi![i] = { ...message, content, details }; },+ });+ });+ candidates.sort((a, b) => a.age - b.age);+ for (const candidate of candidates) {+ if (size <= budget) break;+ if (candidate.saves <= 0) continue;+ candidate.trim();+ size -= candidate.saves;+ }+ // The running total is exact for the parts it replaced; confirm it whole.+ size = bytesOf(assemble());+ if (size <= budget) return assemble() as T;++ // Then the pi context, which the browser still holds for its own resumes.+ pi = undefined;+ size = bytesOf(assemble());+ // Last, the oldest messages — the newest always goes up.+ while (size > budget && messages.length > 1) {+ size -= bytesOf(messages[0]) + 1;+ messages = messages.slice(1);+ }+ return assemble() as T;+}diff --git a/site/src/cut/lib/chatThreads.ts b/site/src/cut/lib/chatThreads.tsindex 5afd569b..c41adea9 100644--- a/site/src/cut/lib/chatThreads.ts+++ b/site/src/cut/lib/chatThreads.ts@@ -122,6 +122,7 @@ export function useThreadStore(store: ThreadStore | null): void {queue = Promise.resolve();flushScheduled = false;goneChecked.clear();+ warned.local = false;setSaveFailure(null);}@@ -358,18 +359,22 @@ export async function flushThreadSaves(): Promise<void> {}// A failed save is told once in the console and shown in the panel until a-// save lands again.-let saveFailure: string | null = null;-let warned = false;+// save lands again — for this browser's copy and for a cloud project's server+// copy alike.+const failures: Record<"local" | "cloud", string | null> = { local: null, cloud: null };+const warned = { local: false, cloud: false };const failureChanges = new Set<() => void>();-function setSaveFailure(message: string | null): void {- if (message === saveFailure) return;- saveFailure = message;+function setFailure(where: "local" | "cloud", message: string | null): void {+ if (message === failures[where]) return;+ failures[where] = message;for (const onChange of failureChanges) onChange();}+function setSaveFailure(message: string | null): void {+ setFailure("local", message);+}function saveFailed(err: unknown): void {- if (!warned) {- warned = true;+ if (!warned.local) {+ warned.local = true;console.warn("[chat] couldn't save chat history in this browser", err);}setSaveFailure(@@ -378,8 +383,20 @@ function saveFailed(err: unknown): void {: "Chat history couldn't be saved in this browser.",);}+/** A cloud project's thread didn't reach the server. It is still in this+ * browser and the next sync retries it. */+export function cloudThreadSaveFailed(err: unknown): void {+ if (!warned.cloud) {+ warned.cloud = true;+ console.warn("[chat] couldn't save chat history to the cloud", err);+ }+ setFailure("cloud", "Chat history couldn't be saved to the cloud — it's kept in this browser and will retry.");+}+export function cloudThreadSaved(): void {+ setFailure("cloud", null);+}export function threadSaveFailure(): string | null {- return saveFailure;+ return failures.local ?? failures.cloud;}export function subscribeThreadSaveFailure(onChange: () => void): () => void {failureChanges.add(onChange);diff --git a/site/src/cut/server/cloud/chats.ts b/site/src/cut/server/cloud/chats.tsindex a2447a75..305fe0d5 100644--- a/site/src/cut/server/cloud/chats.ts+++ b/site/src/cut/server/cloud/chats.ts@@ -7,8 +7,8 @@ import type { Prisma } from "@/generated/prisma/client";import { caught, err } from "./util";-// Well above a slimmed thread (data-URL frames are stripped client-side), well-// below anything that could hurt the database.+// Well below anything that could hurt the database. The client compacts its+// cloud copy under this (chatCompact.ts), trimming the oldest bulky results.const MAX_THREAD_BYTES = 4_000_000;/** Polling reads row versions; transcript bodies are fetched only after a change. */