Open source CapCut alternative.
TypeScript
feat(site): fold mid-turn messages into Claude and Codex chat turns [rebuild]
David Led3d966ebd96a
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsxindex b02a4cb3..cc268425 100644--- a/site/src/cut/components/AiPanel.tsx+++ b/site/src/cut/components/AiPanel.tsx@@ -108,7 +108,7 @@ import { holdEditorChat } from "@/cut/lib/editorWork";import { recoverSceneCall } from "@/cut/lib/chatRecovery";import { replayChatStream } from "@/cut/lib/chatReplay";import { INTERRUPTED_ERROR, isResumeMessage, RESUME_LIMIT, resumePrompt, scrubLostTurn, unfinishedAsk, type ResumeMetadata, type TurnSettled } from "@/cut/lib/chatResume";-import { claimEngineTool, cancelEngineChat } from "@/cut/lib/engineChat";+import { claimEngineTool, cancelEngineChat, foldIntoEngineChat } from "@/cut/lib/engineChat";import { projectBackend } from "@/cut/lib/residency";import { beginBrowserChat, browserChatRunning, cancelBrowserChat, watchBrowserChatTurns } from "@/cut/lib/browserChatTurns";import { putCloudThread } from "@/cut/lib/chatCloud";@@ -1530,8 +1530,7 @@ function ChatSession({// joins the transcript as its own message, a spawn starts a parallel// thread, and the rest wait in the tray. Each verdict is checked against// the live queue before it acts, since a row can be edited, removed, or- // drained while the call is out. A turn on the local engine cannot take a- // fold, so there the row waits as before.+ // drained while the call is out.const triage = async (rows: QueuedMessage[]) => {const ask = runningAskRef.current;if (!ask) return;@@ -1576,14 +1575,21 @@ function ChatSession({if (verdict === "spawn") {const id = onSpawn({ text, attachments: live, model });setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "spawned", threadId: id } : m)));- } else if (verdict === "fold" && provider(model) === "gemini" && cutChatLive(threadId)) {+ } else if (verdict === "fold" && (provider(model) !== "gemini" || cutChatLive(threadId))) {setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "folding" } : m)));- const folded = await foldIntoCutChat({- threadId,- text,- attachments: live,- deps: productionDeps(projectId, turnSignalRef.current),- });+ const folded =+ provider(model) === "gemini"+ ? await foldIntoCutChat({+ threadId,+ text,+ attachments: live,+ deps: productionDeps(projectId, turnSignalRef.current),+ })+ : await foldIntoEngineChat(projectId, threadId, {+ text,+ attachments: live,+ context: productionDeps(projectId).buildContext(),+ }).catch(() => false);if (!folded) {setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "queued" } : m)));continue;diff --git a/site/src/cut/lib/engineChat.ts b/site/src/cut/lib/engineChat.tsindex 62100f90..0437eae1 100644--- a/site/src/cut/lib/engineChat.ts+++ b/site/src/cut/lib/engineChat.ts@@ -15,3 +15,18 @@ export async function cancelEngineChat(projectId: string, threadId: string): Pro});if (!response.ok) throw new Error("Could not stop the chat on this Mac.");}++/** Fold a message into the thread's running turn on this Mac. True once the+ * model has taken it; false when the turn ended first or the engine predates+ * folding — the message then goes out as its own turn. */+export async function foldIntoEngineChat(+ projectId: string,+ threadId: string,+ message: { text: string; attachments: unknown[]; context: unknown },+): Promise<boolean> {+ const response = await localBackend.fetch(`/api/cut/ai/chat/${encodeURIComponent(threadId)}/fold`, {+ method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ projectId, ...message }),+ });+ if (!response.ok) return false;+ return (await response.json() as { folded: boolean }).folded;+}diff --git a/site/src/cut/server/ai/turnFolds.test.ts b/site/src/cut/server/ai/turnFolds.test.tsnew file mode 100644index 00000000..02539c34--- /dev/null+++ b/site/src/cut/server/ai/turnFolds.test.ts@@ -0,0 +1,57 @@+import { describe, expect, test } from "bun:test";+import { FoldInbox, foldIntoTurn, openFoldInbox } from "./turnFolds";++describe("FoldInbox", () => {+ test("a listening run takes each fold as it lands", async () => {+ const inbox = new FoldInbox();+ const taken: string[] = [];+ inbox.listen((fold) => {+ taken.push(fold.text);+ fold.resolve(true);+ });+ expect(await inbox.push("tighter")).toBe(true);+ expect(taken).toEqual(["tighter"]);+ });++ test("folds between runs wait for the next listener", async () => {+ const inbox = new FoldInbox();+ const stop = inbox.listen(() => {});+ stop();+ const answer = inbox.push("later");+ const taken: string[] = [];+ inbox.listen((fold) => {+ taken.push(fold.text);+ fold.resolve(true);+ });+ expect(taken).toEqual(["later"]);+ expect(await answer).toBe(true);+ });++ test("closing hands back what no run took and refuses new folds", async () => {+ const inbox = new FoldInbox();+ const waiting = inbox.push("orphan");+ inbox.close();+ expect(await waiting).toBe(false);+ expect(await inbox.push("after")).toBe(false);+ });++ test("take drains the folds waiting between runs", () => {+ const inbox = new FoldInbox();+ void inbox.push("a");+ void inbox.push("b");+ expect(inbox.take().map((f) => f.text)).toEqual(["a", "b"]);+ expect(inbox.take()).toEqual([]);+ });+});++describe("foldIntoTurn", () => {+ test("reaches only the thread's running turn", async () => {+ expect(await foldIntoTurn("p", "t", "nobody home")).toBe(false);+ const turn = openFoldInbox("p", "t");+ turn.inbox.listen((fold) => fold.resolve(true));+ expect(await foldIntoTurn("p", "t", "hi")).toBe(true);+ expect(await foldIntoTurn("p", "other", "hi")).toBe(false);+ turn.close();+ expect(await foldIntoTurn("p", "t", "late")).toBe(false);+ });+});diff --git a/site/src/cut/server/ai/turnFolds.ts b/site/src/cut/server/ai/turnFolds.tsnew file mode 100644index 00000000..bab5d84f--- /dev/null+++ b/site/src/cut/server/ai/turnFolds.ts@@ -0,0 +1,71 @@+/** A composer message folded into a running engine turn. It resolves true+ * once the provider has taken it into the turn, false when the turn ended+ * first — the page then sends it as its own turn. */+export interface Fold {+ text: string;+ resolve: (taken: boolean) => void;+}++/**+ * The running turn's mailbox for folds. A turn runs as one or more provider+ * runs; the run that is listening gets each fold the moment it lands, and+ * folds that land between runs wait for the next one. Closing the inbox hands+ * every fold no run took back to the page.+ */+export class FoldInbox {+ private waiting: Fold[] = [];+ private listener: ((fold: Fold) => void) | null = null;+ private closed = false;++ push(text: string): Promise<boolean> {+ if (this.closed) return Promise.resolve(false);+ return new Promise((resolve) => {+ const fold = { text, resolve };+ if (this.listener) this.listener(fold);+ else this.waiting.push(fold);+ });+ }++ /** Hand every fold to `take` as it lands, starting with the waiting ones.+ * Returns the call that stops listening. */+ listen(take: (fold: Fold) => void): () => void {+ this.listener = take;+ for (const fold of this.waiting.splice(0)) take(fold);+ return () => {+ if (this.listener === take) this.listener = null;+ };+ }++ /** The folds waiting between runs. */+ take(): Fold[] {+ return this.waiting.splice(0);+ }++ close(): void {+ this.closed = true;+ this.listener = null;+ for (const fold of this.waiting.splice(0)) fold.resolve(false);+ }+}++const inboxes = new Map<string, FoldInbox>();+const keyFor = (projectId: string, threadId: string) => `${projectId}/${threadId}`;++/** The inbox for a turn that is starting; the caller closes it when the turn ends. */+export function openFoldInbox(projectId: string, threadId: string): { inbox: FoldInbox; close: () => void } {+ const key = keyFor(projectId, threadId);+ const inbox = new FoldInbox();+ inboxes.set(key, inbox);+ return {+ inbox,+ close: () => {+ inbox.close();+ if (inboxes.get(key) === inbox) inboxes.delete(key);+ },+ };+}++/** Fold a message into the thread's running turn; false when none is running. */+export function foldIntoTurn(projectId: string, threadId: string, text: string): Promise<boolean> {+ return inboxes.get(keyFor(projectId, threadId))?.push(text) ?? Promise.resolve(false);+}diff --git a/site/src/cut/server/http/ai.ts b/site/src/cut/server/http/ai.tsindex 6cc2ca71..fd8afe1e 100644--- a/site/src/cut/server/http/ai.ts+++ b/site/src/cut/server/http/ai.ts@@ -3,8 +3,9 @@ import os from "node:os";import { z } from "zod";import { SETTINGS } from "@/lib/config/registry";import { startTurnStream, followTurnStream, cancelTurnStream } from "../ai/turnStreams";+import { foldIntoTurn, openFoldInbox, type Fold, type FoldInbox } from "../ai/turnFolds";import path from "node:path";-import { query } from "@anthropic-ai/claude-agent-sdk";+import { query, type SDKUserMessage } from "@anthropic-ai/claude-agent-sdk";import { claudeFailure } from "../ai/claudeFailure";import { createUIMessageStream, type UIMessage } from "ai";@@ -80,6 +81,7 @@ async function runClaude(body: ChatBody,base: string,sessionKey: string,+ inbox: FoldInbox,signal: AbortSignal) {signal.throwIfAborted();@@ -93,11 +95,41 @@ async function runClaude(};/** One run of the query. It ends when the model signs off, when the SDK- * spends the step budget, or when the editor tools failed to bind. */+ * spends the step budget, or when the editor tools failed to bind.+ *+ * The input stays open for the run so folds reach the CLI as they land: it+ * injects each one at the model's next step and acknowledges it then, and+ * one that lands as the model signs off starts a follow-on reply in the same+ * session. The run ends on a result with no fold still unacknowledged. */const pass = async (ask: string): Promise<TurnEnd> => {signal.throwIfAborted();+ const unacked: Fold[] = [];+ const outbox: string[] = [];+ let wake = null as (() => void) | null;+ let open = true;+ const unlisten = inbox.listen((fold) => {+ unacked.push(fold);+ outbox.push(fold.text);+ wake?.();+ });+ const userMessage = (text: string): SDKUserMessage => ({+ type: "user",+ message: { role: "user", content: text },+ parent_tool_use_id: null,+ });+ async function* input(): AsyncGenerator<SDKUserMessage> {+ yield userMessage(ask);+ for (;;) {+ while (outbox.length) yield userMessage(outbox.shift()!);+ if (!open) return;+ await new Promise<void>((resolve) => (wake = resolve));+ wake = null;+ }+ }+ // The CLI echoes each input message as it takes it; the first is the ask.+ let echoes = 0;const q = query({- prompt: ask,+ prompt: input(),options: {model: body.model,...(session ? { resume: session } : {}),@@ -126,6 +158,7 @@ async function runClaude(strictMcpConfig: true,settings: { disableClaudeAiConnectors: true },includePartialMessages: true,+ extraArgs: { "replay-user-messages": null },maxTurns: STEP_BUDGET,cwd: os.tmpdir(),},@@ -160,6 +193,8 @@ async function runClaude(// newest, so an extended turn keeps one continuous history.if (typeof m.session_id === "string") session = m.session_id;emit({ type: "data-session", data: { providerSession: m.session_id }, transient: true });+ } else if (m.type === "user" && m.isReplay === true) {+ if (echoes++ > 0) unacked.shift()?.resolve(true);} else if (m.type === "stream_event") {const ev = m.event as {type: string;@@ -184,6 +219,8 @@ async function runClaude(end = "failed";emit({ type: "error", errorText: stopText(m.result) });}+ // A fold the CLI holds runs next, in this same session.+ if (unacked.length) continue;// The result message is the run's last word — after it the CLI only// tears down. Leaving the loop closes the chat stream now instead of// holding the working indicator open through process exit.@@ -191,6 +228,10 @@ async function runClaude(}}} finally {+ unlisten();+ open = false;+ wake?.();+ for (const fold of unacked.splice(0)) fold.resolve(false);signal.removeEventListener("abort", onAbort);if (textId) emit({ type: "text-end", id: textId });}@@ -217,15 +258,41 @@ async function runCodex(body: ChatBody,base: string,sessionKey: string,+ inbox: FoldInbox,signal: AbortSignal) {+ // `codex exec` reads no input once it starts, so a fold waits for the run+ // to finish and goes in as a resumed run of the same session, inside this+ // turn.+ let session = body.providerSession;+ let ask = prompt;+ for (;;) {+ session = (await codexRun(emit, ask, body.model, session, base, sessionKey, signal)) ?? session;+ if (signal.aborted || !session) return;+ const folds = inbox.take();+ if (!folds.length) return;+ for (const fold of folds) fold.resolve(true);+ ask = folds.map((fold) => fold.text).join("\n\n");+ }+}++/** One `codex exec` run; returns the session it ran in. */+async function codexRun(+ emit: UIChunkWriter["write"],+ prompt: string,+ model: string,+ session: string | undefined,+ base: string,+ sessionKey: string,+ signal: AbortSignal+): Promise<string | undefined> {signal.throwIfAborted();const codex = await codexCommand();const mcp = mcpCommand(base, sessionKey);- const session = body.providerSession;+ let thread = session;const args = ["exec"];if (session) args.push("resume", session);- args.push("--json", "--skip-git-repo-check", "-m", body.model);+ args.push("--json", "--skip-git-repo-check", "-m", model);// `codex exec resume` inherits the original session's sandbox and working// root, and its subcommand parser rejects --sandbox/-C. Pass those only on a// fresh run; appending them to a resume exits the CLI with code 2.@@ -259,8 +326,10 @@ async function runCodex(};let textCount = 0;+ // Part ids stay unique across the runs a turn makes.+ const run = crypto.randomUUID().slice(0, 8);const say = (text: string) => {- const id = `t${++textCount}`;+ const id = `${run}-t${++textCount}`;emit({ type: "text-start", id });emit({ type: "text-delta", id, delta: text });emit({ type: "text-end", id });@@ -282,6 +351,7 @@ async function runCodex(continue;}if (ev.type === "thread.started" && ev.thread_id) {+ thread = ev.thread_id;emit({ type: "data-session", data: { providerSession: ev.thread_id }, transient: true });} else if (ev.type === "item.completed" && ev.item?.type === "agent_message" && ev.item.text) {say(ev.item.text);@@ -324,6 +394,7 @@ async function runCodex(resolve();});});+ return thread;}/**@@ -481,17 +552,19 @@ export const aiApi = {} });// The browser posts tool outputs back to /api/cut/ai/tool-result with this key.emit({ type: "data-session", data: { sessionKey }, transient: true });+ const folds = openFoldInbox(identity.data.context.project.id, body.threadId);try {if (body.model.startsWith("claude")) {- await runClaude(emit, prompt, body, base, sessionKey, signal);+ await runClaude(emit, prompt, body, base, sessionKey, folds.inbox, signal);} else if (body.model === "cut-test") {await runFake(emit, sessionKey, userText);} else {- await runCodex(emit, prompt, body, base, sessionKey, signal);+ await runCodex(emit, prompt, body, base, sessionKey, folds.inbox, signal);}} catch (err) {emit({ type: "error", errorText: errorMessage(err, String(err)) });} finally {+ folds.close();unregisterSession(sessionKey);emit({ type: "finish" });}@@ -505,6 +578,22 @@ export const aiApi = {return followTurnStream(projectId, threadId, req.signal);},+ /** A message sent while the thread's turn runs, folded into that turn.+ * Answers once the provider has taken it, or false when the turn ended+ * first. */+ async foldChat(req: Request, threadId: string) {+ const body = z.object({+ projectId: z.string().min(1).max(200),+ text: z.string().min(1).max(100_000),+ attachments: z.array(z.unknown()).max(100).default([]),+ context: z.unknown(),+ }).safeParse(await req.json().catch(() => null));+ if (!body.success) return Response.json({ error: "A project and a message are required." }, { status: 400 });+ const { projectId, text, attachments, context } = body.data;+ const prompt = `${text}${attachedAssetsBlock(attachments)}\n\n<editor_state>\n${JSON.stringify(context ?? {})}\n</editor_state>`;+ return Response.json({ folded: await foldIntoTurn(projectId, threadId, prompt) });+ },+async cancelChat(req: Request, threadId: string) {const body = z.object({ projectId: z.string().min(1).max(200) }).safeParse(await req.json().catch(() => null));if (!body.success) return Response.json({ error: "A project is required." }, { status: 400 });diff --git a/site/src/cut/server/http/routes.ts b/site/src/cut/server/http/routes.tsindex ec78aa81..2f1d16e4 100644--- a/site/src/cut/server/http/routes.ts+++ b/site/src/cut/server/http/routes.ts@@ -120,6 +120,7 @@ export const CUT_ROUTES: CutRoute[] = [{ method: "POST", path: "/api/cut/projects/:id/scene-lease", handler: (req, p) => localSceneLease(p.id, req) },{ method: "GET", path: "/api/cut/ai/chat/:id/stream", handler: (req, p) => aiApi.resumeChat(req, p.id) },{ method: "POST", path: "/api/cut/ai/chat/:id/cancel", handler: (req, p) => aiApi.cancelChat(req, p.id) },+ { method: "POST", path: "/api/cut/ai/chat/:id/fold", handler: (req, p) => aiApi.foldChat(req, p.id) },{ method: "POST", path: "/api/cut/ai/tool-claim", handler: (req) => aiApi.claimTool(req) },{ method: "POST", path: "/api/cut/ai/captions", handler: (req) => aiApi.captions(req) },{ method: "POST", path: "/api/cut/ai/visual-subtitles", handler: (req) => aiApi.visualSubtitles(req) },