Skip to content

Open source CapCut alternative.

TypeScript

feat(site): fold mid-turn messages into Claude and Codex chat turns [rebuild]

David Led3d966ebd96a
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsx
index b02a4cb3..cc268425 100644
--- a/site/src/cut/components/AiPanel.tsx
+++ b/site/src/cut/components/AiPanel.tsx
@@ -108,7 +108,7 @@ import { holdEditorChat } from "@/cut/lib/editorWork";
import { recoverSceneCall } from "@/cut/lib/chatRecovery";
import { replayChatStream } from "@/cut/lib/chatReplay";
import { INTERRUPTED_ERROR, isResumeMessage, RESUME_LIMIT, resumePrompt, scrubLostTurn, unfinishedAsk, type ResumeMetadata, type TurnSettled } from "@/cut/lib/chatResume";
-import { claimEngineTool, cancelEngineChat } from "@/cut/lib/engineChat";
+import { claimEngineTool, cancelEngineChat, foldIntoEngineChat } from "@/cut/lib/engineChat";
import { projectBackend } from "@/cut/lib/residency";
import { beginBrowserChat, browserChatRunning, cancelBrowserChat, watchBrowserChatTurns } from "@/cut/lib/browserChatTurns";
import { putCloudThread } from "@/cut/lib/chatCloud";
@@ -1530,8 +1530,7 @@ function ChatSession({
// joins the transcript as its own message, a spawn starts a parallel
// thread, and the rest wait in the tray. Each verdict is checked against
// the live queue before it acts, since a row can be edited, removed, or
- // drained while the call is out. A turn on the local engine cannot take a
- // fold, so there the row waits as before.
+ // drained while the call is out.
const triage = async (rows: QueuedMessage[]) => {
const ask = runningAskRef.current;
if (!ask) return;
@@ -1576,14 +1575,21 @@ function ChatSession({
if (verdict === "spawn") {
const id = onSpawn({ text, attachments: live, model });
setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "spawned", threadId: id } : m)));
- } else if (verdict === "fold" && provider(model) === "gemini" && cutChatLive(threadId)) {
+ } else if (verdict === "fold" && (provider(model) !== "gemini" || cutChatLive(threadId))) {
setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "folding" } : m)));
- const folded = await foldIntoCutChat({
- threadId,
- text,
- attachments: live,
- deps: productionDeps(projectId, turnSignalRef.current),
- });
+ const folded =
+ provider(model) === "gemini"
+ ? await foldIntoCutChat({
+ threadId,
+ text,
+ attachments: live,
+ deps: productionDeps(projectId, turnSignalRef.current),
+ })
+ : await foldIntoEngineChat(projectId, threadId, {
+ text,
+ attachments: live,
+ context: productionDeps(projectId).buildContext(),
+ }).catch(() => false);
if (!folded) {
setQueue((q) => q.map((m) => (m.id === row.id ? { ...m, status: "queued" } : m)));
continue;
diff --git a/site/src/cut/lib/engineChat.ts b/site/src/cut/lib/engineChat.ts
index 62100f90..0437eae1 100644
--- a/site/src/cut/lib/engineChat.ts
+++ b/site/src/cut/lib/engineChat.ts
@@ -15,3 +15,18 @@ export async function cancelEngineChat(projectId: string, threadId: string): Pro
});
if (!response.ok) throw new Error("Could not stop the chat on this Mac.");
}
+
+/** Fold a message into the thread's running turn on this Mac. True once the
+ * model has taken it; false when the turn ended first or the engine predates
+ * folding — the message then goes out as its own turn. */
+export async function foldIntoEngineChat(
+ projectId: string,
+ threadId: string,
+ message: { text: string; attachments: unknown[]; context: unknown },
+): Promise<boolean> {
+ const response = await localBackend.fetch(`/api/cut/ai/chat/${encodeURIComponent(threadId)}/fold`, {
+ method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ projectId, ...message }),
+ });
+ if (!response.ok) return false;
+ return (await response.json() as { folded: boolean }).folded;
+}
diff --git a/site/src/cut/server/ai/turnFolds.test.ts b/site/src/cut/server/ai/turnFolds.test.ts
new file mode 100644
index 00000000..02539c34
--- /dev/null
+++ b/site/src/cut/server/ai/turnFolds.test.ts
@@ -0,0 +1,57 @@
+import { describe, expect, test } from "bun:test";
+import { FoldInbox, foldIntoTurn, openFoldInbox } from "./turnFolds";
+
+describe("FoldInbox", () => {
+ test("a listening run takes each fold as it lands", async () => {
+ const inbox = new FoldInbox();
+ const taken: string[] = [];
+ inbox.listen((fold) => {
+ taken.push(fold.text);
+ fold.resolve(true);
+ });
+ expect(await inbox.push("tighter")).toBe(true);
+ expect(taken).toEqual(["tighter"]);
+ });
+
+ test("folds between runs wait for the next listener", async () => {
+ const inbox = new FoldInbox();
+ const stop = inbox.listen(() => {});
+ stop();
+ const answer = inbox.push("later");
+ const taken: string[] = [];
+ inbox.listen((fold) => {
+ taken.push(fold.text);
+ fold.resolve(true);
+ });
+ expect(taken).toEqual(["later"]);
+ expect(await answer).toBe(true);
+ });
+
+ test("closing hands back what no run took and refuses new folds", async () => {
+ const inbox = new FoldInbox();
+ const waiting = inbox.push("orphan");
+ inbox.close();
+ expect(await waiting).toBe(false);
+ expect(await inbox.push("after")).toBe(false);
+ });
+
+ test("take drains the folds waiting between runs", () => {
+ const inbox = new FoldInbox();
+ void inbox.push("a");
+ void inbox.push("b");
+ expect(inbox.take().map((f) => f.text)).toEqual(["a", "b"]);
+ expect(inbox.take()).toEqual([]);
+ });
+});
+
+describe("foldIntoTurn", () => {
+ test("reaches only the thread's running turn", async () => {
+ expect(await foldIntoTurn("p", "t", "nobody home")).toBe(false);
+ const turn = openFoldInbox("p", "t");
+ turn.inbox.listen((fold) => fold.resolve(true));
+ expect(await foldIntoTurn("p", "t", "hi")).toBe(true);
+ expect(await foldIntoTurn("p", "other", "hi")).toBe(false);
+ turn.close();
+ expect(await foldIntoTurn("p", "t", "late")).toBe(false);
+ });
+});
diff --git a/site/src/cut/server/ai/turnFolds.ts b/site/src/cut/server/ai/turnFolds.ts
new file mode 100644
index 00000000..bab5d84f
--- /dev/null
+++ b/site/src/cut/server/ai/turnFolds.ts
@@ -0,0 +1,71 @@
+/** A composer message folded into a running engine turn. It resolves true
+ * once the provider has taken it into the turn, false when the turn ended
+ * first — the page then sends it as its own turn. */
+export interface Fold {
+ text: string;
+ resolve: (taken: boolean) => void;
+}
+
+/**
+ * The running turn's mailbox for folds. A turn runs as one or more provider
+ * runs; the run that is listening gets each fold the moment it lands, and
+ * folds that land between runs wait for the next one. Closing the inbox hands
+ * every fold no run took back to the page.
+ */
+export class FoldInbox {
+ private waiting: Fold[] = [];
+ private listener: ((fold: Fold) => void) | null = null;
+ private closed = false;
+
+ push(text: string): Promise<boolean> {
+ if (this.closed) return Promise.resolve(false);
+ return new Promise((resolve) => {
+ const fold = { text, resolve };
+ if (this.listener) this.listener(fold);
+ else this.waiting.push(fold);
+ });
+ }
+
+ /** Hand every fold to `take` as it lands, starting with the waiting ones.
+ * Returns the call that stops listening. */
+ listen(take: (fold: Fold) => void): () => void {
+ this.listener = take;
+ for (const fold of this.waiting.splice(0)) take(fold);
+ return () => {
+ if (this.listener === take) this.listener = null;
+ };
+ }
+
+ /** The folds waiting between runs. */
+ take(): Fold[] {
+ return this.waiting.splice(0);
+ }
+
+ close(): void {
+ this.closed = true;
+ this.listener = null;
+ for (const fold of this.waiting.splice(0)) fold.resolve(false);
+ }
+}
+
+const inboxes = new Map<string, FoldInbox>();
+const keyFor = (projectId: string, threadId: string) => `${projectId}/${threadId}`;
+
+/** The inbox for a turn that is starting; the caller closes it when the turn ends. */
+export function openFoldInbox(projectId: string, threadId: string): { inbox: FoldInbox; close: () => void } {
+ const key = keyFor(projectId, threadId);
+ const inbox = new FoldInbox();
+ inboxes.set(key, inbox);
+ return {
+ inbox,
+ close: () => {
+ inbox.close();
+ if (inboxes.get(key) === inbox) inboxes.delete(key);
+ },
+ };
+}
+
+/** Fold a message into the thread's running turn; false when none is running. */
+export function foldIntoTurn(projectId: string, threadId: string, text: string): Promise<boolean> {
+ return inboxes.get(keyFor(projectId, threadId))?.push(text) ?? Promise.resolve(false);
+}
diff --git a/site/src/cut/server/http/ai.ts b/site/src/cut/server/http/ai.ts
index 6cc2ca71..fd8afe1e 100644
--- a/site/src/cut/server/http/ai.ts
+++ b/site/src/cut/server/http/ai.ts
@@ -3,8 +3,9 @@ import os from "node:os";
import { z } from "zod";
import { SETTINGS } from "@/lib/config/registry";
import { startTurnStream, followTurnStream, cancelTurnStream } from "../ai/turnStreams";
+import { foldIntoTurn, openFoldInbox, type Fold, type FoldInbox } from "../ai/turnFolds";
import path from "node:path";
-import { query } from "@anthropic-ai/claude-agent-sdk";
+import { query, type SDKUserMessage } from "@anthropic-ai/claude-agent-sdk";
import { claudeFailure } from "../ai/claudeFailure";
import { createUIMessageStream, type UIMessage } from "ai";
@@ -80,6 +81,7 @@ async function runClaude(
body: ChatBody,
base: string,
sessionKey: string,
+ inbox: FoldInbox,
signal: AbortSignal
) {
signal.throwIfAborted();
@@ -93,11 +95,41 @@ async function runClaude(
};
/** One run of the query. It ends when the model signs off, when the SDK
- * spends the step budget, or when the editor tools failed to bind. */
+ * spends the step budget, or when the editor tools failed to bind.
+ *
+ * The input stays open for the run so folds reach the CLI as they land: it
+ * injects each one at the model's next step and acknowledges it then, and
+ * one that lands as the model signs off starts a follow-on reply in the same
+ * session. The run ends on a result with no fold still unacknowledged. */
const pass = async (ask: string): Promise<TurnEnd> => {
signal.throwIfAborted();
+ const unacked: Fold[] = [];
+ const outbox: string[] = [];
+ let wake = null as (() => void) | null;
+ let open = true;
+ const unlisten = inbox.listen((fold) => {
+ unacked.push(fold);
+ outbox.push(fold.text);
+ wake?.();
+ });
+ const userMessage = (text: string): SDKUserMessage => ({
+ type: "user",
+ message: { role: "user", content: text },
+ parent_tool_use_id: null,
+ });
+ async function* input(): AsyncGenerator<SDKUserMessage> {
+ yield userMessage(ask);
+ for (;;) {
+ while (outbox.length) yield userMessage(outbox.shift()!);
+ if (!open) return;
+ await new Promise<void>((resolve) => (wake = resolve));
+ wake = null;
+ }
+ }
+ // The CLI echoes each input message as it takes it; the first is the ask.
+ let echoes = 0;
const q = query({
- prompt: ask,
+ prompt: input(),
options: {
model: body.model,
...(session ? { resume: session } : {}),
@@ -126,6 +158,7 @@ async function runClaude(
strictMcpConfig: true,
settings: { disableClaudeAiConnectors: true },
includePartialMessages: true,
+ extraArgs: { "replay-user-messages": null },
maxTurns: STEP_BUDGET,
cwd: os.tmpdir(),
},
@@ -160,6 +193,8 @@ async function runClaude(
// newest, so an extended turn keeps one continuous history.
if (typeof m.session_id === "string") session = m.session_id;
emit({ type: "data-session", data: { providerSession: m.session_id }, transient: true });
+ } else if (m.type === "user" && m.isReplay === true) {
+ if (echoes++ > 0) unacked.shift()?.resolve(true);
} else if (m.type === "stream_event") {
const ev = m.event as {
type: string;
@@ -184,6 +219,8 @@ async function runClaude(
end = "failed";
emit({ type: "error", errorText: stopText(m.result) });
}
+ // A fold the CLI holds runs next, in this same session.
+ if (unacked.length) continue;
// The result message is the run's last word — after it the CLI only
// tears down. Leaving the loop closes the chat stream now instead of
// holding the working indicator open through process exit.
@@ -191,6 +228,10 @@ async function runClaude(
}
}
} finally {
+ unlisten();
+ open = false;
+ wake?.();
+ for (const fold of unacked.splice(0)) fold.resolve(false);
signal.removeEventListener("abort", onAbort);
if (textId) emit({ type: "text-end", id: textId });
}
@@ -217,15 +258,41 @@ async function runCodex(
body: ChatBody,
base: string,
sessionKey: string,
+ inbox: FoldInbox,
signal: AbortSignal
) {
+ // `codex exec` reads no input once it starts, so a fold waits for the run
+ // to finish and goes in as a resumed run of the same session, inside this
+ // turn.
+ let session = body.providerSession;
+ let ask = prompt;
+ for (;;) {
+ session = (await codexRun(emit, ask, body.model, session, base, sessionKey, signal)) ?? session;
+ if (signal.aborted || !session) return;
+ const folds = inbox.take();
+ if (!folds.length) return;
+ for (const fold of folds) fold.resolve(true);
+ ask = folds.map((fold) => fold.text).join("\n\n");
+ }
+}
+
+/** One `codex exec` run; returns the session it ran in. */
+async function codexRun(
+ emit: UIChunkWriter["write"],
+ prompt: string,
+ model: string,
+ session: string | undefined,
+ base: string,
+ sessionKey: string,
+ signal: AbortSignal
+): Promise<string | undefined> {
signal.throwIfAborted();
const codex = await codexCommand();
const mcp = mcpCommand(base, sessionKey);
- const session = body.providerSession;
+ let thread = session;
const args = ["exec"];
if (session) args.push("resume", session);
- args.push("--json", "--skip-git-repo-check", "-m", body.model);
+ args.push("--json", "--skip-git-repo-check", "-m", model);
// `codex exec resume` inherits the original session's sandbox and working
// root, and its subcommand parser rejects --sandbox/-C. Pass those only on a
// fresh run; appending them to a resume exits the CLI with code 2.
@@ -259,8 +326,10 @@ async function runCodex(
};
let textCount = 0;
+ // Part ids stay unique across the runs a turn makes.
+ const run = crypto.randomUUID().slice(0, 8);
const say = (text: string) => {
- const id = `t${++textCount}`;
+ const id = `${run}-t${++textCount}`;
emit({ type: "text-start", id });
emit({ type: "text-delta", id, delta: text });
emit({ type: "text-end", id });
@@ -282,6 +351,7 @@ async function runCodex(
continue;
}
if (ev.type === "thread.started" && ev.thread_id) {
+ thread = ev.thread_id;
emit({ type: "data-session", data: { providerSession: ev.thread_id }, transient: true });
} else if (ev.type === "item.completed" && ev.item?.type === "agent_message" && ev.item.text) {
say(ev.item.text);
@@ -324,6 +394,7 @@ async function runCodex(
resolve();
});
});
+ return thread;
}
/**
@@ -481,17 +552,19 @@ export const aiApi = {
} });
// The browser posts tool outputs back to /api/cut/ai/tool-result with this key.
emit({ type: "data-session", data: { sessionKey }, transient: true });
+ const folds = openFoldInbox(identity.data.context.project.id, body.threadId);
try {
if (body.model.startsWith("claude")) {
- await runClaude(emit, prompt, body, base, sessionKey, signal);
+ await runClaude(emit, prompt, body, base, sessionKey, folds.inbox, signal);
} else if (body.model === "cut-test") {
await runFake(emit, sessionKey, userText);
} else {
- await runCodex(emit, prompt, body, base, sessionKey, signal);
+ await runCodex(emit, prompt, body, base, sessionKey, folds.inbox, signal);
}
} catch (err) {
emit({ type: "error", errorText: errorMessage(err, String(err)) });
} finally {
+ folds.close();
unregisterSession(sessionKey);
emit({ type: "finish" });
}
@@ -505,6 +578,22 @@ export const aiApi = {
return followTurnStream(projectId, threadId, req.signal);
},
+ /** A message sent while the thread's turn runs, folded into that turn.
+ * Answers once the provider has taken it, or false when the turn ended
+ * first. */
+ async foldChat(req: Request, threadId: string) {
+ const body = z.object({
+ projectId: z.string().min(1).max(200),
+ text: z.string().min(1).max(100_000),
+ attachments: z.array(z.unknown()).max(100).default([]),
+ context: z.unknown(),
+ }).safeParse(await req.json().catch(() => null));
+ if (!body.success) return Response.json({ error: "A project and a message are required." }, { status: 400 });
+ const { projectId, text, attachments, context } = body.data;
+ const prompt = `${text}${attachedAssetsBlock(attachments)}\n\n<editor_state>\n${JSON.stringify(context ?? {})}\n</editor_state>`;
+ return Response.json({ folded: await foldIntoTurn(projectId, threadId, prompt) });
+ },
+
async cancelChat(req: Request, threadId: string) {
const body = z.object({ projectId: z.string().min(1).max(200) }).safeParse(await req.json().catch(() => null));
if (!body.success) return Response.json({ error: "A project is required." }, { status: 400 });
diff --git a/site/src/cut/server/http/routes.ts b/site/src/cut/server/http/routes.ts
index ec78aa81..2f1d16e4 100644
--- a/site/src/cut/server/http/routes.ts
+++ b/site/src/cut/server/http/routes.ts
@@ -120,6 +120,7 @@ export const CUT_ROUTES: CutRoute[] = [
{ method: "POST", path: "/api/cut/projects/:id/scene-lease", handler: (req, p) => localSceneLease(p.id, req) },
{ method: "GET", path: "/api/cut/ai/chat/:id/stream", handler: (req, p) => aiApi.resumeChat(req, p.id) },
{ method: "POST", path: "/api/cut/ai/chat/:id/cancel", handler: (req, p) => aiApi.cancelChat(req, p.id) },
+ { method: "POST", path: "/api/cut/ai/chat/:id/fold", handler: (req, p) => aiApi.foldChat(req, p.id) },
{ method: "POST", path: "/api/cut/ai/tool-claim", handler: (req) => aiApi.claimTool(req) },
{ method: "POST", path: "/api/cut/ai/captions", handler: (req) => aiApi.captions(req) },
{ method: "POST", path: "/api/cut/ai/visual-subtitles", handler: (req) => aiApi.visualSubtitles(req) },