Skip to content

Open source CapCut alternative.

TypeScript

feat(site): engine finishes a turn when its tab is gone [rebuild]

David Le10d2614bd4eb
diff --git a/site/src/cut/server/ai/bridge.ts b/site/src/cut/server/ai/bridge.ts
index 353487d2..841b34b0 100644
--- a/site/src/cut/server/ai/bridge.ts
+++ b/site/src/cut/server/ai/bridge.ts
@@ -25,8 +25,15 @@ interface Session {
const g = globalThis as unknown as { __veditorAiSessions?: Map<string, Session> };
const sessions = (g.__veditorAiSessions ??= new Map<string, Session>());
-export function registerSession(key: string, writer: UIChunkWriter) {
+// The project each chat session edits, kept past the stream's close: a tab
+// that goes away mid-turn leaves the running provider's tool calls a route
+// to the engine's own executor (headlessTools) for that project.
+const g2 = globalThis as unknown as { __veditorAiProjects?: Map<string, string> };
+const sessionProjects = (g2.__veditorAiProjects ??= new Map<string, string>());
+
+export function registerSession(key: string, writer: UIChunkWriter, projectId?: string) {
sessions.set(key, { writer, waiters: new Map() });
+ if (projectId) sessionProjects.set(key, projectId);
}
export function unregisterSession(key: string) {
@@ -38,6 +45,11 @@ export function unregisterSession(key: string) {
}
}
sessions.delete(key);
+ // The chat route unregisters after the provider run settles, so no more
+ // tool calls can arrive under this key. Dropping the project mapping here
+ // keeps the map bounded and keeps a reloaded tab's fresh session from ever
+ // racing a leftover headless executor.
+ sessionProjects.delete(key);
}
const TOOL_TIMEOUT_MS = 120_000; // subtitles generation can take a while
@@ -53,9 +65,14 @@ export function callBrowserTool(
): Promise<{ output?: unknown; errorText?: string }> {
const session = sessions.get(sessionKey);
if (!session) {
- // The proxy called with a key no open chat stream is registered under — a
- // stale/mismatched session. Log which keys ARE live so a recurrence pins
- // the drift instead of only surfacing to the model as "unreachable".
+ // The tab is gone (closed mid-turn) or the key is stale. A session whose
+ // project is known finishes headless: the engine hydrates the doc and
+ // runs the tool itself, so the turn completes and lands on disk.
+ const projectId = sessionProjects.get(sessionKey);
+ if (projectId) {
+ console.log(`[cut-ai] ${toolName}: no editor tab for ${sessionKey}; running headless on ${projectId}`);
+ return import("./headlessTools").then((m) => m.callHeadlessTool(projectId, toolName, input));
+ }
console.warn(
`[cut-ai] ${toolName}: no live editor session for ${sessionKey}; live: ${[...sessions.keys()].join(", ") || "none"}`
);
diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.ts
new file mode 100644
index 00000000..640fc233
--- /dev/null
+++ b/site/src/cut/server/ai/headlessTools.ts
@@ -0,0 +1,109 @@
+import { BROWSER_MEDIA_TOOLS, runAiTool, UI_TOOLS } from "../../lib/aiTools";
+import { serializeDoc, useEditor } from "../../lib/store";
+import type { MediaAsset, ProjectDoc } from "../../lib/types";
+import { readProject, writeProject } from "../projects";
+
+// The engine's own tool executor, for a chat whose editor tab is gone: the
+// project doc hydrates into the same store the page uses, tools run against
+// it, and every mutation lands back on disk. A turn survives its tab.
+//
+// Scope: doc edits. Tools that need the page (preview reads, media decodes)
+// or the page's hosted sign-in (generation, library sync) answer with a
+// typed refusal so the model reports honestly and moves on.
+
+const PAGE_SESSION_TOOLS: ReadonlySet<string> = new Set([
+ "generate_image",
+ "generate_video",
+ "generate_character_video",
+ "generate_scene",
+ "approve_scene",
+ "cancel_scene",
+ "regenerate_shot",
+ "recut_scene",
+ "restyle_scene",
+ "generate_music",
+ "voiceover_generate",
+ "read_subtitles_aloud",
+ "subtitles_translate_track",
+ "wait_for_renders",
+ "library_list",
+ "library_add",
+ "library_organize",
+ "template_add",
+ "file_asset",
+ "import_url",
+]);
+
+interface OpenDoc {
+ projectId: string;
+ stored: ProjectDoc;
+}
+
+let open: OpenDoc | null = null;
+
+async function ensureOpen(projectId: string): Promise<OpenDoc | null> {
+ // Read the doc on every call: the page saves through writeProject between
+ // turns, and flushing a store hydrated from an older read would revert
+ // those edits. The store is reused only while the disk copy still matches
+ // the one it was hydrated from.
+ const stored = await readProject(projectId);
+ if (!stored) return null;
+ const s = useEditor.getState();
+ if (
+ open?.projectId === projectId &&
+ s.projectId === projectId &&
+ s.loaded &&
+ stored.updatedAt === open.stored.updatedAt
+ ) {
+ return open;
+ }
+ const assets: MediaAsset[] = (stored.assets ?? []).map((a) => ({
+ ...a,
+ url: `/api/cut/projects/${encodeURIComponent(projectId)}/media/${encodeURIComponent(a.fileName)}`,
+ }));
+ await useEditor.getState().openProjectDoc(projectId, stored, assets);
+ open = { projectId, stored };
+ return open;
+}
+
+async function flush(doc: OpenDoc): Promise<void> {
+ const merged = { ...doc.stored, ...serializeDoc(useEditor.getState()) } as ProjectDoc;
+ await writeProject(doc.projectId, merged);
+ doc.stored = merged;
+}
+
+/** Run one tool call against the engine-held store for `projectId`,
+ * persisting the doc after it. The bridge calls this when no editor tab is
+ * attached to the chat session. */
+export async function callHeadlessTool(
+ projectId: string,
+ toolName: string,
+ input: unknown
+): Promise<{ output?: unknown; errorText?: string }> {
+ if (UI_TOOLS.has(toolName))
+ return {
+ output: {
+ noEditor: true,
+ note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",
+ },
+ };
+ if (BROWSER_MEDIA_TOOLS.has(toolName))
+ return {
+ errorText:
+ "This tool reads media through the editor page and is unavailable while no tab is attached. Work from the editor state instead.",
+ };
+ if (PAGE_SESSION_TOOLS.has(toolName))
+ return {
+ errorText:
+ "This tool runs with the editor page's hosted sign-in and is unavailable while no tab is attached. Finish the edit from the project state; the user can run it from the editor.",
+ };
+ const doc = await ensureOpen(projectId);
+ if (!doc) return { errorText: "Project not found on this Mac." };
+ try {
+ const output = await runAiTool(toolName, (input ?? {}) as Record<string, unknown>);
+ await flush(doc);
+ return { output: output ?? null };
+ } catch (err) {
+ return { errorText: err instanceof Error ? err.message : String(err) };
+ }
+}
diff --git a/site/src/cut/server/http/ai.ts b/site/src/cut/server/http/ai.ts
index 7b4786a3..944b8471 100644
--- a/site/src/cut/server/http/ai.ts
+++ b/site/src/cut/server/http/ai.ts
@@ -397,7 +397,10 @@ export const aiApi = {
execute: async ({ writer }) => {
const emit: UIChunkWriter["write"] = (chunk) =>
writer.write(chunk as Parameters<typeof writer.write>[0]);
- registerSession(sessionKey, { write: emit });
+ // The editor snapshot names the project; the bridge keeps it so a
+ // turn whose tab closes finishes through the engine's own executor.
+ const projectId = (body.context as { project?: { id?: string } } | undefined)?.project?.id;
+ registerSession(sessionKey, { write: emit }, typeof projectId === "string" ? projectId : undefined);
emit({ type: "start" });
// The browser posts tool outputs back to /api/cut/ai/tool-result with this key.
emit({ type: "data-session", data: { sessionKey }, transient: true });