Open source CapCut alternative.
TypeScript
/**
* Bridge between a running chat request and the browser.
*
* The chat route holds an open UI-message stream per session. When a
* provider calls an MCP tool, the proxy POSTs here; we write the tool call
* into the chat stream (the browser executes it against the editor store
* and POSTs the result back), then hand the result to the provider.
*/
export interface UIChunkWriter {
write: (chunk: Record<string, unknown>) => void;
}
interface Waiter {
resolve: (r: { output?: unknown; errorText?: string }) => void;
timer: ReturnType<typeof setTimeout>;
claimed: boolean;
/** Settles once a page claims the call or it ends unclaimed. */
taken: Promise<void>;
take: () => void;
toolName: string;
input: unknown;
}
interface Session {
writer: UIChunkWriter;
waiters: Map<string, Waiter>;
/** Quality-gate asks the page has not answered yet, by gate id. */
gates: Map<string, (steer: string | null) => void>;
attached: boolean;
/** The page running this turn's tools: the first to claim one. */
owner?: string;
}
// Survives dev-server module reloads.
const g = globalThis as unknown as { __veditorAiSessions?: Map<string, Session> };
const sessions = (g.__veditorAiSessions ??= new Map<string, Session>());
// The project each chat session edits, kept past the stream's close: a tab
// that goes away mid-turn leaves the running provider's tool calls a route
// to the engine's own executor (headlessTools) for that project.
const g2 = globalThis as unknown as { __veditorAiProjects?: Map<string, string> };
const sessionProjects = (g2.__veditorAiProjects ??= new Map<string, string>());
export function registerSession(key: string, writer: UIChunkWriter, projectId?: string) {
sessions.set(key, { writer, waiters: new Map(), gates: new Map(), attached: true });
if (projectId) sessionProjects.set(key, projectId);
}
export function attachSession(key: string): void {
const session = sessions.get(key);
if (session) session.attached = true;
}
export function detachSession(key: string): void {
const session = sessions.get(key);
if (!session) return;
session.attached = false;
for (const waiter of session.waiters.values()) {
clearTimeout(waiter.timer);
waiter.take();
const projectId = sessionProjects.get(key);
if (!waiter.claimed && projectId)
void detachedTool(projectId, waiter.toolName, waiter.input).then(waiter.resolve);
else waiter.resolve({ errorText: "The editor disconnected before recording this result. Check the project state before repeating the operation." });
}
session.waiters.clear();
settleGates(session);
}
export function unregisterSession(key: string) {
const s = sessions.get(key);
if (s) {
for (const w of s.waiters.values()) {
clearTimeout(w.timer);
w.take();
w.resolve({ errorText: "The chat request ended before the tool finished." });
}
settleGates(s);
}
sessions.delete(key);
// The chat route unregisters after the provider run settles, so no more
// tool calls can arrive under this key. Dropping the project mapping here
// keeps the map bounded and keeps a reloaded tab's fresh session from ever
// racing a leftover headless executor.
sessionProjects.delete(key);
}
/** The chat stream carries a tool's output without its frames or sound: the
* model gets them through the returned result, the page never draws or plays
* them, and one watch_video's contact sheets or a few listen_audio stretches
* would otherwise fill the turn journal. */
function streamedOutput(output: unknown): unknown {
if (!output || typeof output !== "object" || Array.isArray(output)) {
return output;
}
const { image, images, audio, ...rest } = output as Record<string, unknown>;
if (image === undefined && images === undefined && audio === undefined) {
return output;
}
return {
...rest,
...(image !== undefined || images !== undefined ? { imagesOmitted: true } : {}),
...(audio !== undefined ? { audioOmitted: true } : {}),
};
}
const TOOL_TIMEOUT_MS = 120_000; // subtitles generation can take a while
function detachedTool(projectId: string, toolName: string, input: unknown): Promise<{ output?: unknown; errorText?: string }> {
return import("./headlessRuntime").then((module) => module.callDetachedTool(projectId, toolName, input)).catch((error: unknown) => ({
errorText: error instanceof Error ? error.message : String(error),
}));
}
/**
* Forward a tool call to the browser via the chat stream and wait for the
* result. Returns { output } or { errorText }.
*/
export function callBrowserTool(
sessionKey: string,
toolName: string,
input: unknown
): Promise<{ output?: unknown; errorText?: string }> {
const session = sessions.get(sessionKey);
if (!session || !session.attached) {
// The tab is gone (closed mid-turn) or the key is stale. A session whose
// project is known finishes headless: the engine hydrates the doc and
// runs the tool itself, so the turn completes and lands on disk.
const projectId = sessionProjects.get(sessionKey);
if (projectId) {
console.log(`[cut-ai] ${toolName}: no editor tab for ${sessionKey}; running headless on ${projectId}`);
const toolCallId = crypto.randomUUID();
session?.writer.write({ type: "tool-input-available", toolCallId, toolName, input });
return detachedTool(projectId, toolName, input).then((result) => {
session?.writer.write(result.errorText !== undefined
? { type: "tool-output-error", toolCallId, errorText: result.errorText }
: { type: "tool-output-available", toolCallId, output: streamedOutput(result.output ?? null) });
return result;
});
}
console.warn(
`[cut-ai] ${toolName}: no live editor session for ${sessionKey}; live: ${[...sessions.keys()].join(", ") || "none"}`
);
return Promise.resolve({ errorText: "No live editor session for this chat." });
}
const toolCallId = crypto.randomUUID().slice(0, 12);
session.writer.write({ type: "tool-input-available", toolCallId, toolName, input });
let take!: () => void;
const taken = new Promise<void>((settle) => { take = settle; });
return new Promise((resolve) => {
const timer = setTimeout(() => {
session.waiters.delete(toolCallId);
take();
const errorText = `The editor did not answer the ${toolName} call in time.`;
console.warn(`[cut-ai] ${toolName}: editor tab did not answer within ${TOOL_TIMEOUT_MS}ms (session ${sessionKey})`);
session.writer.write({ type: "tool-output-error", toolCallId, errorText });
resolve({ errorText });
}, TOOL_TIMEOUT_MS);
session.waiters.set(toolCallId, {
timer,
claimed: false,
taken,
take,
toolName,
input,
resolve: (r) => {
if (r.errorText !== undefined) {
session.writer.write({ type: "tool-output-error", toolCallId, errorText: r.errorText });
} else {
session.writer.write({ type: "tool-output-available", toolCallId, output: streamedOutput(r.output ?? null) });
}
resolve(r);
},
});
});
}
// How long a call waits for the page running its turn before another page
// may take it. That page claims within moments of the call; one that lets
// this pass has closed or left the project.
const OWNER_GRACE_MS = 2_000;
/**
* Give a tool call to the page that asks for it. Every tab following a turn
* asks, and each edits its own copy of the project, so a turn's calls stay in
* the page that claimed its first one: a call landing in another tab would
* miss what the earlier calls made there. The turn moves only when that page
* lets a call go unclaimed. A page that names none (older editors) claims
* call by call.
*/
export async function claimBrowserTool(sessionKey: string, toolCallId: string, page?: string): Promise<boolean> {
const session = sessions.get(sessionKey);
const waiter = session?.waiters.get(toolCallId);
if (!session || !waiter || waiter.claimed) return false;
if (page && session.owner && session.owner !== page) {
let grace: ReturnType<typeof setTimeout> | undefined;
await Promise.race([waiter.taken, new Promise((resolve) => { grace = setTimeout(resolve, OWNER_GRACE_MS); })]);
clearTimeout(grace);
if (waiter.claimed || session.waiters.get(toolCallId) !== waiter) return false;
}
waiter.claimed = true;
waiter.take();
if (page) session.owner = page;
return true;
}
/** Called by /api/cut/ai/tool-result when the browser finishes a tool. */
export function resolveBrowserTool(
sessionKey: string,
toolCallId: string,
result: { output?: unknown; errorText?: string }
): boolean {
const session = sessions.get(sessionKey);
const waiter = session?.waiters.get(toolCallId);
if (!session || !waiter) return false;
session.waiters.delete(toolCallId);
clearTimeout(waiter.timer);
waiter.take();
waiter.resolve(result);
return true;
}
// The page judges the gate with the hosted judge; a page that does not
// answer in this long lets the turn close.
const GATE_TIMEOUT_MS = 20_000;
/** A gate no page will answer lets its turn close. */
function settleGates(session: Session) {
for (const settle of session.gates.values()) settle(null);
session.gates.clear();
}
/**
* Ask the attached page whether a signing-off turn holds up. The page reads
* the turn's record against the ask through the hosted judge and answers
* with the steer for the next pass, or null to let the turn close. A session
* with no attached page closes at once.
*/
export function askPageGate(sessionKey: string, reply: string): Promise<string | null> {
const session = sessions.get(sessionKey);
if (!session?.attached) return Promise.resolve(null);
const gateId = crypto.randomUUID().slice(0, 12);
session.writer.write({ type: "data-gate", data: { gateId, reply }, transient: true });
return new Promise((resolve) => {
const timer = setTimeout(() => settle(null), GATE_TIMEOUT_MS);
const settle = (steer: string | null) => {
clearTimeout(timer);
session.gates.delete(gateId);
resolve(steer);
};
session.gates.set(gateId, settle);
});
}
/** Called by /api/cut/ai/gate with the page's verdict. */
export function resolvePageGate(sessionKey: string, gateId: string, steer: string | null): boolean {
const settle = sessions.get(sessionKey)?.gates.get(gateId);
if (!settle) return false;
settle(steer);
return true;
}