Open source CapCut alternative.
TypeScript
import { parseFolderLink } from "@/cut/lib/folderReference";
import { copyFile } from "node:fs/promises";
import { UI_TOOLS } from "@/cut/lib/projectCommands";
import { PAGE_ONLY_TOOLS, runProjectCommand } from "../../lib/aiTools";
import { projectOperation } from "../../lib/projectOperation";
import { localBackend } from "../../lib/backend/local";
import { DETACHED_MEDIA_ERROR, DETACHED_SESSION_ERROR, DETACHED_UI_NOTE } from "../../lib/chatResume";
import { mp4NameFor, setHostMediaStore } from "../../lib/mediaConvert";
import { setHostDocStore } from "../../lib/projectReference";
import { serializeDoc, useEditor } from "../../lib/store";
import type { MediaAsset, ProjectDoc } from "../../lib/types";
import { convertToMp4 } from "../convert";
import { probeDuration } from "../frames";
import { deleteMedia, listProjects, mediaPath, readProject, writeProject } from "../projects";
import { uniqueName } from "../util";
import { withFileColors } from "../fileColor";
import { drawnLists } from "../../lib/itemKinds";
// The engine's own tool executor, for a chat whose editor tab is gone: the
// project doc hydrates into the same store the page uses, tools run against
// it, and every mutation lands back on disk. A turn survives its tab.
//
// Scope: doc edits. Tools that need the page (preview reads, media decodes)
// or the page's hosted sign-in (generation, library sync) answer with a
// typed refusal so the model reports honestly and moves on.
/** Tools that decode or rasterize media. The engine runs on Bun with the
* bundled tools and no canvas, decoders or Web Audio, so these answer with a
* typed refusal here; the cloud runner installs those primitives and runs the
* same tools for real. */
const PAGE_MEDIA_TOOLS: ReadonlySet<string> = new Set([
"watch_video",
"listen_audio",
"measure_level",
"detect_silence",
"detect_beats",
"refine_speech_cuts",
"sync_audio",
"freeze_frame",
"capture_frame",
"compare_to_source",
"find_picture",
"detect_cuts",
"sample_color",
"track_element",
"track_camera",
"suggest_font",
"render_preview",
"create_sticker",
"subtitles_generate",
"captions_generate",
"subtitles_from_visuals",
"stock_add",
"read_color_stats",
"match_color_grade",
...PAGE_ONLY_TOOLS,
]);
const PAGE_SESSION_TOOLS: ReadonlySet<string> = new Set([
"generate_image",
"generate_video",
"generate_character_video",
"generate_scene",
"approve_scene",
"cancel_scene",
"regenerate_shot",
"recut_scene",
"restyle_scene",
"generate_music",
"voiceover_generate",
"read_subtitles_aloud",
"subtitles_translate_track",
"wait_for_renders",
"library_list",
"library_add",
"library_organize",
"library_share",
"notes_list",
"read_note",
"note_save",
"template_add",
"file_asset",
"import_url",
"check_facts",
"set_interface_language",
]);
// Converting media is the one media job this process can do: ffmpeg is right
// here, and the file is in the project folder. Going through the client's
// backend seam would have the engine call itself over HTTP, so the converter
// works the folder directly and the shared client code never notices.
setHostMediaStore({
async convert(projectId, source, opts) {
const src = mediaPath(projectId, source.fileName);
const fileName = await uniqueName(mp4NameFor(source.fileName), (n) => mediaPath(projectId, n));
const outPath = mediaPath(projectId, fileName);
const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };
const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight, mode: opts.mode });
return {
fileName: outcome.unchanged ? source.fileName : fileName,
...(outcome.unchanged ? { unchanged: true } : {}),
reencoded: outcome.transcodedVideo || outcome.transcodedAudio,
duration: await probeDuration(outcome.unchanged ? src : outPath),
...(outcome.width !== undefined ? { width: outcome.width, height: outcome.height } : {}),
};
},
drop(projectId, fileName) {
return deleteMedia(projectId, fileName);
},
});
// Another project on this Mac, listed and read as a reference: the doc off
// disk and a file copied folder to folder. A cloud or shared reference needs the page's
// sign-in, and the reader says so.
setHostDocStore({
listDocs: () => listProjects(),
readDoc: (projectId) => readProject(projectId),
async copyMedia(srcProjectId, fileName, dstProjectId) {
const target = await uniqueName(fileName, (n) => mediaPath(dstProjectId, n));
await copyFile(mediaPath(srcProjectId, fileName), mediaPath(dstProjectId, target));
return target;
},
});
interface OpenDoc {
projectId: string;
stored: ProjectDoc;
}
let open: OpenDoc | null = null;
async function ensureOpen(projectId: string): Promise<OpenDoc | null> {
// Read the doc on every call: the page saves through writeProject between
// turns, and flushing a store hydrated from an older read would revert
// those edits. The store is reused only while the disk copy still matches
// the one it was hydrated from.
const stored = await readProject(projectId);
if (!stored) return null;
const s = useEditor.getState();
if (
open?.projectId === projectId &&
s.projectId === projectId &&
s.loaded &&
stored.updatedAt === open.stored.updatedAt
) {
return open;
}
// Footage imported before its color was read gets it here, off the file,
// so what a tool renders and what the doc saves both read it right.
const assets: MediaAsset[] = await withFileColors(
(stored.assets ?? []).map((a) => ({
...a,
url: `/api/cut/projects/${encodeURIComponent(projectId)}/media/${encodeURIComponent(a.fileName)}`,
})),
drawnLists(stored),
(fileName) => mediaPath(projectId, fileName)
);
await useEditor.getState().openProjectDoc(projectId, stored, assets);
open = { projectId, stored };
return open;
}
async function flush(doc: OpenDoc): Promise<void> {
const merged = { ...doc.stored, ...serializeDoc(useEditor.getState()) } as ProjectDoc;
await writeProject(doc.projectId, merged);
doc.stored = merged;
}
/** True when a tool input carries svg markup to draw. */
function hasSvgMarkup(input: unknown): boolean {
const svg = (input as { svg?: unknown } | null)?.svg;
return typeof svg === "string" && svg.trim() !== "";
}
/** Run one tool call against the engine-held store for `projectId`,
* persisting the doc after it. The bridge calls this when no editor tab is
* attached to the chat session. */
async function executeHeadlessTool(
projectId: string,
toolName: string,
input: unknown
): Promise<{ output?: unknown; errorText?: string }> {
if (UI_TOOLS.has(toolName))
return {
output: { noEditor: true, note: DETACHED_UI_NOTE },
};
if (PAGE_MEDIA_TOOLS.has(toolName)) return { errorText: DETACHED_MEDIA_ERROR };
// A sticker drawn from svg markup rasterizes it, and the engine has no
// canvas; placing an existing sticker asset is a doc edit and runs here.
if (toolName === "add_sticker" && hasSvgMarkup(input)) {
return { errorText: DETACHED_MEDIA_ERROR };
}
if (toolName === "read_folder") {
const args = (input ?? {}) as { reference?: { scope?: string }; link?: string };
const ref = args.reference ?? (args.link ? parseFolderLink(args.link) : null);
if (ref?.scope !== "project") return { errorText: DETACHED_SESSION_ERROR };
}
if (PAGE_SESSION_TOOLS.has(toolName)) return { errorText: DETACHED_SESSION_ERROR };
const doc = await ensureOpen(projectId);
if (!doc) return { errorText: "Project not found on this Mac." };
try {
const output = await runProjectCommand(projectOperation(projectId, localBackend), toolName, (input ?? {}) as Record<string, unknown>);
await flush(doc);
return { output: output ?? null };
} catch (err) {
return { errorText: err instanceof Error ? err.message : String(err) };
}
}
let toolsIdle: Promise<unknown> = Promise.resolve();
/** The engine's editor store belongs to one tool invocation at a time. */
export function callHeadlessTool(projectId: string, toolName: string, input: unknown): Promise<{ output?: unknown; errorText?: string }> {
const result = toolsIdle.then(() => executeHeadlessTool(projectId, toolName, input));
toolsIdle = result.catch(() => {});
return result;
}