Skip to content

Open source CapCut alternative.

TypeScript

feat(site): chat converts dropped footage and hands back the MP4s [rebuild]

David Lea91d812b0d1e
diff --git a/site/src/cut/components/SidePanel.tools.ts b/site/src/cut/components/SidePanel.tools.ts
index e07c48c9..386d85ad 100644
--- a/site/src/cut/components/SidePanel.tools.ts
+++ b/site/src/cut/components/SidePanel.tools.ts
@@ -32,6 +32,19 @@ export const SIDE_PANEL_TOOLS = [
to: { type: "string", enum: ["media", "library"], description: "Destination" },
}, ["asset_id", "to"]),
},
+ {
+ name: "convert_media",
+ description:
+ "Convert project media to MP4 — H.264 picture, AAC sound — replacing each asset's file in place, so clips already cut from it keep playing. This is the fix for footage that previews black or refuses to play: a phone's HEVC .mov, a screen recorder's ProRes, a camera file with raw PCM sound. It is also the whole answer to \"turn these into mp4\": the user drops the files on this chat, you convert them, and each converted file is on its own card right here for them to download or drag in — nothing to go looking for. A file already in Media stays in Media. Streams that already fit the container are copied packet for packet, so a .mov of H.264 converts in seconds and loses nothing, and a file that is already an MP4 of those streams comes back untouched. `max_height` shrinks the picture at the same time (1080 for a 4K source that only needs to be a timeline clip). It runs wherever the project's bytes already are — this Mac, the render worker, or the tab — so it needs no setup; when nothing available can decode the source the error says what would fix it, and retrying changes nothing.",
+ inputSchema: obj({
+ asset_ids: {
+ type: "array",
+ items: { type: "string" },
+ description: "Project asset ids from `media`, converted in order",
+ },
+ max_height: num("Cap the output height in pixels (e.g. 1080); omit to keep the source size"),
+ }, ["asset_ids"]),
+ },
{
name: "delete_asset",
description:
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 17540932..5324b33b 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -64,6 +64,7 @@ import {
renderAudioSpanWav,
sampleWatchFrames,
} from "./media";
+import { convertAssetToMp4 } from "./mediaConvert";
import { isLottieAsset } from "./lottieAssets";
import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";
import { requestSidePanel } from "./panelRequest";
@@ -1949,6 +1950,60 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
throw new ToolError('`to` must be "media" or "library".');
},
+ convert_media: async (s, input) => {
+ const projectId = s.projectId;
+ if (!projectId) throw new ToolError("No project open.");
+ const ids = Array.isArray(input.asset_ids) ? input.asset_ids.map((v) => String(v)) : [];
+ if (ids.length === 0) throw new ToolError("Pass the asset ids to convert.");
+ const maxHeight = isNum(input.max_height) ? Math.round(input.max_height) : undefined;
+ // Named `assets` so each converted file renders as its own card in the
+ // thread (ToolOutputAssets keys off assetId) — the user drops files on
+ // the chat and takes the MP4s back from the same place.
+ const converted: Record<string, unknown>[] = [];
+ const failed: { assetId: string; name?: string; error: string }[] = [];
+ for (const id of ids) {
+ // Read live: each conversion rewrites the asset it just finished.
+ const asset = useEditor.getState().assets.find((a) => a.id === id);
+ if (!asset) {
+ failed.push({ assetId: id, error: `No project asset with id ${id}.` });
+ continue;
+ }
+ try {
+ const out = await convertAssetToMp4(projectId, asset, { maxHeight });
+ converted.push({
+ assetId: out.assetId,
+ // The card now reads `name`; `from` is what the user dropped.
+ name: out.name,
+ ...(out.name !== asset.name ? { from: asset.name } : {}),
+ file: out.fileName,
+ // False means the packets were copied — same picture, new container.
+ reencoded: out.reencoded,
+ ...(out.unchanged ? { unchanged: true } : {}),
+ duration: round2(out.duration),
+ ...(out.width !== undefined ? { width: out.width, height: out.height } : {}),
+ // The file stayed where it already lived: a chat attachment on its
+ // card in this thread, an import in the Media panel.
+ where: asset.origin === "chat" ? "chat card" : asset.origin ?? "media panel",
+ });
+ } catch (e) {
+ failed.push({
+ assetId: id,
+ name: asset.name,
+ error: e instanceof Error ? e.message : "Could not convert that file.",
+ });
+ }
+ }
+ if (converted.length === 0) throw new ToolError(failed[0]?.error ?? "Nothing was converted.");
+ const untouched = converted.every((c) => c.unchanged);
+ return {
+ assets: converted,
+ ...(failed.length > 0 ? { failed } : {}),
+ ...(untouched
+ ? { note: "These were already MP4 with H.264 picture and AAC sound, so they were left as they are — say so." }
+ : {}),
+ };
+ },
+
delete_asset: (s, input) => {
const asset = requireItem(s.assets, input.asset_id, "project asset");
const uses =
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index dc0e767d..378cf5c2 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -193,6 +193,7 @@ Project media (\`media\` in editor_state) is every file in the open project. The
- add_clip / add_overlay_video place a project asset on the timeline; listen_audio plays an asset's sound to you — an audio asset, or a video/clip's own track; watch_video is the visual sibling.
- file_asset moves a created asset into Media (to:"media", clears its origin) or copies any asset into the Library (to:"library").
- delete_asset removes an asset from the project plus its timeline clips; the media does not come back with undo, so only on an explicit ask, and say what went with it.
+- convert_media rewrites project media as MP4 (H.264/AAC) in place, one call for a whole list of asset ids. Reach for it when footage plays black or shows a format warning — phone HEVC, ProRes, raw PCM sound — and whenever someone asks for mp4 files. That ask is usually files dropped straight on this chat: convert them and each one is on its own card in the thread, converted, ready to download or drag onto the timeline — close by saying they're on their cards. An asset that came from the Media panel stays there. Streams that already fit are copied, so most .mov files convert in seconds, and a file that is already H.264/AAC MP4 comes back untouched (say so instead of claiming a conversion); max_height shrinks an oversized source at the same time. Clips already cut from the asset keep playing. It runs on whichever machine already holds the bytes — this Mac, the render worker, or the tab — and works with no editor tab attached; when nothing available can decode the source the error names the fix (the Mac app, another browser, or moving the project to the cloud), so report it and move on.
The Library is shared across every project: folders, reusable assets, and templates — a template is a saved arrangement (clips, overlays, titles, captions, by reference) that comes back editable.
- library_list browses it; library_add copies an asset into the project (the import step "library"-scope attachments need); template_add re-materializes a template; save_template saves timeline items as one.
- library_organize handles folders (create/rename/delete), filing (move_asset), and deletes (permanent — explicit ask only).
diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.ts
index 94a81846..246541b1 100644
--- a/site/src/cut/server/ai/headlessTools.ts
+++ b/site/src/cut/server/ai/headlessTools.ts
@@ -1,7 +1,11 @@
import { runAiTool, UI_TOOLS } from "../../lib/aiTools";
+import { mp4NameFor, setHostMediaStore } from "../../lib/mediaConvert";
import { serializeDoc, useEditor } from "../../lib/store";
import type { MediaAsset, ProjectDoc } from "../../lib/types";
-import { readProject, writeProject } from "../projects";
+import { convertToMp4 } from "../convert";
+import { probeDuration } from "../frames";
+import { deleteMedia, mediaPath, readProject, writeProject } from "../projects";
+import { uniqueName } from "../util";
// The engine's own tool executor, for a chat whose editor tab is gone: the
// project doc hydrates into the same store the page uses, tools run against
@@ -52,6 +56,29 @@ const PAGE_SESSION_TOOLS: ReadonlySet<string> = new Set([
"import_url",
]);
+// Converting media is the one media job this process can do: ffmpeg is right
+// here, and the file is in the project folder. Going through the client's
+// backend seam would have the engine call itself over HTTP, so the converter
+// works the folder directly and the shared client code never notices.
+setHostMediaStore({
+ async convert(projectId, source, opts) {
+ const src = mediaPath(projectId, source.fileName);
+ const fileName = await uniqueName(mp4NameFor(source.fileName), (n) => mediaPath(projectId, n));
+ const outPath = mediaPath(projectId, fileName);
+ const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };
+ const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight });
+ return {
+ fileName,
+ reencoded: outcome.transcodedVideo || outcome.transcodedAudio,
+ duration: await probeDuration(outPath),
+ ...(outcome.width !== undefined ? { width: outcome.width, height: outcome.height } : {}),
+ };
+ },
+ drop(projectId, fileName) {
+ return deleteMedia(projectId, fileName);
+ },
+});
+
interface OpenDoc {
projectId: string;
stored: ProjectDoc;