Skip to content

Open source CapCut alternative.

TypeScript

feat(site): land media drops instantly and copy bytes behind the editor

David Le12ddc250392a
diff --git a/site/src/cut/components/StockMusicPanel.tsx b/site/src/cut/components/StockMusicPanel.tsx
index 075acfed..49f59330 100644
--- a/site/src/cut/components/StockMusicPanel.tsx
+++ b/site/src/cut/components/StockMusicPanel.tsx
@@ -79,7 +79,11 @@ export function SampleLibrary({ projectId }: { projectId: string }) {
playing={playingUrl === s.file}
onTogglePlay={() => togglePlay(s.file)}
onAdd={() =>
- void importStockMusic(projectId, { url: s.file, name: sampleName(s.id) })
+ void importStockMusic(projectId, {
+ url: s.file,
+ name: sampleName(s.id),
+ duration: s.duration,
+ })
.then((a) => useEditor.getState().addAudioFromAsset(a.id))
.catch(() => {})
}
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 8cb23fc9..7001f312 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -33,7 +33,7 @@ import {
import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";
import { requestSidePanel, SIDE_PANEL_TABS } from "./panelRequest";
import { blobToInlineAudio, refToInlineAudio, visualRefs, type InlineImage } from "./refMedia";
-import { characterPrompt, stockTitle } from "./stock";
+import { characterPrompt, stockAspectDims, stockTitle } from "./stock";
import { STOCK_IMAGES } from "./stockManifest";
import { STOCK_VIDEOS } from "./stockVideoManifest";
import { applyOverlayPatchSettled, track0Clips, trackGapAt, getClipSpans, nextFreeStart, overlayLayers, TIMELINE_H_MAX, TIMELINE_H_MIN, totalDuration, useEditor } from "./store";
@@ -1260,7 +1260,12 @@ export async function runAiTool(
// asked, even if the user switches threads while it downloads.
const chatId = chatOwner();
const asset = vid
- ? await importStockVideo(projectId, { url: vid.file, name: stockTitle(vid.id) })
+ ? await importStockVideo(projectId, {
+ url: vid.file,
+ name: stockTitle(vid.id),
+ duration: vid.duration,
+ ...stockAspectDims(vid.aspect),
+ })
: await importImage(projectId, { url: img!.file, name: stockTitle(img!.id) });
tagChatAsset(asset.id, chatId);
const addToTimeline = wantsTimeline(input, "start");
diff --git a/site/src/cut/lib/importQueue.ts b/site/src/cut/lib/importQueue.ts
index c77585e1..487bf54c 100644
--- a/site/src/cut/lib/importQueue.ts
+++ b/site/src/cut/lib/importQueue.ts
@@ -1,9 +1,10 @@
"use client";
// Imports that are on screen before they are stored. A dropped file is probed
-// from its own bytes and placed straight away; the upload runs here, behind the
-// editor, and the asset swaps from its local object URL to the stored one when
-// the bytes land.
+// from its own bytes and placed straight away; a stock tile or library clip is
+// registered against its source URL. The copy into project storage runs here,
+// behind the editor, and the asset swaps from the URL it arrived on to the
+// stored one when the bytes land.
//
// Three things keep that honest:
// - Nothing about a pending asset reaches the saved document (see storedAssets
@@ -81,7 +82,7 @@ async function run(job: Job) {
try {
if (!wanted(job)) return cancelUpload(asset.id);
let shown = -1;
- await send({
+ const fileName = await send({
signal: job.controller.signal,
onProgress: (fraction) => {
// The progress stream is the cheapest place to notice the asset is
@@ -95,11 +96,14 @@ async function run(job: Job) {
});
if (!wanted(job)) return cancelUpload(asset.id);
jobs.delete(asset.id);
- // The stored file takes over from the local bytes. Filmstrips are
- // self-contained frames and survive the swap; a still's single "frame" is
- // the source itself, so it repoints with the asset.
- const url = mediaUrl(job.projectId, asset.fileName);
+ // The stored file takes over from the source bytes, under the name the
+ // copy resolved to (a dropped file reserved it up front; a remote import
+ // learns it here). Filmstrips are self-contained frames and survive the
+ // swap; a still's single "frame" is the source itself, so it repoints
+ // with the asset.
+ const url = mediaUrl(job.projectId, fileName);
useEditor.getState().updateAsset(asset.id, {
+ fileName,
url,
upload: undefined,
...(asset.type === "image" ? { thumbs: [url] } : {}),
diff --git a/site/src/cut/lib/library.ts b/site/src/cut/lib/library.ts
index fa782f1b..d510a177 100644
--- a/site/src/cut/lib/library.ts
+++ b/site/src/cut/lib/library.ts
@@ -2,7 +2,13 @@
import { apiJson, getBackend } from "./backend";
import { readSnapshot, writeSnapshot } from "./cache";
-import { enrichAsset, presignedUpload, probeFileMeta, uploadProjectMediaTo } from "./media";
+import {
+ enrichAsset,
+ importRemote,
+ presignedUpload,
+ probeFileMeta,
+ uploadProjectMediaTo,
+} from "./media";
import {
activeResidency,
availableResidencies,
@@ -223,7 +229,8 @@ async function copyLibraryFileToProject(
projectId: string,
residency: Residency,
fileName: string,
- assetId?: string
+ assetId?: string,
+ opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }
): Promise<string> {
const target = getBackend();
if (assetId && residency === target.kind) {
@@ -231,41 +238,42 @@ async function copyLibraryFileToProject(
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ assetId, projectId }),
+ signal: opts?.signal,
});
const body = await apiJson<{ fileName?: string }>(res);
if (!res.ok || !body.fileName) throw new Error(body.error ?? "Could not add from library.");
return body.fileName;
}
const bytes = await backendFor(residency).fetch(
- `/api/cut/library/media/${encodeURIComponent(fileName)}`
+ `/api/cut/library/media/${encodeURIComponent(fileName)}`,
+ { signal: opts?.signal }
);
if (!bytes.ok) throw new Error("Could not read that library file.");
- return uploadProjectMediaTo(target, projectId, await bytes.blob(), fileName);
+ return uploadProjectMediaTo(target, projectId, await bytes.blob(), fileName, opts);
}
-/** Copy a library asset into the open project's media and register it, without
- * placing it on the timeline. Callers choose where it lands. */
+/** Register a library asset in the open project's media, without placing it on
+ * the timeline. Callers choose where it lands. Usable immediately: it plays
+ * from the library's own route while the copy into the project runs behind
+ * the editor (server-side on the asset's own shelf, download-and-upload
+ * across residencies). */
export async function importLibraryAsset(
projectId: string,
lib: LibraryAsset
): Promise<MediaAsset> {
- const fileName = await copyLibraryFileToProject(projectId, lib.residency, lib.fileName, lib.id);
-
- const asset: MediaAsset = {
- id: crypto.randomUUID().slice(0, 8),
- fileName,
- name: lib.name,
- type: lib.type,
- duration: lib.duration,
- width: lib.width,
- height: lib.height,
- url: mediaUrl(projectId, fileName),
- };
- useEditor.getState().addAsset(asset);
- // Generate filmstrip thumbnails / waveform peaks so the clip renders like any
- // other on the timeline (project uploads get this via the editor on add).
- void enrichAsset(asset);
- return asset;
+ return importRemote(
+ projectId,
+ {
+ url: libraryMediaUrl(lib.fileName, lib.residency),
+ name: lib.name,
+ fileName: lib.fileName,
+ type: lib.type,
+ duration: lib.duration,
+ width: lib.width,
+ height: lib.height,
+ },
+ (opts) => copyLibraryFileToProject(projectId, lib.residency, lib.fileName, lib.id, opts)
+ );
}
/** Copy a library asset into the open project and append it to the timeline. */
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 681c72a2..b4663cf6 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -4,6 +4,7 @@ import { scanSilence, type PcmChunk } from "./audioScan";
import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";
import { quotaErrorMessage } from "./backend/cloud";
import { encodeWav } from "./cloudTranscribe";
+import { startUpload } from "./importQueue";
import {
audioChunks,
audioPeaks,
@@ -185,7 +186,8 @@ export async function uploadProjectMediaTo(
backend: CutBackend,
projectId: string,
file: Blob,
- name: string
+ name: string,
+ opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }
): Promise<string> {
if (backend.kind !== "cloud") {
const form = new FormData();
@@ -193,17 +195,20 @@ export async function uploadProjectMediaTo(
const res = await backend.fetch(`/api/cut/projects/${projectId}/media`, {
method: "POST",
body: form,
+ signal: opts?.signal,
});
const body = await apiJson<{ fileName?: string }>(res);
if (!res.ok || !body.fileName) throw new Error(body.error ?? "Upload failed.");
return body.fileName;
}
- const key = await presignedUpload(
+ const signed = await presignUpload(
`/api/cut/projects/${projectId}/media/presign`,
file,
name,
backend
);
+ await putSigned(signed.url, file, file.type || "application/octet-stream", opts);
+ const key = signed.key;
const res = await backend.fetch(`/api/cut/projects/${projectId}/media/complete`, {
method: "POST",
headers: { "Content-Type": "application/json" },
@@ -329,10 +334,12 @@ export type PendingImport = {
/** Ready to add to the project: plays from `localUrl`, carries its final
* `fileName`, and is marked `upload` until the bytes land. */
asset: MediaAsset;
- /** The object URL the asset plays from until the stored file takes over. */
+ /** The URL the asset plays from until the stored file takes over — a local
+ * object URL for a dropped file, the source URL for remote media. */
localUrl: string;
- /** Send the bytes and mark the object complete. */
- send: (opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }) => Promise<void>;
+ /** Send the bytes and mark the object complete; resolves to the stored
+ * (deduped) file name the asset swaps to. */
+ send: (opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }) => Promise<string>;
};
/** Prepare a dropped file so it can appear instantly: reserve its stored name
@@ -385,6 +392,7 @@ export async function prepareImport(
});
const body = await apiJson<{ fileName?: string }>(res);
if (!res.ok || !body.fileName) throw new Error(body.error ?? "Upload failed.");
+ return body.fileName;
};
return { asset, localUrl, send };
} catch (err) {
@@ -453,49 +461,137 @@ function loadImageMeta(url: string): Promise<{ width: number; height: number }>
});
}
+/** What `importRemote` needs to register a fetchable source as an asset the
+ * user can edit with right away. Duration and dimensions come from the
+ * caller's catalog entry, so nothing has to be read before the asset exists. */
+type RemoteImportInit = {
+ url: string;
+ name: string;
+ /** Stored name to claim; defaults to the URL's basename. */
+ fileName?: string;
+ type: AssetType;
+ duration: number;
+ width?: number;
+ height?: number;
+ origin?: StoredAsset["origin"];
+};
+
+/** Copy attempts ride out transient blips before the queue marks the asset
+ * failed; an abort (asset deleted, project left) stops immediately. */
+async function withRetries(
+ copy: (opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }) => Promise<string>,
+ opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }
+): Promise<string> {
+ for (let attempt = 0; ; attempt++) {
+ try {
+ return await copy(opts);
+ } catch (err) {
+ if (opts?.signal?.aborted || attempt >= 2) throw err;
+ await new Promise((r) => setTimeout(r, 1500 * (attempt + 1)));
+ }
+ }
+}
+
+/** Register media that lives at a fetchable URL — a stock tile, a library
+ * file — as a project asset that is usable the moment this returns. The asset
+ * plays from the source URL while `copy` moves the bytes into project storage
+ * behind the editor (the import queue); when they land it swaps to the stored
+ * file and joins the saved document, exactly like a dropped OS file. `copy`
+ * defaults to download-and-upload; callers with a server-side copy (the
+ * library's same-shelf route) pass their own and skip the round trip. */
+export function importRemote(
+ projectId: string,
+ init: RemoteImportInit,
+ copy?: (opts?: { onProgress?: (fraction: number) => void; signal?: AbortSignal }) => Promise<string>
+): MediaAsset {
+ const fileName =
+ init.fileName || init.url.split("/").pop()?.split("?")[0] || `${init.type}-${uid()}`;
+ const asset: MediaAsset = {
+ id: uid(),
+ fileName,
+ name: init.name,
+ type: init.type,
+ duration: init.duration,
+ ...(init.width !== undefined ? { width: init.width } : {}),
+ ...(init.height !== undefined ? { height: init.height } : {}),
+ ...(init.origin ? { origin: init.origin } : {}),
+ url: init.url,
+ upload: { progress: 0 },
+ };
+ useEditor.getState().addAsset(asset);
+ void enrichAsset(asset);
+ // Catalog dimensions are nominal (an aspect, not the file's pixels): read
+ // the real ones behind the placement so the doc stores the truth.
+ if (init.type === "video") {
+ void probeMediaFile(init.url)
+ .then((m) => {
+ if (m.hasVideo) {
+ useEditor.getState().updateAsset(asset.id, { width: m.width, height: m.height });
+ }
+ })
+ .catch(() => {});
+ }
+ const download = async (opts?: {
+ onProgress?: (fraction: number) => void;
+ signal?: AbortSignal;
+ }) => {
+ const dl = await fetch(init.url, { signal: opts?.signal });
+ if (!dl.ok) throw new Error("Could not read the media.");
+ const blob = await dl.blob();
+ return uploadProjectMediaTo(getBackend(), projectId, blob, fileName, opts);
+ };
+ startUpload(projectId, {
+ asset,
+ localUrl: init.url,
+ send: (opts) => withRetries(copy ?? download, opts),
+ });
+ return asset;
+}
+
/** Store a fetchable image (a stock tile) in the project's media as a
* first-class image asset at its native resolution — no video baking — and
* register it, without placing it on the timeline. Callers choose where it
- * lands. */
+ * lands. Ready as soon as the pixel size is read; the bytes copy in behind
+ * the editor. */
export async function importImage(
projectId: string,
image: { url: string; name: string }
): Promise<MediaAsset> {
- const dl = await fetch(image.url);
- if (!dl.ok) throw new Error("Could not read the image.");
- const blob = await dl.blob();
- const body = await uploadProjectImage(projectId, blob, image.url.split("/").pop() || "image.png", {
- name: image.name,
- });
+ // The size frames the still (and the first-asset aspect guess); the source
+ // is a same-origin file, so this is one cached header read, not a download.
+ const dims = await loadImageMeta(image.url);
// A stock image lands on the timeline where the caller places it, not in the
// Media panel — tag it so it stays out.
- const asset: MediaAsset = { ...body, origin: "stock" };
- useEditor.getState().addAsset(asset);
- void enrichAsset(asset);
- return asset;
+ return importRemote(projectId, {
+ url: image.url,
+ name: image.name,
+ type: "image",
+ duration: 0,
+ width: dims.width,
+ height: dims.height,
+ origin: "stock",
+ });
}
/** Store a fetchable video (a stock clip) in the project's media as a regular
* video asset and register it, without placing it on the timeline. Callers
- * choose where it lands. */
+ * choose where it lands. Instant when the catalog supplies the duration;
+ * otherwise one metadata read stands between the call and the asset. */
export async function importStockVideo(
projectId: string,
- video: { url: string; name: string }
+ video: { url: string; name: string; duration?: number; width?: number; height?: number }
): Promise<MediaAsset> {
- const dl = await fetch(video.url);
- if (!dl.ok) throw new Error("Could not read the video.");
- const blob = await dl.blob();
- const file = new File([blob], video.url.split("/").pop() || "video.mp4", {
- type: blob.type || "video/mp4",
- });
- const asset = await importFileToProject(projectId, file);
- if (!asset) throw new Error("Could not add the video.");
- asset.name = video.name;
+ const duration = video.duration ?? (await probeMediaFile(video.url)).duration;
// Like a stock image, it lands where the caller places it, not in Media.
- asset.origin = "stock";
- useEditor.getState().addAsset(asset);
- void enrichAsset(asset);
- return asset;
+ return importRemote(projectId, {
+ url: video.url,
+ name: video.name,
+ type: "video",
+ duration,
+ width: video.width,
+ height: video.height,
+ origin: "stock",
+ });
}
/** Store a bundled stock-music bed in the project's media as a regular audio
@@ -503,21 +599,16 @@ export async function importStockVideo(
* where it lands (the soundtrack). Tagged "stock" so it stays out of Media. */
export async function importStockMusic(
projectId: string,
- music: { url: string; name: string }
+ music: { url: string; name: string; duration?: number }
): Promise<MediaAsset> {
- const dl = await fetch(music.url);
- if (!dl.ok) throw new Error("Could not read the music.");
- const blob = await dl.blob();
- const file = new File([blob], music.url.split("/").pop() || "music.mp3", {
- type: blob.type || "audio/mpeg",
+ const duration = music.duration ?? (await probeMediaFile(music.url)).duration;
+ return importRemote(projectId, {
+ url: music.url,
+ name: music.name,
+ type: "audio",
+ duration,
+ origin: "stock",
});
- const asset = await importFileToProject(projectId, file);
- if (!asset) throw new Error("Could not add the music.");
- asset.name = music.name;
- asset.origin = "stock";
- useEditor.getState().addAsset(asset);
- void enrichAsset(asset);
- return asset;
}
/** Download a URL (TikTok, YouTube, an X post, an article, …) into the project
diff --git a/site/src/cut/lib/stock.ts b/site/src/cut/lib/stock.ts
index e63eee55..7579e289 100644
--- a/site/src/cut/lib/stock.ts
+++ b/site/src/cut/lib/stock.ts
@@ -64,6 +64,13 @@ export interface StockMusic {
peaks: number[];
}
+/** Nominal pixel size for a stock clip's aspect. The catalog stores only the
+ * ratio; these stand in until the file's real dimensions are probed, and all
+ * that reads them before then is orientation (the first-asset aspect guess)
+ * and framing math, which only use the ratio. */
+export const stockAspectDims = (aspect: StockVideoAspect): { width: number; height: number } =>
+ aspect === "9:16" ? { width: 1080, height: 1920 } : { width: 1920, height: 1080 };
+
export const STOCK_MUSIC_CATEGORIES = [
"Songs",
"Cinematic",