Skip to content

Open source CapCut alternative.

TypeScript

feat(site): video clips carry their sound as a band under the filmstrip

David Le7b161bc69986
diff --git a/site/src/cut/components/Timeline.tsx b/site/src/cut/components/Timeline.tsx
index 8a0edaff..cd2be92a 100644
--- a/site/src/cut/components/Timeline.tsx
+++ b/site/src/cut/components/Timeline.tsx
@@ -76,7 +76,8 @@ const TRANSITION_ICONS: Record<TransitionStyle, LucideIcon> = {
splitclose: FoldHorizontal,
};
-const VIDEO_H = 64;
+/** Video row height: the picture band plus the sound band under it. */
+const VIDEO_H = 88;
/** How far past the top or bottom of the video stack a drag has to carry a
* clip to open a new track. Every drop inside the stack lands on a row, which
* opens room for it, so this reach is the one gesture that adds a track — and
@@ -90,6 +91,12 @@ const OVERLAY_H = VIDEO_H; // every video track shares the same row height
* and the next row keeps its usual distance below the line. */
const KEYRAIL_EXTRA = 20;
const AUDIO_H = 44;
+/** The sound band along a video clip box's bottom edge. */
+const WAVE_H = 24;
+/** The picture band above it — the filmstrip's height. A clip with nothing to
+ * play in the band (an image, a muted or silent video) drops the band and its
+ * box stops at the filmstrip's edge. */
+const FILM_H = VIDEO_H - 4 - WAVE_H;
/** The `mt-1.5` every row carries, so a row drag knows what one row's worth of
* travel is. */
const ROW_GAP = 6;
@@ -1605,7 +1612,9 @@ export function Timeline() {
style={{ left: t * pps, width: Math.max(10, len * pps - CLIP_GAP), height: h }}
>
{ghost && (
- <DropGhostFilm ghost={ghost} w={Math.max(10, len * pps - CLIP_GAP)} h={h} pps={pps} />
+ <div className="pointer-events-none absolute inset-x-0 top-0" style={{ height: h - WAVE_H }}>
+ <DropGhostFilm ghost={ghost} w={Math.max(10, len * pps - CLIP_GAP)} h={h - WAVE_H} pps={pps} />
+ </div>
)}
<span className="absolute top-1 left-1 rounded-[5px] bg-black/65 px-1.5 py-px font-mono text-[10px] tabular-nums text-white">
{len.toFixed(1)}s
@@ -3514,11 +3523,15 @@ function ClipView({
() => false
);
const boxRef = useRef<HTMLDivElement>(null);
+ // The clip's own sound, drawn in the box's band under the picture. Muting —
+ // detach included — takes the sound off the clip, so the band drops and the
+ // box stops at the filmstrip (a detached copy draws on the soundtrack row).
+ const hasWave = asset.type === "video" && !clip.muted && !!asset.peaks?.length;
// Bumped as tile captures land, so the memo re-plans and picks them up.
const [filmGen, onTileFrame] = useReducer((x: number) => x + 1, 0);
const filmstrip = useMemo(
() =>
- filmstripFrames(asset, filmIn, w, pps, speed, VIDEO_H - 4, 26, {
+ filmstripFrames(asset, filmIn, w, pps, speed, FILM_H, 26, {
start: startFrame,
end: endFrame,
}),
@@ -3563,7 +3576,7 @@ function ClipView({
left: drag ? drag.ghostX : (partAt ?? span.start) * pps,
top: drag ? 2 + drag.ghostY : undefined,
width: Math.max(10, w - CLIP_GAP),
- height: VIDEO_H - 4,
+ height: hasWave ? VIDEO_H - 4 : FILM_H,
// Inline so it beats SELECTED_SHADOW's z-10 class on the same element.
zIndex: drag ? 20 : undefined,
}}
@@ -3580,7 +3593,17 @@ function ClipView({
});
}}
>
- <Filmstrip frames={filmstrip} grade={clip.grade} failed={stripFailed} />
+ <div className="pointer-events-none absolute inset-x-0 top-0" style={{ height: FILM_H }}>
+ <Filmstrip frames={filmstrip} grade={clip.grade} failed={stripFailed} />
+ </div>
+ {hasWave && (
+ <div
+ className="tl-clip-wave pointer-events-none absolute inset-x-0 bottom-0 bg-gradient-to-b from-[#59c09d] to-[#43b18d]"
+ style={{ height: WAVE_H }}
+ >
+ <WaveformCanvas asset={asset} from={filmIn} to={filmOut} w={w} h={WAVE_H - 4} className="inset-x-0 inset-y-0.5" />
+ </div>
+ )}
{selected && (
// A blue wash over the whole clip so a multi-selection reads at a
// glance, not just from the thin border.
@@ -4323,6 +4346,55 @@ function MuteChip({
);
}
+/** Waveform bars for the slice of `asset` between source times `from` and
+ * `to`, stretched across a `w`px bar. Each bar folds the max of every peak
+ * bucket it covers, so a zoomed-out strip keeps its transients. */
+function WaveformCanvas({
+ asset,
+ from,
+ to,
+ w,
+ h,
+ className,
+}: {
+ asset: MediaAsset;
+ from: number;
+ to: number;
+ w: number;
+ h: number;
+ className?: string;
+}) {
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const peaks = asset.peaks;
+ useEffect(() => {
+ const canvas = canvasRef.current;
+ if (!canvas || !peaks?.length) return;
+ // Cap the backing store below browser canvas limits; the element is
+ // CSS-stretched to the bar, so past the cap bars widen instead of the
+ // tail going bare (a canvas keeps its intrinsic width under inset-x-0).
+ const width = Math.min(16384, Math.round(w));
+ canvas.width = width;
+ canvas.height = h;
+ const ctx = canvas.getContext("2d")!;
+ ctx.clearRect(0, 0, width, h);
+ ctx.fillStyle = "rgba(255, 255, 255, 0.85)";
+ const n = peaks.length;
+ const bars = Math.max(1, Math.floor(width / 3));
+ const start = (from / asset.duration) * n;
+ const perBar = (((to - from) / asset.duration) * n) / bars;
+ for (let i = 0; i < bars; i++) {
+ const a = Math.max(0, Math.min(n - 1, Math.floor(start + i * perBar)));
+ const b = Math.max(a + 1, Math.ceil(start + (i + 1) * perBar));
+ let p = 0;
+ for (let j = a; j < b && j < n; j++) if (peaks[j] > p) p = peaks[j];
+ const bh = Math.max(1.5, p * (h - 2));
+ ctx.fillRect(i * 3, (h - bh) / 2, 2, bh);
+ }
+ }, [asset.duration, peaks, from, to, w, h]);
+ if (!peaks?.length) return null;
+ return <canvas ref={canvasRef} className={cn("pointer-events-none absolute w-full", className)} />;
+}
+
function AudioView({
clip,
asset,
@@ -4361,36 +4433,10 @@ function AudioView({
/** Paint (or clear) the snap guide at this stage-x pixel. */
onSnap: (x: number | null) => void;
}) {
- const canvasRef = useRef<HTMLCanvasElement>(null);
const loading = useEditor((s) => (asset ? s.loadingMedia.has(asset.fileName) : false));
const len = clipLen(clip);
const w = Math.max(10, len * pps);
- useEffect(() => {
- const canvas = canvasRef.current;
- if (!canvas || !asset?.peaks) return;
- // Cap the backing store below browser canvas limits; the element is
- // CSS-stretched to the bar, so past the cap bars widen instead of the
- // tail going bare (a canvas keeps its intrinsic width under inset-x-0).
- const width = Math.min(16384, Math.round(w));
- const height = AUDIO_H - 8;
- canvas.width = width;
- canvas.height = height;
- const ctx = canvas.getContext("2d")!;
- ctx.clearRect(0, 0, width, height);
- ctx.fillStyle = "rgba(255, 255, 255, 0.85)";
- const peaks = asset.peaks;
- const n = peaks.length;
- const from = (clip.in / asset.duration) * n;
- const span = ((clip.out - clip.in) / asset.duration) * n;
- const bars = Math.max(1, Math.floor(width / 3));
- for (let i = 0; i < bars; i++) {
- const p = peaks[Math.min(n - 1, Math.floor(from + (i / bars) * span))] ?? 0;
- const h = Math.max(1.5, p * (height - 2));
- ctx.fillRect(i * 3, (height - h) / 2, 2, h);
- }
- }, [asset, clip.in, clip.out, w]);
-
if (!asset) return null;
const ui = { pps, rowH: AUDIO_H, laneCount, homeRow, onDrag, onSnap };
@@ -4419,7 +4465,7 @@ function AudioView({
data-tl-sel={`audio:${clip.id}`}
onPointerDown={(e) => startLaneMove(e, "audio", clip.id, ui)}
>
- <canvas ref={canvasRef} className="pointer-events-none absolute inset-x-0 inset-y-1 w-full" />
+ <WaveformCanvas asset={asset} from={clip.in} to={clip.out} w={w} h={AUDIO_H - 8} className="inset-x-0 inset-y-1" />
{(clip.fadeIn ?? 0) > 0 && (
<div
className="tl-fade-in pointer-events-none absolute inset-y-0 left-0 bg-gradient-to-r from-black/45 to-transparent"
diff --git a/site/src/cut/components/editor-mock/MockTimeline.tsx b/site/src/cut/components/editor-mock/MockTimeline.tsx
index 25c14d39..5c958ed3 100644
--- a/site/src/cut/components/editor-mock/MockTimeline.tsx
+++ b/site/src/cut/components/editor-mock/MockTimeline.tsx
@@ -12,9 +12,10 @@ import type { MockProject } from "@/cut/components/editor-mock/mockData";
// usable track width is the frame minus that column and some gutter.
const TRACK_W = 800;
const CLIP_GAP = 4;
-const PANEL_H = 170;
+const PANEL_H = 190;
const RULER_H = 26;
-const VIDEO_H = 56;
+const VIDEO_H = 76;
+const WAVE_H = 20;
const AUDIO_H = 40;
const SUB_H = 18;
const SFX_H = 20;
@@ -71,17 +72,33 @@ export function MockTimeline({ project }: { project: MockProject }) {
"absolute top-0.5 overflow-hidden rounded-lg bg-black shadow-[inset_0_0_0_1px_rgba(0,0,0,0.12)]",
c.selected && "z-10 ring-2 ring-[#0a84ff]"
)}
- style={{
- left: c.left,
- width: c.width,
- height: VIDEO_H - 4,
- // The editor draws a clip as a strip of frames repeated across
- // its width; one tile at native aspect gives the same read.
- backgroundImage: `url(${c.thumb})`,
- backgroundSize: "auto 100%",
- backgroundRepeat: "repeat-x",
- }}
+ style={{ left: c.left, width: c.width, height: c.muted ? VIDEO_H - 4 - WAVE_H : VIDEO_H - 4 }}
>
+ <div
+ className="absolute inset-x-0 top-0"
+ style={{
+ height: VIDEO_H - 4 - WAVE_H,
+ // The editor draws a clip as a strip of frames repeated across
+ // its width; one tile at native aspect gives the same read.
+ backgroundImage: `url(${c.thumb})`,
+ backgroundSize: "auto 100%",
+ backgroundRepeat: "repeat-x",
+ }}
+ />
+ {!c.muted && (
+ <div
+ className="absolute inset-x-0 bottom-0 flex items-center justify-between bg-gradient-to-b from-[#59c09d] to-[#43b18d] px-1"
+ style={{ height: WAVE_H }}
+ >
+ {Array.from({ length: Math.max(8, Math.floor(c.width / 5)) }, (_, i) => (
+ <span
+ key={i}
+ className="w-[2px] rounded-full bg-white/85"
+ style={{ height: `${WAVE[(i + c.label.length) % WAVE.length] * 0.7}%` }}
+ />
+ ))}
+ </div>
+ )}
{c.muted && (
<span className="absolute bottom-1 left-1 grid size-[18px] place-items-center rounded-[5px] bg-black/70 text-white">
<VolumeX className="size-3" />
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 69876ae3..6f0d4756 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -391,7 +391,7 @@ async function probeMedia(type: AssetType, src: string | Blob): Promise<ProbedMe
if (!meta.hasVideo) {
// The waveform rides along with the probe: the file is open and its audio
// is being read either way, so enrichAsset has nothing left to do.
- return { type: "audio", duration: meta.duration, peaks: await audioPeaks(src, PEAK_BUCKETS) };
+ return { type: "audio", duration: meta.duration, peaks: await makePeaks(src, meta.duration) };
}
return { type: "video", duration: meta.duration, width: meta.width, height: meta.height };
}
@@ -1449,7 +1449,10 @@ function scheduleEnrichRetry(assetId: string) {
enrichRetryTimers.delete(assetId);
const asset = useEditor.getState().assets.find((a) => a.id === assetId);
if (!asset) return;
- const done = asset.type === "audio" ? !!asset.peaks?.length : !!asset.thumbs?.length;
+ const done =
+ asset.type === "audio"
+ ? !!asset.peaks?.length
+ : !!asset.thumbs?.length && (asset.type !== "video" || asset.peaks !== undefined);
if (done) return;
void enrichAsset(asset);
}, ENRICH_RETRY_BASE_MS * 3 ** n);
@@ -1467,28 +1470,43 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
if (!asset.thumbs?.length) {
useEditor.getState().updateAsset(asset.id, { thumbs: [src], thumbStep: IMAGE_CLIP_SECONDS });
}
- } else if (asset.type === "video" && !asset.thumbs?.length) {
- setStripFailed(asset.id, false);
- const key = stripCacheKey(useEditor.getState().projectId, asset.fileName);
- const cached = await readCachedStrip(key, asset.duration);
- if (cached) {
- useEditor.getState().updateAsset(asset.id, {
- thumbs: cached.thumbs,
- thumbStep: cached.thumbStep,
- sceneCuts: cached.cuts ?? [],
- });
- } else {
- const { thumbs, thumbStep, cuts } = await makeThumbs(src, asset.duration);
- // An empty strip is a failure, not a result: persisting it would
- // leave the asset looking permanently mid-load.
- if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");
- useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep, sceneCuts: cuts });
- writeCachedStrip(key, { thumbs, thumbStep, cuts, duration: asset.duration, at: Date.now() });
+ } else if (asset.type === "video") {
+ if (!asset.thumbs?.length) {
+ setStripFailed(asset.id, false);
+ const key = stripCacheKey(useEditor.getState().projectId, asset.fileName);
+ const cached = await readCachedStrip(key, asset.duration);
+ if (cached) {
+ useEditor.getState().updateAsset(asset.id, {
+ thumbs: cached.thumbs,
+ thumbStep: cached.thumbStep,
+ sceneCuts: cached.cuts ?? [],
+ });
+ } else {
+ const { thumbs, thumbStep, cuts } = await makeThumbs(src, asset.duration);
+ // An empty strip is a failure, not a result: persisting it would
+ // leave the asset looking permanently mid-load.
+ if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");
+ useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep, sceneCuts: cuts });
+ writeCache(key, { thumbs, thumbStep, cuts, duration: asset.duration, at: Date.now() });
+ }
+ }
+ // The clip box draws the sound under the picture, so a video enriches
+ // waveform peaks alongside its filmstrip. `[]` is an answer too — a
+ // silent file — and stops the strip from re-decoding every open.
+ if (asset.peaks === undefined) {
+ try {
+ const peaks = await loadPeaks(asset, src);
+ if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });
+ } catch (err) {
+ // The strip above already painted; a peaks failure (an expired
+ // signed URL, an undecodable track) never marks it failed.
+ console.error(`[cut] peaks failed for ${asset.fileName}`, err);
+ }
}
enrichRetries.delete(asset.id);
} else if (asset.type === "audio" && !asset.peaks?.length) {
- const peaks = await makePeaks(src);
- useEditor.getState().updateAsset(asset.id, { peaks });
+ const peaks = await loadPeaks(asset, src);
+ if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });
enrichRetries.delete(asset.id);
}
} catch (err) {
@@ -1511,8 +1529,8 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
export async function ensurePeaks(asset: MediaAsset) {
try {
if (!asset.peaks?.length) {
- const peaks = await makePeaks(asset.url);
- useEditor.getState().updateAsset(asset.id, { peaks });
+ const peaks = await loadPeaks(asset);
+ if (peaks?.length) useEditor.getState().updateAsset(asset.id, { peaks });
}
} catch (err) {
// Waveforms are decorative; editing works without them. Error level
@@ -1546,6 +1564,13 @@ function stripCacheKey(projectId: string | null, fileName: string) {
return `${projectId ?? ""}/${fileName}@${THUMB_H}s3`;
}
+type CachedPeaks = { peaks: number[]; duration: number; at: number };
+
+function peaksCacheKey(projectId: string | null, fileName: string) {
+ // The density marker versions the cache; peaks sampled differently regenerate.
+ return `${projectId ?? ""}/${fileName}@peaks${PEAKS_PER_SECOND}`;
+}
+
function openStripDb(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const req = indexedDB.open(STRIP_DB, 1);
@@ -1575,13 +1600,44 @@ async function readCachedStrip(key: string, duration: number): Promise<CachedStr
}
}
-function writeCachedStrip(key: string, strip: CachedStrip) {
+async function readCachedPeaks(key: string, duration: number): Promise<number[] | null> {
+ try {
+ const db = await openStripDb();
+ const entry = await new Promise<CachedPeaks | undefined>((resolve, reject) => {
+ const req = db.transaction(STRIP_STORE).objectStore(STRIP_STORE).get(key);
+ req.onsuccess = () => resolve(req.result as CachedPeaks | undefined);
+ req.onerror = () => reject(req.error);
+ });
+ db.close();
+ // A same-path file with a different duration was rewritten; regenerate.
+ if (!Array.isArray(entry?.peaks) || Math.abs(entry.duration - duration) > 0.25) return null;
+ return entry.peaks;
+ } catch {
+ return null;
+ }
+}
+
+/** Waveform peaks for an asset, from the cache when this file was already
+ * decoded — a real audio decode otherwise. Empty peaks are an answer (a file
+ * with no audible track) and cache like any other. A headless runtime returns
+ * undefined so the browser decodes on its next open. */
+async function loadPeaks(asset: MediaAsset, src = asset.url): Promise<number[] | undefined> {
+ if (typeof AudioDecoder === "undefined") return undefined;
+ const key = peaksCacheKey(useEditor.getState().projectId, asset.fileName);
+ const cached = await readCachedPeaks(key, asset.duration);
+ if (cached) return cached;
+ const peaks = await makePeaks(src, asset.duration);
+ writeCache(key, { peaks, duration: asset.duration, at: Date.now() });
+ return peaks;
+}
+
+function writeCache(key: string, entry: CachedStrip | CachedPeaks) {
void (async () => {
try {
const db = await openStripDb();
const tx = db.transaction(STRIP_STORE, "readwrite");
const store = tx.objectStore(STRIP_STORE);
- store.put(strip, key);
+ store.put(entry, key);
const count = await new Promise<number>((resolve, reject) => {
const req = store.count();
req.onsuccess = () => resolve(req.result);
@@ -1897,8 +1953,13 @@ async function pumpEdgeFrames() {
}
}
-const PEAK_BUCKETS = 1600;
+/** Waveform buckets per second of media. Density scaling with length keeps a
+ * zoomed-in strip honest; the bounds keep tiny stings drawable and hour-long
+ * files a bounded array. */
+const PEAKS_PER_SECOND = 50;
-function makePeaks(url: string): Promise<number[]> {
- return audioPeaks(url, PEAK_BUCKETS);
+function makePeaks(src: string | Blob, duration: number): Promise<number[]> {
+ const buckets =
+ duration > 0 ? Math.min(30_000, Math.max(400, Math.round(duration * PEAKS_PER_SECOND))) : 1600;
+ return audioPeaks(src, buckets);
}
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 41d2270d..9ffe3c8b 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -181,8 +181,8 @@ function placedState(
};
}
-export const TIMELINE_H_DEFAULT = 248;
-export const TIMELINE_H_MIN = 170;
+export const TIMELINE_H_DEFAULT = 272;
+export const TIMELINE_H_MIN = 194;
/** Tallest the timeline may grow: the window height less room for the top bar
* and a usable preview. The constant covers code running without a window. */
export const timelineHMax = () =>