Open source CapCut alternative.
TypeScript
feat(site): video clips carry their sound as a band under the filmstrip
David Le7b161bc69986
diff --git a/site/src/cut/components/Timeline.tsx b/site/src/cut/components/Timeline.tsxindex 8a0edaff..cd2be92a 100644--- a/site/src/cut/components/Timeline.tsx+++ b/site/src/cut/components/Timeline.tsx@@ -76,7 +76,8 @@ const TRANSITION_ICONS: Record<TransitionStyle, LucideIcon> = {splitclose: FoldHorizontal,};-const VIDEO_H = 64;+/** Video row height: the picture band plus the sound band under it. */+const VIDEO_H = 88;/** How far past the top or bottom of the video stack a drag has to carry a* clip to open a new track. Every drop inside the stack lands on a row, which* opens room for it, so this reach is the one gesture that adds a track — and@@ -90,6 +91,12 @@ const OVERLAY_H = VIDEO_H; // every video track shares the same row height* and the next row keeps its usual distance below the line. */const KEYRAIL_EXTRA = 20;const AUDIO_H = 44;+/** The sound band along a video clip box's bottom edge. */+const WAVE_H = 24;+/** The picture band above it — the filmstrip's height. A clip with nothing to+ * play in the band (an image, a muted or silent video) drops the band and its+ * box stops at the filmstrip's edge. */+const FILM_H = VIDEO_H - 4 - WAVE_H;/** The `mt-1.5` every row carries, so a row drag knows what one row's worth of* travel is. */const ROW_GAP = 6;@@ -1605,7 +1612,9 @@ export function Timeline() {style={{ left: t * pps, width: Math.max(10, len * pps - CLIP_GAP), height: h }}>{ghost && (- <DropGhostFilm ghost={ghost} w={Math.max(10, len * pps - CLIP_GAP)} h={h} pps={pps} />+ <div className="pointer-events-none absolute inset-x-0 top-0" style={{ height: h - WAVE_H }}>+ <DropGhostFilm ghost={ghost} w={Math.max(10, len * pps - CLIP_GAP)} h={h - WAVE_H} pps={pps} />+ </div>)}<span className="absolute top-1 left-1 rounded-[5px] bg-black/65 px-1.5 py-px font-mono text-[10px] tabular-nums text-white">{len.toFixed(1)}s@@ -3514,11 +3523,15 @@ function ClipView({() => false);const boxRef = useRef<HTMLDivElement>(null);+ // The clip's own sound, drawn in the box's band under the picture. Muting —+ // detach included — takes the sound off the clip, so the band drops and the+ // box stops at the filmstrip (a detached copy draws on the soundtrack row).+ const hasWave = asset.type === "video" && !clip.muted && !!asset.peaks?.length;// Bumped as tile captures land, so the memo re-plans and picks them up.const [filmGen, onTileFrame] = useReducer((x: number) => x + 1, 0);const filmstrip = useMemo(() =>- filmstripFrames(asset, filmIn, w, pps, speed, VIDEO_H - 4, 26, {+ filmstripFrames(asset, filmIn, w, pps, speed, FILM_H, 26, {start: startFrame,end: endFrame,}),@@ -3563,7 +3576,7 @@ function ClipView({left: drag ? drag.ghostX : (partAt ?? span.start) * pps,top: drag ? 2 + drag.ghostY : undefined,width: Math.max(10, w - CLIP_GAP),- height: VIDEO_H - 4,+ height: hasWave ? VIDEO_H - 4 : FILM_H,// Inline so it beats SELECTED_SHADOW's z-10 class on the same element.zIndex: drag ? 20 : undefined,}}@@ -3580,7 +3593,17 @@ function ClipView({});}}>- <Filmstrip frames={filmstrip} grade={clip.grade} failed={stripFailed} />+ <div className="pointer-events-none absolute inset-x-0 top-0" style={{ height: FILM_H }}>+ <Filmstrip frames={filmstrip} grade={clip.grade} failed={stripFailed} />+ </div>+ {hasWave && (+ <div+ className="tl-clip-wave pointer-events-none absolute inset-x-0 bottom-0 bg-gradient-to-b from-[#59c09d] to-[#43b18d]"+ style={{ height: WAVE_H }}+ >+ <WaveformCanvas asset={asset} from={filmIn} to={filmOut} w={w} h={WAVE_H - 4} className="inset-x-0 inset-y-0.5" />+ </div>+ )}{selected && (// A blue wash over the whole clip so a multi-selection reads at a// glance, not just from the thin border.@@ -4323,6 +4346,55 @@ function MuteChip({);}+/** Waveform bars for the slice of `asset` between source times `from` and+ * `to`, stretched across a `w`px bar. Each bar folds the max of every peak+ * bucket it covers, so a zoomed-out strip keeps its transients. */+function WaveformCanvas({+ asset,+ from,+ to,+ w,+ h,+ className,+}: {+ asset: MediaAsset;+ from: number;+ to: number;+ w: number;+ h: number;+ className?: string;+}) {+ const canvasRef = useRef<HTMLCanvasElement>(null);+ const peaks = asset.peaks;+ useEffect(() => {+ const canvas = canvasRef.current;+ if (!canvas || !peaks?.length) return;+ // Cap the backing store below browser canvas limits; the element is+ // CSS-stretched to the bar, so past the cap bars widen instead of the+ // tail going bare (a canvas keeps its intrinsic width under inset-x-0).+ const width = Math.min(16384, Math.round(w));+ canvas.width = width;+ canvas.height = h;+ const ctx = canvas.getContext("2d")!;+ ctx.clearRect(0, 0, width, h);+ ctx.fillStyle = "rgba(255, 255, 255, 0.85)";+ const n = peaks.length;+ const bars = Math.max(1, Math.floor(width / 3));+ const start = (from / asset.duration) * n;+ const perBar = (((to - from) / asset.duration) * n) / bars;+ for (let i = 0; i < bars; i++) {+ const a = Math.max(0, Math.min(n - 1, Math.floor(start + i * perBar)));+ const b = Math.max(a + 1, Math.ceil(start + (i + 1) * perBar));+ let p = 0;+ for (let j = a; j < b && j < n; j++) if (peaks[j] > p) p = peaks[j];+ const bh = Math.max(1.5, p * (h - 2));+ ctx.fillRect(i * 3, (h - bh) / 2, 2, bh);+ }+ }, [asset.duration, peaks, from, to, w, h]);+ if (!peaks?.length) return null;+ return <canvas ref={canvasRef} className={cn("pointer-events-none absolute w-full", className)} />;+}+function AudioView({clip,asset,@@ -4361,36 +4433,10 @@ function AudioView({/** Paint (or clear) the snap guide at this stage-x pixel. */onSnap: (x: number | null) => void;}) {- const canvasRef = useRef<HTMLCanvasElement>(null);const loading = useEditor((s) => (asset ? s.loadingMedia.has(asset.fileName) : false));const len = clipLen(clip);const w = Math.max(10, len * pps);- useEffect(() => {- const canvas = canvasRef.current;- if (!canvas || !asset?.peaks) return;- // Cap the backing store below browser canvas limits; the element is- // CSS-stretched to the bar, so past the cap bars widen instead of the- // tail going bare (a canvas keeps its intrinsic width under inset-x-0).- const width = Math.min(16384, Math.round(w));- const height = AUDIO_H - 8;- canvas.width = width;- canvas.height = height;- const ctx = canvas.getContext("2d")!;- ctx.clearRect(0, 0, width, height);- ctx.fillStyle = "rgba(255, 255, 255, 0.85)";- const peaks = asset.peaks;- const n = peaks.length;- const from = (clip.in / asset.duration) * n;- const span = ((clip.out - clip.in) / asset.duration) * n;- const bars = Math.max(1, Math.floor(width / 3));- for (let i = 0; i < bars; i++) {- const p = peaks[Math.min(n - 1, Math.floor(from + (i / bars) * span))] ?? 0;- const h = Math.max(1.5, p * (height - 2));- ctx.fillRect(i * 3, (height - h) / 2, 2, h);- }- }, [asset, clip.in, clip.out, w]);-if (!asset) return null;const ui = { pps, rowH: AUDIO_H, laneCount, homeRow, onDrag, onSnap };@@ -4419,7 +4465,7 @@ function AudioView({data-tl-sel={`audio:${clip.id}`}onPointerDown={(e) => startLaneMove(e, "audio", clip.id, ui)}>- <canvas ref={canvasRef} className="pointer-events-none absolute inset-x-0 inset-y-1 w-full" />+ <WaveformCanvas asset={asset} from={clip.in} to={clip.out} w={w} h={AUDIO_H - 8} className="inset-x-0 inset-y-1" />{(clip.fadeIn ?? 0) > 0 && (<divclassName="tl-fade-in pointer-events-none absolute inset-y-0 left-0 bg-gradient-to-r from-black/45 to-transparent"diff --git a/site/src/cut/components/editor-mock/MockTimeline.tsx b/site/src/cut/components/editor-mock/MockTimeline.tsxindex 25c14d39..5c958ed3 100644--- a/site/src/cut/components/editor-mock/MockTimeline.tsx+++ b/site/src/cut/components/editor-mock/MockTimeline.tsx@@ -12,9 +12,10 @@ import type { MockProject } from "@/cut/components/editor-mock/mockData";// usable track width is the frame minus that column and some gutter.const TRACK_W = 800;const CLIP_GAP = 4;-const PANEL_H = 170;+const PANEL_H = 190;const RULER_H = 26;-const VIDEO_H = 56;+const VIDEO_H = 76;+const WAVE_H = 20;const AUDIO_H = 40;const SUB_H = 18;const SFX_H = 20;@@ -71,17 +72,33 @@ export function MockTimeline({ project }: { project: MockProject }) {"absolute top-0.5 overflow-hidden rounded-lg bg-black shadow-[inset_0_0_0_1px_rgba(0,0,0,0.12)]",c.selected && "z-10 ring-2 ring-[#0a84ff]")}- style={{- left: c.left,- width: c.width,- height: VIDEO_H - 4,- // The editor draws a clip as a strip of frames repeated across- // its width; one tile at native aspect gives the same read.- backgroundImage: `url(${c.thumb})`,- backgroundSize: "auto 100%",- backgroundRepeat: "repeat-x",- }}+ style={{ left: c.left, width: c.width, height: c.muted ? VIDEO_H - 4 - WAVE_H : VIDEO_H - 4 }}>+ <div+ className="absolute inset-x-0 top-0"+ style={{+ height: VIDEO_H - 4 - WAVE_H,+ // The editor draws a clip as a strip of frames repeated across+ // its width; one tile at native aspect gives the same read.+ backgroundImage: `url(${c.thumb})`,+ backgroundSize: "auto 100%",+ backgroundRepeat: "repeat-x",+ }}+ />+ {!c.muted && (+ <div+ className="absolute inset-x-0 bottom-0 flex items-center justify-between bg-gradient-to-b from-[#59c09d] to-[#43b18d] px-1"+ style={{ height: WAVE_H }}+ >+ {Array.from({ length: Math.max(8, Math.floor(c.width / 5)) }, (_, i) => (+ <span+ key={i}+ className="w-[2px] rounded-full bg-white/85"+ style={{ height: `${WAVE[(i + c.label.length) % WAVE.length] * 0.7}%` }}+ />+ ))}+ </div>+ )}{c.muted && (<span className="absolute bottom-1 left-1 grid size-[18px] place-items-center rounded-[5px] bg-black/70 text-white"><VolumeX className="size-3" />diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 69876ae3..6f0d4756 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -391,7 +391,7 @@ async function probeMedia(type: AssetType, src: string | Blob): Promise<ProbedMeif (!meta.hasVideo) {// The waveform rides along with the probe: the file is open and its audio// is being read either way, so enrichAsset has nothing left to do.- return { type: "audio", duration: meta.duration, peaks: await audioPeaks(src, PEAK_BUCKETS) };+ return { type: "audio", duration: meta.duration, peaks: await makePeaks(src, meta.duration) };}return { type: "video", duration: meta.duration, width: meta.width, height: meta.height };}@@ -1449,7 +1449,10 @@ function scheduleEnrichRetry(assetId: string) {enrichRetryTimers.delete(assetId);const asset = useEditor.getState().assets.find((a) => a.id === assetId);if (!asset) return;- const done = asset.type === "audio" ? !!asset.peaks?.length : !!asset.thumbs?.length;+ const done =+ asset.type === "audio"+ ? !!asset.peaks?.length+ : !!asset.thumbs?.length && (asset.type !== "video" || asset.peaks !== undefined);if (done) return;void enrichAsset(asset);}, ENRICH_RETRY_BASE_MS * 3 ** n);@@ -1467,28 +1470,43 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {if (!asset.thumbs?.length) {useEditor.getState().updateAsset(asset.id, { thumbs: [src], thumbStep: IMAGE_CLIP_SECONDS });}- } else if (asset.type === "video" && !asset.thumbs?.length) {- setStripFailed(asset.id, false);- const key = stripCacheKey(useEditor.getState().projectId, asset.fileName);- const cached = await readCachedStrip(key, asset.duration);- if (cached) {- useEditor.getState().updateAsset(asset.id, {- thumbs: cached.thumbs,- thumbStep: cached.thumbStep,- sceneCuts: cached.cuts ?? [],- });- } else {- const { thumbs, thumbStep, cuts } = await makeThumbs(src, asset.duration);- // An empty strip is a failure, not a result: persisting it would- // leave the asset looking permanently mid-load.- if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");- useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep, sceneCuts: cuts });- writeCachedStrip(key, { thumbs, thumbStep, cuts, duration: asset.duration, at: Date.now() });+ } else if (asset.type === "video") {+ if (!asset.thumbs?.length) {+ setStripFailed(asset.id, false);+ const key = stripCacheKey(useEditor.getState().projectId, asset.fileName);+ const cached = await readCachedStrip(key, asset.duration);+ if (cached) {+ useEditor.getState().updateAsset(asset.id, {+ thumbs: cached.thumbs,+ thumbStep: cached.thumbStep,+ sceneCuts: cached.cuts ?? [],+ });+ } else {+ const { thumbs, thumbStep, cuts } = await makeThumbs(src, asset.duration);+ // An empty strip is a failure, not a result: persisting it would+ // leave the asset looking permanently mid-load.+ if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");+ useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep, sceneCuts: cuts });+ writeCache(key, { thumbs, thumbStep, cuts, duration: asset.duration, at: Date.now() });+ }+ }+ // The clip box draws the sound under the picture, so a video enriches+ // waveform peaks alongside its filmstrip. `[]` is an answer too — a+ // silent file — and stops the strip from re-decoding every open.+ if (asset.peaks === undefined) {+ try {+ const peaks = await loadPeaks(asset, src);+ if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });+ } catch (err) {+ // The strip above already painted; a peaks failure (an expired+ // signed URL, an undecodable track) never marks it failed.+ console.error(`[cut] peaks failed for ${asset.fileName}`, err);+ }}enrichRetries.delete(asset.id);} else if (asset.type === "audio" && !asset.peaks?.length) {- const peaks = await makePeaks(src);- useEditor.getState().updateAsset(asset.id, { peaks });+ const peaks = await loadPeaks(asset, src);+ if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });enrichRetries.delete(asset.id);}} catch (err) {@@ -1511,8 +1529,8 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {export async function ensurePeaks(asset: MediaAsset) {try {if (!asset.peaks?.length) {- const peaks = await makePeaks(asset.url);- useEditor.getState().updateAsset(asset.id, { peaks });+ const peaks = await loadPeaks(asset);+ if (peaks?.length) useEditor.getState().updateAsset(asset.id, { peaks });}} catch (err) {// Waveforms are decorative; editing works without them. Error level@@ -1546,6 +1564,13 @@ function stripCacheKey(projectId: string | null, fileName: string) {return `${projectId ?? ""}/${fileName}@${THUMB_H}s3`;}+type CachedPeaks = { peaks: number[]; duration: number; at: number };++function peaksCacheKey(projectId: string | null, fileName: string) {+ // The density marker versions the cache; peaks sampled differently regenerate.+ return `${projectId ?? ""}/${fileName}@peaks${PEAKS_PER_SECOND}`;+}+function openStripDb(): Promise<IDBDatabase> {return new Promise((resolve, reject) => {const req = indexedDB.open(STRIP_DB, 1);@@ -1575,13 +1600,44 @@ async function readCachedStrip(key: string, duration: number): Promise<CachedStr}}-function writeCachedStrip(key: string, strip: CachedStrip) {+async function readCachedPeaks(key: string, duration: number): Promise<number[] | null> {+ try {+ const db = await openStripDb();+ const entry = await new Promise<CachedPeaks | undefined>((resolve, reject) => {+ const req = db.transaction(STRIP_STORE).objectStore(STRIP_STORE).get(key);+ req.onsuccess = () => resolve(req.result as CachedPeaks | undefined);+ req.onerror = () => reject(req.error);+ });+ db.close();+ // A same-path file with a different duration was rewritten; regenerate.+ if (!Array.isArray(entry?.peaks) || Math.abs(entry.duration - duration) > 0.25) return null;+ return entry.peaks;+ } catch {+ return null;+ }+}++/** Waveform peaks for an asset, from the cache when this file was already+ * decoded — a real audio decode otherwise. Empty peaks are an answer (a file+ * with no audible track) and cache like any other. A headless runtime returns+ * undefined so the browser decodes on its next open. */+async function loadPeaks(asset: MediaAsset, src = asset.url): Promise<number[] | undefined> {+ if (typeof AudioDecoder === "undefined") return undefined;+ const key = peaksCacheKey(useEditor.getState().projectId, asset.fileName);+ const cached = await readCachedPeaks(key, asset.duration);+ if (cached) return cached;+ const peaks = await makePeaks(src, asset.duration);+ writeCache(key, { peaks, duration: asset.duration, at: Date.now() });+ return peaks;+}++function writeCache(key: string, entry: CachedStrip | CachedPeaks) {void (async () => {try {const db = await openStripDb();const tx = db.transaction(STRIP_STORE, "readwrite");const store = tx.objectStore(STRIP_STORE);- store.put(strip, key);+ store.put(entry, key);const count = await new Promise<number>((resolve, reject) => {const req = store.count();req.onsuccess = () => resolve(req.result);@@ -1897,8 +1953,13 @@ async function pumpEdgeFrames() {}}-const PEAK_BUCKETS = 1600;+/** Waveform buckets per second of media. Density scaling with length keeps a+ * zoomed-in strip honest; the bounds keep tiny stings drawable and hour-long+ * files a bounded array. */+const PEAKS_PER_SECOND = 50;-function makePeaks(url: string): Promise<number[]> {- return audioPeaks(url, PEAK_BUCKETS);+function makePeaks(src: string | Blob, duration: number): Promise<number[]> {+ const buckets =+ duration > 0 ? Math.min(30_000, Math.max(400, Math.round(duration * PEAKS_PER_SECOND))) : 1600;+ return audioPeaks(src, buckets);}diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex 41d2270d..9ffe3c8b 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -181,8 +181,8 @@ function placedState(};}-export const TIMELINE_H_DEFAULT = 248;-export const TIMELINE_H_MIN = 170;+export const TIMELINE_H_DEFAULT = 272;+export const TIMELINE_H_MIN = 194;/** Tallest the timeline may grow: the window height less room for the top bar* and a usable preview. The constant covers code running without a window. */export const timelineHMax = () =>