The open-source CapCut alternative
feat: make timeline renderer 15x faster
Maze Wintherd747fb458d2f
diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.tsindex 6241a7d9..08785369 100644--- a/apps/web/src/lib/timeline-renderer.ts+++ b/apps/web/src/lib/timeline-renderer.ts@@ -1,6 +1,6 @@-import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";import type { TimelineTrack } from "@/types/timeline";import type { MediaFile } from "@/types/media";+import { videoCache } from "./video-cache";export interface RenderContext {ctx: CanvasRenderingContext2D;@@ -65,31 +65,34 @@ export async function renderTimelineFrame({for (const { element, mediaItem } of active) {if (element.type === "media" && mediaItem) {if (mediaItem.type === "video") {- const input = new Input({- source: new BlobSource(mediaItem.file),- formats: ALL_FORMATS,- });- const track = await input.getPrimaryVideoTrack();- if (!track) continue;- const decodable = await track.canDecode();- if (!decodable) continue;- const sink = new VideoSampleSink(track);+ try {+ const localTime = time - element.startTime + element.trimStart;- const localTime = time - element.startTime + element.trimStart;- const sample = await sink.getSample(localTime);- if (!sample) continue;+ const frame = await videoCache.getFrameAt(+ mediaItem.id,+ mediaItem.file,+ localTime+ );+ if (!frame) continue;- const mediaW = Math.max(1, mediaItem.width || canvasWidth);- const mediaH = Math.max(1, mediaItem.height || canvasHeight);- const containScale = Math.min(- canvasWidth / mediaW,- canvasHeight / mediaH- );- const drawW = mediaW * containScale;- const drawH = mediaH * containScale;- const drawX = (canvasWidth - drawW) / 2;- const drawY = (canvasHeight - drawH) / 2;- sample.draw(ctx, drawX, drawY, drawW, drawH);+ const mediaW = Math.max(1, mediaItem.width || canvasWidth);+ const mediaH = Math.max(1, mediaItem.height || canvasHeight);+ const containScale = Math.min(+ canvasWidth / mediaW,+ canvasHeight / mediaH+ );+ const drawW = mediaW * containScale;+ const drawH = mediaH * containScale;+ const drawX = (canvasWidth - drawW) / 2;+ const drawY = (canvasHeight - drawH) / 2;++ ctx.drawImage(frame.canvas, drawX, drawY, drawW, drawH);+ } catch (error) {+ console.warn(+ `Failed to render video frame for ${mediaItem.name}:`,+ error+ );+ }}if (mediaItem.type === "image") {const img = new Image();diff --git a/apps/web/src/lib/video-cache.ts b/apps/web/src/lib/video-cache.tsnew file mode 100644index 00000000..aeb62d1c--- /dev/null+++ b/apps/web/src/lib/video-cache.ts@@ -0,0 +1,186 @@+import {+ Input,+ ALL_FORMATS,+ BlobSource,+ CanvasSink,+ WrappedCanvas,+} from "mediabunny";++interface VideoSinkData {+ sink: CanvasSink;+ iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;+ currentFrame: WrappedCanvas | null;+ lastTime: number;+}+export class VideoCache {+ private sinks = new Map<string, VideoSinkData>();+ private initPromises = new Map<string, Promise<void>>();++ async getFrameAt(+ mediaId: string,+ file: File,+ time: number+ ): Promise<WrappedCanvas | null> {+ await this.ensureSink(mediaId, file);++ const sinkData = this.sinks.get(mediaId);+ if (!sinkData) return null;++ if (+ sinkData.currentFrame &&+ this.isFrameValid(sinkData.currentFrame, time)+ ) {+ return sinkData.currentFrame;+ }++ if (+ sinkData.iterator &&+ sinkData.currentFrame &&+ time >= sinkData.lastTime &&+ time < sinkData.lastTime + 2.0+ ) {+ const frame = await this.iterateToTime(sinkData, time);+ if (frame) return frame;+ }++ return await this.seekToTime(sinkData, time);+ }++ private isFrameValid(frame: WrappedCanvas, time: number): boolean {+ return time >= frame.timestamp && time < frame.timestamp + frame.duration;+ }+ private async iterateToTime(+ sinkData: VideoSinkData,+ targetTime: number+ ): Promise<WrappedCanvas | null> {+ if (!sinkData.iterator) return null;++ try {+ while (true) {+ const { value: frame, done } = await sinkData.iterator.next();++ if (done || !frame) break;++ sinkData.currentFrame = frame;+ sinkData.lastTime = frame.timestamp;++ if (this.isFrameValid(frame, targetTime)) {+ return frame;+ }++ if (frame.timestamp > targetTime + 1.0) break;+ }+ } catch (error) {+ console.warn("Iterator failed, will restart:", error);+ sinkData.iterator = null;+ }++ return null;+ }+ private async seekToTime(+ sinkData: VideoSinkData,+ time: number+ ): Promise<WrappedCanvas | null> {+ try {+ if (sinkData.iterator) {+ await sinkData.iterator.return();+ sinkData.iterator = null;+ }++ sinkData.iterator = sinkData.sink.canvases(time);+ sinkData.lastTime = time;++ const { value: frame } = await sinkData.iterator.next();++ if (frame) {+ sinkData.currentFrame = frame;+ return frame;+ }+ } catch (error) {+ console.warn("Failed to seek video:", error);+ }++ return null;+ }+ private async ensureSink(mediaId: string, file: File): Promise<void> {+ if (this.sinks.has(mediaId)) return;++ if (this.initPromises.has(mediaId)) {+ await this.initPromises.get(mediaId);+ return;+ }++ const initPromise = this.initializeSink(mediaId, file);+ this.initPromises.set(mediaId, initPromise);++ try {+ await initPromise;+ } finally {+ this.initPromises.delete(mediaId);+ }+ }+ private async initializeSink(mediaId: string, file: File): Promise<void> {+ try {+ const input = new Input({+ source: new BlobSource(file),+ formats: ALL_FORMATS,+ });++ const videoTrack = await input.getPrimaryVideoTrack();+ if (!videoTrack) {+ throw new Error("No video track found");+ }++ const canDecode = await videoTrack.canDecode();+ if (!canDecode) {+ throw new Error("Video codec not supported for decoding");+ }++ const sink = new CanvasSink(videoTrack, {+ poolSize: 3,+ fit: "contain",+ });++ this.sinks.set(mediaId, {+ sink,+ iterator: null,+ currentFrame: null,+ lastTime: -1,+ });+ } catch (error) {+ console.error(`Failed to initialize video sink for ${mediaId}:`, error);+ throw error;+ }+ }++ clearVideo(mediaId: string): void {+ const sinkData = this.sinks.get(mediaId);+ if (sinkData) {+ if (sinkData.iterator) {+ sinkData.iterator.return();+ }++ this.sinks.delete(mediaId);+ }++ this.initPromises.delete(mediaId);+ }++ clearAll(): void {+ for (const [mediaId] of this.sinks) {+ this.clearVideo(mediaId);+ }+ }++ getStats() {+ return {+ totalSinks: this.sinks.size,+ activeSinks: Array.from(this.sinks.values()).filter((s) => s.iterator)+ .length,+ cachedFrames: Array.from(this.sinks.values()).filter(+ (s) => s.currentFrame+ ).length,+ };+ }+}+export const videoCache = new VideoCache();diff --git a/apps/web/src/stores/media-store.ts b/apps/web/src/stores/media-store.tsindex 2b7cb62e..c9e29fff 100644--- a/apps/web/src/stores/media-store.ts+++ b/apps/web/src/stores/media-store.ts@@ -3,6 +3,7 @@ import { storageService } from "@/lib/storage/storage-service";import { useTimelineStore } from "./timeline-store";import { generateUUID } from "@/lib/utils";import { MediaType, MediaFile } from "@/types/media";+import { videoCache } from "@/lib/video-cache";interface MediaStore {mediaFiles: MediaFile[];@@ -165,6 +166,8 @@ export const useMediaStore = create<MediaStore>((set, get) => ({const state = get();const item = state.mediaFiles.find((media) => media.id === id);+ videoCache.clearVideo(id);+// Cleanup object URLs to prevent memory leaksif (item?.url) {URL.revokeObjectURL(item.url);@@ -289,6 +292,8 @@ export const useMediaStore = create<MediaStore>((set, get) => ({clearAllMedia: () => {const state = get();+ videoCache.clearAll();+// Cleanup all object URLsstate.mediaFiles.forEach((item) => {if (item.url) {