The open-source CapCut alternative
feat: mediabunny + preview canvas refactor
Maze Winthera4f2273e2599
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsxindex 4fe2cbd7..6e18781c 100644--- a/apps/web/src/components/editor/media-panel/views/captions.tsx+++ b/apps/web/src/components/editor/media-panel/views/captions.tsx@@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item";import { BaseView } from "./base-view";import { Language, LanguageSelect } from "@/components/language-select";import { useState, useRef, useEffect } from "react";-import { extractTimelineAudio } from "@/lib/ffmpeg-utils";+import { extractTimelineAudio } from "@/lib/mediabunny-utils";import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption";import { useTimelineStore } from "@/stores/timeline-store";import { Loader2, Shield, Trash2, Upload } from "lucide-react";diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsxindex 42c4becc..99c4edef 100644--- a/apps/web/src/components/editor/preview-panel.tsx+++ b/apps/web/src/components/editor/preview-panel.tsx@@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline";import { useMediaStore, type MediaItem } from "@/stores/media-store";import { usePlaybackStore } from "@/stores/playback-store";import { useEditorStore } from "@/stores/editor-store";-import { VideoPlayer } from "@/components/ui/video-player";-import { AudioPlayer } from "@/components/ui/audio-player";import { Button } from "@/components/ui/button";import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react";import { useState, useRef, useEffect, useCallback } from "react";+import { VideoSampleSink } from "mediabunny";+import { renderTimelineFrame } from "@/lib/timeline-renderer";import { cn } from "@/lib/utils";import { formatTimeCode } from "@/lib/time";import { EditableTimecode } from "@/components/ui/editable-timecode";-import { FONT_CLASS_MAP } from "@/lib/font-config";-import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store";+import {+ DEFAULT_CANVAS_SIZE,+ DEFAULT_FPS,+ useProjectStore,+} from "@/stores/project-store";import { TextElementDragState } from "@/types/editor";import {Popover,@@ -37,8 +40,20 @@ export function PreviewPanel() {const { tracks, getTotalDuration, updateTextElement } = useTimelineStore();const { mediaItems } = useMediaStore();const { currentTime, toggle, setCurrentTime } = usePlaybackStore();+ const { isPlaying, volume, muted } = usePlaybackStore();const { activeProject } = useProjectStore();const previewRef = useRef<HTMLDivElement>(null);+ const canvasRef = useRef<HTMLCanvasElement>(null);+ const lastFrameTimeRef = useRef(0);+ const renderSeqRef = useRef(0);+ const offscreenCanvasRef = useRef<OffscreenCanvas | HTMLCanvasElement | null>(+ null+ );++ const audioContextRef = useRef<AudioContext | null>(null);+ const audioGainRef = useRef<GainNode | null>(null);+ const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map());+ const playingSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());const containerRef = useRef<HTMLDivElement>(null);const [previewDimensions, setPreviewDimensions] = useState({width: 0,@@ -262,227 +277,315 @@ export function PreviewPanel() {const activeElements = getActiveElements();- // Get media elements for blur background (video/image only)- const getBlurBackgroundElements = (): ActiveElement[] => {- return activeElements.filter(- ({ element, mediaItem }) =>- element.type === "media" &&- mediaItem &&- (mediaItem.type === "video" || mediaItem.type === "image") &&- element.mediaId !== "test" // Exclude test elements- );- };-- const blurBackgroundElements = getBlurBackgroundElements();-- // Render blur background layer- const renderBlurBackground = () => {- if (- !activeProject?.backgroundType ||- activeProject.backgroundType !== "blur" ||- blurBackgroundElements.length === 0- ) {- return null;- }-- // Use the first media element for background (could be enhanced to use primary/focused element)- const backgroundElement = blurBackgroundElements[0];- const { element, mediaItem } = backgroundElement;-- if (!mediaItem) return null;-- const blurIntensity = activeProject.blurIntensity || 8;+ // Ensure first frame after mount/seek renders immediately+ useEffect(() => {+ const onSeek = () => {+ lastFrameTimeRef.current = -Infinity;+ renderSeqRef.current++;+ };+ window.addEventListener("playback-seek", onSeek as EventListener);+ lastFrameTimeRef.current = -Infinity;+ return () => {+ window.removeEventListener("playback-seek", onSeek as EventListener);+ };+ }, []);- if (mediaItem.type === "video") {- return (- <div- key={`blur-${element.id}`}- className="absolute inset-0 overflow-hidden"- style={{- filter: `blur(${blurIntensity}px)`,- transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts- transformOrigin: "center",- }}- >- <VideoPlayer- src={mediaItem.url!}- poster={mediaItem.thumbnailUrl}- clipStartTime={element.startTime}- trimStart={element.trimStart}- trimEnd={element.trimEnd}- clipDuration={element.duration}- className="w-full h-full object-cover"- trackMuted={true}- />- </div>- );- }+ // Web Audio: schedule only on play/pause/seek/volume/mute changes+ useEffect(() => {+ const stopAll = () => {+ for (const src of playingSourcesRef.current) {+ try {+ src.stop();+ } catch {}+ }+ playingSourcesRef.current.clear();+ };- if (mediaItem.type === "image") {- return (- <div- key={`blur-${element.id}`}- className="absolute inset-0 overflow-hidden"- style={{- filter: `blur(${blurIntensity}px)`,- transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts- transformOrigin: "center",- }}- >- <img- src={mediaItem.url!}- alt={mediaItem.name}- className="w-full h-full object-cover"- draggable={false}- />- </div>+ type WebAudioWindow = Window & {+ AudioContext?: typeof AudioContext;+ webkitAudioContext?: typeof AudioContext;+ };+ const ensureAudioGraph = async () => {+ if (!audioContextRef.current) {+ const win = window as WebAudioWindow;+ const Ctx = win.AudioContext ?? win.webkitAudioContext;+ if (!Ctx) return;+ audioContextRef.current = new Ctx();+ }+ if (!audioGainRef.current) {+ audioGainRef.current = audioContextRef.current!.createGain();+ audioGainRef.current.connect(audioContextRef.current!.destination);+ }+ if (audioContextRef.current!.state === "suspended") {+ try {+ await audioContextRef.current!.resume();+ } catch {}+ }+ const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume));+ audioGainRef.current!.gain.setValueAtTime(+ gainValue,+ audioContextRef.current!.currentTime);- }+ };- return null;- };+ const scheduleNow = async () => {+ await ensureAudioGraph();+ const audioCtx = audioContextRef.current!;+ const gain = audioGainRef.current!;++ const tracksSnapshot = useTimelineStore.getState().tracks;+ const mediaList = useMediaStore.getState().mediaItems;+ const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const));+ const playbackNow = usePlaybackStore.getState().currentTime;++ const audible: Array<{+ id: string;+ elementStart: number;+ trimStart: number;+ trimEnd: number;+ duration: number;+ muted: boolean;+ trackMuted: boolean;+ }> = [];+ const uniqueIds = new Set<string>();+ for (const track of tracksSnapshot) {+ for (const element of track.elements) {+ if (element.type !== "media") continue;+ const media = idToMedia.get(element.mediaId);+ if (!media || media.type !== "audio") continue;+ const visibleDuration =+ element.duration - element.trimStart - element.trimEnd;+ if (visibleDuration <= 0) continue;+ const localTime = playbackNow - element.startTime + element.trimStart;+ if (localTime < 0 || localTime >= visibleDuration) continue;+ audible.push({+ id: media.id,+ elementStart: element.startTime,+ trimStart: element.trimStart,+ trimEnd: element.trimEnd,+ duration: element.duration,+ muted: !!element.muted,+ trackMuted: !!track.muted,+ });+ uniqueIds.add(media.id);+ }+ }- // Render an element- const renderElement = (elementData: ActiveElement, index: number) => {- const { element, mediaItem } = elementData;+ if (audible.length === 0) return;++ // Decode buffers as needed+ const decodePromises: Array<Promise<void>> = [];+ for (const id of uniqueIds) {+ if (!audioBuffersRef.current.has(id)) {+ const mediaItem = idToMedia.get(id);+ if (!mediaItem) continue;+ const p = (async () => {+ const arr = await mediaItem.file.arrayBuffer();+ const buf = await audioCtx.decodeAudioData(arr.slice(0));+ audioBuffersRef.current.set(id, buf);+ })();+ decodePromises.push(p);+ }+ }+ await Promise.all(decodePromises);++ const startAt = audioCtx.currentTime + 0.02;+ for (const entry of audible) {+ if (entry.muted || entry.trackMuted) continue;+ const buffer = audioBuffersRef.current.get(entry.id);+ if (!buffer) continue;+ const visibleDuration =+ entry.duration - entry.trimStart - entry.trimEnd;+ const localTime = Math.max(+ 0,+ playbackNow - entry.elementStart + entry.trimStart+ );+ const playDuration = Math.max(0, visibleDuration - localTime);+ if (playDuration <= 0) continue;+ const src = audioCtx.createBufferSource();+ src.buffer = buffer;+ src.connect(gain);+ try {+ src.start(startAt, localTime, playDuration);+ playingSourcesRef.current.add(src);+ } catch {}+ }+ };- // Text elements- if (element.type === "text") {- const fontClassName =- FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || "";+ const onSeek = () => {+ if (!isPlaying) return;+ for (const src of playingSourcesRef.current) {+ try {+ src.stop();+ } catch {}+ }+ playingSourcesRef.current.clear();+ void scheduleNow();+ };- const scaleRatio = previewDimensions.width / canvasSize.width;+ // Apply volume/mute changes immediately+ void ensureAudioGraph();- return (- <div- key={element.id}- className="absolute cursor-grab"- onMouseDown={(e) =>- handleTextMouseDown(e, element, elementData.track.id)- }- style={{- left: `${- 50 +- ((dragState.isDragging && dragState.elementId === element.id- ? dragState.currentX- : element.x) /- canvasSize.width) *- 100- }%`,- top: `${- 50 +- ((dragState.isDragging && dragState.elementId === element.id- ? dragState.currentY- : element.y) /- canvasSize.height) *- 100- }%`,- transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`,- opacity: element.opacity,- zIndex: 100 + index, // Text elements on top- }}- >- <div- className={fontClassName}- style={{- fontSize: `${element.fontSize * scaleRatio}px`,- color: element.color,- backgroundColor: element.backgroundColor,- textAlign: element.textAlign,- fontWeight: element.fontWeight,- fontStyle: element.fontStyle,- textDecoration: element.textDecoration,- padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`,- borderRadius: `${2 * scaleRatio}px`,- whiteSpace: "nowrap",- // Fallback for system fonts that don't have classes- ...(fontClassName === "" && { fontFamily: element.fontFamily }),- }}- >- {element.content}- </div>- </div>- );+ // Start/stop on play state changes+ for (const src of playingSourcesRef.current) {+ try {+ src.stop();+ } catch {}+ }+ playingSourcesRef.current.clear();+ if (isPlaying) {+ void scheduleNow();}- // Media elements- if (element.type === "media") {- // Test elements- if (!mediaItem || element.mediaId === "test") {- return (- <div- key={element.id}- className="absolute inset-0 bg-linear-to-br from-blue-500/20 to-purple-500/20 flex items-center justify-center"- >- <div className="text-center">- <div className="text-2xl mb-2">🎬</div>- <p className="text-xs text-foreground">{element.name}</p>- </div>- </div>- );+ window.addEventListener("playback-seek", onSeek as EventListener);+ return () => {+ window.removeEventListener("playback-seek", onSeek as EventListener);+ for (const src of playingSourcesRef.current) {+ try {+ src.stop();+ } catch {}}+ playingSourcesRef.current.clear();+ };+ }, [isPlaying, volume, muted, mediaItems]);- // Video elements- if (mediaItem.type === "video") {- return (- <div- key={element.id}- className="absolute inset-0 flex items-center justify-center"- >- <VideoPlayer- src={mediaItem.url!}- poster={mediaItem.thumbnailUrl}- clipStartTime={element.startTime}- trimStart={element.trimStart}- trimEnd={element.trimEnd}- clipDuration={element.duration}- trackMuted={element.muted || elementData.track.muted}- />- </div>- );+ // Canvas: draw current frame for visible elements using offscreen compositing+ useEffect(() => {+ const draw = async () => {+ const canvas = canvasRef.current;+ if (!canvas) return;+ const mainCtx = canvas.getContext("2d");+ if (!mainCtx) return;++ // Set canvas internal resolution to avoid blurry scaling+ const displayWidth = Math.max(1, Math.floor(previewDimensions.width));+ const displayHeight = Math.max(1, Math.floor(previewDimensions.height));+ if (canvas.width !== displayWidth || canvas.height !== displayHeight) {+ canvas.width = displayWidth;+ canvas.height = displayHeight;}- // Image elements- if (mediaItem.type === "image") {- return (- <div- key={element.id}- className="absolute inset-0 flex items-center justify-center"- >- <img- src={mediaItem.url!}- alt={mediaItem.name}- className="max-w-full max-h-full object-contain"- draggable={false}- />- </div>- );+ // Throttle rendering to project FPS during playback only+ const fps = activeProject?.fps || DEFAULT_FPS;+ const minDelta = 1 / fps;+ if (isPlaying) {+ if (currentTime - lastFrameTimeRef.current < minDelta) {+ return;+ }+ lastFrameTimeRef.current = currentTime;}- // Audio elements (no visual representation)- if (mediaItem.type === "audio") {- return (- <div- key={element.id}- className="absolute inset-0"- style={{ pointerEvents: "none" }}- >- <AudioPlayer- src={mediaItem.url!}- clipStartTime={element.startTime}- trimStart={element.trimStart}- trimEnd={element.trimEnd}- clipDuration={element.duration}- trackMuted={element.muted || elementData.track.muted}- />- </div>+ // Invalidate older async renders when user scrubs rapidly+ const mySeq = (renderSeqRef.current += 1);++ // Offscreen buffer to avoid flicker (reuse canvas)+ if (!offscreenCanvasRef.current) {+ const hasOffscreen =+ typeof (globalThis as unknown as { OffscreenCanvas?: unknown })+ .OffscreenCanvas !== "undefined";+ if (hasOffscreen) {+ // eslint-disable-next-line @typescript-eslint/no-explicit-any+ offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas(+ displayWidth,+ displayHeight+ ) as OffscreenCanvas;+ } else {+ const c = document.createElement("canvas");+ c.width = displayWidth;+ c.height = displayHeight;+ offscreenCanvasRef.current = c;+ }+ }+ // Ensure size matches+ if (+ offscreenCanvasRef.current &&+ (offscreenCanvasRef.current as HTMLCanvasElement).getContext+ ) {+ const c = offscreenCanvasRef.current as HTMLCanvasElement;+ if (c.width !== displayWidth || c.height !== displayHeight) {+ c.width = displayWidth;+ c.height = displayHeight;+ }+ } else {+ const c = offscreenCanvasRef.current as OffscreenCanvas;+ // @ts-ignore width/height exist on OffscreenCanvas in modern browsers+ if (+ (c as unknown as { width: number }).width !== displayWidth ||+ (c as unknown as { height: number }).height !== displayHeight+ ) {+ // @ts-ignore+ (c as unknown as { width: number }).width = displayWidth;+ // @ts-ignore+ (c as unknown as { height: number }).height = displayHeight;+ }+ }+ const offscreenCanvas = offscreenCanvasRef.current as+ | HTMLCanvasElement+ | OffscreenCanvas;+ const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext+ ? (offscreenCanvas as HTMLCanvasElement).getContext("2d")+ : (offscreenCanvas as OffscreenCanvas).getContext("2d");+ if (!offCtx) return;++ await renderTimelineFrame({+ ctx: offCtx as CanvasRenderingContext2D,+ time: currentTime,+ canvasWidth: displayWidth,+ canvasHeight: displayHeight,+ tracks,+ mediaItems,+ backgroundColor:+ activeProject?.backgroundType === "blur"+ ? "transparent"+ : activeProject?.backgroundColor || "#000000",+ useCache: false,+ fps: activeProject?.fps || DEFAULT_FPS,+ });++ // Blit offscreen to visible canvas+ mainCtx.clearRect(0, 0, displayWidth, displayHeight);+ if ((offscreenCanvas as HTMLCanvasElement).getContext) {+ mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0);+ } else {+ mainCtx.drawImage(+ offscreenCanvas as unknown as CanvasImageSource,+ 0,+ 0);}- }+ };++ void draw();+ }, [+ activeElements,+ currentTime,+ previewDimensions.width,+ previewDimensions.height,+ canvasSize.width,+ canvasSize.height,+ activeProject?.backgroundType,+ activeProject?.backgroundColor,+ ]);- return null;+ // Get media elements for blur background (video/image only)+ const getBlurBackgroundElements = (): ActiveElement[] => {+ return activeElements.filter(+ ({ element, mediaItem }) =>+ element.type === "media" &&+ mediaItem &&+ (mediaItem.type === "video" || mediaItem.type === "image") &&+ element.mediaId !== "test" // Exclude test elements+ );};+ const blurBackgroundElements = getBlurBackgroundElements();++ // Render blur background layer (handled by canvas now)+ const renderBlurBackground = () => null;++ // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio.+ const renderElement = (_elementData: ActiveElement) => null;+return (<><div className="h-full w-full flex flex-col min-h-0 min-w-0 bg-panel rounded-sm relative">@@ -505,14 +608,23 @@ export function PreviewPanel() {}}>{renderBlurBackground()}+ <canvas+ ref={canvasRef}+ style={{+ position: "absolute",+ left: 0,+ top: 0,+ width: previewDimensions.width,+ height: previewDimensions.height,+ }}+ aria-label="Video preview canvas"+ />{activeElements.length === 0 ? (<div className="absolute inset-0 flex items-center justify-center text-muted-foreground">No elements at current time</div>) : (- activeElements.map((elementData, index) =>- renderElement(elementData, index)- )+ activeElements.map((elementData) => renderElement(elementData)))}<LayoutGuideOverlay /></div>diff --git a/apps/web/src/lib/media-processing.ts b/apps/web/src/lib/media-processing.tsindex 353c3113..15c71626 100644--- a/apps/web/src/lib/media-processing.ts+++ b/apps/web/src/lib/media-processing.ts@@ -6,7 +6,7 @@ import {getImageDimensions,type MediaItem,} from "@/stores/media-store";-import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils";+import { generateThumbnail, getVideoInfo } from "./mediabunny-utils";export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {}@@ -37,33 +37,23 @@ export async function processMediaFiles(try {if (fileType === "image") {- // Get image dimensionsconst dimensions = await getImageDimensions(file);width = dimensions.width;height = dimensions.height;} else if (fileType === "video") {try {- // Use FFmpeg for comprehensive video info extraction- const videoInfo = await getVideoInfo(file);+ const videoInfo = await getVideoInfo({ videoFile: file });duration = videoInfo.duration;width = videoInfo.width;height = videoInfo.height;fps = videoInfo.fps;- // Generate thumbnail using FFmpeg- thumbnailUrl = await generateThumbnail(file, 1);+ thumbnailUrl = await generateThumbnail({+ videoFile: file,+ timeInSeconds: 1,+ });} catch (error) {- console.warn(- "FFmpeg processing failed, falling back to basic processing:",- error- );- // Fallback to basic processing- const videoResult = await generateVideoThumbnail(file);- thumbnailUrl = videoResult.thumbnailUrl;- width = videoResult.width;- height = videoResult.height;- duration = await getMediaDuration(file);- // FPS will remain undefined for fallback+ console.warn("Video processing failed", error);}} else if (fileType === "audio") {// For audio, we don't set width/height/fps (they'll be undefined)@@ -82,7 +72,6 @@ export async function processMediaFiles(fps,});- // Yield back to the event loop to keep the UI responsiveawait new Promise((resolve) => setTimeout(resolve, 0));completed += 1;diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/mediabunny-utils.tssimilarity index 53%rename from apps/web/src/lib/ffmpeg-utils.tsrename to apps/web/src/lib/mediabunny-utils.tsindex 0643ecb9..28e6b97f 100644--- a/apps/web/src/lib/ffmpeg-utils.ts+++ b/apps/web/src/lib/mediabunny-utils.ts@@ -1,7 +1,7 @@import { FFmpeg } from "@ffmpeg/ffmpeg";-import { toBlobURL } from "@ffmpeg/util";import { useTimelineStore } from "@/stores/timeline-store";import { useMediaStore } from "@/stores/media-store";+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";let ffmpeg: FFmpeg | null = null;@@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {return ffmpeg;};-export const generateThumbnail = async (- videoFile: File,- timeInSeconds = 1-): Promise<string> => {- const ffmpeg = await initFFmpeg();+export async function generateThumbnail({+ videoFile,+ timeInSeconds,+}: {+ videoFile: File;+ timeInSeconds: number;+}): Promise<string> {+ const input = new Input({+ source: new BlobSource(videoFile),+ formats: ALL_FORMATS,+ });++ const videoTrack = await input.getPrimaryVideoTrack();+ if (!videoTrack) {+ throw new Error("No video track found in the file");+ }- const inputName = "input.mp4";- const outputName = "thumbnail.jpg";-- // Write input file- await ffmpeg.writeFile(- inputName,- new Uint8Array(await videoFile.arrayBuffer())- );-- // Generate thumbnail at specific time- await ffmpeg.exec([- "-i",- inputName,- "-ss",- timeInSeconds.toString(),- "-vframes",- "1",- "-vf",- "scale=320:240",- "-q:v",- "2",- outputName,- ]);-- // Read output file- const data = await ffmpeg.readFile(outputName);- const blob = new Blob([data], { type: "image/jpeg" });-- // Cleanup- await ffmpeg.deleteFile(inputName);- await ffmpeg.deleteFile(outputName);-- return URL.createObjectURL(blob);-};+ // Check if we can decode this video+ const canDecode = await videoTrack.canDecode();+ if (!canDecode) {+ throw new Error("Video codec not supported for decoding");+ }-export const trimVideo = async (- videoFile: File,- startTime: number,- endTime: number,- onProgress?: (progress: number) => void-): Promise<Blob> => {- const ffmpeg = await initFFmpeg();+ const sink = new VideoSampleSink(videoTrack);- const inputName = "input.mp4";- const outputName = "output.mp4";+ const frame = await sink.getSample(timeInSeconds);- // Set up progress callback- if (onProgress) {- ffmpeg.on("progress", ({ progress }) => {- onProgress(progress * 100);- });+ if (!frame) {+ throw new Error("Could not get frame at specified time");}- // Write input file- await ffmpeg.writeFile(- inputName,- new Uint8Array(await videoFile.arrayBuffer())- );-- const duration = endTime - startTime;-- // Trim video- await ffmpeg.exec([- "-i",- inputName,- "-ss",- startTime.toString(),- "-t",- duration.toString(),- "-c",- "copy", // Use stream copy for faster processing- outputName,- ]);-- // Read output file- const data = await ffmpeg.readFile(outputName);- const blob = new Blob([data], { type: "video/mp4" });-- // Cleanup- await ffmpeg.deleteFile(inputName);- await ffmpeg.deleteFile(outputName);-- return blob;-};+ const canvas = document.createElement("canvas");+ canvas.width = 320;+ canvas.height = 240;+ const ctx = canvas.getContext("2d");-export const getVideoInfo = async (- videoFile: File-): Promise<{+ if (!ctx) {+ throw new Error("Could not get canvas context");+ }++ frame.draw(ctx, 0, 0, 320, 240);++ return new Promise((resolve, reject) => {+ canvas.toBlob(+ (blob) => {+ if (blob) {+ resolve(URL.createObjectURL(blob));+ } else {+ reject(new Error("Failed to create thumbnail blob"));+ }+ },+ "image/jpeg",+ 0.8+ );+ });+}++export async function getVideoInfo({+ videoFile,+}: {+ videoFile: File;+}): Promise<{duration: number;width: number;height: number;fps: number;-}> => {- const ffmpeg = await initFFmpeg();-- const inputName = "input.mp4";+}> {+ const input = new Input({+ source: new BlobSource(videoFile),+ formats: ALL_FORMATS,+ });- // Write input file- await ffmpeg.writeFile(- inputName,- new Uint8Array(await videoFile.arrayBuffer())- );+ const duration = await input.computeDuration();+ const videoTrack = await input.getPrimaryVideoTrack();- // Capture FFmpeg stderr output with a one-time listener pattern- let ffmpegOutput = "";- let listening = true;- const listener = (data: string) => {- if (listening) ffmpegOutput += data;- };- ffmpeg.on("log", ({ message }) => listener(message));-- // Run ffmpeg to get info (stderr will contain the info)- try {- await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]);- } catch (error) {- listening = false;- await ffmpeg.deleteFile(inputName);- console.error("FFmpeg execution failed:", error);- throw new Error(- "Failed to extract video info. The file may be corrupted or in an unsupported format."- );+ if (!videoTrack) {+ throw new Error("No video track found in the file");}- // Disable listener after exec completes- listening = false;-- // Cleanup- await ffmpeg.deleteFile(inputName);-- // Parse output for duration, resolution, and fps- // Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s- // Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc-- const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/);- let duration = 0;- if (durationMatch) {- const [, h, m, s] = durationMatch;- duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);- }-- const videoStreamMatch = ffmpegOutput.match(- /Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/- );- let width = 0,- height = 0,- fps = 0;- if (videoStreamMatch) {- width = parseInt(videoStreamMatch[1]);- height = parseInt(videoStreamMatch[2]);- fps = parseFloat(videoStreamMatch[3]);- }+ // Get frame rate from packet statistics+ const packetStats = await videoTrack.computePacketStats(100);+ const fps = packetStats.averagePacketRate;return {duration,- width,- height,+ width: videoTrack.displayWidth,+ height: videoTrack.displayHeight,fps,};-};--export const convertToWebM = async (- videoFile: File,- onProgress?: (progress: number) => void-): Promise<Blob> => {- const ffmpeg = await initFFmpeg();-- const inputName = "input.mp4";- const outputName = "output.webm";-- // Set up progress callback- if (onProgress) {- ffmpeg.on("progress", ({ progress }) => {- onProgress(progress * 100);- });- }-- // Write input file- await ffmpeg.writeFile(- inputName,- new Uint8Array(await videoFile.arrayBuffer())- );-- // Convert to WebM- await ffmpeg.exec([- "-i",- inputName,- "-c:v",- "libvpx-vp9",- "-crf",- "30",- "-b:v",- "0",- "-c:a",- "libopus",- outputName,- ]);-- // Read output file- const data = await ffmpeg.readFile(outputName);- const blob = new Blob([data], { type: "video/webm" });-- // Cleanup- await ffmpeg.deleteFile(inputName);- await ffmpeg.deleteFile(outputName);-- return blob;-};--export const extractAudio = async (- videoFile: File,- format: "mp3" | "wav" = "mp3"-): Promise<Blob> => {- const ffmpeg = await initFFmpeg();-- const inputName = "input.mp4";- const outputName = `output.${format}`;-- // Write input file- await ffmpeg.writeFile(- inputName,- new Uint8Array(await videoFile.arrayBuffer())- );-- // Extract audio- await ffmpeg.exec([- "-i",- inputName,- "-vn", // Disable video- "-acodec",- format === "mp3" ? "libmp3lame" : "pcm_s16le",- outputName,- ]);-- // Read output file- const data = await ffmpeg.readFile(outputName);- const blob = new Blob([data], { type: `audio/${format}` });-- // Cleanup- await ffmpeg.deleteFile(inputName);- await ffmpeg.deleteFile(outputName);-- return blob;-};+}+// Audio mixing for timeline - keeping FFmpeg for now due to complexity+// TODO: Replace with Mediabunny audio processing when implementing canvas previewexport const extractTimelineAudio = async (onProgress?: (progress: number) => void): Promise<Blob> => {diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.tsnew file mode 100644index 00000000..1c1eafcd--- /dev/null+++ b/apps/web/src/lib/timeline-renderer.ts@@ -0,0 +1,179 @@+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";+import type { TimelineTrack } from "@/types/timeline";+import type { MediaItem } from "@/stores/media-store";++export interface RenderContext {+ ctx: CanvasRenderingContext2D;+ time: number;+ canvasWidth: number;+ canvasHeight: number;+ tracks: TimelineTrack[];+ mediaItems: MediaItem[];+ backgroundColor?: string;+ useCache?: boolean;+ cache?: Map<+ string,+ Map<+ number,+ {+ draw: (+ c: CanvasRenderingContext2D,+ x: number,+ y: number,+ w: number,+ h: number+ ) => void;+ }+ >+ >;+ fps: number;+}++export async function renderTimelineFrame({+ ctx,+ time,+ canvasWidth,+ canvasHeight,+ tracks,+ mediaItems,+ backgroundColor,+ useCache = true,+ cache,+ fps,+}: RenderContext): Promise<void> {+ // Background+ ctx.clearRect(0, 0, canvasWidth, canvasHeight);+ if (backgroundColor && backgroundColor !== "transparent") {+ ctx.fillStyle = backgroundColor;+ ctx.fillRect(0, 0, canvasWidth, canvasHeight);+ }++ const scaleX = 1;+ const scaleY = 1;+ const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));+ const active: Array<{+ track: TimelineTrack;+ element: TimelineTrack["elements"][number];+ mediaItem: MediaItem | null;+ }> = [];++ for (let t = tracks.length - 1; t >= 0; t -= 1) {+ const track = tracks[t];+ for (const element of track.elements) {+ if ((element as any).hidden) continue;+ const elementStart = element.startTime;+ const elementEnd =+ element.startTime ++ (element.duration - element.trimStart - element.trimEnd);+ if (time >= elementStart && time < elementEnd) {+ let mediaItem: MediaItem | null = null;+ if (element.type === "media") {+ mediaItem =+ element.mediaId === "test"+ ? null+ : idToMedia.get(element.mediaId) || null;+ }+ active.push({ track, element, mediaItem });+ }+ }+ }++ for (const { element, mediaItem } of active) {+ if (element.type === "media" && mediaItem) {+ if (mediaItem.type === "video") {+ const input = new Input({+ source: new BlobSource(mediaItem.file),+ formats: ALL_FORMATS,+ });+ const track = await input.getPrimaryVideoTrack();+ if (!track) continue;+ const decodable = await track.canDecode();+ if (!decodable) continue;+ const sink = new VideoSampleSink(track);++ const localTime = time - element.startTime + element.trimStart;+ const sample = await sink.getSample(localTime);+ if (!sample) continue;++ const mediaW = Math.max(1, mediaItem.width || canvasWidth);+ const mediaH = Math.max(1, mediaItem.height || canvasHeight);+ const containScale = Math.min(+ canvasWidth / mediaW,+ canvasHeight / mediaH+ );+ const drawW = mediaW * containScale;+ const drawH = mediaH * containScale;+ const drawX = (canvasWidth - drawW) / 2;+ const drawY = (canvasHeight - drawH) / 2;+ sample.draw(ctx, drawX, drawY, drawW, drawH);+ }+ if (mediaItem.type === "image") {+ const img = new Image();+ await new Promise<void>((resolve, reject) => {+ img.onload = () => resolve();+ img.onerror = () => reject(new Error("Image load failed"));+ img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);+ });+ const mediaW = Math.max(+ 1,+ mediaItem.width || img.naturalWidth || canvasWidth+ );+ const mediaH = Math.max(+ 1,+ mediaItem.height || img.naturalHeight || canvasHeight+ );+ const containScale = Math.min(+ canvasWidth / mediaW,+ canvasHeight / mediaH+ );+ const drawW = mediaW * containScale;+ const drawH = mediaH * containScale;+ const drawX = (canvasWidth - drawW) / 2;+ const drawY = (canvasHeight - drawH) / 2;+ ctx.drawImage(img, drawX, drawY, drawW, drawH);+ }+ }+ if (element.type === "text") {+ const posX = canvasWidth / 2 + (element as any).x * scaleX;+ const posY = canvasHeight / 2 + (element as any).y * scaleY;+ ctx.save();+ ctx.translate(posX, posY);+ ctx.rotate(((element as any).rotation * Math.PI) / 180);+ ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));+ const px = (element as any).fontSize * scaleX;+ const weight = (element as any).fontWeight === "bold" ? "bold " : "";+ const style = (element as any).fontStyle === "italic" ? "italic " : "";+ ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;+ ctx.fillStyle = (element as any).color;+ ctx.textAlign = (element as any).textAlign;+ ctx.textBaseline = "middle";+ const metrics = ctx.measureText((element as any).content);+ const ascent =+ (metrics as unknown as { actualBoundingBoxAscent?: number })+ .actualBoundingBoxAscent ?? px * 0.8;+ const descent =+ (metrics as unknown as { actualBoundingBoxDescent?: number })+ .actualBoundingBoxDescent ?? px * 0.2;+ const textW = metrics.width;+ const textH = ascent + descent;+ const padX = 8 * scaleX;+ const padY = 4 * scaleX;+ if ((element as any).backgroundColor) {+ ctx.save();+ ctx.fillStyle = (element as any).backgroundColor;+ let bgLeft = -textW / 2;+ if (ctx.textAlign === "left") bgLeft = 0;+ if (ctx.textAlign === "right") bgLeft = -textW;+ ctx.fillRect(+ bgLeft - padX,+ -textH / 2 - padY,+ textW + padX * 2,+ textH + padY * 2+ );+ ctx.restore();+ }+ ctx.fillText((element as any).content, 0, 0);+ ctx.restore();+ }+ }+}diff --git a/bun.lock b/bun.lockindex ea7c8c9a..95f7c874 100644--- a/bun.lock+++ b/bun.lock@@ -4,6 +4,7 @@"": {"name": "opencut","dependencies": {+ "mediabunny": "^1.9.3","next": "^15.3.4","react-country-flag": "^3.1.0","wavesurfer.js": "^7.9.8",@@ -558,6 +559,10 @@"@types/deep-eql": ["@types/[email protected]", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="],+ "@types/dom-mediacapture-transform": ["@types/[email protected]", "", { "dependencies": { "@types/dom-webcodecs": "*" } }, "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ=="],++ "@types/dom-webcodecs": ["@types/[email protected]", "", {}, "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ=="],+"@types/estree": ["@types/[email protected]", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],"@types/estree-jsx": ["@types/[email protected]", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="],@@ -896,6 +901,8 @@"mdast-util-to-string": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],+ "mediabunny": ["[email protected]", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-uVspdzrpwvW8NxtGspHPOJvdNSY0dNFEmOQngSheX9PYX9oXijZENBjfYWOgm9c9IgSSlN2bdJF9iyPyY3f+fQ=="],+"micromark": ["[email protected]", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="],"micromark-core-commonmark": ["[email protected]", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="],@@ -1320,10 +1327,10 @@"motion/framer-motion/motion-utils": ["[email protected]", "", {}, "sha512-eAWoPgr4eFEOFfg2WjIsMoqJTW6Z8MTUCgn/GZ3VRpClWBdnbjryiA3ZSNLyxCTmCQx4RmYX6jX1iWHbenUPNQ=="],- "opencut/@types/node/undici-types": ["[email protected]", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],-"next/postcss/nanoid": ["[email protected]", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],+ "opencut/@types/node/undici-types": ["[email protected]", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],+"opencut/next/@next/env": ["@next/[email protected]", "", {}, "sha512-ruM+q2SCOVCepUiERoxOmZY9ZVoecR3gcXNwCYZRvQQWRjhOiPJGmQ2fAiLR6YKWXcSAh7G79KEFxN3rwhs4LQ=="],"opencut/next/@next/swc-darwin-arm64": ["@next/[email protected]", "", { "os": "darwin", "cpu": "arm64" }, "sha512-84dAN4fkfdC7nX6udDLz9GzQlMUwEMKD7zsseXrl7FTeIItF8vpk1lhLEnsotiiDt+QFu3O1FVWnqwcRD2U3KA=="],diff --git a/package.json b/package.jsonindex 45a98282..616c97f7 100644--- a/package.json+++ b/package.json@@ -20,6 +20,7 @@"format": "npx ultracite@latest format"},"dependencies": {+ "mediabunny": "^1.9.3","next": "^15.3.4","react-country-flag": "^3.1.0","wavesurfer.js": "^7.9.8"