The open-source CapCut alternative
feat: add audio support for export
Maze Winther70a9bfebb196
diff --git a/apps/web/src/components/editor/export-button.tsx b/apps/web/src/components/editor/export-button.tsxindex e24d20df..c9738e81 100644--- a/apps/web/src/components/editor/export-button.tsx+++ b/apps/web/src/components/editor/export-button.tsx@@ -7,6 +7,7 @@ import { Button } from "../ui/button";import { Label } from "../ui/label";import { RadioGroup, RadioGroupItem } from "../ui/radio-group";import { Progress } from "../ui/progress";+import { Checkbox } from "../ui/checkbox";import { cn } from "@/lib/utils";import {exportProject,@@ -75,6 +76,9 @@ function ExportPopover({const [quality, setQuality] = useState<ExportQuality>(DEFAULT_EXPORT_OPTIONS.quality);+ const [includeAudio, setIncludeAudio] = useState<boolean>(+ DEFAULT_EXPORT_OPTIONS.includeAudio || true+ );const [isExporting, setIsExporting] = useState(false);const [progress, setProgress] = useState(0);const [exportResult, setExportResult] = useState<ExportResult | null>(null);@@ -90,6 +94,7 @@ function ExportPopover({format,quality,fps: activeProject.fps,+ includeAudio,onProgress: setProgress,onCancel: () => false, // TODO: Add cancel functionality});@@ -197,6 +202,23 @@ function ExportPopover({</div></RadioGroup></PropertyGroup>++ <PropertyGroup+ title="Audio"+ titleClassName="text-sm"+ defaultExpanded={false}+ >+ <div className="flex items-center space-x-2">+ <Checkbox+ id="include-audio"+ checked={includeAudio}+ onCheckedChange={(checked) => setIncludeAudio(!!checked)}+ />+ <Label htmlFor="include-audio">+ Include audio in export+ </Label>+ </div>+ </PropertyGroup></div><Button onClick={handleExport} className="w-full gap-2">diff --git a/apps/web/src/lib/export.ts b/apps/web/src/lib/export.tsindex 69bc8d1d..f25f5f6b 100644--- a/apps/web/src/lib/export.ts+++ b/apps/web/src/lib/export.ts@@ -1,147 +1,296 @@-import {- Output,- Mp4OutputFormat,- WebMOutputFormat,- BufferTarget,- CanvasSource,- QUALITY_LOW,- QUALITY_MEDIUM,- QUALITY_HIGH,- QUALITY_VERY_HIGH,-} from "mediabunny";-import { renderTimelineFrame } from "./timeline-renderer";-import { useTimelineStore } from "@/stores/timeline-store";-import { useMediaStore } from "@/stores/media-store";-import { useProjectStore } from "@/stores/project-store";-import { DEFAULT_FPS } from "@/stores/project-store";-import { ExportOptions, ExportResult } from "@/types/export";--export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {- format: "mp4",- quality: "high",-};--const qualityMap = {- low: QUALITY_LOW,- medium: QUALITY_MEDIUM,- high: QUALITY_HIGH,- very_high: QUALITY_VERY_HIGH,-};--export async function exportProject(- options: ExportOptions-): Promise<ExportResult> {- const { format, quality, fps, onProgress, onCancel } = options;-- try {- const timelineStore = useTimelineStore.getState();- const mediaStore = useMediaStore.getState();- const projectStore = useProjectStore.getState();-- const { tracks, getTotalDuration } = timelineStore;- const { mediaFiles } = mediaStore;- const { activeProject } = projectStore;-- if (!activeProject) {- return { success: false, error: "No active project" };- }-- const duration = getTotalDuration();- if (duration === 0) {- return { success: false, error: "Project is empty" };- }-- const exportFps = fps || activeProject.fps || DEFAULT_FPS;- const canvasSize = activeProject.canvasSize;-- const outputFormat =- format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();-- // BufferTarget for smaller files, StreamTarget for larger ones- // TODO: Implement StreamTarget- const output = new Output({- format: outputFormat,- target: new BufferTarget(),- });-- // Canvas for rendering- const canvas = document.createElement("canvas");- canvas.width = canvasSize.width;- canvas.height = canvasSize.height;- const ctx = canvas.getContext("2d");-- if (!ctx) {- return { success: false, error: "Failed to create canvas context" };- }-- const videoSource = new CanvasSource(canvas, {- codec: format === "webm" ? "vp9" : "avc", // VP9 for WebM, H.264 for MP4- bitrate: qualityMap[quality],- });-- output.addVideoTrack(videoSource, { frameRate: exportFps });-- // Start the output- await output.start();-- const totalFrames = Math.ceil(duration * exportFps);- let cancelled = false;-- // Render each frame- for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) {- // Check for cancellation- if (onCancel?.()) {- cancelled = true;- break;- }-- const time = frameIndex / exportFps;-- await renderTimelineFrame({- ctx,- time,- canvasWidth: canvas.width,- canvasHeight: canvas.height,- tracks,- mediaFiles,- backgroundColor:- activeProject.backgroundType === "blur"- ? "transparent"- : activeProject.backgroundColor || "#000000",- projectCanvasSize: canvasSize,- });-- const frameDuration = 1 / exportFps;- await videoSource.add(time, frameDuration);-- onProgress?.(frameIndex / totalFrames);- }-- if (cancelled) {- await output.cancel();- return { success: false, cancelled: true };- }- videoSource.close();- await output.finalize();- onProgress?.(1);-- return {- success: true,- buffer: output.target.buffer || undefined,- };- } catch (error) {- console.error("Export failed:", error);- return {- success: false,- error: error instanceof Error ? error.message : "Unknown export error",- };- }-}--export function getExportMimeType(format: "mp4" | "webm"): string {- return format === "webm" ? "video/webm" : "video/mp4";-}--export function getExportFileExtension(format: "mp4" | "webm"): string {- return `.${format}`;-}+import {+ Output,+ Mp4OutputFormat,+ WebMOutputFormat,+ BufferTarget,+ CanvasSource,+ AudioBufferSource,+ QUALITY_LOW,+ QUALITY_MEDIUM,+ QUALITY_HIGH,+ QUALITY_VERY_HIGH,+} from "mediabunny";+import { renderTimelineFrame } from "./timeline-renderer";+import { useTimelineStore } from "@/stores/timeline-store";+import { useMediaStore } from "@/stores/media-store";+import { useProjectStore } from "@/stores/project-store";+import { DEFAULT_FPS } from "@/stores/project-store";+import { ExportOptions, ExportResult } from "@/types/export";++export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {+ format: "mp4",+ quality: "high",+ includeAudio: true,+};++const qualityMap = {+ low: QUALITY_LOW,+ medium: QUALITY_MEDIUM,+ high: QUALITY_HIGH,+ very_high: QUALITY_VERY_HIGH,+};++interface AudioElement {+ buffer: AudioBuffer;+ startTime: number;+ duration: number;+ trimStart: number;+ trimEnd: number;+ muted: boolean;+}++async function createTimelineAudioBuffer(+ tracks: any[],+ mediaFiles: any[],+ duration: number,+ sampleRate: number = 44100+): Promise<AudioBuffer | null> {+ // Get Web Audio context+ const audioContext = new (window.AudioContext ||+ (window as any).webkitAudioContext)();++ // Collect all audio elements from timeline+ const audioElements: AudioElement[] = [];+ const mediaMap = new Map(mediaFiles.map((m) => [m.id, m]));++ for (const track of tracks) {+ if (track.muted) continue;++ for (const element of track.elements) {+ if (element.type !== "media") continue;++ const mediaItem = mediaMap.get(element.mediaId);+ if (!mediaItem || mediaItem.type !== "audio") continue;++ const visibleDuration =+ element.duration - element.trimStart - element.trimEnd;+ if (visibleDuration <= 0) continue;++ try {+ // Decode audio file+ const arrayBuffer = await mediaItem.file.arrayBuffer();+ const audioBuffer = await audioContext.decodeAudioData(+ arrayBuffer.slice(0)+ );++ audioElements.push({+ buffer: audioBuffer,+ startTime: element.startTime,+ duration: element.duration,+ trimStart: element.trimStart,+ trimEnd: element.trimEnd,+ muted: element.muted || track.muted || false,+ });+ } catch (error) {+ console.warn(`Failed to decode audio file ${mediaItem.name}:`, error);+ }+ }+ }++ if (audioElements.length === 0) {+ return null; // No audio to mix+ }++ // Create output buffer+ const outputChannels = 2; // Stereo+ const outputLength = Math.ceil(duration * sampleRate);+ const outputBuffer = audioContext.createBuffer(+ outputChannels,+ outputLength,+ sampleRate+ );++ // Mix all audio elements+ for (const element of audioElements) {+ if (element.muted) continue;++ const {+ buffer,+ startTime,+ trimStart,+ trimEnd,+ duration: elementDuration,+ } = element;++ // Calculate timing+ const sourceStartSample = Math.floor(trimStart * buffer.sampleRate);+ const sourceDuration = elementDuration - trimStart - trimEnd;+ const sourceLengthSamples = Math.floor(sourceDuration * buffer.sampleRate);+ const outputStartSample = Math.floor(startTime * sampleRate);++ // Resample if needed (simple approach)+ const resampleRatio = sampleRate / buffer.sampleRate;+ const resampledLength = Math.floor(sourceLengthSamples * resampleRatio);++ // Mix each channel+ for (let channel = 0; channel < outputChannels; channel++) {+ const outputData = outputBuffer.getChannelData(channel);+ const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1);+ const sourceData = buffer.getChannelData(sourceChannel);++ for (let i = 0; i < resampledLength; i++) {+ const outputIndex = outputStartSample + i;+ if (outputIndex >= outputLength) break;++ // Simple resampling (could be improved with proper interpolation)+ const sourceIndex = sourceStartSample + Math.floor(i / resampleRatio);+ if (sourceIndex >= sourceData.length) break;++ outputData[outputIndex] += sourceData[sourceIndex];+ }+ }+ }++ return outputBuffer;+}++export async function exportProject(+ options: ExportOptions+): Promise<ExportResult> {+ const { format, quality, fps, includeAudio, onProgress, onCancel } = options;++ try {+ const timelineStore = useTimelineStore.getState();+ const mediaStore = useMediaStore.getState();+ const projectStore = useProjectStore.getState();++ const { tracks, getTotalDuration } = timelineStore;+ const { mediaFiles } = mediaStore;+ const { activeProject } = projectStore;++ if (!activeProject) {+ return { success: false, error: "No active project" };+ }++ const duration = getTotalDuration();+ if (duration === 0) {+ return { success: false, error: "Project is empty" };+ }++ const exportFps = fps || activeProject.fps || DEFAULT_FPS;+ const canvasSize = activeProject.canvasSize;++ const outputFormat =+ format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();++ // BufferTarget for smaller files, StreamTarget for larger ones+ // TODO: Implement StreamTarget+ const output = new Output({+ format: outputFormat,+ target: new BufferTarget(),+ });++ // Canvas for rendering+ const canvas = document.createElement("canvas");+ canvas.width = canvasSize.width;+ canvas.height = canvasSize.height;+ const ctx = canvas.getContext("2d");++ if (!ctx) {+ return { success: false, error: "Failed to create canvas context" };+ }++ const videoSource = new CanvasSource(canvas, {+ codec: format === "webm" ? "vp9" : "avc", // VP9 for WebM, H.264 for MP4+ bitrate: qualityMap[quality],+ });++ output.addVideoTrack(videoSource, { frameRate: exportFps });++ // Add audio track if requested (but don't add data yet)+ let audioSource: AudioBufferSource | null = null;+ let audioBuffer: AudioBuffer | null = null;++ if (includeAudio) {+ onProgress?.(0.05); // 5% for audio processing++ audioBuffer = await createTimelineAudioBuffer(+ tracks,+ mediaFiles,+ duration+ );++ if (audioBuffer) {+ audioSource = new AudioBufferSource({+ codec: format === "webm" ? "opus" : "aac", // Opus for WebM, AAC for MP4+ bitrate: qualityMap[quality], // Use same quality for audio+ });++ output.addAudioTrack(audioSource);+ }+ }++ // Start the output (after all tracks are added)+ await output.start();++ // Now add audio data after starting+ if (audioSource && audioBuffer) {+ await audioSource.add(audioBuffer);+ audioSource.close();+ }++ const totalFrames = Math.ceil(duration * exportFps);+ let cancelled = false;++ // Render each frame+ for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) {+ // Check for cancellation+ if (onCancel?.()) {+ cancelled = true;+ break;+ }++ const time = frameIndex / exportFps;++ await renderTimelineFrame({+ ctx,+ time,+ canvasWidth: canvas.width,+ canvasHeight: canvas.height,+ tracks,+ mediaFiles,+ backgroundColor:+ activeProject.backgroundType === "blur"+ ? "transparent"+ : activeProject.backgroundColor || "#000000",+ projectCanvasSize: canvasSize,+ });++ const frameDuration = 1 / exportFps;+ await videoSource.add(time, frameDuration);++ // Adjust progress to account for audio processing (5% at start)+ const videoProgress = includeAudio+ ? 0.05 + (frameIndex / totalFrames) * 0.95+ : frameIndex / totalFrames;+ onProgress?.(videoProgress);+ }++ if (cancelled) {+ await output.cancel();+ return { success: false, cancelled: true };+ }+ videoSource.close();+ await output.finalize();+ onProgress?.(1);++ return {+ success: true,+ buffer: output.target.buffer || undefined,+ };+ } catch (error) {+ console.error("Export failed:", error);+ return {+ success: false,+ error: error instanceof Error ? error.message : "Unknown export error",+ };+ }+}++export function getExportMimeType(format: "mp4" | "webm"): string {+ return format === "webm" ? "video/webm" : "video/mp4";+}++export function getExportFileExtension(format: "mp4" | "webm"): string {+ return `.${format}`;+}diff --git a/apps/web/src/types/export.ts b/apps/web/src/types/export.tsindex 2fa10a50..093268ce 100644--- a/apps/web/src/types/export.ts+++ b/apps/web/src/types/export.ts@@ -1,17 +1,18 @@-export type ExportFormat = "mp4" | "webm";-export type ExportQuality = "low" | "medium" | "high" | "very_high";--export interface ExportOptions {- format: ExportFormat;- quality: ExportQuality;- fps?: number;- onProgress?: (progress: number) => void;- onCancel?: () => boolean;-}--export interface ExportResult {- success: boolean;- buffer?: ArrayBuffer;- error?: string;- cancelled?: boolean;-}+export type ExportFormat = "mp4" | "webm";+export type ExportQuality = "low" | "medium" | "high" | "very_high";++export interface ExportOptions {+ format: ExportFormat;+ quality: ExportQuality;+ fps?: number;+ includeAudio?: boolean;+ onProgress?: (progress: number) => void;+ onCancel?: () => boolean;+}++export interface ExportResult {+ success: boolean;+ buffer?: ArrayBuffer;+ error?: string;+ cancelled?: boolean;+}