The open-source CapCut alternative
fix: surface unsupported video codec on import, fix Input leaks
Maze Winther0dffefdd59d8
diff --git a/apps/web/src/media/mediabunny.ts b/apps/web/src/media/mediabunny.tsindex 47ffaeb1..3c949b35 100644--- a/apps/web/src/media/mediabunny.ts+++ b/apps/web/src/media/mediabunny.ts@@ -1,43 +1,81 @@-import { Input, ALL_FORMATS, BlobSource } from "mediabunny";+import {+ Input,+ ALL_FORMATS,+ BlobSource,+ VideoSampleSink,+ type VideoCodec,+} from "mediabunny";import { createTimelineAudioBuffer } from "@/media/audio";import type { SceneTracks } from "@/timeline";import type { MediaAsset } from "@/media/types";import { TICKS_PER_SECOND } from "@/wasm";+import { renderThumbnailDataUrl } from "./thumbnail";-export async function getVideoInfo({- videoFile,-}: {- videoFile: File;-}): Promise<{+export type VideoFileData = {duration: number;width: number;height: number;fps: number;hasAudio: boolean;-}> {+ codec: VideoCodec | null;+ canDecode: boolean;+ thumbnailUrl: string | null;+};++export async function readVideoFile({+ file,+}: {+ file: File;+}): Promise<VideoFileData> {const input = new Input({- source: new BlobSource(videoFile),+ source: new BlobSource(file),formats: ALL_FORMATS,});- const duration = await input.computeDuration();- const videoTrack = await input.getPrimaryVideoTrack();+ try {+ const duration = await input.computeDuration();+ const videoTrack = await input.getPrimaryVideoTrack();- if (!videoTrack) {- throw new Error("No video track found in the file");- }+ if (!videoTrack) {+ throw new Error("No video track found in the file");+ }++ const canDecode = await videoTrack.canDecode();+ const packetStats = await videoTrack.computePacketStats(100);+ const audioTrack = await input.getPrimaryAudioTrack();++ let thumbnailUrl: string | null = null;+ if (canDecode) {+ const sink = new VideoSampleSink(videoTrack);+ const frame = await sink.getSample(1);+ if (frame) {+ try {+ thumbnailUrl = renderThumbnailDataUrl({+ width: videoTrack.displayWidth,+ height: videoTrack.displayHeight,+ draw: ({ context, width, height }) => {+ frame.draw(context, 0, 0, width, height);+ },+ });+ } finally {+ frame.close();+ }+ }+ }- const packetStats = await videoTrack.computePacketStats(100);- const fps = packetStats.averagePacketRate;- const audioTrack = await input.getPrimaryAudioTrack();-- return {- duration,- width: videoTrack.displayWidth,- height: videoTrack.displayHeight,- fps,- hasAudio: audioTrack !== null,- };+ return {+ duration,+ width: videoTrack.displayWidth,+ height: videoTrack.displayHeight,+ fps: packetStats.averagePacketRate,+ hasAudio: audioTrack !== null,+ codec: videoTrack.codec,+ canDecode,+ thumbnailUrl,+ };+ } finally {+ input.dispose();+ }}const SAMPLE_RATE = 44100;diff --git a/apps/web/src/media/processing.ts b/apps/web/src/media/processing.tsindex d8d483f8..34ccadbc 100644--- a/apps/web/src/media/processing.ts+++ b/apps/web/src/media/processing.ts@@ -1,15 +1,25 @@-import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";import { toast } from "sonner";import { getMediaTypeFromFile } from "@/media/media-utils";import { formatStorageBytes } from "@/services/storage/quota";import { storageService } from "@/services/storage/service";import type { MediaAsset } from "@/media/types";-import { getVideoInfo } from "./mediabunny";+import { readVideoFile } from "./mediabunny";+import type { VideoFileData } from "./mediabunny";+import { renderThumbnailDataUrl } from "./thumbnail";export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}-const THUMBNAIL_MAX_WIDTH = 1280;-const THUMBNAIL_MAX_HEIGHT = 720;+const getUnsupportedVideoDescription = ({+ codec,+}: {+ codec: VideoFileData["codec"];+}): string => {+ const codecLabel = codec ? codec.toUpperCase() : "this video codec";++ return codec === "hevc"+ ? `${codecLabel} cannot be decoded in this browser, so this clip may not preview correctly. Convert it to H.264 MP4 or try importing it in Safari.`+ : `${codecLabel} cannot be decoded in this browser, so this clip may not preview correctly. Convert it to H.264 MP4 and reimport it.`;+};const getStorageLimitDescription = ({fileSize,@@ -29,103 +39,6 @@ const getStorageLimitDescription = ({})} is safely available in browser storage.`;};-const getThumbnailSize = ({- width,- height,-}: {- width: number;- height: number;-}): { width: number; height: number } => {- const aspectRatio = width / height;- let targetWidth = width;- let targetHeight = height;-- if (targetWidth > THUMBNAIL_MAX_WIDTH) {- targetWidth = THUMBNAIL_MAX_WIDTH;- targetHeight = Math.round(targetWidth / aspectRatio);- }- if (targetHeight > THUMBNAIL_MAX_HEIGHT) {- targetHeight = THUMBNAIL_MAX_HEIGHT;- targetWidth = Math.round(targetHeight * aspectRatio);- }-- return { width: targetWidth, height: targetHeight };-};--const renderToThumbnailDataUrl = ({- width,- height,- draw,-}: {- width: number;- height: number;- draw: ({- context,- width,- height,- }: {- context: CanvasRenderingContext2D;- width: number;- height: number;- }) => void;-}): string => {- const size = getThumbnailSize({ width, height });- const canvas = document.createElement("canvas");- canvas.width = size.width;- canvas.height = size.height;- const context = canvas.getContext("2d");-- if (!context) {- throw new Error("Could not get canvas context");- }-- draw({ context, width: size.width, height: size.height });- return canvas.toDataURL("image/jpeg", 0.8);-};--async function generateThumbnail({- videoFile,- timeInSeconds,-}: {- videoFile: File;- timeInSeconds: number;-}): Promise<string> {- const input = new Input({- source: new BlobSource(videoFile),- formats: ALL_FORMATS,- });-- const videoTrack = await input.getPrimaryVideoTrack();- if (!videoTrack) {- throw new Error("No video track found in the file");- }-- const canDecode = await videoTrack.canDecode();- if (!canDecode) {- throw new Error("Video codec not supported for decoding");- }-- const sink = new VideoSampleSink(videoTrack);-- const frame = await sink.getSample(timeInSeconds);-- if (!frame) {- throw new Error("Could not get frame at specified time");- }-- try {- return renderToThumbnailDataUrl({- width: videoTrack.displayWidth,- height: videoTrack.displayHeight,- draw: ({ context, width, height }) => {- frame.draw(context, 0, 0, width, height);- },- });- } finally {- frame.close();- }-}-async function generateImageThumbnail({imageFile,}: {@@ -137,7 +50,7 @@ async function generateImageThumbnail({image.addEventListener("load", () => {try {- const thumbnailUrl = renderToThumbnailDataUrl({+ const thumbnailUrl = renderThumbnailDataUrl({width: image.naturalWidth,height: image.naturalHeight,draw: ({ context, width, height }) => {@@ -220,24 +133,34 @@ export async function processMediaAssets({height = result.height;} else if (fileType === "video") {try {- const videoInfo = await getVideoInfo({ videoFile: file });- duration = videoInfo.duration;- width = videoInfo.width;- height = videoInfo.height;- fps = Number.isFinite(videoInfo.fps)- ? Math.round(videoInfo.fps)+ const videoData = await readVideoFile({ file });+ duration = videoData.duration;+ width = videoData.width;+ height = videoData.height;+ fps = Number.isFinite(videoData.fps)+ ? Math.round(videoData.fps): undefined;- hasAudio = videoInfo.hasAudio;+ hasAudio = videoData.hasAudio;+ thumbnailUrl = videoData.thumbnailUrl ?? undefined;++ if (!videoData.canDecode) {+ toast.error(`Can't preview ${file.name}`, {+ description: getUnsupportedVideoDescription({+ codec: videoData.codec,+ }),+ });+ }+ } catch (error) {+ const message =+ error instanceof Error+ ? error.message+ : "Could not process video";- thumbnailUrl = await generateThumbnail({- videoFile: file,- timeInSeconds: 1,+ toast.error(`Couldn't process ${file.name}`, {+ description: message,});- } catch (error) {- console.warn("Video processing failed", error);}} else if (fileType === "audio") {- // For audio, we don't set width/height/fps (they'll be undefined)duration = await getMediaDuration({ file });}@@ -264,7 +187,7 @@ export async function processMediaAssets({} catch (error) {console.error("Error processing file:", file.name, error);toast.error(`Failed to process ${file.name}`);- URL.revokeObjectURL(url); // Clean up on error+ URL.revokeObjectURL(url);}}diff --git a/apps/web/src/media/thumbnail.ts b/apps/web/src/media/thumbnail.tsnew file mode 100644index 00000000..e71077d9--- /dev/null+++ b/apps/web/src/media/thumbnail.ts@@ -0,0 +1,56 @@+const THUMBNAIL_MAX_WIDTH = 1280;+const THUMBNAIL_MAX_HEIGHT = 720;++export function thumbnailSize({+ width,+ height,+}: {+ width: number;+ height: number;+}): { width: number; height: number } {+ const aspectRatio = width / height;+ let targetWidth = width;+ let targetHeight = height;++ if (targetWidth > THUMBNAIL_MAX_WIDTH) {+ targetWidth = THUMBNAIL_MAX_WIDTH;+ targetHeight = Math.round(targetWidth / aspectRatio);+ }+ if (targetHeight > THUMBNAIL_MAX_HEIGHT) {+ targetHeight = THUMBNAIL_MAX_HEIGHT;+ targetWidth = Math.round(targetHeight * aspectRatio);+ }++ return { width: targetWidth, height: targetHeight };+}++export function renderThumbnailDataUrl({+ width,+ height,+ draw,+}: {+ width: number;+ height: number;+ draw: ({+ context,+ width,+ height,+ }: {+ context: CanvasRenderingContext2D;+ width: number;+ height: number;+ }) => void;+}): string {+ const size = thumbnailSize({ width, height });+ const canvas = document.createElement("canvas");+ canvas.width = size.width;+ canvas.height = size.height;+ const context = canvas.getContext("2d");++ if (!context) {+ throw new Error("Could not get canvas context");+ }++ draw({ context, width: size.width, height: size.height });+ return canvas.toDataURL("image/jpeg", 0.8);+}diff --git a/apps/web/src/services/video-cache/service.ts b/apps/web/src/services/video-cache/service.tsindex e5f3991d..31467065 100644--- a/apps/web/src/services/video-cache/service.ts+++ b/apps/web/src/services/video-cache/service.ts@@ -7,6 +7,7 @@ import {} from "mediabunny";interface VideoSinkData {+ input: Input;sink: CanvasSink;iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null;currentFrame: WrappedCanvas | null;@@ -262,12 +263,12 @@ export class VideoCache {mediaId: string;file: File;}): Promise<void> {- try {- const input = new Input({- source: new BlobSource(file),- formats: ALL_FORMATS,- });+ const input = new Input({+ source: new BlobSource(file),+ formats: ALL_FORMATS,+ });+ try {const videoTrack = await input.getPrimaryVideoTrack();if (!videoTrack) {throw new Error("No video track found");@@ -284,6 +285,7 @@ export class VideoCache {});this.sinks.set(mediaId, {+ input,sink,iterator: null,currentFrame: null,@@ -293,6 +295,7 @@ export class VideoCache {prefetchPromise: null,});} catch (error) {+ input.dispose();console.error(`Failed to initialize video sink for ${mediaId}:`, error);throw error;}@@ -305,6 +308,7 @@ export class VideoCache {void sinkData.iterator.return();}+ sinkData.input.dispose();this.sinks.delete(mediaId);}