🎥 Make videos programmatically with React
Preserve original media format detection errors through fallback handling
Jonny Burgerefb18488ea7c
diff --git a/packages/media/src/audio/audio-for-preview.tsx b/packages/media/src/audio/audio-for-preview.tsxindex e01457f545..5f1922b3d8 100644--- a/packages/media/src/audio/audio-for-preview.tsx+++ b/packages/media/src/audio/audio-for-preview.tsx@@ -296,20 +296,12 @@ const AudioForPreviewAssertedShowing: React.FC<NewAudioForPreviewProps> = ({}};- if (result.type === 'unknown-container-format') {- handleError(- new Error(`Unknown container format ${preloadedSrc}.`),- `Unknown container format for ${preloadedSrc} (Supported formats: https://www.remotion.dev/docs/mediabunny/formats), falling back to <Html5Audio>`,- );- return;- }-- if (result.type === 'network-error') {- handleError(- new Error(`Network error fetching ${preloadedSrc}.`),- `Network error fetching ${preloadedSrc}, falling back to <Html5Audio>`,- );- return;+ if (+ result.type === 'unknown-container-format' ||+ result.type === 'network-error'+ ) {+ // The catch below applies onError and the existing fallback policy.+ throw result.error;}if (result.type === 'cannot-decode') {diff --git a/packages/media/src/audio/audio-for-rendering.tsx b/packages/media/src/audio/audio-for-rendering.tsxindex 93439ea759..b6b23b2839 100644--- a/packages/media/src/audio/audio-for-rendering.tsx+++ b/packages/media/src/audio/audio-for-rendering.tsx@@ -166,6 +166,19 @@ export const AudioForRendering: React.FC<AudioProps> = ({setReplaceWithHtml5Audio(true);};+ if (+ (result.type === 'unknown-container-format' ||+ result.type === 'network-error') &&+ result.error+ ) {+ handleError(+ result.error,+ result.error,+ `Failed to read ${src}: ${result.error.message}, falling back to <Html5Audio>`,+ );+ return;+ }+if (result.type === 'unknown-container-format') {handleError(new Error(`Unknown container format ${src}.`),diff --git a/packages/media/src/extract-frame-and-audio.ts b/packages/media/src/extract-frame-and-audio.tsindex 413db2282b..f4a596f3c7 100644--- a/packages/media/src/extract-frame-and-audio.ts+++ b/packages/media/src/extract-frame-and-audio.ts@@ -45,6 +45,24 @@ export const extractFrameAndAudio = async ({mediaCache: MediaCache;}): Promise<ExtractFrameViaBroadcastChannelResult> => {try {+ if (includeVideo || includeAudio) {+ const sink = await mediaCache.sinkManager.getSink(+ src,+ logLevel,+ credentials,+ requestInit,+ );+ // Handle format detection before extraction, including duration reads for loops.+ if (sink.formatDetectionError) {+ return {+ type: isNetworkError(sink.formatDetectionError)+ ? 'network-error'+ : 'unknown-container-format',+ error: sink.formatDetectionError,+ };+ }+ }+const [video, audio] = await Promise.all([includeVideo? extractFrame({@@ -98,7 +116,7 @@ export const extractFrameAndAudio = async ({}if (video?.type === 'unknown-container-format') {- return {type: 'unknown-container-format'};+ return {type: 'unknown-container-format', error: null};}if (video?.type === 'cannot-decode-alpha') {@@ -109,15 +127,15 @@ export const extractFrameAndAudio = async ({}if (video?.type === 'network-error') {- return {type: 'network-error'};+ return {type: 'network-error', error: null};}if (audio === 'unknown-container-format') {- return {type: 'unknown-container-format'};+ return {type: 'unknown-container-format', error: null};}if (audio === 'network-error') {- return {type: 'network-error'};+ return {type: 'network-error', error: null};}if (audio === 'cannot-decode') {@@ -142,7 +160,7 @@ export const extractFrameAndAudio = async ({} catch (err) {const error = err as Error;if (isNetworkError(error)) {- return {type: 'network-error'};+ return {type: 'network-error', error};}throw err;diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.tsindex 1218095cff..b06748a9b7 100644--- a/packages/media/src/media-player.ts+++ b/packages/media/src/media-player.ts@@ -34,10 +34,10 @@ import type {MaxCanvasSinkFrameSize} from './video/props';export type MediaPlayerInitResult =| {type: 'success'; durationInSeconds: number}- | {type: 'unknown-container-format'}+ | {type: 'unknown-container-format'; error: Error}| {type: 'cannot-decode'}| {type: 'cannot-decode-prores'}- | {type: 'network-error'}+ | {type: 'network-error'; error: Error}| {type: 'no-tracks'}| {type: 'disposed'};@@ -324,7 +324,7 @@ export class MediaPlayer {error,);- return {type: 'unknown-container-format'};+ return {type: 'unknown-container-format', error: err};}const [durationInSeconds, videoTrack, audioTracks] = await Promise.all([@@ -490,7 +490,7 @@ export class MediaPlayer {`[MediaPlayer] Network/CORS error for ${this.src}`,err,);- return {type: 'network-error'};+ return {type: 'network-error', error: err};}Internals.Log.error(diff --git a/packages/media/src/video-extraction/add-broadcast-channel-listener.ts b/packages/media/src/video-extraction/add-broadcast-channel-listener.tsindex 8578809181..7426a6df09 100644--- a/packages/media/src/video-extraction/add-broadcast-channel-listener.ts+++ b/packages/media/src/video-extraction/add-broadcast-channel-listener.ts@@ -35,10 +35,12 @@ export type MessageFromMainTab =| {type: 'response-network-error';id: string;+ error: {name: string; message: string; stack: string | null} | null;}| {type: 'response-unknown-container-format';id: string;+ error: {name: string; message: string; stack: string | null} | null;}| {type: 'main-tab-ready';@@ -155,25 +157,26 @@ export const addBroadcastChannelListener = () => {return;}- if (result.type === 'network-error') {- const networkErrorResponse: MessageFromMainTab = {- type: 'response-network-error',+ if (+ result.type === 'network-error' ||+ result.type === 'unknown-container-format'+ ) {+ const mediaErrorResponse: MessageFromMainTab = {+ type:+ result.type === 'network-error'+ ? 'response-network-error'+ : 'response-unknown-container-format',id: data.id,+ error: result.error+ ? {+ name: result.error.name,+ message: result.error.message,+ stack: result.error.stack ?? null,+ }+ : null,};- window.remotion_broadcastChannel!.postMessage(networkErrorResponse);- return;- }-- if (result.type === 'unknown-container-format') {- const unknownContainerFormatResponse: MessageFromMainTab = {- type: 'response-unknown-container-format',- id: data.id,- };-- window.remotion_broadcastChannel!.postMessage(- unknownContainerFormatResponse,- );+ window.remotion_broadcastChannel!.postMessage(mediaErrorResponse);return;}diff --git a/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts b/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.tsindex 46642327fb..b10e2d8d65 100644--- a/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts+++ b/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts@@ -25,8 +25,8 @@ export type ExtractFrameViaBroadcastChannelResult =| {type: 'cannot-decode'; durationInSeconds: number | null}| {type: 'cannot-decode-prores'; durationInSeconds: number | null}| {type: 'cannot-decode-alpha'; durationInSeconds: number | null}- | {type: 'network-error'}- | {type: 'unknown-container-format'};+ | {type: 'network-error'; error: Error | null}+ | {type: 'unknown-container-format'; error: Error | null};addBroadcastChannelListener();@@ -105,8 +105,8 @@ export const extractFrameViaBroadcastChannel = async ({| {type: 'cannot-decode'; durationInSeconds: number | null}| {type: 'cannot-decode-prores'; durationInSeconds: number | null}| {type: 'cannot-decode-alpha'; durationInSeconds: number | null}- | {type: 'network-error'}- | {type: 'unknown-container-format'}+ | {type: 'network-error'; error: Error | null}+ | {type: 'unknown-container-format'; error: Error | null}>((resolve, reject) => {const onMessage = (event: MessageEvent) => {const data = event.data as MessageFromMainTab;@@ -172,17 +172,23 @@ export const extractFrameViaBroadcastChannel = async ({return;}- if (data.type === 'response-network-error') {- resolve({type: 'network-error'});- window.remotion_broadcastChannel!.removeEventListener(- 'message',- onMessage,- );- return;- }+ if (+ data.type === 'response-network-error' ||+ data.type === 'response-unknown-container-format'+ ) {+ const error = data.error ? new Error(data.error.message) : null;+ if (error && data.error) {+ error.name = data.error.name;+ error.stack = data.error.stack ?? undefined;+ }- if (data.type === 'response-unknown-container-format') {- resolve({type: 'unknown-container-format'});+ resolve({+ type:+ data.type === 'response-network-error'+ ? 'network-error'+ : 'unknown-container-format',+ error,+ });window.remotion_broadcastChannel!.removeEventListener('message',onMessage,diff --git a/packages/media/src/video-extraction/get-frames-since-keyframe.ts b/packages/media/src/video-extraction/get-frames-since-keyframe.tsindex 12ace17ba4..7d5631297c 100644--- a/packages/media/src/video-extraction/get-frames-since-keyframe.ts+++ b/packages/media/src/video-extraction/get-frames-since-keyframe.ts@@ -45,20 +45,6 @@ export type VideoSinkResult =| 'unknown-container-format'| 'network-error';-const getFormatOrNullOrNetworkError = async (- input: Input,-): Promise<InputFormat | 'network-error' | null> => {- try {- return await input.getFormat();- } catch (err) {- if (isNetworkError(err as Error)) {- return 'network-error';- }-- return null;- }-};-export const makeSinks = (src: string,logLevel: LogLevel,@@ -81,11 +67,20 @@ export const makeSinks = (}),});const getSinks = async () => {- const format = await getFormatOrNullOrNetworkError(input);+ let format: InputFormat | null = null;+ let formatDetectionError: Error | null = null;+ try {+ format = await input.getFormat();+ } catch (error) {+ formatDetectionError = error as Error;+ }++ const isNetworkFailure =+ formatDetectionError !== null && isNetworkError(formatDetectionError);const isMatroska = format === MATROSKA || format === WEBM;const getVideoSinks = async (): Promise<VideoSinkResult> => {- if (format === 'network-error') {+ if (isNetworkFailure) {return 'network-error';}@@ -164,12 +159,12 @@ export const makeSinks = (const getAudioSinks = async (index: number | null,): Promise<AudioSinkResult> => {- if (format === null) {- return 'unknown-container-format';+ if (isNetworkFailure) {+ return 'network-error';}- if (format === 'network-error') {- return 'network-error';+ if (format === null) {+ return 'unknown-container-format';}const [videoTrack, audioTracks] = await Promise.all([@@ -211,6 +206,7 @@ export const makeSinks = (};return {+ formatDetectionError,getVideo: () => getVideoSinksPromise(),getAudio: (index: number | null) => getAudioSinksPromise(index),actualMatroskaTimestamps: rememberActualMatroskaTimestamps(isMatroska),diff --git a/packages/media/src/video/video-for-preview.tsx b/packages/media/src/video/video-for-preview.tsxindex fb35cba543..b44a47ae47 100644--- a/packages/media/src/video/video-for-preview.tsx+++ b/packages/media/src/video/video-for-preview.tsx@@ -366,20 +366,12 @@ const VideoForPreviewAssertedShowing: React.FC<setShouldFallbackToNativeVideo(true);};- if (result.type === 'unknown-container-format') {- handleError(- new Error(`Unknown container format ${preloadedSrc}.`),- `Unknown container format for ${preloadedSrc} (Supported formats: https://www.remotion.dev/docs/mediabunny/formats), falling back to <OffthreadVideo>`,- );- return;- }-- if (result.type === 'network-error') {- handleError(- new Error(`Network error fetching ${preloadedSrc}.`),- `Network error fetching ${preloadedSrc}, falling back to <OffthreadVideo>`,- );- return;+ if (+ result.type === 'unknown-container-format' ||+ result.type === 'network-error'+ ) {+ // The catch below applies onError and the existing fallback policy.+ throw result.error;}if (result.type === 'cannot-decode') {diff --git a/packages/media/src/video/video-for-rendering.tsx b/packages/media/src/video/video-for-rendering.tsxindex 8a0751e2b6..ab4a1164f9 100644--- a/packages/media/src/video/video-for-rendering.tsx+++ b/packages/media/src/video/video-for-rendering.tsx@@ -252,6 +252,20 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({});};+ if (+ (result.type === 'unknown-container-format' ||+ result.type === 'network-error') &&+ result.error+ ) {+ handleError(+ result.error,+ result.error,+ `Failed to read ${src}: ${result.error.message}, falling back to <OffthreadVideo>`,+ null,+ );+ return;+ }+if (result.type === 'unknown-container-format') {handleError(new Error(`Unknown container format ${src}.`),