Skip to content

🎥 Make videos programmatically with React

Preserve original media format detection errors through fallback handling

Jonny Burgerefb18488ea7c
diff --git a/packages/media/src/audio/audio-for-preview.tsx b/packages/media/src/audio/audio-for-preview.tsx
index e01457f545..5f1922b3d8 100644
--- a/packages/media/src/audio/audio-for-preview.tsx
+++ b/packages/media/src/audio/audio-for-preview.tsx
@@ -296,20 +296,12 @@ const AudioForPreviewAssertedShowing: React.FC<NewAudioForPreviewProps> = ({
}
};
- if (result.type === 'unknown-container-format') {
- handleError(
- new Error(`Unknown container format ${preloadedSrc}.`),
- `Unknown container format for ${preloadedSrc} (Supported formats: https://www.remotion.dev/docs/mediabunny/formats), falling back to <Html5Audio>`,
- );
- return;
- }
-
- if (result.type === 'network-error') {
- handleError(
- new Error(`Network error fetching ${preloadedSrc}.`),
- `Network error fetching ${preloadedSrc}, falling back to <Html5Audio>`,
- );
- return;
+ if (
+ result.type === 'unknown-container-format' ||
+ result.type === 'network-error'
+ ) {
+ // The catch below applies onError and the existing fallback policy.
+ throw result.error;
}
if (result.type === 'cannot-decode') {
diff --git a/packages/media/src/audio/audio-for-rendering.tsx b/packages/media/src/audio/audio-for-rendering.tsx
index 93439ea759..b6b23b2839 100644
--- a/packages/media/src/audio/audio-for-rendering.tsx
+++ b/packages/media/src/audio/audio-for-rendering.tsx
@@ -166,6 +166,19 @@ export const AudioForRendering: React.FC<AudioProps> = ({
setReplaceWithHtml5Audio(true);
};
+ if (
+ (result.type === 'unknown-container-format' ||
+ result.type === 'network-error') &&
+ result.error
+ ) {
+ handleError(
+ result.error,
+ result.error,
+ `Failed to read ${src}: ${result.error.message}, falling back to <Html5Audio>`,
+ );
+ return;
+ }
+
if (result.type === 'unknown-container-format') {
handleError(
new Error(`Unknown container format ${src}.`),
diff --git a/packages/media/src/extract-frame-and-audio.ts b/packages/media/src/extract-frame-and-audio.ts
index 413db2282b..f4a596f3c7 100644
--- a/packages/media/src/extract-frame-and-audio.ts
+++ b/packages/media/src/extract-frame-and-audio.ts
@@ -45,6 +45,24 @@ export const extractFrameAndAudio = async ({
mediaCache: MediaCache;
}): Promise<ExtractFrameViaBroadcastChannelResult> => {
try {
+ if (includeVideo || includeAudio) {
+ const sink = await mediaCache.sinkManager.getSink(
+ src,
+ logLevel,
+ credentials,
+ requestInit,
+ );
+ // Handle format detection before extraction, including duration reads for loops.
+ if (sink.formatDetectionError) {
+ return {
+ type: isNetworkError(sink.formatDetectionError)
+ ? 'network-error'
+ : 'unknown-container-format',
+ error: sink.formatDetectionError,
+ };
+ }
+ }
+
const [video, audio] = await Promise.all([
includeVideo
? extractFrame({
@@ -98,7 +116,7 @@ export const extractFrameAndAudio = async ({
}
if (video?.type === 'unknown-container-format') {
- return {type: 'unknown-container-format'};
+ return {type: 'unknown-container-format', error: null};
}
if (video?.type === 'cannot-decode-alpha') {
@@ -109,15 +127,15 @@ export const extractFrameAndAudio = async ({
}
if (video?.type === 'network-error') {
- return {type: 'network-error'};
+ return {type: 'network-error', error: null};
}
if (audio === 'unknown-container-format') {
- return {type: 'unknown-container-format'};
+ return {type: 'unknown-container-format', error: null};
}
if (audio === 'network-error') {
- return {type: 'network-error'};
+ return {type: 'network-error', error: null};
}
if (audio === 'cannot-decode') {
@@ -142,7 +160,7 @@ export const extractFrameAndAudio = async ({
} catch (err) {
const error = err as Error;
if (isNetworkError(error)) {
- return {type: 'network-error'};
+ return {type: 'network-error', error};
}
throw err;
diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.ts
index 1218095cff..b06748a9b7 100644
--- a/packages/media/src/media-player.ts
+++ b/packages/media/src/media-player.ts
@@ -34,10 +34,10 @@ import type {MaxCanvasSinkFrameSize} from './video/props';
export type MediaPlayerInitResult =
| {type: 'success'; durationInSeconds: number}
- | {type: 'unknown-container-format'}
+ | {type: 'unknown-container-format'; error: Error}
| {type: 'cannot-decode'}
| {type: 'cannot-decode-prores'}
- | {type: 'network-error'}
+ | {type: 'network-error'; error: Error}
| {type: 'no-tracks'}
| {type: 'disposed'};
@@ -324,7 +324,7 @@ export class MediaPlayer {
error,
);
- return {type: 'unknown-container-format'};
+ return {type: 'unknown-container-format', error: err};
}
const [durationInSeconds, videoTrack, audioTracks] = await Promise.all([
@@ -490,7 +490,7 @@ export class MediaPlayer {
`[MediaPlayer] Network/CORS error for ${this.src}`,
err,
);
- return {type: 'network-error'};
+ return {type: 'network-error', error: err};
}
Internals.Log.error(
diff --git a/packages/media/src/video-extraction/add-broadcast-channel-listener.ts b/packages/media/src/video-extraction/add-broadcast-channel-listener.ts
index 8578809181..7426a6df09 100644
--- a/packages/media/src/video-extraction/add-broadcast-channel-listener.ts
+++ b/packages/media/src/video-extraction/add-broadcast-channel-listener.ts
@@ -35,10 +35,12 @@ export type MessageFromMainTab =
| {
type: 'response-network-error';
id: string;
+ error: {name: string; message: string; stack: string | null} | null;
}
| {
type: 'response-unknown-container-format';
id: string;
+ error: {name: string; message: string; stack: string | null} | null;
}
| {
type: 'main-tab-ready';
@@ -155,25 +157,26 @@ export const addBroadcastChannelListener = () => {
return;
}
- if (result.type === 'network-error') {
- const networkErrorResponse: MessageFromMainTab = {
- type: 'response-network-error',
+ if (
+ result.type === 'network-error' ||
+ result.type === 'unknown-container-format'
+ ) {
+ const mediaErrorResponse: MessageFromMainTab = {
+ type:
+ result.type === 'network-error'
+ ? 'response-network-error'
+ : 'response-unknown-container-format',
id: data.id,
+ error: result.error
+ ? {
+ name: result.error.name,
+ message: result.error.message,
+ stack: result.error.stack ?? null,
+ }
+ : null,
};
- window.remotion_broadcastChannel!.postMessage(networkErrorResponse);
- return;
- }
-
- if (result.type === 'unknown-container-format') {
- const unknownContainerFormatResponse: MessageFromMainTab = {
- type: 'response-unknown-container-format',
- id: data.id,
- };
-
- window.remotion_broadcastChannel!.postMessage(
- unknownContainerFormatResponse,
- );
+ window.remotion_broadcastChannel!.postMessage(mediaErrorResponse);
return;
}
diff --git a/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts b/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts
index 46642327fb..b10e2d8d65 100644
--- a/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts
+++ b/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts
@@ -25,8 +25,8 @@ export type ExtractFrameViaBroadcastChannelResult =
| {type: 'cannot-decode'; durationInSeconds: number | null}
| {type: 'cannot-decode-prores'; durationInSeconds: number | null}
| {type: 'cannot-decode-alpha'; durationInSeconds: number | null}
- | {type: 'network-error'}
- | {type: 'unknown-container-format'};
+ | {type: 'network-error'; error: Error | null}
+ | {type: 'unknown-container-format'; error: Error | null};
addBroadcastChannelListener();
@@ -105,8 +105,8 @@ export const extractFrameViaBroadcastChannel = async ({
| {type: 'cannot-decode'; durationInSeconds: number | null}
| {type: 'cannot-decode-prores'; durationInSeconds: number | null}
| {type: 'cannot-decode-alpha'; durationInSeconds: number | null}
- | {type: 'network-error'}
- | {type: 'unknown-container-format'}
+ | {type: 'network-error'; error: Error | null}
+ | {type: 'unknown-container-format'; error: Error | null}
>((resolve, reject) => {
const onMessage = (event: MessageEvent) => {
const data = event.data as MessageFromMainTab;
@@ -172,17 +172,23 @@ export const extractFrameViaBroadcastChannel = async ({
return;
}
- if (data.type === 'response-network-error') {
- resolve({type: 'network-error'});
- window.remotion_broadcastChannel!.removeEventListener(
- 'message',
- onMessage,
- );
- return;
- }
+ if (
+ data.type === 'response-network-error' ||
+ data.type === 'response-unknown-container-format'
+ ) {
+ const error = data.error ? new Error(data.error.message) : null;
+ if (error && data.error) {
+ error.name = data.error.name;
+ error.stack = data.error.stack ?? undefined;
+ }
- if (data.type === 'response-unknown-container-format') {
- resolve({type: 'unknown-container-format'});
+ resolve({
+ type:
+ data.type === 'response-network-error'
+ ? 'network-error'
+ : 'unknown-container-format',
+ error,
+ });
window.remotion_broadcastChannel!.removeEventListener(
'message',
onMessage,
diff --git a/packages/media/src/video-extraction/get-frames-since-keyframe.ts b/packages/media/src/video-extraction/get-frames-since-keyframe.ts
index 12ace17ba4..7d5631297c 100644
--- a/packages/media/src/video-extraction/get-frames-since-keyframe.ts
+++ b/packages/media/src/video-extraction/get-frames-since-keyframe.ts
@@ -45,20 +45,6 @@ export type VideoSinkResult =
| 'unknown-container-format'
| 'network-error';
-const getFormatOrNullOrNetworkError = async (
- input: Input,
-): Promise<InputFormat | 'network-error' | null> => {
- try {
- return await input.getFormat();
- } catch (err) {
- if (isNetworkError(err as Error)) {
- return 'network-error';
- }
-
- return null;
- }
-};
-
export const makeSinks = (
src: string,
logLevel: LogLevel,
@@ -81,11 +67,20 @@ export const makeSinks = (
}),
});
const getSinks = async () => {
- const format = await getFormatOrNullOrNetworkError(input);
+ let format: InputFormat | null = null;
+ let formatDetectionError: Error | null = null;
+ try {
+ format = await input.getFormat();
+ } catch (error) {
+ formatDetectionError = error as Error;
+ }
+
+ const isNetworkFailure =
+ formatDetectionError !== null && isNetworkError(formatDetectionError);
const isMatroska = format === MATROSKA || format === WEBM;
const getVideoSinks = async (): Promise<VideoSinkResult> => {
- if (format === 'network-error') {
+ if (isNetworkFailure) {
return 'network-error';
}
@@ -164,12 +159,12 @@ export const makeSinks = (
const getAudioSinks = async (
index: number | null,
): Promise<AudioSinkResult> => {
- if (format === null) {
- return 'unknown-container-format';
+ if (isNetworkFailure) {
+ return 'network-error';
}
- if (format === 'network-error') {
- return 'network-error';
+ if (format === null) {
+ return 'unknown-container-format';
}
const [videoTrack, audioTracks] = await Promise.all([
@@ -211,6 +206,7 @@ export const makeSinks = (
};
return {
+ formatDetectionError,
getVideo: () => getVideoSinksPromise(),
getAudio: (index: number | null) => getAudioSinksPromise(index),
actualMatroskaTimestamps: rememberActualMatroskaTimestamps(isMatroska),
diff --git a/packages/media/src/video/video-for-preview.tsx b/packages/media/src/video/video-for-preview.tsx
index fb35cba543..b44a47ae47 100644
--- a/packages/media/src/video/video-for-preview.tsx
+++ b/packages/media/src/video/video-for-preview.tsx
@@ -366,20 +366,12 @@ const VideoForPreviewAssertedShowing: React.FC<
setShouldFallbackToNativeVideo(true);
};
- if (result.type === 'unknown-container-format') {
- handleError(
- new Error(`Unknown container format ${preloadedSrc}.`),
- `Unknown container format for ${preloadedSrc} (Supported formats: https://www.remotion.dev/docs/mediabunny/formats), falling back to <OffthreadVideo>`,
- );
- return;
- }
-
- if (result.type === 'network-error') {
- handleError(
- new Error(`Network error fetching ${preloadedSrc}.`),
- `Network error fetching ${preloadedSrc}, falling back to <OffthreadVideo>`,
- );
- return;
+ if (
+ result.type === 'unknown-container-format' ||
+ result.type === 'network-error'
+ ) {
+ // The catch below applies onError and the existing fallback policy.
+ throw result.error;
}
if (result.type === 'cannot-decode') {
diff --git a/packages/media/src/video/video-for-rendering.tsx b/packages/media/src/video/video-for-rendering.tsx
index 8a0751e2b6..ab4a1164f9 100644
--- a/packages/media/src/video/video-for-rendering.tsx
+++ b/packages/media/src/video/video-for-rendering.tsx
@@ -252,6 +252,20 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({
});
};
+ if (
+ (result.type === 'unknown-container-format' ||
+ result.type === 'network-error') &&
+ result.error
+ ) {
+ handleError(
+ result.error,
+ result.error,
+ `Failed to read ${src}: ${result.error.message}, falling back to <OffthreadVideo>`,
+ null,
+ );
+ return;
+ }
+
if (result.type === 'unknown-container-format') {
handleError(
new Error(`Unknown container format ${src}.`),