Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

Recover Lightning decoder failures with a software retry

webadderall3aa683970189
diff --git a/src/lib/exporter/streamingDecodePipeline.ts b/src/lib/exporter/streamingDecodePipeline.ts
index acd24c80..d3ed0390 100644
--- a/src/lib/exporter/streamingDecodePipeline.ts
+++ b/src/lib/exporter/streamingDecodePipeline.ts
@@ -39,12 +39,70 @@ const STARTUP_MAX_PENDING_FRAMES = 28;
export async function decodeVideoStream(
context: StreamingDecodeContext,
+ targetFrameRate: number,
+ trimRegions: TrimRegion[] | undefined,
+ speedRegions: SpeedRegion[] | undefined,
+ onFrame: OnFrameCallback,
+ segmentsOverride?: VideoSegment[],
+): Promise<void> {
+ let emittedFrames = 0;
+ try {
+ await decodeVideoStreamAttempt(
+ context,
+ targetFrameRate,
+ trimRegions,
+ speedRegions,
+ async (...args) => {
+ await onFrame(...args);
+ emittedFrames++;
+ },
+ segmentsOverride,
+ );
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ const codec = context.metadata.codec.toLowerCase();
+ // Only retry failures that completed the decode-error cleanup. Configuration
+ // and renderer/encoder callback failures must propagate without replay.
+ if (
+ context.cancelled ||
+ context.decoder !== null ||
+ codec.includes("av01") ||
+ codec.includes("av1") ||
+ !/^\[VIDEO_DECODE_ENCODING_ERROR\]|^\[VIDEO_DECODER_RESOURCE_EXHAUSTED\]/.test(message)
+ ) {
+ throw error;
+ }
+ console.warn(
+ "[StreamingVideoDecoder] Decoder failed; replaying with software decoding",
+ error,
+ );
+ // Replay from a keyframe to rebuild decoder state. Output already handed to
+ // the renderer/encoder must not be submitted again.
+ let replayedFrames = 0;
+ await decodeVideoStreamAttempt(
+ context,
+ targetFrameRate,
+ trimRegions,
+ speedRegions,
+ async (...args) => {
+ if (replayedFrames++ < emittedFrames) return;
+ await onFrame(...args);
+ },
+ segmentsOverride,
+ true,
+ );
+ }
+}
+
+async function decodeVideoStreamAttempt(
+ context: StreamingDecodeContext,
targetFrameRate: number,
trimRegions: TrimRegion[] | undefined,
speedRegions: SpeedRegion[] | undefined,
onFrame: OnFrameCallback,
segmentsOverride?: VideoSegment[],
+ forceSoftwareDecode = false,
): Promise<void> {
if (!context.demuxer || !context.metadata) {
throw new Error("Must call loadMetadata() before decodeAll()");
@@ -52,7 +110,8 @@ export async function decodeVideoStream(
const decoderConfig = await context.demuxer.getDecoderConfig("video");
const codec = context.metadata.codec.toLowerCase();
- const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
+ const shouldPreferSoftwareDecode =
+ forceSoftwareDecode || codec.includes("av01") || codec.includes("av1");
const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
duration: context.metadata.duration,
streamDuration: context.metadata.streamDuration,
@@ -157,7 +216,7 @@ export async function decodeVideoStream(
try {
context.decoder.configure(preferredDecoderConfig);
} catch (error) {
- if (!shouldPreferSoftwareDecode) {
+ if (!shouldPreferSoftwareDecode || forceSoftwareDecode) {
throw buildVideoDecodeFailure(error, getDecoderFailureContext());
}
// Fall back to default decoder config if software preference is unsupported.
diff --git a/src/lib/exporter/streamingDecoder.test.ts b/src/lib/exporter/streamingDecoder.test.ts
index d8d2c9e5..e11c6e61 100644
--- a/src/lib/exporter/streamingDecoder.test.ts
+++ b/src/lib/exporter/streamingDecoder.test.ts
@@ -139,18 +139,19 @@ describe("StreamingVideoDecoder decode failures", () => {
codedWidth: 1920,
codedHeight: 1080,
});
- mockDemuxerRead.mockReturnValue(
- new ReadableStream({
- start(controller) {
- controller.enqueue({
- type: "key",
- timestamp: 0,
- duration: 33_333,
- byteLength: 4,
- });
- controller.close();
- },
- }),
+ mockDemuxerRead.mockImplementation(
+ () =>
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue({
+ type: "key",
+ timestamp: 0,
+ duration: 33_333,
+ byteLength: 4,
+ });
+ controller.close();
+ },
+ }),
);
Object.assign(globalThis, {
window: {
@@ -168,7 +169,7 @@ describe("StreamingVideoDecoder decode failures", () => {
});
it("does not emit frozen remaining frames after a decoder error", async () => {
- const frame = { timestamp: 0, close: vi.fn() } as unknown as VideoFrame;
+ const frames: Array<{ timestamp: number; close: ReturnType<typeof vi.fn> }> = [];
class FailingVideoDecoder {
state: CodecState = "unconfigured";
decodeQueueSize = 0;
@@ -182,7 +183,9 @@ describe("StreamingVideoDecoder decode failures", () => {
this.state = "configured";
}
decode() {
- this.callbacks.output(frame);
+ const frame = { timestamp: 0, close: vi.fn() };
+ frames.push(frame);
+ this.callbacks.output(frame as unknown as VideoFrame);
this.callbacks.error(new DOMException("bad frame", "EncodingError"));
}
async flush() {}
@@ -200,7 +203,131 @@ describe("StreamingVideoDecoder decode failures", () => {
"[VIDEO_DECODE_ENCODING_ERROR]",
);
expect(onFrame).not.toHaveBeenCalled();
- expect(frame.close).toHaveBeenCalledTimes(1);
+ expect(frames).toHaveLength(2);
+ expect(mockDemuxerRead).toHaveBeenCalledTimes(2);
+ for (const frame of frames) expect(frame.close).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([
+ "EncodingError",
+ "QuotaExceededError",
+ ])("replays %s in software without duplicating exported frames", async (errorName) => {
+ mockDemuxerRead.mockImplementation(
+ () =>
+ new ReadableStream({
+ start(controller) {
+ for (let i = 0; i < 120; i++)
+ controller.enqueue({ timestamp: (i * 1_000_000) / 30 });
+ controller.close();
+ },
+ }),
+ );
+ const configs: VideoDecoderConfig[] = [];
+ const timestamps: number[] = [];
+ let framesBeforeRetry = 0;
+ const closedFrames: ReturnType<typeof vi.fn>[] = [];
+ class RecoveringDecoder {
+ state: CodecState = "unconfigured";
+ decodeQueueSize = 0;
+ software = false;
+ constructor(
+ private callbacks: {
+ output: (frame: VideoFrame) => void;
+ error: (error: DOMException) => void;
+ },
+ ) {}
+ configure(config: VideoDecoderConfig) {
+ configs.push(config);
+ this.software = config.hardwareAcceleration === "prefer-software";
+ if (this.software) framesBeforeRetry = timestamps.length;
+ this.state = "configured";
+ }
+ decode(chunk: EncodedVideoChunk) {
+ if (!this.software && chunk.timestamp >= 500_000) {
+ this.state = "closed";
+ this.callbacks.error(new DOMException("driver decode failed", errorName));
+ return;
+ }
+ const close = vi.fn();
+ closedFrames.push(close);
+ this.callbacks.output({
+ timestamp: chunk.timestamp,
+ close,
+ } as unknown as VideoFrame);
+ }
+ async flush() {}
+ close() {
+ this.state = "closed";
+ }
+ }
+ vi.stubGlobal("VideoDecoder", RecoveringDecoder);
+ const decoder = new StreamingVideoDecoder();
+ await decoder.loadMetadata("/tmp/driver-failure.mp4");
+ await decoder.decodeAll(30, undefined, undefined, async (_frame, timestamp) => {
+ timestamps.push(timestamp);
+ });
+ expect(configs).toHaveLength(2);
+ expect(configs[1].hardwareAcceleration).toBe("prefer-software");
+ expect(framesBeforeRetry).toBeGreaterThan(0);
+ expect(timestamps).toHaveLength(120);
+ for (let i = 0; i < timestamps.length; i++)
+ expect(timestamps[i]).toBeCloseTo((i * 1_000_000) / 30);
+ expect(mockDemuxerRead).toHaveBeenCalledTimes(2);
+ for (const close of closedFrames) expect(close).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([
+ {
+ codec: "av01.0.04M.08",
+ errorName: "EncodingError",
+ cancelled: false,
+ setupFailure: false,
+ },
+ { codec: "avc1.640034", errorName: "DataError", cancelled: false, setupFailure: false },
+ { codec: "avc1.640034", errorName: "EncodingError", cancelled: true, setupFailure: false },
+ { codec: "avc1.640034", errorName: "EncodingError", cancelled: false, setupFailure: true },
+ ])("does not retry $codec / $errorName (cancelled: $cancelled, setup: $setupFailure)", async ({
+ codec,
+ errorName,
+ cancelled,
+ setupFailure,
+ }) => {
+ const info = await mockDemuxerGetMediaInfo();
+ mockDemuxerGetMediaInfo.mockResolvedValueOnce({
+ ...info,
+ streams: info.streams.map((stream: Record<string, unknown>) => ({
+ ...stream,
+ codec_string: codec,
+ })),
+ });
+ const configure = vi.fn();
+ const decoder = new StreamingVideoDecoder();
+ class NonRetryDecoder {
+ state: CodecState = "unconfigured";
+ decodeQueueSize = 0;
+ constructor(private callbacks: { error: (error: DOMException) => void }) {}
+ configure() {
+ configure();
+ if (setupFailure) throw new DOMException("setup failed", errorName);
+ this.state = "configured";
+ }
+ decode() {
+ if (cancelled) decoder.cancel();
+ this.state = "closed";
+ this.callbacks.error(new DOMException("decode failed", errorName));
+ }
+ async flush() {}
+ close() {
+ this.state = "closed";
+ }
+ }
+ vi.stubGlobal("VideoDecoder", NonRetryDecoder);
+ await decoder.loadMetadata("/tmp/no-retry.mp4");
+ await expect(decoder.decodeAll(30, undefined, undefined, vi.fn())).rejects.toThrow(
+ setupFailure ? "setup failed" : "decode failed",
+ );
+ expect(configure).toHaveBeenCalledTimes(1);
+ decoder.destroy();
});
it.each([