Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

14.5 KB
Raw Blame
import type { WebDemuxer } from "web-demuxer";
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
import {
	buildVideoDecodeFailure,
	buildVideoDecodeRecoveryFailure,
	type DecodedVideoInfo,
	getDecodedFrameTimelineOffsetUs,
	preserveFirstVideoDecodeFailure,
	type VideoDecodeFailureContext,
} from "./streamingDecoderSupport";
import {
	computeVideoSegments,
	segmentFrameCount,
	segmentSourceTime,
	splitVideoSegmentsBySpeed,
	type VideoSegment,
} from "./videoTimelineSegments";

type OnFrameCallback = (
	frame: VideoFrame,
	exportTimestampUs: number,
	sourceTimestampMs: number,
	cursorTimestampMs: number,
) => Promise<void>;

interface StreamingDecodeContext {
	demuxer: WebDemuxer;
	decoder: VideoDecoder | null;
	readonly cancelled: boolean;
	readonly metadata: DecodedVideoInfo;
	pendingFrames: VideoFrame[];
	readonly maxDecodeQueue: number;
	readonly maxPendingFrames: number;
}

const STARTUP_STABILIZATION_SECONDS = 1.25;
const STARTUP_MAX_DECODE_QUEUE = 12;
const STARTUP_MAX_PENDING_FRAMES = 28;

export async function decodeVideoStream(
	context: StreamingDecodeContext,
	targetFrameRate: number,
	trimRegions: TrimRegion[] | undefined,
	speedRegions: SpeedRegion[] | undefined,
	onFrame: OnFrameCallback,
	segmentsOverride?: VideoSegment[],
): Promise<void> {
	let emittedFrames = 0;
	try {
		await decodeVideoStreamAttempt(
			context,
			targetFrameRate,
			trimRegions,
			speedRegions,
			async (...args) => {
				await onFrame(...args);
				emittedFrames++;
			},
			segmentsOverride,
		);
	} catch (error) {
		const message = error instanceof Error ? error.message : String(error);
		const codec = context.metadata.codec.toLowerCase();
		// Only retry failures that completed the decode-error cleanup. Configuration
		// and renderer/encoder callback failures must propagate without replay.
		if (
			context.cancelled ||
			context.decoder !== null ||
			codec.includes("av01") ||
			codec.includes("av1") ||
			!/^\[VIDEO_DECODE_ENCODING_ERROR\]|^\[VIDEO_DECODER_RESOURCE_EXHAUSTED\]/.test(message)
		) {
			throw error;
		}
		console.warn(
			"[StreamingVideoDecoder] Decoder failed; replaying with software decoding",
			error,
		);
		// Replay from a keyframe to rebuild decoder state. Output already handed to
		// the renderer/encoder must not be submitted again.
		let replayedFrames = 0;
		try {
			await decodeVideoStreamAttempt(
				context,
				targetFrameRate,
				trimRegions,
				speedRegions,
				async (...args) => {
					if (replayedFrames++ < emittedFrames) return;
					await onFrame(...args);
				},
				segmentsOverride,
				true,
			);
		} catch (softwareError) {
			const softwareMessage =
				softwareError instanceof Error ? softwareError.message : String(softwareError);
			if (!/^\[VIDEO_(?:DECODE|DECODER|CODEC)_/.test(softwareMessage)) throw softwareError;
			throw buildVideoDecodeRecoveryFailure(error, softwareError);
		}
	}
}

async function decodeVideoStreamAttempt(
	context: StreamingDecodeContext,

	targetFrameRate: number,
	trimRegions: TrimRegion[] | undefined,
	speedRegions: SpeedRegion[] | undefined,
	onFrame: OnFrameCallback,
	segmentsOverride?: VideoSegment[],
	forceSoftwareDecode = false,
): Promise<void> {
	if (!context.demuxer || !context.metadata) {
		throw new Error("Must call loadMetadata() before decodeAll()");
	}

	const decoderConfig = await context.demuxer.getDecoderConfig("video");
	const codec = context.metadata.codec.toLowerCase();
	const shouldPreferSoftwareDecode =
		forceSoftwareDecode || codec.includes("av01") || codec.includes("av1");
	const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
		duration: context.metadata.duration,
		streamDuration: context.metadata.streamDuration,
	});
	const segments =
		segmentsOverride ??
		splitVideoSegmentsBySpeed(
			computeVideoSegments(effectiveVideoDuration, trimRegions),
			speedRegions,
		);
	const segmentOutputFrameCounts = segments.map((segment) =>
		segmentFrameCount(segment, targetFrameRate),
	);
	const expectedOutputFrames = segmentOutputFrameCounts.reduce((sum, count) => sum + count, 0);
	const frameDurationUs = 1_000_000 / targetFrameRate;
	const epsilonSec = 0.001;
	const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
	const startupFrameBudget = Math.max(
		1,
		Math.round(targetFrameRate * startupStabilizationSeconds),
	);
	let exportFrameIndex = 0;
	let loggedSteadyStateBackpressure = false;
	const backpressureWaiters = new Set<() => void>();

	const notifyBackpressureProgress = () => {
		if (backpressureWaiters.size === 0) {
			return;
		}

		const waiters = [...backpressureWaiters];
		backpressureWaiters.clear();
		for (const resolve of waiters) {
			resolve();
		}
	};

	const waitForBackpressureProgress = () =>
		new Promise<void>((resolve) => {
			backpressureWaiters.add(resolve);
		});

	console.log(
		`[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
	);

	// Async frame queue — decoder pushes, consumer pulls
	context.pendingFrames.length = 0;
	const pendingFrames = context.pendingFrames;
	let frameResolve: ((frame: VideoFrame | null) => void) | null = null;
	let decodeError: Error | null = null;
	let decodeDone = false;
	let firstDecodedFrameTimestampUs: number | null = null;
	let decodedFrameTimelineOffsetUs = 0;
	let submittedChunkCount = 0;
	let lastSubmittedChunk: EncodedVideoChunk | undefined;
	let lastSubmittedChunkIndex: number | undefined;
	const preferredDecoderConfig = shouldPreferSoftwareDecode
		? {
				...decoderConfig,
				hardwareAcceleration: "prefer-software" as const,
			}
		: decoderConfig;
	let activeDecoderConfig = preferredDecoderConfig;
	const getDecoderFailureContext = (): VideoDecodeFailureContext => ({
		decoderConfig: activeDecoderConfig,
		sourceMetadata: context.metadata ?? undefined,
		chunkIndex: lastSubmittedChunkIndex,
		chunk: lastSubmittedChunk,
		decoderState: context.decoder?.state,
		decodeQueueSize: context.decoder?.decodeQueueSize,
	});
	const recordFirstDecodeError = (error: unknown) => {
		decodeError = preserveFirstVideoDecodeFailure(
			decodeError,
			error,
			getDecoderFailureContext(),
		);
	};

	context.decoder = new VideoDecoder({
		output: (frame: VideoFrame) => {
			if (frameResolve) {
				const resolve = frameResolve;
				frameResolve = null;
				resolve(frame);
			} else {
				pendingFrames.push(frame);
			}
			notifyBackpressureProgress();
		},
		error: (e: DOMException) => {
			recordFirstDecodeError(e);
			if (frameResolve) {
				const resolve = frameResolve;
				frameResolve = null;
				resolve(null);
			}
			notifyBackpressureProgress();
		},
	});
	try {
		context.decoder.configure(preferredDecoderConfig);
	} catch (error) {
		if (!shouldPreferSoftwareDecode || forceSoftwareDecode) {
			throw buildVideoDecodeFailure(error, getDecoderFailureContext());
		}
		// Fall back to default decoder config if software preference is unsupported.
		activeDecoderConfig = decoderConfig;
		try {
			context.decoder.configure(decoderConfig);
		} catch (fallbackError) {
			throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext());
		}
	}

	const getNextFrame = (): Promise<VideoFrame | null> => {
		if (decodeError) return Promise.resolve(null);
		if (pendingFrames.length > 0) {
			const frame = pendingFrames.shift()!;
			notifyBackpressureProgress();
			return Promise.resolve(frame);
		}
		if (decodeDone) return Promise.resolve(null);
		return new Promise((resolve) => {
			frameResolve = resolve;
		});
	};

	// One forward stream through the whole file.
	// Pass explicit range because some containers are truncated when no end is provided.
	const readEndSec =
		Math.max(
			context.metadata.duration + (context.metadata.mediaStartTime ?? 0),
			(context.metadata.streamDuration ?? context.metadata.duration) +
				(context.metadata.streamStartTime ?? context.metadata.mediaStartTime ?? 0),
		) + 0.5;
	const reader = context.demuxer.read("video", 0, readEndSec).getReader();

	// Feed chunks to decoder in background with backpressure
	const feedPromise = (async () => {
		try {
			while (!context.cancelled && !decodeError) {
				const { done, value: chunk } = await reader.read();
				if (done || !chunk) break;

				if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) {
					loggedSteadyStateBackpressure = true;
					console.log(
						"[StreamingVideoDecoder] Switched to steady-state decode backpressure",
					);
				}

				const decodeQueueLimit =
					exportFrameIndex < startupFrameBudget
						? Math.min(context.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
						: context.maxDecodeQueue;
				const pendingFrameLimit =
					exportFrameIndex < startupFrameBudget
						? Math.min(context.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
						: context.maxPendingFrames;

				// Backpressure on both decode queue and decoded frame backlog.
				while (
					!decodeError &&
					context.decoder!.state === "configured" &&
					(context.decoder!.decodeQueueSize > decodeQueueLimit ||
						pendingFrames.length > pendingFrameLimit) &&
					!context.cancelled
				) {
					await waitForBackpressureProgress();
				}
				if (context.cancelled || decodeError) break;
				if (context.decoder!.state !== "configured") {
					recordFirstDecodeError(
						new DOMException(
							"Decoder closed before the next video chunk was submitted.",
							"InvalidStateError",
						),
					);
					break;
				}

				lastSubmittedChunk = chunk;
				lastSubmittedChunkIndex = submittedChunkCount;
				context.decoder!.decode(chunk);
				submittedChunkCount++;
			}

			if (!context.cancelled && context.decoder!.state === "configured") {
				await context.decoder!.flush();
			}
		} catch (e) {
			recordFirstDecodeError(e);
		} finally {
			decodeDone = true;
			if (frameResolve) {
				const resolve = frameResolve;
				frameResolve = null;
				resolve(null);
			}
			notifyBackpressureProgress();
		}
	})();

	// Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling
	let segmentIdx = 0;
	let segmentFrameIndex = 0;
	let lastDecodedFrameSec: number | null = null;
	let heldFrame: VideoFrame | null = null;
	let heldFrameSec = 0;

	const emitHeldFrameForTarget = async (segment: VideoSegment) => {
		if (!heldFrame) return false;
		const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
		if (segmentFrameIndex >= segmentFrameCount) return false;

		const sourceTimeSec = segmentSourceTime(segment, segmentFrameIndex, targetFrameRate);

		const sourceTimestampMs = sourceTimeSec * 1000;
		await onFrame(
			heldFrame,
			(segment.outputStartSec === undefined
				? exportFrameIndex
				: Math.ceil(segment.outputStartSec * targetFrameRate) + segmentFrameIndex) *
				frameDurationUs,
			sourceTimestampMs,
			sourceTimestampMs,
		);
		segmentFrameIndex++;
		exportFrameIndex++;
		return true;
	};

	while (!context.cancelled && segmentIdx < segments.length) {
		const frame = await getNextFrame();
		if (!frame) break;

		if (firstDecodedFrameTimestampUs === null) {
			firstDecodedFrameTimestampUs = frame.timestamp;
			decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs(
				firstDecodedFrameTimestampUs,
				context.metadata,
			);
		}

		const normalizedFrameTimeSec = Math.max(
			0,
			(frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) /
				1_000_000,
		);
		const frameTimeSec: number =
			lastDecodedFrameSec === null
				? normalizedFrameTimeSec
				: Math.max(lastDecodedFrameSec, normalizedFrameTimeSec);
		lastDecodedFrameSec = frameTimeSec;

		// Finalize completed segments before handling this frame.
		while (
			segmentIdx < segments.length &&
			frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
		) {
			const segment = segments[segmentIdx];
			while (!context.cancelled && (await emitHeldFrameForTarget(segment))) {
				// Keep emitting remaining output frames for this segment from the last known frame.
			}

			segmentIdx++;
			segmentFrameIndex = 0;
		}

		if (segmentIdx >= segments.length) {
			frame.close();
			continue;
		}

		const currentSegment = segments[segmentIdx];

		// Keep the nearest pre-roll frame for cuts between source frames.
		if (frameTimeSec < currentSegment.startSec - epsilonSec) {
			heldFrame?.close();
			heldFrame = frame;
			heldFrameSec = frameTimeSec;
			continue;
		}

		if (!heldFrame) {
			heldFrame = frame;
			heldFrameSec = frameTimeSec;
			continue;
		}

		// Any target timestamp before this midpoint is closer to heldFrame than current frame.
		const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2;
		while (!context.cancelled) {
			const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
			if (segmentFrameIndex >= segmentFrameCount) {
				break;
			}

			const sourceTimeSec = segmentSourceTime(
				currentSegment,
				segmentFrameIndex,
				targetFrameRate,
			);
			if (sourceTimeSec > handoffBoundarySec) {
				break;
			}

			await emitHeldFrameForTarget(currentSegment);
		}

		heldFrame.close();
		heldFrame = frame;
		heldFrameSec = frameTimeSec;
	}

	// Flush remaining output frames for the last decoded frame.
	if (!decodeError && heldFrame && segmentIdx < segments.length) {
		while (!context.cancelled && segmentIdx < segments.length) {
			const segment = segments[segmentIdx];
			if (heldFrameSec < segment.startSec - epsilonSec) {
				break;
			}

			while (!context.cancelled && (await emitHeldFrameForTarget(segment))) {
				// Keep emitting output frames for the active segment.
			}

			segmentIdx++;
			segmentFrameIndex = 0;
			if (
				segmentIdx < segments.length &&
				heldFrameSec < segments[segmentIdx].startSec - epsilonSec
			) {
				break;
			}
		}
		heldFrame.close();
		heldFrame = null;
	} else if (heldFrame) {
		heldFrame.close();
		heldFrame = null;
	}

	// Drain leftover decoded frames
	while (!decodeDone && !decodeError) {
		const frame = await getNextFrame();
		if (!frame) break;
		frame.close();
	}

	try {
		reader.cancel();
	} catch {
		/* already closed */
	}
	await feedPromise;
	for (const f of pendingFrames) f.close();
	pendingFrames.length = 0;

	if (context.decoder?.state === "configured") {
		context.decoder.close();
	}
	context.decoder = null;

	if (decodeError) {
		throw decodeError;
	}

	const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
	if (
		!context.cancelled &&
		lastDecodedFrameSec !== null &&
		requiredEndSec - lastDecodedFrameSec > 1 &&
		exportFrameIndex < expectedOutputFrames
	) {
		throw buildVideoDecodeFailure(
			new Error(
				`Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`,
			),
			getDecoderFailureContext(),
		);
	}
}