Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

Refactor audio and streaming exporters into focused modules

webadderall679d8f676c2b
diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts
index 6f1ca075..5cda0a88 100644
--- a/src/lib/exporter/audioEncoder.ts
+++ b/src/lib/exporter/audioEncoder.ts
@@ -1,158 +1,28 @@
-import { WebDemuxer } from "web-demuxer";
-import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
+import type { WebDemuxer } from "web-demuxer";
import type {
AudioRegion,
ClipRegion,
SourceAudioTrackSettings,
SpeedRegion,
- TrimRegion,
} from "@/components/video-editor/types";
-import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine";
-import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
-import { resolveMediaElementSource } from "./localMediaSource";
-import type { VideoMuxer } from "./muxer";
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
+import { AudioTranscodeProcessor } from "./audioTranscodeProcessor";
+import {
+ hasNonDefaultSourceTrackSettings,
+ MIN_SPEED_REGION_DELTA_MS,
+ MP4_AUDIO_CODEC,
+ type TrimLikeRegion,
+} from "./audioProcessorShared";
+import type { VideoMuxer } from "./muxer";
-const AUDIO_BITRATE = 128_000;
-const DECODE_BACKPRESSURE_LIMIT = 20;
-const ENCODE_BACKPRESSURE_LIMIT = 20;
-const MIN_SPEED_REGION_DELTA_MS = 0.0001;
-const MP4_AUDIO_CODEC = "mp4a.40.2";
-const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
-const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
-const OFFLINE_CHUNK_DURATION_SEC = 30;
-const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9;
-const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985;
-
-function softLimitSample(sample: number): number {
- const magnitude = Math.abs(sample);
- if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) {
- return sample;
- }
-
- const sign = sample < 0 ? -1 : 1;
- const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD;
- const limitedMagnitude =
- OFFLINE_MIX_SOFT_LIMITER_THRESHOLD +
- kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange);
- return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude);
-}
-
-export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean {
- let changed = false;
- for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) {
- const data = buffer.getChannelData(channel);
- for (let index = 0; index < data.length; index += 1) {
- const sample = data[index];
- if (!Number.isFinite(sample)) {
- data[index] = 0;
- changed = true;
- continue;
- }
-
- const limited = softLimitSample(sample);
- if (limited !== sample) {
- data[index] = limited;
- changed = true;
- }
- }
- }
- return changed;
-}
-
-function resolveSourceTrackGain(
- sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
- trackId: "mic" | "system" | "mixed",
-) {
- const settings = sourceAudioTrackSettings?.[trackId];
- if (!settings) {
- return 1;
- }
- const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
- return Math.max(0, Math.min(2, settings.volume * normalizeGain));
-}
-
-export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
- const normalized = audioPath.toLowerCase();
- // Check for common patterns like .mic., -mic., mic.mp4, etc.
- if (
- normalized.includes(".mic.") ||
- normalized.includes("-mic.") ||
- normalized.includes("_mic_") ||
- normalized.includes("/mic.") ||
- normalized.includes("\\mic.") ||
- normalized.endsWith("mic.mp4") ||
- normalized.endsWith("mic.m4a") ||
- normalized.endsWith("mic.wav")
- ) {
- return "mic";
- }
- if (
- normalized.includes(".system.") ||
- normalized.includes("-system.") ||
- normalized.includes("_system_") ||
- normalized.includes("/system.") ||
- normalized.includes("\\system.") ||
- normalized.endsWith("system.mp4") ||
- normalized.endsWith("system.m4a") ||
- normalized.endsWith("system.wav")
- ) {
- return "system";
- }
- return "mixed";
-}
-
-export function hasNonDefaultSourceTrackSettings(
- sourceAudioTrackSettings?: SourceAudioTrackSettings,
-) {
- if (!sourceAudioTrackSettings) {
- return false;
- }
- return Object.values(sourceAudioTrackSettings).some(
- (settings) =>
- Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
- );
-}
-
-interface TimelineSlice {
- sourceStartMs: number;
- sourceEndMs: number;
- speed: number;
-}
-
-interface PreparedOfflineRender {
- mainBufferEntry: { buffer: AudioBuffer; gain: number } | null;
- companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>;
- regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>;
- mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>;
- slices: TimelineSlice[];
- outputDurationMs: number;
- numChannels: number;
-}
-
-export async function isAacAudioEncodingSupported(
- sampleRate = 48_000,
- numberOfChannels = 2,
-): Promise<boolean> {
- try {
- const support = await AudioEncoder.isConfigSupported({
- codec: MP4_AUDIO_CODEC,
- sampleRate,
- numberOfChannels,
- bitrate: AUDIO_BITRATE,
- });
- return support.supported === true;
- } catch {
- return false;
- }
-}
-
-type TrimLikeRegion = TrimRegion | ClipRegion;
-
-export class AudioProcessor {
- private cancelled = false;
- private onProgress?: (progress: number) => void;
+export {
+ getSourceTrackIdFromPath,
+ hasNonDefaultSourceTrackSettings,
+ isAacAudioEncodingSupported,
+ softLimitOfflineMixPeaksInPlace,
+} from "./audioProcessorShared";
+export class AudioProcessor extends AudioTranscodeProcessor {
private isPassthroughAudioCodec(codec: string | undefined): boolean {
if (!codec) {
return false;
@@ -166,7 +36,7 @@ export class AudioProcessor {
);
}
- private async passthroughAudioStream(
+ protected async passthroughAudioStream(
audioStream: ReadableStream<EncodedAudioChunk>,
audioConfig: AudioDecoderConfig,
muxer: VideoMuxer,
@@ -407,1450 +277,4 @@ export class AudioProcessor {
}
// Legacy trim-only path used when no speed regions are configured.
- private async processTrimOnlyAudio(
- demuxer: WebDemuxer,
- muxer: VideoMuxer,
- sortedTrims: TrimLikeRegion[],
- readEndSec?: number,
- ): Promise<void> {
- let audioConfig: AudioDecoderConfig;
- try {
- audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
- } catch {
- console.warn("[AudioProcessor] No audio track found, skipping");
- return;
- }
-
- const codecCheck = await AudioDecoder.isConfigSupported(audioConfig);
- if (!codecCheck.supported) {
- console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec);
- return;
- }
-
- const audioStream =
- typeof readEndSec === "number"
- ? demuxer.read("audio", 0, readEndSec)
- : demuxer.read("audio");
-
- let sourceTimestampOffsetUs: number | null = null;
-
- await this.transcodeAudioStream(
- audioStream as ReadableStream<EncodedAudioChunk>,
- audioConfig,
- muxer,
- {
- observeChunkTimestampUs: (timestampUs) => {
- if (sourceTimestampOffsetUs === null) {
- sourceTimestampOffsetUs = timestampUs;
- }
- },
- shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims),
- transformAudioData: (data) => {
- const timestampMs = data.timestamp / 1000;
- const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims);
- const adjustedTimestampUs =
- data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000;
- return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs));
- },
- },
- );
- }
-
- private async transcodeAudioStream(
- audioStream: ReadableStream<EncodedAudioChunk>,
- audioConfig: AudioDecoderConfig,
- muxer: VideoMuxer,
- options: {
- observeChunkTimestampUs?: (timestampUs: number) => void;
- shouldSkipChunk?: (timestampMs: number) => boolean;
- transformAudioData?: (data: AudioData) => AudioData | null;
- } = {},
- ): Promise<void> {
- const pendingFrames: AudioData[] = [];
- let decodeError: Error | null = null;
- let encodeError: Error | null = null;
- let muxError: Error | null = null;
- let pendingMuxing = Promise.resolve();
- const capacityWaiters = new Set<() => void>();
-
- const notifyCapacityAvailable = () => {
- if (capacityWaiters.size === 0) {
- return;
- }
-
- const waiters = [...capacityWaiters];
- capacityWaiters.clear();
- for (const resolve of waiters) {
- resolve();
- }
- };
-
- const waitForCapacity = () =>
- new Promise<void>((resolve) => {
- capacityWaiters.add(resolve);
- });
-
- const failIfNeeded = () => {
- if (decodeError) throw decodeError;
- if (encodeError) throw encodeError;
- if (muxError) throw muxError;
- };
-
- const pumpEncodedFrames = () => {
- while (!this.cancelled && pendingFrames.length > 0) {
- if (encodeError || muxError) {
- break;
- }
- if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) {
- break;
- }
-
- const frame = pendingFrames.shift();
- if (!frame) {
- break;
- }
-
- encoder.encode(frame);
- frame.close();
- notifyCapacityAvailable();
- }
- };
-
- const cleanupPendingFrames = () => {
- for (const frame of pendingFrames) {
- frame.close();
- }
- pendingFrames.length = 0;
- };
-
- const sampleRate = audioConfig.sampleRate || 48_000;
- const channels = audioConfig.numberOfChannels || 2;
- const encodeConfig: AudioEncoderConfig = {
- codec: MP4_AUDIO_CODEC,
- sampleRate,
- numberOfChannels: channels,
- bitrate: AUDIO_BITRATE,
- };
-
- const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig);
- if (!encodeSupport.supported) {
- console.warn("[AudioProcessor] AAC encoding not supported, skipping audio");
- return;
- }
-
- const encoder = new AudioEncoder({
- output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => {
- pendingMuxing = pendingMuxing
- .then(async () => {
- if (this.cancelled) {
- return;
- }
- await muxer.addAudioChunk(chunk, meta);
- })
- .catch((error) => {
- muxError = error instanceof Error ? error : new Error(String(error));
- notifyCapacityAvailable();
- });
- notifyCapacityAvailable();
- },
- error: (error: DOMException) => {
- encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`);
- notifyCapacityAvailable();
- },
- });
-
- encoder.configure(encodeConfig);
-
- const decoder = new AudioDecoder({
- output: (data: AudioData) => {
- if (this.cancelled || encodeError || muxError) {
- data.close();
- return;
- }
-
- const transformed = options.transformAudioData
- ? options.transformAudioData(data)
- : data;
-
- if (transformed !== data) {
- data.close();
- }
-
- if (!transformed) {
- return;
- }
-
- pendingFrames.push(transformed);
- notifyCapacityAvailable();
- },
- error: (error: DOMException) => {
- decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`);
- notifyCapacityAvailable();
- },
- });
- decoder.configure(audioConfig);
-
- let reader: ReadableStreamDefaultReader<EncodedAudioChunk> | null = null;
-
- try {
- reader = audioStream.getReader();
- while (!this.cancelled) {
- failIfNeeded();
-
- const { done, value: chunk } = await reader.read();
- if (done || !chunk) break;
-
- options.observeChunkTimestampUs?.(chunk.timestamp);
- const timestampMs = chunk.timestamp / 1000;
- if (options.shouldSkipChunk?.(timestampMs)) continue;
-
- decoder.decode(chunk);
- pumpEncodedFrames();
-
- while (
- !this.cancelled &&
- (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT ||
- pendingFrames.length > DECODE_BACKPRESSURE_LIMIT ||
- encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT)
- ) {
- failIfNeeded();
- pumpEncodedFrames();
- await waitForCapacity();
- }
- }
-
- if (decoder.state === "configured") {
- await decoder.flush();
- }
-
- while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) {
- failIfNeeded();
- pumpEncodedFrames();
- if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) {
- await waitForCapacity();
- }
- }
-
- failIfNeeded();
-
- if (encoder.state === "configured") {
- await encoder.flush();
- }
-
- await pendingMuxing;
- failIfNeeded();
- } finally {
- notifyCapacityAvailable();
- if (reader) {
- try {
- await reader.cancel();
- } catch {
- // reader already closed
- }
- }
-
- cleanupPendingFrames();
-
- if (decoder.state === "configured") {
- decoder.close();
- }
-
- if (encoder.state === "configured") {
- encoder.close();
- }
- }
-
- if (this.cancelled) {
- return;
- }
- }
-
- // ---------- Offline audio rendering pipeline ----------
- // Replaces the old real-time MediaElement+MediaRecorder approach with
- // OfflineAudioContext, which renders as fast as the CPU allows instead of
- // waiting for 1× real-time playback.
-
- private async renderAndMuxOfflineAudio(
- videoUrl: string,
- trimRegions: TrimLikeRegion[],
- speedRegions: SpeedRegion[],
- audioRegions: AudioRegion[],
- sourceAudioFallbackPaths: string[],
- sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
- sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
- clipRegions: ClipRegion[] | undefined,
- muxer: VideoMuxer,
- ): Promise<void> {
- const prepared = await this.prepareOfflineRender(
- videoUrl,
- trimRegions,
- speedRegions,
- audioRegions,
- sourceAudioFallbackPaths,
- sourceAudioFallbackStartDelayMsByPath,
- sourceAudioTrackSettings,
- clipRegions,
- );
- if (this.cancelled) return;
- await this.renderAndEncodeChunked(prepared, muxer);
- }
-
- private async prepareOfflineRender(
- videoUrl: string,
- trimRegions: TrimLikeRegion[],
- speedRegions: SpeedRegion[],
- audioRegions: AudioRegion[],
- sourceAudioFallbackPaths: string[],
- sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
- sourceAudioTrackSettings?: SourceAudioTrackSettings,
- clipRegions?: ClipRegion[],
- ): Promise<PreparedOfflineRender> {
- if (this.cancelled) throw new Error("Export cancelled");
- this.onProgress?.(0);
-
- const resolvedPlan = buildResolvedAudioPlan({
- videoResource: videoUrl,
- sourceAudioFallbackPaths,
- audioRegions,
- sourceTrackGainById: {
- mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"),
- system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"),
- mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"),
- },
- embeddedGain: Math.max(
- 0,
- Math.min(
- 2,
- sourceAudioTrackSettings?.mixed
- ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed")
- : sourceAudioTrackSettings?.system
- ? resolveSourceTrackGain(sourceAudioTrackSettings, "system")
- : 1,
- ),
- ),
- });
-
- // Decode embedded source audio separately from companion sidecars.
- const mainBuffer = resolvedPlan.includeEmbeddedInExport
- ? await this.decodeAudioFromUrl(videoUrl)
- : null;
- const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed");
- const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null;
- if (this.cancelled) throw new Error("Export cancelled");
-
- // Decode companion / sidecar audio files
- const companionEntries: Array<{
- buffer: AudioBuffer;
- startDelaySec: number;
- gain: number;
- }> = [];
- const refDuration =
- mainBuffer?.duration ??
- (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
- for (const audioPath of resolvedPlan.playbackPaths) {
- if (this.cancelled) throw new Error("Export cancelled");
- const buffer = await this.decodeAudioFromUrl(audioPath);
- if (!buffer) continue;
-
- companionEntries.push({
- buffer,
- gain: resolveSourceTrackGain(
- sourceAudioTrackSettings,
- getSourceTrackIdFromPath(audioPath),
- ),
- startDelaySec: estimateCompanionAudioStartDelaySeconds(
- refDuration,
- buffer.duration,
- sourceAudioFallbackStartDelayMsByPath?.[audioPath],
- ),
- });
- }
- if (this.cancelled) throw new Error("Export cancelled");
-
- // Decode audio region overlay files
- const regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }> = [];
- for (const region of audioRegions) {
- if (this.cancelled) throw new Error("Export cancelled");
- const buffer = await this.decodeAudioFromUrl(region.audioPath);
- if (buffer) regionEntries.push({ buffer, region });
- }
-
- this.onProgress?.(0.2);
-
- // Determine source duration for timeline calculation
- const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null;
- if (!primaryBuffer && regionEntries.length === 0) {
- throw new Error("No decodable audio sources found");
- }
-
- let sourceDurationSec: number;
- if (mainBufferEntry?.buffer) {
- sourceDurationSec = mainBufferEntry.buffer.duration;
- } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) {
- sourceDurationSec = await this.getMediaDurationSec(videoUrl);
- } else {
- sourceDurationSec = primaryBuffer?.duration ?? 0;
- }
- const sourceDurationMs = sourceDurationSec * 1000;
-
- // Build timeline slices (non-trimmed segments with speed info)
- const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions);
-
- let outputDurationMs = 0;
- for (const slice of slices) {
- outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed;
- }
-
- // Extend for audio regions that might exceed the video timeline
- for (const { region } of regionEntries) {
- const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices);
- outputDurationMs = Math.max(outputDurationMs, regionEndOutput);
- }
-
- const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2);
- const mutedSourceOutputRangesSec = (clipRegions ?? [])
- .filter(
- (clip) =>
- Boolean(clip.muted) &&
- Number.isFinite(clip.startMs) &&
- Number.isFinite(clip.endMs) &&
- clip.endMs > clip.startMs,
- )
- .map((clip) => ({
- startSec: Math.max(0, clip.startMs / 1000),
- endSec: Math.max(0, clip.endMs / 1000),
- }));
-
- return {
- mainBufferEntry,
- companionEntries,
- regionEntries,
- mutedSourceOutputRangesSec,
- slices,
- outputDurationMs,
- numChannels,
- };
- }
-
- // Render timeline in chunks and encode each chunk to the muxer immediately.
- // Memory is bounded to ~OFFLINE_CHUNK_DURATION_SEC of PCM per chunk
- // instead of holding the entire output buffer in memory.
- private async renderAndEncodeChunked(
- prepared: PreparedOfflineRender,
- muxer: VideoMuxer,
- ): Promise<void> {
- const { numChannels } = prepared;
- const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
-
- let encodeError: Error | null = null;
- let muxError: Error | null = null;
- let pendingMuxing = Promise.resolve();
- let wroteFirstChunk = false;
-
- const encodeConfig: AudioEncoderConfig = {
- codec: MP4_AUDIO_CODEC,
- sampleRate: OFFLINE_AUDIO_SAMPLE_RATE,
- numberOfChannels: numChannels,
- bitrate: AUDIO_BITRATE,
- };
-
- const supported = await AudioEncoder.isConfigSupported(encodeConfig);
- if (!supported.supported) {
- console.warn("[AudioProcessor] AAC encoding not supported for offline audio");
- return;
- }
-
- const encoder = new AudioEncoder({
- output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => {
- pendingMuxing = pendingMuxing
- .then(async () => {
- if (this.cancelled) return;
- await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined);
- wroteFirstChunk = true;
- })
- .catch((error) => {
- muxError = error instanceof Error ? error : new Error(String(error));
- });
- },
- error: (error: DOMException) => {
- encodeError = new Error(`Audio encode error: ${error.message}`);
- },
- });
- encoder.configure(encodeConfig);
-
- try {
- await this.renderChunked(
- prepared,
- totalOutputSec,
- async (rendered, outputOffsetSec) => {
- if (encodeError) throw encodeError;
- if (muxError) throw muxError;
- await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec);
- },
- );
-
- if (encodeError) throw encodeError;
- if (muxError) throw muxError;
-
- if (encoder.state === "configured") {
- await encoder.flush();
- }
-
- await pendingMuxing;
-
- if (encodeError) throw encodeError;
- if (muxError) throw muxError;
- } finally {
- if (encoder.state === "configured") {
- encoder.close();
- }
- }
- }
-
- // Render timeline to a WAV blob for the native/FFmpeg export path.
- // Processes in chunks to avoid holding the entire output in memory.
- private async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise<Blob> {
- const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
- const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE);
- const numChannels = prepared.numChannels;
-
- const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames);
- const pcmParts: ArrayBuffer[] = [header];
-
- await this.renderChunked(prepared, totalOutputSec, async (rendered) => {
- pcmParts.push(...this.audioBufferToPcmParts(rendered));
- });
-
- return new Blob(pcmParts, { type: "audio/wav" });
- }
-
- // Shared chunked rendering loop. Processes the timeline in
- // OFFLINE_CHUNK_DURATION_SEC segments, calling onChunk for each rendered buffer.
- private async renderChunked(
- prepared: PreparedOfflineRender,
- totalOutputSec: number,
- onChunk: (
- rendered: AudioBuffer,
- outputOffsetSec: number,
- chunkIndex: number,
- ) => Promise<void>,
- ): Promise<void> {
- const { slices, numChannels } = prepared;
- let outputOffsetSec = 0;
- const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC);
-
- for (let i = 0; i < chunkCount && !this.cancelled; i++) {
- const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec);
- const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE);
-
- const offlineCtx = new OfflineAudioContext(
- numChannels,
- chunkFrames,
- OFFLINE_AUDIO_SAMPLE_RATE,
- );
-
- // Schedule main audio
- if (prepared.mainBufferEntry) {
- this.scheduleBufferThroughTimeline(
- offlineCtx,
- prepared.mainBufferEntry.buffer,
- slices,
- 0,
- prepared.mainBufferEntry.gain,
- outputOffsetSec,
- chunkSec,
- prepared.mutedSourceOutputRangesSec,
- );
- }
-
- // Schedule companion/sidecar audio
- for (const entry of prepared.companionEntries) {
- this.scheduleBufferThroughTimeline(
- offlineCtx,
- entry.buffer,
- slices,
- entry.startDelaySec,
- entry.gain,
- outputOffsetSec,
- chunkSec,
- prepared.mutedSourceOutputRangesSec,
- );
- }
-
- // Schedule audio region overlays
- for (const { buffer, region } of prepared.regionEntries) {
- this.scheduleRegionForChunk(
- offlineCtx,
- buffer,
- region,
- slices,
- outputOffsetSec,
- chunkSec,
- );
- }
-
- const rendered = await offlineCtx.startRendering();
- if (this.cancelled) break;
- softLimitOfflineMixPeaksInPlace(rendered);
-
- await onChunk(rendered, outputOffsetSec, i);
-
- outputOffsetSec += chunkSec;
- this.onProgress?.(0.3 + (outputOffsetSec / totalOutputSec) * 0.7);
- }
- }
-
- // Schedule an audio region overlay clipped to a specific chunk window.
- private scheduleRegionForChunk(
- ctx: OfflineAudioContext,
- buffer: AudioBuffer,
- region: AudioRegion,
- slices: TimelineSlice[],
- chunkOutputStartSec: number,
- chunkDurationSec: number,
- ): void {
- const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices);
- const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices);
-
- let localStartSec = outputStartMs / 1000 - chunkOutputStartSec;
- let localEndSec = outputEndMs / 1000 - chunkOutputStartSec;
-
- // Skip if region doesn't overlap with this chunk
- if (localEndSec <= 0 || localStartSec >= chunkDurationSec) return;
-
- // Clip to chunk bounds
- let bufferOffsetSec = 0;
- if (localStartSec < 0) {
- bufferOffsetSec = -localStartSec;
- localStartSec = 0;
- }
- if (localEndSec > chunkDurationSec) {
- localEndSec = chunkDurationSec;
- }
-
- const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec);
- if (duration <= 0.001) return;
-
- const gainNode = ctx.createGain();
- const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
- gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain));
- gainNode.connect(ctx.destination);
-
- const source = ctx.createBufferSource();
- source.buffer = buffer;
- source.connect(gainNode);
- source.start(localStartSec, bufferOffsetSec, duration);
- }
-
- // Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset.
- private async feedBufferToEncoder(
- encoder: AudioEncoder,
- buffer: AudioBuffer,
- timestampOffsetSec: number,
- ): Promise<void> {
- const sampleRate = buffer.sampleRate;
- const numChannels = buffer.numberOfChannels;
- const totalFrames = buffer.length;
-
- for (
- let offset = 0;
- offset < totalFrames && !this.cancelled;
- offset += OFFLINE_ENCODE_CHUNK_FRAMES
- ) {
- const frameCount = Math.min(OFFLINE_ENCODE_CHUNK_FRAMES, totalFrames - offset);
-
- const planarData = new Float32Array(frameCount * numChannels);
- for (let ch = 0; ch < numChannels; ch++) {
- const channelData = buffer.getChannelData(ch);
- planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount);
- }
-
- const audioData = new AudioData({
- format: "f32-planar",
- sampleRate,
- numberOfFrames: frameCount,
- numberOfChannels: numChannels,
- timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000),
- data: planarData,
- });
-
- encoder.encode(audioData);
- audioData.close();
-
- while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) {
- await new Promise((r) => setTimeout(r, 1));
- }
- }
- }
-
- // Decode audio from a URL using streaming WebCodecs decode with bulk fallback.
- // Streaming decode avoids holding the full compressed file in memory alongside
- // the decoded AudioBuffer, reducing peak memory for large recordings.
- private async decodeAudioFromUrl(url: string): Promise<AudioBuffer | null> {
- try {
- const buffer = await this.streamDecodeFromUrl(url);
- if (buffer) return buffer;
- } catch (error) {
- console.warn(
- "[AudioProcessor] Streaming decode failed, falling back to bulk decode:",
- url,
- error,
- );
- }
- return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE);
- }
-
- // Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk
- // without loading the entire compressed file into a contiguous ArrayBuffer.
- private async streamDecodeFromUrl(url: string): Promise<AudioBuffer | null> {
- const source = await resolveMediaElementSource(url);
- let demuxer: WebDemuxer | null = null;
-
- try {
- const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
- demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
- await demuxer.load(source.src);
-
- let audioConfig: AudioDecoderConfig;
- try {
- audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
- } catch {
- return null; // No audio track
- }
-
- const sampleRate = audioConfig.sampleRate || 48_000;
- const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2);
-
- // Accumulate decoded PCM per channel
- const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
- let totalFrames = 0;
- let decodeError: Error | null = null;
- const decodeCapacityWaiters = new Set<() => void>();
-
- const notifyDecodeCapacityAvailable = () => {
- if (decodeCapacityWaiters.size === 0) {
- return;
- }
-
- const waiters = [...decodeCapacityWaiters];
- decodeCapacityWaiters.clear();
- for (const resolve of waiters) {
- resolve();
- }
- };
-
- const waitForDecodeCapacity = () =>
- new Promise<void>((resolve) => {
- decodeCapacityWaiters.add(resolve);
- });
-
- const decoder = new AudioDecoder({
- output: (data: AudioData) => {
- try {
- const frames = data.numberOfFrames;
- const dataChannels = Math.min(data.numberOfChannels, numChannels);
- const format = data.format;
-
- if (format?.includes("planar")) {
- for (let ch = 0; ch < dataChannels; ch++) {
- const size = data.allocationSize({
- planeIndex: ch,
- });
- const bytes = new ArrayBuffer(size);
- data.copyTo(bytes, { planeIndex: ch });
- channelChunks[ch].push(this.rawToFloat32(bytes, format, frames));
- }
- } else if (format) {
- // Interleaved format — deinterleave into per-channel arrays.
- // Use data.numberOfChannels as stride (not capped dataChannels)
- // since the raw buffer contains all source channels.
- const srcChannels = data.numberOfChannels;
- const size = data.allocationSize({ planeIndex: 0 });
- const bytes = new ArrayBuffer(size);
- data.copyTo(bytes, { planeIndex: 0 });
- const interleaved = this.rawToFloat32(
- bytes,
- format,
- frames * srcChannels,
- );
- for (let ch = 0; ch < dataChannels; ch++) {
- const chData = new Float32Array(frames);
- for (let i = 0; i < frames; i++) {
- chData[i] = interleaved[i * srcChannels + ch];
- }
- channelChunks[ch].push(chData);
- }
- }
-
- // Fill missing channels with silence
- for (let ch = dataChannels; ch < numChannels; ch++) {
- channelChunks[ch].push(new Float32Array(frames));
- }
-
- totalFrames += frames;
- } finally {
- data.close();
- notifyDecodeCapacityAvailable();
- }
- },
- error: (err: DOMException) => {
- decodeError = new Error(`Streaming audio decode error: ${err.message}`);
- notifyDecodeCapacityAvailable();
- },
- });
-
- decoder.configure(audioConfig);
-
- const audioStream = demuxer.read("audio");
- const reader = (audioStream as ReadableStream<EncodedAudioChunk>).getReader();
-
- try {
- while (!this.cancelled) {
- if (decodeError) throw decodeError;
- const { done, value: chunk } = await reader.read();
- if (done || !chunk) break;
-
- decoder.decode(chunk);
-
- while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
- if (decodeError) throw decodeError;
- await waitForDecodeCapacity();
- }
- }
-
- if (decoder.state === "configured") {
- await decoder.flush();
- }
- if (decodeError) throw decodeError;
- } finally {
- notifyDecodeCapacityAvailable();
- try {
- await reader.cancel();
- } catch {
- /* reader already closed */
- }
- if (decoder.state === "configured") {
- decoder.close();
- }
- }
-
- if (totalFrames === 0) return null;
-
- // Build AudioBuffer from accumulated chunks
- const audioBuffer = new AudioBuffer({
- length: totalFrames,
- numberOfChannels: numChannels,
- sampleRate,
- });
- for (let ch = 0; ch < numChannels; ch++) {
- const channelData = audioBuffer.getChannelData(ch);
- let writeOffset = 0;
- for (const chunk of channelChunks[ch]) {
- channelData.set(chunk, writeOffset);
- writeOffset += chunk.length;
- }
- }
-
- return audioBuffer;
- } finally {
- source.revoke();
- try {
- demuxer?.destroy();
- } catch {
- /* cleanup */
- }
- }
- }
-
- // Convert raw bytes from AudioData to Float32Array based on the sample format.
- private rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array {
- if (format.startsWith("f32")) {
- return new Float32Array(bytes);
- }
- if (format.startsWith("s16")) {
- const int16 = new Int16Array(bytes);
- const f32 = new Float32Array(sampleCount);
- for (let i = 0; i < sampleCount; i++) {
- f32[i] = int16[i] / 0x8000;
- }
- return f32;
- }
- if (format.startsWith("s32")) {
- const int32 = new Int32Array(bytes);
- const f32 = new Float32Array(sampleCount);
- for (let i = 0; i < sampleCount; i++) {
- f32[i] = int32[i] / 0x80000000;
- }
- return f32;
- }
- if (format.startsWith("u8")) {
- const uint8 = new Uint8Array(bytes);
- const f32 = new Float32Array(sampleCount);
- for (let i = 0; i < sampleCount; i++) {
- f32[i] = (uint8[i] - 128) / 128;
- }
- return f32;
- }
- // Unknown format — attempt float32 interpretation
- return new Float32Array(bytes);
- }
-
- // Bulk decode fallback: loads entire file into memory and uses decodeAudioData.
- private async bulkDecodeFromUrl(url: string, sampleRate: number): Promise<AudioBuffer | null> {
- try {
- const source = await resolveMediaElementSource(url);
- try {
- const response = await fetch(source.src);
- const arrayBuffer = await response.arrayBuffer();
- const tempCtx = new OfflineAudioContext(2, 1, sampleRate);
- return await tempCtx.decodeAudioData(arrayBuffer);
- } finally {
- source.revoke();
- }
- } catch (error) {
- console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error);
- return null;
- }
- }
-
- // Get the duration of a media file by loading only its metadata.
- private async getMediaDurationSec(url: string): Promise<number> {
- const source = await resolveMediaElementSource(url);
- try {
- const media = document.createElement("video");
- media.preload = "metadata";
- media.src = source.src;
-
- return await new Promise<number>((resolve, reject) => {
- const timeout = setTimeout(() => {
- cleanup();
- media.src = "";
- media.load();
- reject(new Error("Timed out getting media duration (30s)"));
- }, 30_000);
-
- const onLoaded = () => {
- cleanup();
- const duration = media.duration;
- media.src = "";
- media.load();
- resolve(Number.isFinite(duration) ? duration : 0);
- };
- const onError = () => {
- cleanup();
- media.src = "";
- media.load();
- reject(new Error("Failed to get media duration"));
- };
- const cleanup = () => {
- clearTimeout(timeout);
- media.removeEventListener("loadedmetadata", onLoaded);
- media.removeEventListener("error", onError);
- };
-
- media.addEventListener("loadedmetadata", onLoaded);
- media.addEventListener("error", onError, { once: true });
- });
- } finally {
- source.revoke();
- }
- }
-
- // Build non-overlapping timeline slices from the source timeline, excluding
- // trimmed regions and tagging each slice with its playback speed.
- private buildTimelineSlices(
- sourceDurationMs: number,
- trimRegions: TrimLikeRegion[],
- speedRegions: SpeedRegion[],
- ): TimelineSlice[] {
- const boundaries = new Set<number>();
- boundaries.add(0);
- boundaries.add(sourceDurationMs);
-
- for (const trim of trimRegions) {
- if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs);
- if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs);
- }
- for (const speed of speedRegions) {
- if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs)
- boundaries.add(speed.startMs);
- if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs);
- }
-
- const sorted = [...boundaries].sort((a, b) => a - b);
- const slices: TimelineSlice[] = [];
-
- for (let i = 0; i < sorted.length - 1; i++) {
- const start = sorted[i];
- const end = sorted[i + 1];
- if (end - start < 0.001) continue;
-
- // Skip segments entirely inside a trim region
- const midpoint = (start + end) / 2;
- if (this.isInTrimRegion(midpoint, trimRegions)) continue;
-
- const speedRegion = speedRegions.find(
- (s) => midpoint >= s.startMs && midpoint < s.endMs,
- );
-
- slices.push({
- sourceStartMs: start,
- sourceEndMs: end,
- speed: speedRegion?.speed ?? 1,
- });
- }
-
- return slices;
- }
-
- // Map a source-timeline timestamp to the corresponding output-timeline timestamp.
- private sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number {
- let outputMs = 0;
-
- for (const slice of slices) {
- if (sourceMs <= slice.sourceStartMs) {
- return outputMs;
- }
- const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs;
- if (sourceMs >= slice.sourceEndMs) {
- outputMs += sliceDurationMs / slice.speed;
- continue;
- }
- // Source time falls within this slice
- outputMs += (sourceMs - slice.sourceStartMs) / slice.speed;
- return outputMs;
- }
-
- return outputMs;
- }
-
- // Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext.
- // Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate
- // playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are
- // provided, only sources overlapping the chunk window are scheduled.
- private scheduleBufferThroughTimeline(
- ctx: OfflineAudioContext,
- buffer: AudioBuffer,
- slices: TimelineSlice[],
- sourceStartDelaySec: number,
- gain = 1,
- chunkOutputStartSec = 0,
- chunkDurationSec = Number.POSITIVE_INFINITY,
- mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [],
- ): void {
- let outputOffsetSec = 0;
-
- for (const slice of slices) {
- const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000;
- const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed;
-
- // Where in the buffer does this slice read from?
- const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec;
-
- // Skip if slice doesn't overlap with the buffer at all
- if (
- bufferOffsetSec + sliceSourceDurationSec <= 0 ||
- bufferOffsetSec >= buffer.duration
- ) {
- outputOffsetSec += sliceOutputDurationSec;
- continue;
- }
-
- // Clamp to buffer bounds
- let effectiveBufferStartSec = Math.max(0, bufferOffsetSec);
- const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec;
- let effectiveSourceDurationSec = Math.min(
- sliceSourceDurationSec - trimmedFromStartSec,
- buffer.duration - effectiveBufferStartSec,
- );
-
- if (effectiveSourceDurationSec <= 0.001) {
- outputOffsetSec += sliceOutputDurationSec;
- continue;
- }
-
- // Calculate output position (global then chunk-local)
- let localOutputStartSec =
- outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec;
- let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed;
-
- // Skip if entirely outside chunk window
- if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) {
- outputOffsetSec += sliceOutputDurationSec;
- continue;
- }
-
- // Clip to chunk start
- if (localOutputStartSec < 0) {
- const skipOutputSec = -localOutputStartSec;
- const skipSourceSec = skipOutputSec * slice.speed;
- effectiveBufferStartSec += skipSourceSec;
- effectiveSourceDurationSec -= skipSourceSec;
- localOutputStartSec = 0;
- }
-
- // Clip to chunk end
- if (localOutputEndSec > chunkDurationSec) {
- const excessOutputSec = localOutputEndSec - chunkDurationSec;
- effectiveSourceDurationSec -= excessOutputSec * slice.speed;
- }
-
- if (effectiveSourceDurationSec <= 0.001) {
- outputOffsetSec += sliceOutputDurationSec;
- continue;
- }
-
- const audibleRanges: Array<{ startSec: number; endSec: number }> = [
- {
- startSec: localOutputStartSec + chunkOutputStartSec,
- endSec:
- localOutputStartSec +
- chunkOutputStartSec +
- effectiveSourceDurationSec / slice.speed,
- },
- ];
- for (const mutedRange of mutedOutputRangesSec) {
- for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) {
- const current = audibleRanges[rangeIndex];
- const overlapStart = Math.max(current.startSec, mutedRange.startSec);
- const overlapEnd = Math.min(current.endSec, mutedRange.endSec);
- if (overlapEnd <= overlapStart) {
- continue;
- }
- audibleRanges.splice(rangeIndex, 1);
- if (current.startSec < overlapStart) {
- audibleRanges.push({ startSec: current.startSec, endSec: overlapStart });
- }
- if (overlapEnd < current.endSec) {
- audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec });
- }
- }
- }
-
- for (const audibleRange of audibleRanges) {
- const audibleDurationSec = audibleRange.endSec - audibleRange.startSec;
- if (audibleDurationSec <= 0.001) {
- continue;
- }
- const source = ctx.createBufferSource();
- const gainNode = ctx.createGain();
- gainNode.gain.value = Math.max(0, Math.min(2, gain));
-
- const sourceOffsetSec =
- effectiveBufferStartSec +
- (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
- slice.speed;
- const localStartSec = audibleRange.startSec - chunkOutputStartSec;
- const sourceDurationSec = audibleDurationSec * slice.speed;
-
- const stretchedBuffer = this.stretchAudioBuffer(
- buffer,
- slice.speed,
- sourceOffsetSec,
- sourceDurationSec,
- audibleDurationSec,
- ctx,
- );
-
- source.buffer = stretchedBuffer;
- source.playbackRate.value = 1;
- source.connect(gainNode);
- gainNode.connect(ctx.destination);
-
- source.start(localStartSec);
- }
-
- outputOffsetSec += sliceOutputDurationSec;
- }
- }
-
- private stretchAudioBuffer(
- originalBuffer: AudioBuffer,
- speed: number,
- sourceOffsetSec: number,
- sourceDurationSec: number,
- audibleDurationSec: number,
- ctx: BaseAudioContext,
- ): AudioBuffer {
- const sampleRate = originalBuffer.sampleRate;
- const channels = originalBuffer.numberOfChannels;
-
- const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate));
- const sourceSamples = Math.floor(sourceDurationSec * sampleRate);
- const endSample = Math.min(originalBuffer.length, startSample + sourceSamples);
-
- const outSamples = Math.floor(audibleDurationSec * sampleRate);
- if (outSamples <= 0 || startSample >= originalBuffer.length) {
- return ctx.createBuffer(channels, 1, sampleRate);
- }
-
- const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate);
-
- if (Math.abs(speed - 1) < 0.001) {
- const copyLength = Math.min(endSample - startSample, outSamples);
- if (copyLength > 0) {
- for (let c = 0; c < channels; c++) {
- outBuffer.copyToChannel(
- originalBuffer
- .getChannelData(c)
- .subarray(startSample, startSample + copyLength),
- c,
- );
- }
- }
- return outBuffer;
- }
-
- // WSOLA uses windowing which causes fade-in at the start and fade-out at the end.
- // To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it.
- const paddingSec = 0.1;
- const paddingOutSamples = Math.floor(sampleRate * paddingSec);
- const paddingInSamples = Math.floor(paddingOutSamples * speed);
-
- const workStartIn = Math.max(0, startSample - paddingInSamples);
- const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);
-
- const actualPaddingInStart = startSample - workStartIn;
- // We expect the output offset for the requested start to be roughly:
- const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);
-
- const windowSize = Math.floor(sampleRate * 0.04);
- const hopOut = Math.floor(windowSize * 0.5);
- const hopIn = Math.floor(hopOut * speed);
- const searchRange = Math.floor(sampleRate * 0.015);
-
- const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
- const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);
-
- const inDataByChannel = Array.from({ length: channels }, (_, c) =>
- originalBuffer.getChannelData(c),
- );
- const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
- workOutBuffer.getChannelData(c),
- );
-
- const window = new Float32Array(windowSize);
- for (let i = 0; i < windowSize; i++) {
- window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1)));
- }
-
- let inOffset = workStartIn;
- let outOffset = 0;
-
- // Initial window
- for (let i = 0; i < windowSize; i++) {
- if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
- for (let c = 0; c < channels; c++) {
- workOutDataByChannel[c][outOffset + i] +=
- inDataByChannel[c][inOffset + i] * window[i];
- }
- }
- }
-
- outOffset += hopOut;
- inOffset += hopIn;
-
- while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) {
- let bestOffset = inOffset;
- const minSearch = Math.max(workStartIn, inOffset - searchRange);
- const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange);
-
- if (maxSearch > minSearch) {
- let maxCorr = -Infinity;
- let bestDelta = 0;
-
- for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) {
- let corr = 0;
- for (let i = 0; i < hopOut; i += 4) {
- if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
- for (let c = 0; c < channels; c++) {
- corr +=
- workOutDataByChannel[c][outOffset + i] *
- inDataByChannel[c][testOffset + i];
- }
- }
- }
- if (corr > maxCorr) {
- maxCorr = corr;
- bestDelta = testOffset - inOffset;
- }
- }
- bestOffset = inOffset + bestDelta;
- }
-
- for (let i = 0; i < windowSize; i++) {
- if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
- for (let c = 0; c < channels; c++) {
- workOutDataByChannel[c][outOffset + i] +=
- inDataByChannel[c][bestOffset + i] * window[i];
- }
- }
- }
-
- outOffset += hopOut;
- inOffset += hopIn;
- }
-
- // Transfer the stable middle portion to the final buffer
- for (let c = 0; c < channels; c++) {
- const finalData = outBuffer.getChannelData(c);
- const tempData = workOutBuffer.getChannelData(c);
- for (let i = 0; i < outSamples; i++) {
- const srcIdx = actualPaddingOutStart + i;
- if (srcIdx < workOutSamples) {
- finalData[i] = tempData[srcIdx];
- }
- }
- }
-
- return outBuffer;
- }
-
- // Create a WAV file header for the given audio parameters.
- private createWavHeader(
- sampleRate: number,
- numChannels: number,
- totalFrames: number,
- ): ArrayBuffer {
- const bytesPerSample = 2; // 16-bit PCM
- const dataSize = totalFrames * numChannels * bytesPerSample;
- const headerSize = 44;
- const header = new ArrayBuffer(headerSize);
- const view = new DataView(header);
-
- const writeString = (offset: number, str: string) => {
- for (let i = 0; i < str.length; i++) {
- view.setUint8(offset + i, str.charCodeAt(i));
- }
- };
-
- writeString(0, "RIFF");
- view.setUint32(4, headerSize - 8 + dataSize, true);
- writeString(8, "WAVE");
- writeString(12, "fmt ");
- view.setUint32(16, 16, true);
- view.setUint16(20, 1, true); // PCM format
- view.setUint16(22, numChannels, true);
- view.setUint32(24, sampleRate, true);
- view.setUint32(28, sampleRate * numChannels * bytesPerSample, true);
- view.setUint16(32, numChannels * bytesPerSample, true);
- view.setUint16(34, bytesPerSample * 8, true);
- writeString(36, "data");
- view.setUint32(40, dataSize, true);
-
- return header;
- }
-
- // Convert an AudioBuffer to chunked 16-bit PCM ArrayBuffers.
- // Returns small (~256KB) pieces instead of one massive allocation.
- private audioBufferToPcmParts(buffer: AudioBuffer): ArrayBuffer[] {
- const PCM_CHUNK_FRAMES = 65536;
- const numChannels = buffer.numberOfChannels;
- const numFrames = buffer.length;
- const bytesPerSample = 2;
- const parts: ArrayBuffer[] = [];
-
- const channels: Float32Array[] = [];
- for (let ch = 0; ch < numChannels; ch++) {
- channels.push(buffer.getChannelData(ch));
- }
-
- for (let frameOffset = 0; frameOffset < numFrames; frameOffset += PCM_CHUNK_FRAMES) {
- const chunkFrames = Math.min(PCM_CHUNK_FRAMES, numFrames - frameOffset);
- const chunkBuffer = new ArrayBuffer(chunkFrames * numChannels * bytesPerSample);
- const view = new DataView(chunkBuffer);
-
- let byteOffset = 0;
- for (let i = 0; i < chunkFrames; i++) {
- for (let ch = 0; ch < numChannels; ch++) {
- const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i]));
- view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
- byteOffset += 2;
- }
- }
-
- parts.push(chunkBuffer);
- }
-
- return parts;
- }
-
- // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering).
- private async loadAudioFileDemuxer(audioPath: string): Promise<WebDemuxer | null> {
- try {
- const source = await resolveMediaElementSource(audioPath);
- try {
- const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
- const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
- await demuxer.load(source.src);
- return demuxer;
- } finally {
- source.revoke();
- }
- } catch (error) {
- console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error);
- return null;
- }
- }
-
- private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData {
- const isPlanar = src.format?.includes("planar") ?? false;
- const numPlanes = isPlanar ? src.numberOfChannels : 1;
-
- let totalSize = 0;
- for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
- totalSize += src.allocationSize({ planeIndex });
- }
-
- const buffer = new ArrayBuffer(totalSize);
- let offset = 0;
-
- for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
- const planeSize = src.allocationSize({ planeIndex });
- src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex });
- offset += planeSize;
- }
-
- return new AudioData({
- format: src.format!,
- sampleRate: src.sampleRate,
- numberOfFrames: src.numberOfFrames,
- numberOfChannels: src.numberOfChannels,
- timestamp: newTimestamp,
- data: buffer,
- });
- }
-
- private cloneEncodedAudioChunkWithTimestamp(
- src: EncodedAudioChunk,
- newTimestamp: number,
- ): EncodedAudioChunk {
- const data = new Uint8Array(src.byteLength);
- src.copyTo(data);
-
- return new EncodedAudioChunk({
- type: src.type,
- timestamp: newTimestamp,
- duration: src.duration ?? undefined,
- data,
- });
- }
-
- private isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) {
- return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs);
- }
-
- private computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) {
- let offset = 0;
- for (const trim of trims) {
- if (trim.endMs <= timestampMs) {
- offset += trim.endMs - trim.startMs;
- }
- }
- return offset;
- }
-
- cancel() {
- this.cancelled = true;
- }
}
diff --git a/src/lib/exporter/audioMediaProcessor.ts b/src/lib/exporter/audioMediaProcessor.ts
new file mode 100644
index 00000000..3359873a
--- /dev/null
+++ b/src/lib/exporter/audioMediaProcessor.ts
@@ -0,0 +1,282 @@
+import { WebDemuxer } from "web-demuxer";
+import { AudioTimelineProcessor } from "./audioTimelineProcessor";
+import { DECODE_BACKPRESSURE_LIMIT, OFFLINE_AUDIO_SAMPLE_RATE } from "./audioProcessorShared";
+import { resolveMediaElementSource } from "./localMediaSource";
+
+export class AudioMediaProcessor extends AudioTimelineProcessor {
+ protected async decodeAudioFromUrl(url: string): Promise<AudioBuffer | null> {
+ try {
+ const buffer = await this.streamDecodeFromUrl(url);
+ if (buffer) return buffer;
+ } catch (error) {
+ console.warn(
+ "[AudioProcessor] Streaming decode failed, falling back to bulk decode:",
+ url,
+ error,
+ );
+ }
+ return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE);
+ }
+
+ // Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk
+ // without loading the entire compressed file into a contiguous ArrayBuffer.
+ protected async streamDecodeFromUrl(url: string): Promise<AudioBuffer | null> {
+ const source = await resolveMediaElementSource(url);
+ let demuxer: WebDemuxer | null = null;
+
+ try {
+ const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
+ demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
+ await demuxer.load(source.src);
+
+ let audioConfig: AudioDecoderConfig;
+ try {
+ audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
+ } catch {
+ return null; // No audio track
+ }
+
+ const sampleRate = audioConfig.sampleRate || 48_000;
+ const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2);
+
+ // Accumulate decoded PCM per channel
+ const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
+ let totalFrames = 0;
+ let decodeError: Error | null = null;
+ const decodeCapacityWaiters = new Set<() => void>();
+
+ const notifyDecodeCapacityAvailable = () => {
+ if (decodeCapacityWaiters.size === 0) {
+ return;
+ }
+
+ const waiters = [...decodeCapacityWaiters];
+ decodeCapacityWaiters.clear();
+ for (const resolve of waiters) {
+ resolve();
+ }
+ };
+
+ const waitForDecodeCapacity = () =>
+ new Promise<void>((resolve) => {
+ decodeCapacityWaiters.add(resolve);
+ });
+
+ const decoder = new AudioDecoder({
+ output: (data: AudioData) => {
+ try {
+ const frames = data.numberOfFrames;
+ const dataChannels = Math.min(data.numberOfChannels, numChannels);
+ const format = data.format;
+
+ if (format?.includes("planar")) {
+ for (let ch = 0; ch < dataChannels; ch++) {
+ const size = data.allocationSize({
+ planeIndex: ch,
+ });
+ const bytes = new ArrayBuffer(size);
+ data.copyTo(bytes, { planeIndex: ch });
+ channelChunks[ch].push(this.rawToFloat32(bytes, format, frames));
+ }
+ } else if (format) {
+ // Interleaved format — deinterleave into per-channel arrays.
+ // Use data.numberOfChannels as stride (not capped dataChannels)
+ // since the raw buffer contains all source channels.
+ const srcChannels = data.numberOfChannels;
+ const size = data.allocationSize({ planeIndex: 0 });
+ const bytes = new ArrayBuffer(size);
+ data.copyTo(bytes, { planeIndex: 0 });
+ const interleaved = this.rawToFloat32(
+ bytes,
+ format,
+ frames * srcChannels,
+ );
+ for (let ch = 0; ch < dataChannels; ch++) {
+ const chData = new Float32Array(frames);
+ for (let i = 0; i < frames; i++) {
+ chData[i] = interleaved[i * srcChannels + ch];
+ }
+ channelChunks[ch].push(chData);
+ }
+ }
+
+ // Fill missing channels with silence
+ for (let ch = dataChannels; ch < numChannels; ch++) {
+ channelChunks[ch].push(new Float32Array(frames));
+ }
+
+ totalFrames += frames;
+ } finally {
+ data.close();
+ notifyDecodeCapacityAvailable();
+ }
+ },
+ error: (err: DOMException) => {
+ decodeError = new Error(`Streaming audio decode error: ${err.message}`);
+ notifyDecodeCapacityAvailable();
+ },
+ });
+
+ decoder.configure(audioConfig);
+
+ const audioStream = demuxer.read("audio");
+ const reader = (audioStream as ReadableStream<EncodedAudioChunk>).getReader();
+
+ try {
+ while (!this.cancelled) {
+ if (decodeError) throw decodeError;
+ const { done, value: chunk } = await reader.read();
+ if (done || !chunk) break;
+
+ decoder.decode(chunk);
+
+ while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
+ if (decodeError) throw decodeError;
+ await waitForDecodeCapacity();
+ }
+ }
+
+ if (decoder.state === "configured") {
+ await decoder.flush();
+ }
+ if (decodeError) throw decodeError;
+ } finally {
+ notifyDecodeCapacityAvailable();
+ try {
+ await reader.cancel();
+ } catch {
+ /* reader already closed */
+ }
+ if (decoder.state === "configured") {
+ decoder.close();
+ }
+ }
+
+ if (totalFrames === 0) return null;
+
+ // Build AudioBuffer from accumulated chunks
+ const audioBuffer = new AudioBuffer({
+ length: totalFrames,
+ numberOfChannels: numChannels,
+ sampleRate,
+ });
+ for (let ch = 0; ch < numChannels; ch++) {
+ const channelData = audioBuffer.getChannelData(ch);
+ let writeOffset = 0;
+ for (const chunk of channelChunks[ch]) {
+ channelData.set(chunk, writeOffset);
+ writeOffset += chunk.length;
+ }
+ }
+
+ return audioBuffer;
+ } finally {
+ source.revoke();
+ try {
+ demuxer?.destroy();
+ } catch {
+ /* cleanup */
+ }
+ }
+ }
+
+ // Convert raw bytes from AudioData to Float32Array based on the sample format.
+ protected rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array {
+ if (format.startsWith("f32")) {
+ return new Float32Array(bytes);
+ }
+ if (format.startsWith("s16")) {
+ const int16 = new Int16Array(bytes);
+ const f32 = new Float32Array(sampleCount);
+ for (let i = 0; i < sampleCount; i++) {
+ f32[i] = int16[i] / 0x8000;
+ }
+ return f32;
+ }
+ if (format.startsWith("s32")) {
+ const int32 = new Int32Array(bytes);
+ const f32 = new Float32Array(sampleCount);
+ for (let i = 0; i < sampleCount; i++) {
+ f32[i] = int32[i] / 0x80000000;
+ }
+ return f32;
+ }
+ if (format.startsWith("u8")) {
+ const uint8 = new Uint8Array(bytes);
+ const f32 = new Float32Array(sampleCount);
+ for (let i = 0; i < sampleCount; i++) {
+ f32[i] = (uint8[i] - 128) / 128;
+ }
+ return f32;
+ }
+ // Unknown format — attempt float32 interpretation
+ return new Float32Array(bytes);
+ }
+
+ // Bulk decode fallback: loads entire file into memory and uses decodeAudioData.
+ protected async bulkDecodeFromUrl(
+ url: string,
+ sampleRate: number,
+ ): Promise<AudioBuffer | null> {
+ try {
+ const source = await resolveMediaElementSource(url);
+ try {
+ const response = await fetch(source.src);
+ const arrayBuffer = await response.arrayBuffer();
+ const tempCtx = new OfflineAudioContext(2, 1, sampleRate);
+ return await tempCtx.decodeAudioData(arrayBuffer);
+ } finally {
+ source.revoke();
+ }
+ } catch (error) {
+ console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error);
+ return null;
+ }
+ }
+
+ // Get the duration of a media file by loading only its metadata.
+ protected async getMediaDurationSec(url: string): Promise<number> {
+ const source = await resolveMediaElementSource(url);
+ try {
+ const media = document.createElement("video");
+ media.preload = "metadata";
+ media.src = source.src;
+
+ return await new Promise<number>((resolve, reject) => {
+ const timeout = setTimeout(() => {
+ cleanup();
+ media.src = "";
+ media.load();
+ reject(new Error("Timed out getting media duration (30s)"));
+ }, 30_000);
+
+ const onLoaded = () => {
+ cleanup();
+ const duration = media.duration;
+ media.src = "";
+ media.load();
+ resolve(Number.isFinite(duration) ? duration : 0);
+ };
+ const onError = () => {
+ cleanup();
+ media.src = "";
+ media.load();
+ reject(new Error("Failed to get media duration"));
+ };
+ const cleanup = () => {
+ clearTimeout(timeout);
+ media.removeEventListener("loadedmetadata", onLoaded);
+ media.removeEventListener("error", onError);
+ };
+
+ media.addEventListener("loadedmetadata", onLoaded);
+ media.addEventListener("error", onError, { once: true });
+ });
+ } finally {
+ source.revoke();
+ }
+ }
+
+ // Build non-overlapping timeline slices from the source timeline, excluding
+ // trimmed regions and tagging each slice with its playback speed.
+}
diff --git a/src/lib/exporter/audioProcessorBase.ts b/src/lib/exporter/audioProcessorBase.ts
new file mode 100644
index 00000000..07842504
--- /dev/null
+++ b/src/lib/exporter/audioProcessorBase.ts
@@ -0,0 +1,154 @@
+import { WebDemuxer } from "web-demuxer";
+import { resolveMediaElementSource } from "./localMediaSource";
+import type { TrimLikeRegion } from "./audioProcessorShared";
+
+export class AudioProcessorBase {
+ protected cancelled = false;
+ protected onProgress?: (progress: number) => void;
+ protected createWavHeader(
+ sampleRate: number,
+ numChannels: number,
+ totalFrames: number,
+ ): ArrayBuffer {
+ const bytesPerSample = 2; // 16-bit PCM
+ const dataSize = totalFrames * numChannels * bytesPerSample;
+ const headerSize = 44;
+ const header = new ArrayBuffer(headerSize);
+ const view = new DataView(header);
+
+ const writeString = (offset: number, str: string) => {
+ for (let i = 0; i < str.length; i++) {
+ view.setUint8(offset + i, str.charCodeAt(i));
+ }
+ };
+
+ writeString(0, "RIFF");
+ view.setUint32(4, headerSize - 8 + dataSize, true);
+ writeString(8, "WAVE");
+ writeString(12, "fmt ");
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true); // PCM format
+ view.setUint16(22, numChannels, true);
+ view.setUint32(24, sampleRate, true);
+ view.setUint32(28, sampleRate * numChannels * bytesPerSample, true);
+ view.setUint16(32, numChannels * bytesPerSample, true);
+ view.setUint16(34, bytesPerSample * 8, true);
+ writeString(36, "data");
+ view.setUint32(40, dataSize, true);
+
+ return header;
+ }
+
+ // Convert an AudioBuffer to chunked 16-bit PCM ArrayBuffers.
+ // Returns small (~256KB) pieces instead of one massive allocation.
+ protected audioBufferToPcmParts(buffer: AudioBuffer): ArrayBuffer[] {
+ const PCM_CHUNK_FRAMES = 65536;
+ const numChannels = buffer.numberOfChannels;
+ const numFrames = buffer.length;
+ const bytesPerSample = 2;
+ const parts: ArrayBuffer[] = [];
+
+ const channels: Float32Array[] = [];
+ for (let ch = 0; ch < numChannels; ch++) {
+ channels.push(buffer.getChannelData(ch));
+ }
+
+ for (let frameOffset = 0; frameOffset < numFrames; frameOffset += PCM_CHUNK_FRAMES) {
+ const chunkFrames = Math.min(PCM_CHUNK_FRAMES, numFrames - frameOffset);
+ const chunkBuffer = new ArrayBuffer(chunkFrames * numChannels * bytesPerSample);
+ const view = new DataView(chunkBuffer);
+
+ let byteOffset = 0;
+ for (let i = 0; i < chunkFrames; i++) {
+ for (let ch = 0; ch < numChannels; ch++) {
+ const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i]));
+ view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+ byteOffset += 2;
+ }
+ }
+
+ parts.push(chunkBuffer);
+ }
+
+ return parts;
+ }
+
+ // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering).
+ protected async loadAudioFileDemuxer(audioPath: string): Promise<WebDemuxer | null> {
+ try {
+ const source = await resolveMediaElementSource(audioPath);
+ try {
+ const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
+ const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
+ await demuxer.load(source.src);
+ return demuxer;
+ } finally {
+ source.revoke();
+ }
+ } catch (error) {
+ console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error);
+ return null;
+ }
+ }
+
+ protected cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData {
+ const isPlanar = src.format?.includes("planar") ?? false;
+ const numPlanes = isPlanar ? src.numberOfChannels : 1;
+
+ let totalSize = 0;
+ for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
+ totalSize += src.allocationSize({ planeIndex });
+ }
+
+ const buffer = new ArrayBuffer(totalSize);
+ let offset = 0;
+
+ for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
+ const planeSize = src.allocationSize({ planeIndex });
+ src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex });
+ offset += planeSize;
+ }
+
+ return new AudioData({
+ format: src.format!,
+ sampleRate: src.sampleRate,
+ numberOfFrames: src.numberOfFrames,
+ numberOfChannels: src.numberOfChannels,
+ timestamp: newTimestamp,
+ data: buffer,
+ });
+ }
+
+ protected cloneEncodedAudioChunkWithTimestamp(
+ src: EncodedAudioChunk,
+ newTimestamp: number,
+ ): EncodedAudioChunk {
+ const data = new Uint8Array(src.byteLength);
+ src.copyTo(data);
+
+ return new EncodedAudioChunk({
+ type: src.type,
+ timestamp: newTimestamp,
+ duration: src.duration ?? undefined,
+ data,
+ });
+ }
+
+ protected isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) {
+ return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs);
+ }
+
+ protected computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) {
+ let offset = 0;
+ for (const trim of trims) {
+ if (trim.endMs <= timestampMs) {
+ offset += trim.endMs - trim.startMs;
+ }
+ }
+ return offset;
+ }
+
+ cancel() {
+ this.cancelled = true;
+ }
+}
diff --git a/src/lib/exporter/audioProcessorShared.ts b/src/lib/exporter/audioProcessorShared.ts
new file mode 100644
index 00000000..ca6e35e9
--- /dev/null
+++ b/src/lib/exporter/audioProcessorShared.ts
@@ -0,0 +1,144 @@
+import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
+import type {
+ AudioRegion,
+ ClipRegion,
+ SourceAudioTrackSettings,
+ TrimRegion,
+} from "@/components/video-editor/types";
+import type { SourceTrackId } from "@/lib/exporter/audioRoutingEngine";
+
+export const AUDIO_BITRATE = 128_000;
+export const DECODE_BACKPRESSURE_LIMIT = 20;
+export const ENCODE_BACKPRESSURE_LIMIT = 20;
+export const MIN_SPEED_REGION_DELTA_MS = 0.0001;
+export const MP4_AUDIO_CODEC = "mp4a.40.2";
+export const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
+export const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
+export const OFFLINE_CHUNK_DURATION_SEC = 30;
+const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9;
+const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985;
+
+function softLimitSample(sample: number): number {
+ const magnitude = Math.abs(sample);
+ if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) {
+ return sample;
+ }
+
+ const sign = sample < 0 ? -1 : 1;
+ const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD;
+ const limitedMagnitude =
+ OFFLINE_MIX_SOFT_LIMITER_THRESHOLD +
+ kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange);
+ return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude);
+}
+
+export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean {
+ let changed = false;
+ for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) {
+ const data = buffer.getChannelData(channel);
+ for (let index = 0; index < data.length; index += 1) {
+ const sample = data[index];
+ if (!Number.isFinite(sample)) {
+ data[index] = 0;
+ changed = true;
+ continue;
+ }
+
+ const limited = softLimitSample(sample);
+ if (limited !== sample) {
+ data[index] = limited;
+ changed = true;
+ }
+ }
+ }
+ return changed;
+}
+
+export function resolveSourceTrackGain(
+ sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
+ trackId: "mic" | "system" | "mixed",
+) {
+ const settings = sourceAudioTrackSettings?.[trackId];
+ if (!settings) {
+ return 1;
+ }
+ const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
+ return Math.max(0, Math.min(2, settings.volume * normalizeGain));
+}
+
+export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
+ const normalized = audioPath.toLowerCase();
+ // Check for common patterns like .mic., -mic., mic.mp4, etc.
+ if (
+ normalized.includes(".mic.") ||
+ normalized.includes("-mic.") ||
+ normalized.includes("_mic_") ||
+ normalized.includes("/mic.") ||
+ normalized.includes("\\mic.") ||
+ normalized.endsWith("mic.mp4") ||
+ normalized.endsWith("mic.m4a") ||
+ normalized.endsWith("mic.wav")
+ ) {
+ return "mic";
+ }
+ if (
+ normalized.includes(".system.") ||
+ normalized.includes("-system.") ||
+ normalized.includes("_system_") ||
+ normalized.includes("/system.") ||
+ normalized.includes("\\system.") ||
+ normalized.endsWith("system.mp4") ||
+ normalized.endsWith("system.m4a") ||
+ normalized.endsWith("system.wav")
+ ) {
+ return "system";
+ }
+ return "mixed";
+}
+
+export function hasNonDefaultSourceTrackSettings(
+ sourceAudioTrackSettings?: SourceAudioTrackSettings,
+) {
+ if (!sourceAudioTrackSettings) {
+ return false;
+ }
+ return Object.values(sourceAudioTrackSettings).some(
+ (settings) =>
+ Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
+ );
+}
+
+export interface TimelineSlice {
+ sourceStartMs: number;
+ sourceEndMs: number;
+ speed: number;
+}
+
+export interface PreparedOfflineRender {
+ mainBufferEntry: { buffer: AudioBuffer; gain: number } | null;
+ companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>;
+ regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>;
+ mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>;
+ slices: TimelineSlice[];
+ outputDurationMs: number;
+ numChannels: number;
+}
+
+export async function isAacAudioEncodingSupported(
+ sampleRate = 48_000,
+ numberOfChannels = 2,
+): Promise<boolean> {
+ try {
+ const support = await AudioEncoder.isConfigSupported({
+ codec: MP4_AUDIO_CODEC,
+ sampleRate,
+ numberOfChannels,
+ bitrate: AUDIO_BITRATE,
+ });
+ return support.supported === true;
+ } catch {
+ return false;
+ }
+}
+
+export type TrimLikeRegion = TrimRegion | ClipRegion;
diff --git a/src/lib/exporter/audioTimelineProcessor.ts b/src/lib/exporter/audioTimelineProcessor.ts
new file mode 100644
index 00000000..44253799
--- /dev/null
+++ b/src/lib/exporter/audioTimelineProcessor.ts
@@ -0,0 +1,355 @@
+import type { SpeedRegion } from "@/components/video-editor/types";
+import { AudioProcessorBase } from "./audioProcessorBase";
+import type { TimelineSlice, TrimLikeRegion } from "./audioProcessorShared";
+
+export class AudioTimelineProcessor extends AudioProcessorBase {
+ protected buildTimelineSlices(
+ sourceDurationMs: number,
+ trimRegions: TrimLikeRegion[],
+ speedRegions: SpeedRegion[],
+ ): TimelineSlice[] {
+ const boundaries = new Set<number>();
+ boundaries.add(0);
+ boundaries.add(sourceDurationMs);
+
+ for (const trim of trimRegions) {
+ if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs);
+ if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs);
+ }
+ for (const speed of speedRegions) {
+ if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs)
+ boundaries.add(speed.startMs);
+ if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs);
+ }
+
+ const sorted = [...boundaries].sort((a, b) => a - b);
+ const slices: TimelineSlice[] = [];
+
+ for (let i = 0; i < sorted.length - 1; i++) {
+ const start = sorted[i];
+ const end = sorted[i + 1];
+ if (end - start < 0.001) continue;
+
+ // Skip segments entirely inside a trim region
+ const midpoint = (start + end) / 2;
+ if (this.isInTrimRegion(midpoint, trimRegions)) continue;
+
+ const speedRegion = speedRegions.find(
+ (s) => midpoint >= s.startMs && midpoint < s.endMs,
+ );
+
+ slices.push({
+ sourceStartMs: start,
+ sourceEndMs: end,
+ speed: speedRegion?.speed ?? 1,
+ });
+ }
+
+ return slices;
+ }
+
+ // Map a source-timeline timestamp to the corresponding output-timeline timestamp.
+ protected sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number {
+ let outputMs = 0;
+
+ for (const slice of slices) {
+ if (sourceMs <= slice.sourceStartMs) {
+ return outputMs;
+ }
+ const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs;
+ if (sourceMs >= slice.sourceEndMs) {
+ outputMs += sliceDurationMs / slice.speed;
+ continue;
+ }
+ // Source time falls within this slice
+ outputMs += (sourceMs - slice.sourceStartMs) / slice.speed;
+ return outputMs;
+ }
+
+ return outputMs;
+ }
+
+ // Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext.
+ // Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate
+ // playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are
+ // provided, only sources overlapping the chunk window are scheduled.
+ protected scheduleBufferThroughTimeline(
+ ctx: OfflineAudioContext,
+ buffer: AudioBuffer,
+ slices: TimelineSlice[],
+ sourceStartDelaySec: number,
+ gain = 1,
+ chunkOutputStartSec = 0,
+ chunkDurationSec = Number.POSITIVE_INFINITY,
+ mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [],
+ ): void {
+ let outputOffsetSec = 0;
+
+ for (const slice of slices) {
+ const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000;
+ const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed;
+
+ // Where in the buffer does this slice read from?
+ const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec;
+
+ // Skip if slice doesn't overlap with the buffer at all
+ if (
+ bufferOffsetSec + sliceSourceDurationSec <= 0 ||
+ bufferOffsetSec >= buffer.duration
+ ) {
+ outputOffsetSec += sliceOutputDurationSec;
+ continue;
+ }
+
+ // Clamp to buffer bounds
+ let effectiveBufferStartSec = Math.max(0, bufferOffsetSec);
+ const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec;
+ let effectiveSourceDurationSec = Math.min(
+ sliceSourceDurationSec - trimmedFromStartSec,
+ buffer.duration - effectiveBufferStartSec,
+ );
+
+ if (effectiveSourceDurationSec <= 0.001) {
+ outputOffsetSec += sliceOutputDurationSec;
+ continue;
+ }
+
+ // Calculate output position (global then chunk-local)
+ let localOutputStartSec =
+ outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec;
+ let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed;
+
+ // Skip if entirely outside chunk window
+ if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) {
+ outputOffsetSec += sliceOutputDurationSec;
+ continue;
+ }
+
+ // Clip to chunk start
+ if (localOutputStartSec < 0) {
+ const skipOutputSec = -localOutputStartSec;
+ const skipSourceSec = skipOutputSec * slice.speed;
+ effectiveBufferStartSec += skipSourceSec;
+ effectiveSourceDurationSec -= skipSourceSec;
+ localOutputStartSec = 0;
+ }
+
+ // Clip to chunk end
+ if (localOutputEndSec > chunkDurationSec) {
+ const excessOutputSec = localOutputEndSec - chunkDurationSec;
+ effectiveSourceDurationSec -= excessOutputSec * slice.speed;
+ }
+
+ if (effectiveSourceDurationSec <= 0.001) {
+ outputOffsetSec += sliceOutputDurationSec;
+ continue;
+ }
+
+ const audibleRanges: Array<{ startSec: number; endSec: number }> = [
+ {
+ startSec: localOutputStartSec + chunkOutputStartSec,
+ endSec:
+ localOutputStartSec +
+ chunkOutputStartSec +
+ effectiveSourceDurationSec / slice.speed,
+ },
+ ];
+ for (const mutedRange of mutedOutputRangesSec) {
+ for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) {
+ const current = audibleRanges[rangeIndex];
+ const overlapStart = Math.max(current.startSec, mutedRange.startSec);
+ const overlapEnd = Math.min(current.endSec, mutedRange.endSec);
+ if (overlapEnd <= overlapStart) {
+ continue;
+ }
+ audibleRanges.splice(rangeIndex, 1);
+ if (current.startSec < overlapStart) {
+ audibleRanges.push({ startSec: current.startSec, endSec: overlapStart });
+ }
+ if (overlapEnd < current.endSec) {
+ audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec });
+ }
+ }
+ }
+
+ for (const audibleRange of audibleRanges) {
+ const audibleDurationSec = audibleRange.endSec - audibleRange.startSec;
+ if (audibleDurationSec <= 0.001) {
+ continue;
+ }
+ const source = ctx.createBufferSource();
+ const gainNode = ctx.createGain();
+ gainNode.gain.value = Math.max(0, Math.min(2, gain));
+
+ const sourceOffsetSec =
+ effectiveBufferStartSec +
+ (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
+ slice.speed;
+ const localStartSec = audibleRange.startSec - chunkOutputStartSec;
+ const sourceDurationSec = audibleDurationSec * slice.speed;
+
+ const stretchedBuffer = this.stretchAudioBuffer(
+ buffer,
+ slice.speed,
+ sourceOffsetSec,
+ sourceDurationSec,
+ audibleDurationSec,
+ ctx,
+ );
+
+ source.buffer = stretchedBuffer;
+ source.playbackRate.value = 1;
+ source.connect(gainNode);
+ gainNode.connect(ctx.destination);
+
+ source.start(localStartSec);
+ }
+
+ outputOffsetSec += sliceOutputDurationSec;
+ }
+ }
+
+ protected stretchAudioBuffer(
+ originalBuffer: AudioBuffer,
+ speed: number,
+ sourceOffsetSec: number,
+ sourceDurationSec: number,
+ audibleDurationSec: number,
+ ctx: BaseAudioContext,
+ ): AudioBuffer {
+ const sampleRate = originalBuffer.sampleRate;
+ const channels = originalBuffer.numberOfChannels;
+
+ const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate));
+ const sourceSamples = Math.floor(sourceDurationSec * sampleRate);
+ const endSample = Math.min(originalBuffer.length, startSample + sourceSamples);
+
+ const outSamples = Math.floor(audibleDurationSec * sampleRate);
+ if (outSamples <= 0 || startSample >= originalBuffer.length) {
+ return ctx.createBuffer(channels, 1, sampleRate);
+ }
+
+ const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate);
+
+ if (Math.abs(speed - 1) < 0.001) {
+ const copyLength = Math.min(endSample - startSample, outSamples);
+ if (copyLength > 0) {
+ for (let c = 0; c < channels; c++) {
+ outBuffer.copyToChannel(
+ originalBuffer
+ .getChannelData(c)
+ .subarray(startSample, startSample + copyLength),
+ c,
+ );
+ }
+ }
+ return outBuffer;
+ }
+
+ // WSOLA uses windowing which causes fade-in at the start and fade-out at the end.
+ // To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it.
+ const paddingSec = 0.1;
+ const paddingOutSamples = Math.floor(sampleRate * paddingSec);
+ const paddingInSamples = Math.floor(paddingOutSamples * speed);
+
+ const workStartIn = Math.max(0, startSample - paddingInSamples);
+ const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);
+
+ const actualPaddingInStart = startSample - workStartIn;
+ // We expect the output offset for the requested start to be roughly:
+ const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);
+
+ const windowSize = Math.floor(sampleRate * 0.04);
+ const hopOut = Math.floor(windowSize * 0.5);
+ const hopIn = Math.floor(hopOut * speed);
+ const searchRange = Math.floor(sampleRate * 0.015);
+
+ const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
+ const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);
+
+ const inDataByChannel = Array.from({ length: channels }, (_, c) =>
+ originalBuffer.getChannelData(c),
+ );
+ const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
+ workOutBuffer.getChannelData(c),
+ );
+
+ const window = new Float32Array(windowSize);
+ for (let i = 0; i < windowSize; i++) {
+ window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1)));
+ }
+
+ let inOffset = workStartIn;
+ let outOffset = 0;
+
+ // Initial window
+ for (let i = 0; i < windowSize; i++) {
+ if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
+ for (let c = 0; c < channels; c++) {
+ workOutDataByChannel[c][outOffset + i] +=
+ inDataByChannel[c][inOffset + i] * window[i];
+ }
+ }
+ }
+
+ outOffset += hopOut;
+ inOffset += hopIn;
+
+ while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) {
+ let bestOffset = inOffset;
+ const minSearch = Math.max(workStartIn, inOffset - searchRange);
+ const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange);
+
+ if (maxSearch > minSearch) {
+ let maxCorr = -Infinity;
+ let bestDelta = 0;
+
+ for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) {
+ let corr = 0;
+ for (let i = 0; i < hopOut; i += 4) {
+ if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
+ for (let c = 0; c < channels; c++) {
+ corr +=
+ workOutDataByChannel[c][outOffset + i] *
+ inDataByChannel[c][testOffset + i];
+ }
+ }
+ }
+ if (corr > maxCorr) {
+ maxCorr = corr;
+ bestDelta = testOffset - inOffset;
+ }
+ }
+ bestOffset = inOffset + bestDelta;
+ }
+
+ for (let i = 0; i < windowSize; i++) {
+ if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
+ for (let c = 0; c < channels; c++) {
+ workOutDataByChannel[c][outOffset + i] +=
+ inDataByChannel[c][bestOffset + i] * window[i];
+ }
+ }
+ }
+
+ outOffset += hopOut;
+ inOffset += hopIn;
+ }
+
+ // Transfer the stable middle portion to the final buffer
+ for (let c = 0; c < channels; c++) {
+ const finalData = outBuffer.getChannelData(c);
+ const tempData = workOutBuffer.getChannelData(c);
+ for (let i = 0; i < outSamples; i++) {
+ const srcIdx = actualPaddingOutStart + i;
+ if (srcIdx < workOutSamples) {
+ finalData[i] = tempData[srcIdx];
+ }
+ }
+ }
+
+ return outBuffer;
+ }
+
+ // Create a WAV file header for the given audio parameters.
+}
diff --git a/src/lib/exporter/audioTranscodeProcessor.ts b/src/lib/exporter/audioTranscodeProcessor.ts
new file mode 100644
index 00000000..209a280a
--- /dev/null
+++ b/src/lib/exporter/audioTranscodeProcessor.ts
@@ -0,0 +1,275 @@
+import type { WebDemuxer } from "web-demuxer";
+import { OfflineAudioProcessor } from "./offlineAudioProcessor";
+import {
+ AUDIO_BITRATE,
+ DECODE_BACKPRESSURE_LIMIT,
+ ENCODE_BACKPRESSURE_LIMIT,
+ MP4_AUDIO_CODEC,
+ type TrimLikeRegion,
+} from "./audioProcessorShared";
+import type { VideoMuxer } from "./muxer";
+
+export class AudioTranscodeProcessor extends OfflineAudioProcessor {
+ protected async processTrimOnlyAudio(
+ demuxer: WebDemuxer,
+ muxer: VideoMuxer,
+ sortedTrims: TrimLikeRegion[],
+ readEndSec?: number,
+ ): Promise<void> {
+ let audioConfig: AudioDecoderConfig;
+ try {
+ audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
+ } catch {
+ console.warn("[AudioProcessor] No audio track found, skipping");
+ return;
+ }
+
+ const codecCheck = await AudioDecoder.isConfigSupported(audioConfig);
+ if (!codecCheck.supported) {
+ console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec);
+ return;
+ }
+
+ const audioStream =
+ typeof readEndSec === "number"
+ ? demuxer.read("audio", 0, readEndSec)
+ : demuxer.read("audio");
+
+ let sourceTimestampOffsetUs: number | null = null;
+
+ await this.transcodeAudioStream(
+ audioStream as ReadableStream<EncodedAudioChunk>,
+ audioConfig,
+ muxer,
+ {
+ observeChunkTimestampUs: (timestampUs) => {
+ if (sourceTimestampOffsetUs === null) {
+ sourceTimestampOffsetUs = timestampUs;
+ }
+ },
+ shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims),
+ transformAudioData: (data) => {
+ const timestampMs = data.timestamp / 1000;
+ const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims);
+ const adjustedTimestampUs =
+ data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000;
+ return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs));
+ },
+ },
+ );
+ }
+
+ protected async transcodeAudioStream(
+ audioStream: ReadableStream<EncodedAudioChunk>,
+ audioConfig: AudioDecoderConfig,
+ muxer: VideoMuxer,
+ options: {
+ observeChunkTimestampUs?: (timestampUs: number) => void;
+ shouldSkipChunk?: (timestampMs: number) => boolean;
+ transformAudioData?: (data: AudioData) => AudioData | null;
+ } = {},
+ ): Promise<void> {
+ const pendingFrames: AudioData[] = [];
+ let decodeError: Error | null = null;
+ let encodeError: Error | null = null;
+ let muxError: Error | null = null;
+ let pendingMuxing = Promise.resolve();
+ const capacityWaiters = new Set<() => void>();
+
+ const notifyCapacityAvailable = () => {
+ if (capacityWaiters.size === 0) {
+ return;
+ }
+
+ const waiters = [...capacityWaiters];
+ capacityWaiters.clear();
+ for (const resolve of waiters) {
+ resolve();
+ }
+ };
+
+ const waitForCapacity = () =>
+ new Promise<void>((resolve) => {
+ capacityWaiters.add(resolve);
+ });
+
+ const failIfNeeded = () => {
+ if (decodeError) throw decodeError;
+ if (encodeError) throw encodeError;
+ if (muxError) throw muxError;
+ };
+
+ const pumpEncodedFrames = () => {
+ while (!this.cancelled && pendingFrames.length > 0) {
+ if (encodeError || muxError) {
+ break;
+ }
+ if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) {
+ break;
+ }
+
+ const frame = pendingFrames.shift();
+ if (!frame) {
+ break;
+ }
+
+ encoder.encode(frame);
+ frame.close();
+ notifyCapacityAvailable();
+ }
+ };
+
+ const cleanupPendingFrames = () => {
+ for (const frame of pendingFrames) {
+ frame.close();
+ }
+ pendingFrames.length = 0;
+ };
+
+ const sampleRate = audioConfig.sampleRate || 48_000;
+ const channels = audioConfig.numberOfChannels || 2;
+ const encodeConfig: AudioEncoderConfig = {
+ codec: MP4_AUDIO_CODEC,
+ sampleRate,
+ numberOfChannels: channels,
+ bitrate: AUDIO_BITRATE,
+ };
+
+ const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig);
+ if (!encodeSupport.supported) {
+ console.warn("[AudioProcessor] AAC encoding not supported, skipping audio");
+ return;
+ }
+
+ const encoder = new AudioEncoder({
+ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => {
+ pendingMuxing = pendingMuxing
+ .then(async () => {
+ if (this.cancelled) {
+ return;
+ }
+ await muxer.addAudioChunk(chunk, meta);
+ })
+ .catch((error) => {
+ muxError = error instanceof Error ? error : new Error(String(error));
+ notifyCapacityAvailable();
+ });
+ notifyCapacityAvailable();
+ },
+ error: (error: DOMException) => {
+ encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`);
+ notifyCapacityAvailable();
+ },
+ });
+
+ encoder.configure(encodeConfig);
+
+ const decoder = new AudioDecoder({
+ output: (data: AudioData) => {
+ if (this.cancelled || encodeError || muxError) {
+ data.close();
+ return;
+ }
+
+ const transformed = options.transformAudioData
+ ? options.transformAudioData(data)
+ : data;
+
+ if (transformed !== data) {
+ data.close();
+ }
+
+ if (!transformed) {
+ return;
+ }
+
+ pendingFrames.push(transformed);
+ notifyCapacityAvailable();
+ },
+ error: (error: DOMException) => {
+ decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`);
+ notifyCapacityAvailable();
+ },
+ });
+ decoder.configure(audioConfig);
+
+ let reader: ReadableStreamDefaultReader<EncodedAudioChunk> | null = null;
+
+ try {
+ reader = audioStream.getReader();
+ while (!this.cancelled) {
+ failIfNeeded();
+
+ const { done, value: chunk } = await reader.read();
+ if (done || !chunk) break;
+
+ options.observeChunkTimestampUs?.(chunk.timestamp);
+ const timestampMs = chunk.timestamp / 1000;
+ if (options.shouldSkipChunk?.(timestampMs)) continue;
+
+ decoder.decode(chunk);
+ pumpEncodedFrames();
+
+ while (
+ !this.cancelled &&
+ (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT ||
+ pendingFrames.length > DECODE_BACKPRESSURE_LIMIT ||
+ encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT)
+ ) {
+ failIfNeeded();
+ pumpEncodedFrames();
+ await waitForCapacity();
+ }
+ }
+
+ if (decoder.state === "configured") {
+ await decoder.flush();
+ }
+
+ while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) {
+ failIfNeeded();
+ pumpEncodedFrames();
+ if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) {
+ await waitForCapacity();
+ }
+ }
+
+ failIfNeeded();
+
+ if (encoder.state === "configured") {
+ await encoder.flush();
+ }
+
+ await pendingMuxing;
+ failIfNeeded();
+ } finally {
+ notifyCapacityAvailable();
+ if (reader) {
+ try {
+ await reader.cancel();
+ } catch {
+ // reader already closed
+ }
+ }
+
+ cleanupPendingFrames();
+
+ if (decoder.state === "configured") {
+ decoder.close();
+ }
+
+ if (encoder.state === "configured") {
+ encoder.close();
+ }
+ }
+
+ if (this.cancelled) {
+ return;
+ }
+ }
+
+ // ---------- Offline audio rendering pipeline ----------
+ // Replaces the old real-time MediaElement+MediaRecorder approach with
+ // OfflineAudioContext, which renders as fast as the CPU allows instead of
+ // waiting for 1× real-time playback.
+}
diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts
new file mode 100644
index 00000000..7526db08
--- /dev/null
+++ b/src/lib/exporter/offlineAudioProcessor.ts
@@ -0,0 +1,444 @@
+import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
+import type {
+ AudioRegion,
+ ClipRegion,
+ SourceAudioTrackSettings,
+ SpeedRegion,
+} from "@/components/video-editor/types";
+import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine";
+import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
+import { AudioMediaProcessor } from "./audioMediaProcessor";
+import {
+ AUDIO_BITRATE,
+ ENCODE_BACKPRESSURE_LIMIT,
+ getSourceTrackIdFromPath,
+ MP4_AUDIO_CODEC,
+ OFFLINE_AUDIO_SAMPLE_RATE,
+ OFFLINE_CHUNK_DURATION_SEC,
+ OFFLINE_ENCODE_CHUNK_FRAMES,
+ type PreparedOfflineRender,
+ resolveSourceTrackGain,
+ softLimitOfflineMixPeaksInPlace,
+ type TimelineSlice,
+ type TrimLikeRegion,
+} from "./audioProcessorShared";
+import type { VideoMuxer } from "./muxer";
+
+export class OfflineAudioProcessor extends AudioMediaProcessor {
+ protected async renderAndMuxOfflineAudio(
+ videoUrl: string,
+ trimRegions: TrimLikeRegion[],
+ speedRegions: SpeedRegion[],
+ audioRegions: AudioRegion[],
+ sourceAudioFallbackPaths: string[],
+ sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
+ sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
+ clipRegions: ClipRegion[] | undefined,
+ muxer: VideoMuxer,
+ ): Promise<void> {
+ const prepared = await this.prepareOfflineRender(
+ videoUrl,
+ trimRegions,
+ speedRegions,
+ audioRegions,
+ sourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
+ sourceAudioTrackSettings,
+ clipRegions,
+ );
+ if (this.cancelled) return;
+ await this.renderAndEncodeChunked(prepared, muxer);
+ }
+
+ protected async prepareOfflineRender(
+ videoUrl: string,
+ trimRegions: TrimLikeRegion[],
+ speedRegions: SpeedRegion[],
+ audioRegions: AudioRegion[],
+ sourceAudioFallbackPaths: string[],
+ sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
+ sourceAudioTrackSettings?: SourceAudioTrackSettings,
+ clipRegions?: ClipRegion[],
+ ): Promise<PreparedOfflineRender> {
+ if (this.cancelled) throw new Error("Export cancelled");
+ this.onProgress?.(0);
+
+ const resolvedPlan = buildResolvedAudioPlan({
+ videoResource: videoUrl,
+ sourceAudioFallbackPaths,
+ audioRegions,
+ sourceTrackGainById: {
+ mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"),
+ system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"),
+ mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"),
+ },
+ embeddedGain: Math.max(
+ 0,
+ Math.min(
+ 2,
+ sourceAudioTrackSettings?.mixed
+ ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed")
+ : sourceAudioTrackSettings?.system
+ ? resolveSourceTrackGain(sourceAudioTrackSettings, "system")
+ : 1,
+ ),
+ ),
+ });
+
+ // Decode embedded source audio separately from companion sidecars.
+ const mainBuffer = resolvedPlan.includeEmbeddedInExport
+ ? await this.decodeAudioFromUrl(videoUrl)
+ : null;
+ const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed");
+ const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null;
+ if (this.cancelled) throw new Error("Export cancelled");
+
+ // Decode companion / sidecar audio files
+ const companionEntries: Array<{
+ buffer: AudioBuffer;
+ startDelaySec: number;
+ gain: number;
+ }> = [];
+ const refDuration =
+ mainBuffer?.duration ??
+ (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
+ for (const audioPath of resolvedPlan.playbackPaths) {
+ if (this.cancelled) throw new Error("Export cancelled");
+ const buffer = await this.decodeAudioFromUrl(audioPath);
+ if (!buffer) continue;
+
+ companionEntries.push({
+ buffer,
+ gain: resolveSourceTrackGain(
+ sourceAudioTrackSettings,
+ getSourceTrackIdFromPath(audioPath),
+ ),
+ startDelaySec: estimateCompanionAudioStartDelaySeconds(
+ refDuration,
+ buffer.duration,
+ sourceAudioFallbackStartDelayMsByPath?.[audioPath],
+ ),
+ });
+ }
+ if (this.cancelled) throw new Error("Export cancelled");
+
+ // Decode audio region overlay files
+ const regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }> = [];
+ for (const region of audioRegions) {
+ if (this.cancelled) throw new Error("Export cancelled");
+ const buffer = await this.decodeAudioFromUrl(region.audioPath);
+ if (buffer) regionEntries.push({ buffer, region });
+ }
+
+ this.onProgress?.(0.2);
+
+ // Determine source duration for timeline calculation
+ const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null;
+ if (!primaryBuffer && regionEntries.length === 0) {
+ throw new Error("No decodable audio sources found");
+ }
+
+ let sourceDurationSec: number;
+ if (mainBufferEntry?.buffer) {
+ sourceDurationSec = mainBufferEntry.buffer.duration;
+ } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) {
+ sourceDurationSec = await this.getMediaDurationSec(videoUrl);
+ } else {
+ sourceDurationSec = primaryBuffer?.duration ?? 0;
+ }
+ const sourceDurationMs = sourceDurationSec * 1000;
+
+ // Build timeline slices (non-trimmed segments with speed info)
+ const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions);
+
+ let outputDurationMs = 0;
+ for (const slice of slices) {
+ outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed;
+ }
+
+ // Extend for audio regions that might exceed the video timeline
+ for (const { region } of regionEntries) {
+ const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices);
+ outputDurationMs = Math.max(outputDurationMs, regionEndOutput);
+ }
+
+ const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2);
+ const mutedSourceOutputRangesSec = (clipRegions ?? [])
+ .filter(
+ (clip) =>
+ Boolean(clip.muted) &&
+ Number.isFinite(clip.startMs) &&
+ Number.isFinite(clip.endMs) &&
+ clip.endMs > clip.startMs,
+ )
+ .map((clip) => ({
+ startSec: Math.max(0, clip.startMs / 1000),
+ endSec: Math.max(0, clip.endMs / 1000),
+ }));
+
+ return {
+ mainBufferEntry,
+ companionEntries,
+ regionEntries,
+ mutedSourceOutputRangesSec,
+ slices,
+ outputDurationMs,
+ numChannels,
+ };
+ }
+
+ // Render timeline in chunks and encode each chunk to the muxer immediately.
+ // Memory is bounded to ~OFFLINE_CHUNK_DURATION_SEC of PCM per chunk
+ // instead of holding the entire output buffer in memory.
+ protected async renderAndEncodeChunked(
+ prepared: PreparedOfflineRender,
+ muxer: VideoMuxer,
+ ): Promise<void> {
+ const { numChannels } = prepared;
+ const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
+
+ let encodeError: Error | null = null;
+ let muxError: Error | null = null;
+ let pendingMuxing = Promise.resolve();
+ let wroteFirstChunk = false;
+
+ const encodeConfig: AudioEncoderConfig = {
+ codec: MP4_AUDIO_CODEC,
+ sampleRate: OFFLINE_AUDIO_SAMPLE_RATE,
+ numberOfChannels: numChannels,
+ bitrate: AUDIO_BITRATE,
+ };
+
+ const supported = await AudioEncoder.isConfigSupported(encodeConfig);
+ if (!supported.supported) {
+ console.warn("[AudioProcessor] AAC encoding not supported for offline audio");
+ return;
+ }
+
+ const encoder = new AudioEncoder({
+ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => {
+ pendingMuxing = pendingMuxing
+ .then(async () => {
+ if (this.cancelled) return;
+ await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined);
+ wroteFirstChunk = true;
+ })
+ .catch((error) => {
+ muxError = error instanceof Error ? error : new Error(String(error));
+ });
+ },
+ error: (error: DOMException) => {
+ encodeError = new Error(`Audio encode error: ${error.message}`);
+ },
+ });
+ encoder.configure(encodeConfig);
+
+ try {
+ await this.renderChunked(
+ prepared,
+ totalOutputSec,
+ async (rendered, outputOffsetSec) => {
+ if (encodeError) throw encodeError;
+ if (muxError) throw muxError;
+ await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec);
+ },
+ );
+
+ if (encodeError) throw encodeError;
+ if (muxError) throw muxError;
+
+ if (encoder.state === "configured") {
+ await encoder.flush();
+ }
+
+ await pendingMuxing;
+
+ if (encodeError) throw encodeError;
+ if (muxError) throw muxError;
+ } finally {
+ if (encoder.state === "configured") {
+ encoder.close();
+ }
+ }
+ }
+
+ // Render timeline to a WAV blob for the native/FFmpeg export path.
+ // Processes in chunks to avoid holding the entire output in memory.
+ protected async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise<Blob> {
+ const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01);
+ const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE);
+ const numChannels = prepared.numChannels;
+
+ const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames);
+ const pcmParts: ArrayBuffer[] = [header];
+
+ await this.renderChunked(prepared, totalOutputSec, async (rendered) => {
+ pcmParts.push(...this.audioBufferToPcmParts(rendered));
+ });
+
+ return new Blob(pcmParts, { type: "audio/wav" });
+ }
+
+ // Shared chunked rendering loop. Processes the timeline in
+ // OFFLINE_CHUNK_DURATION_SEC segments, calling onChunk for each rendered buffer.
+ protected async renderChunked(
+ prepared: PreparedOfflineRender,
+ totalOutputSec: number,
+ onChunk: (
+ rendered: AudioBuffer,
+ outputOffsetSec: number,
+ chunkIndex: number,
+ ) => Promise<void>,
+ ): Promise<void> {
+ const { slices, numChannels } = prepared;
+ let outputOffsetSec = 0;
+ const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC);
+
+ for (let i = 0; i < chunkCount && !this.cancelled; i++) {
+ const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec);
+ const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE);
+
+ const offlineCtx = new OfflineAudioContext(
+ numChannels,
+ chunkFrames,
+ OFFLINE_AUDIO_SAMPLE_RATE,
+ );
+
+ // Schedule main audio
+ if (prepared.mainBufferEntry) {
+ this.scheduleBufferThroughTimeline(
+ offlineCtx,
+ prepared.mainBufferEntry.buffer,
+ slices,
+ 0,
+ prepared.mainBufferEntry.gain,
+ outputOffsetSec,
+ chunkSec,
+ prepared.mutedSourceOutputRangesSec,
+ );
+ }
+
+ // Schedule companion/sidecar audio
+ for (const entry of prepared.companionEntries) {
+ this.scheduleBufferThroughTimeline(
+ offlineCtx,
+ entry.buffer,
+ slices,
+ entry.startDelaySec,
+ entry.gain,
+ outputOffsetSec,
+ chunkSec,
+ prepared.mutedSourceOutputRangesSec,
+ );
+ }
+
+ // Schedule audio region overlays
+ for (const { buffer, region } of prepared.regionEntries) {
+ this.scheduleRegionForChunk(
+ offlineCtx,
+ buffer,
+ region,
+ slices,
+ outputOffsetSec,
+ chunkSec,
+ );
+ }
+
+ const rendered = await offlineCtx.startRendering();
+ if (this.cancelled) break;
+ softLimitOfflineMixPeaksInPlace(rendered);
+
+ await onChunk(rendered, outputOffsetSec, i);
+
+ outputOffsetSec += chunkSec;
+ this.onProgress?.(0.3 + (outputOffsetSec / totalOutputSec) * 0.7);
+ }
+ }
+
+ // Schedule an audio region overlay clipped to a specific chunk window.
+ protected scheduleRegionForChunk(
+ ctx: OfflineAudioContext,
+ buffer: AudioBuffer,
+ region: AudioRegion,
+ slices: TimelineSlice[],
+ chunkOutputStartSec: number,
+ chunkDurationSec: number,
+ ): void {
+ const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices);
+ const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices);
+
+ let localStartSec = outputStartMs / 1000 - chunkOutputStartSec;
+ let localEndSec = outputEndMs / 1000 - chunkOutputStartSec;
+
+ // Skip if region doesn't overlap with this chunk
+ if (localEndSec <= 0 || localStartSec >= chunkDurationSec) return;
+
+ // Clip to chunk bounds
+ let bufferOffsetSec = 0;
+ if (localStartSec < 0) {
+ bufferOffsetSec = -localStartSec;
+ localStartSec = 0;
+ }
+ if (localEndSec > chunkDurationSec) {
+ localEndSec = chunkDurationSec;
+ }
+
+ const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec);
+ if (duration <= 0.001) return;
+
+ const gainNode = ctx.createGain();
+ const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
+ gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain));
+ gainNode.connect(ctx.destination);
+
+ const source = ctx.createBufferSource();
+ source.buffer = buffer;
+ source.connect(gainNode);
+ source.start(localStartSec, bufferOffsetSec, duration);
+ }
+
+ // Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset.
+ protected async feedBufferToEncoder(
+ encoder: AudioEncoder,
+ buffer: AudioBuffer,
+ timestampOffsetSec: number,
+ ): Promise<void> {
+ const sampleRate = buffer.sampleRate;
+ const numChannels = buffer.numberOfChannels;
+ const totalFrames = buffer.length;
+
+ for (
+ let offset = 0;
+ offset < totalFrames && !this.cancelled;
+ offset += OFFLINE_ENCODE_CHUNK_FRAMES
+ ) {
+ const frameCount = Math.min(OFFLINE_ENCODE_CHUNK_FRAMES, totalFrames - offset);
+
+ const planarData = new Float32Array(frameCount * numChannels);
+ for (let ch = 0; ch < numChannels; ch++) {
+ const channelData = buffer.getChannelData(ch);
+ planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount);
+ }
+
+ const audioData = new AudioData({
+ format: "f32-planar",
+ sampleRate,
+ numberOfFrames: frameCount,
+ numberOfChannels: numChannels,
+ timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000),
+ data: planarData,
+ });
+
+ encoder.encode(audioData);
+ audioData.close();
+
+ while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) {
+ await new Promise((r) => setTimeout(r, 1));
+ }
+ }
+ }
+
+ // Decode audio from a URL using streaming WebCodecs decode with bulk fallback.
+ // Streaming decode avoids holding the full compressed file in memory alongside
+ // the decoded AudioBuffer, reducing peak memory for large recordings.
+}
diff --git a/src/lib/exporter/streamingDecodePipeline.ts b/src/lib/exporter/streamingDecodePipeline.ts
new file mode 100644
index 00000000..289bcf38
--- /dev/null
+++ b/src/lib/exporter/streamingDecodePipeline.ts
@@ -0,0 +1,448 @@
+import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
+import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
+import type { WebDemuxer } from "web-demuxer";
+import {
+ buildVideoDecodeFailure,
+ getDecodedFrameTimelineOffsetUs,
+ preserveFirstVideoDecodeFailure,
+ type DecodedVideoInfo,
+ type VideoDecodeFailureContext,
+} from "./streamingDecoderSupport";
+import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments";
+
+type OnFrameCallback = (
+ frame: VideoFrame,
+ exportTimestampUs: number,
+ sourceTimestampMs: number,
+ cursorTimestampMs: number,
+) => Promise<void>;
+
+interface StreamingDecodeContext {
+ demuxer: WebDemuxer;
+ decoder: VideoDecoder | null;
+ readonly cancelled: boolean;
+ readonly metadata: DecodedVideoInfo;
+ pendingFrames: VideoFrame[];
+ readonly maxDecodeQueue: number;
+ readonly maxPendingFrames: number;
+}
+
+const STARTUP_STABILIZATION_SECONDS = 1.25;
+const STARTUP_MAX_DECODE_QUEUE = 12;
+const STARTUP_MAX_PENDING_FRAMES = 28;
+
+export async function decodeVideoStream(
+ context: StreamingDecodeContext,
+
+ targetFrameRate: number,
+ trimRegions: TrimRegion[] | undefined,
+ speedRegions: SpeedRegion[] | undefined,
+ onFrame: OnFrameCallback,
+): Promise<void> {
+ if (!context.demuxer || !context.metadata) {
+ throw new Error("Must call loadMetadata() before decodeAll()");
+ }
+
+ const decoderConfig = await context.demuxer.getDecoderConfig("video");
+ const codec = context.metadata.codec.toLowerCase();
+ const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
+ const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
+ duration: context.metadata.duration,
+ streamDuration: context.metadata.streamDuration,
+ });
+ const segments = splitVideoSegmentsBySpeed(
+ computeVideoSegments(effectiveVideoDuration, trimRegions),
+ speedRegions,
+ );
+ const segmentOutputFrameCounts = segments.map((segment) =>
+ Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate),
+ );
+ const expectedOutputFrames = segmentOutputFrameCounts.reduce((sum, count) => sum + count, 0);
+ const frameDurationUs = 1_000_000 / targetFrameRate;
+ const epsilonSec = 0.001;
+ const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
+ const startupFrameBudget = Math.max(
+ 1,
+ Math.round(targetFrameRate * startupStabilizationSeconds),
+ );
+ let exportFrameIndex = 0;
+ let loggedSteadyStateBackpressure = false;
+ const backpressureWaiters = new Set<() => void>();
+
+ const notifyBackpressureProgress = () => {
+ if (backpressureWaiters.size === 0) {
+ return;
+ }
+
+ const waiters = [...backpressureWaiters];
+ backpressureWaiters.clear();
+ for (const resolve of waiters) {
+ resolve();
+ }
+ };
+
+ const waitForBackpressureProgress = () =>
+ new Promise<void>((resolve) => {
+ backpressureWaiters.add(resolve);
+ });
+
+ console.log(
+ `[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
+ );
+
+ // Async frame queue — decoder pushes, consumer pulls
+ context.pendingFrames.length = 0;
+ const pendingFrames = context.pendingFrames;
+ let frameResolve: ((frame: VideoFrame | null) => void) | null = null;
+ let decodeError: Error | null = null;
+ let decodeDone = false;
+ let firstDecodedFrameTimestampUs: number | null = null;
+ let decodedFrameTimelineOffsetUs = 0;
+ let submittedChunkCount = 0;
+ let lastSubmittedChunk: EncodedVideoChunk | undefined;
+ let lastSubmittedChunkIndex: number | undefined;
+ const preferredDecoderConfig = shouldPreferSoftwareDecode
+ ? {
+ ...decoderConfig,
+ hardwareAcceleration: "prefer-software" as const,
+ }
+ : decoderConfig;
+ let activeDecoderConfig = preferredDecoderConfig;
+ const getDecoderFailureContext = (): VideoDecodeFailureContext => ({
+ decoderConfig: activeDecoderConfig,
+ sourceMetadata: context.metadata ?? undefined,
+ chunkIndex: lastSubmittedChunkIndex,
+ chunk: lastSubmittedChunk,
+ decoderState: context.decoder?.state,
+ decodeQueueSize: context.decoder?.decodeQueueSize,
+ });
+ const recordFirstDecodeError = (error: unknown) => {
+ decodeError = preserveFirstVideoDecodeFailure(
+ decodeError,
+ error,
+ getDecoderFailureContext(),
+ );
+ };
+
+ context.decoder = new VideoDecoder({
+ output: (frame: VideoFrame) => {
+ if (frameResolve) {
+ const resolve = frameResolve;
+ frameResolve = null;
+ resolve(frame);
+ } else {
+ pendingFrames.push(frame);
+ }
+ notifyBackpressureProgress();
+ },
+ error: (e: DOMException) => {
+ recordFirstDecodeError(e);
+ if (frameResolve) {
+ const resolve = frameResolve;
+ frameResolve = null;
+ resolve(null);
+ }
+ notifyBackpressureProgress();
+ },
+ });
+ try {
+ context.decoder.configure(preferredDecoderConfig);
+ } catch (error) {
+ if (!shouldPreferSoftwareDecode) {
+ throw buildVideoDecodeFailure(error, getDecoderFailureContext());
+ }
+ // Fall back to default decoder config if software preference is unsupported.
+ activeDecoderConfig = decoderConfig;
+ try {
+ context.decoder.configure(decoderConfig);
+ } catch (fallbackError) {
+ throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext());
+ }
+ }
+
+ const getNextFrame = (): Promise<VideoFrame | null> => {
+ if (decodeError) return Promise.resolve(null);
+ if (pendingFrames.length > 0) {
+ const frame = pendingFrames.shift()!;
+ notifyBackpressureProgress();
+ return Promise.resolve(frame);
+ }
+ if (decodeDone) return Promise.resolve(null);
+ return new Promise((resolve) => {
+ frameResolve = resolve;
+ });
+ };
+
+ // One forward stream through the whole file.
+ // Pass explicit range because some containers are truncated when no end is provided.
+ const readEndSec =
+ Math.max(
+ context.metadata.duration + (context.metadata.mediaStartTime ?? 0),
+ (context.metadata.streamDuration ?? context.metadata.duration) +
+ (context.metadata.streamStartTime ?? context.metadata.mediaStartTime ?? 0),
+ ) + 0.5;
+ const reader = context.demuxer.read("video", 0, readEndSec).getReader();
+
+ // Feed chunks to decoder in background with backpressure
+ const feedPromise = (async () => {
+ try {
+ while (!context.cancelled && !decodeError) {
+ const { done, value: chunk } = await reader.read();
+ if (done || !chunk) break;
+
+ if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) {
+ loggedSteadyStateBackpressure = true;
+ console.log(
+ "[StreamingVideoDecoder] Switched to steady-state decode backpressure",
+ );
+ }
+
+ const decodeQueueLimit =
+ exportFrameIndex < startupFrameBudget
+ ? Math.min(context.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
+ : context.maxDecodeQueue;
+ const pendingFrameLimit =
+ exportFrameIndex < startupFrameBudget
+ ? Math.min(context.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
+ : context.maxPendingFrames;
+
+ // Backpressure on both decode queue and decoded frame backlog.
+ while (
+ !decodeError &&
+ context.decoder!.state === "configured" &&
+ (context.decoder!.decodeQueueSize > decodeQueueLimit ||
+ pendingFrames.length > pendingFrameLimit) &&
+ !context.cancelled
+ ) {
+ await waitForBackpressureProgress();
+ }
+ if (context.cancelled || decodeError) break;
+ if (context.decoder!.state !== "configured") {
+ recordFirstDecodeError(
+ new DOMException(
+ "Decoder closed before the next video chunk was submitted.",
+ "InvalidStateError",
+ ),
+ );
+ break;
+ }
+
+ lastSubmittedChunk = chunk;
+ lastSubmittedChunkIndex = submittedChunkCount;
+ context.decoder!.decode(chunk);
+ submittedChunkCount++;
+ }
+
+ if (!context.cancelled && context.decoder!.state === "configured") {
+ await context.decoder!.flush();
+ }
+ } catch (e) {
+ recordFirstDecodeError(e);
+ } finally {
+ decodeDone = true;
+ if (frameResolve) {
+ const resolve = frameResolve;
+ frameResolve = null;
+ resolve(null);
+ }
+ notifyBackpressureProgress();
+ }
+ })();
+
+ // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling
+ let segmentIdx = 0;
+ let segmentFrameIndex = 0;
+ let lastDecodedFrameSec: number | null = null;
+ let heldFrame: VideoFrame | null = null;
+ let heldFrameSec = 0;
+
+ const emitHeldFrameForTarget = async (segment: {
+ startSec: number;
+ endSec: number;
+ speed: number;
+ }) => {
+ if (!heldFrame) return false;
+ const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
+ if (segmentFrameIndex >= segmentFrameCount) return false;
+
+ const segmentDurationSec = segment.endSec - segment.startSec;
+ const sourceTimeSec =
+ segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
+ if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
+
+ const sourceTimestampMs = sourceTimeSec * 1000;
+ await onFrame(
+ heldFrame,
+ exportFrameIndex * frameDurationUs,
+ sourceTimestampMs,
+ sourceTimestampMs,
+ );
+ segmentFrameIndex++;
+ exportFrameIndex++;
+ return true;
+ };
+
+ while (!context.cancelled && segmentIdx < segments.length) {
+ const frame = await getNextFrame();
+ if (!frame) break;
+
+ if (firstDecodedFrameTimestampUs === null) {
+ firstDecodedFrameTimestampUs = frame.timestamp;
+ decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs(
+ firstDecodedFrameTimestampUs,
+ context.metadata,
+ );
+ }
+
+ const normalizedFrameTimeSec = Math.max(
+ 0,
+ (frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) /
+ 1_000_000,
+ );
+ const frameTimeSec: number =
+ lastDecodedFrameSec === null
+ ? normalizedFrameTimeSec
+ : Math.max(lastDecodedFrameSec, normalizedFrameTimeSec);
+ lastDecodedFrameSec = frameTimeSec;
+
+ // Finalize completed segments before handling this frame.
+ while (
+ segmentIdx < segments.length &&
+ frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
+ ) {
+ const segment = segments[segmentIdx];
+ while (!context.cancelled && (await emitHeldFrameForTarget(segment))) {
+ // Keep emitting remaining output frames for this segment from the last known frame.
+ }
+
+ segmentIdx++;
+ segmentFrameIndex = 0;
+ if (
+ heldFrame &&
+ segmentIdx < segments.length &&
+ heldFrameSec < segments[segmentIdx].startSec - epsilonSec
+ ) {
+ heldFrame.close();
+ heldFrame = null;
+ }
+ }
+
+ if (segmentIdx >= segments.length) {
+ frame.close();
+ continue;
+ }
+
+ const currentSegment = segments[segmentIdx];
+
+ // Before current segment (trimmed region or pre-roll).
+ if (frameTimeSec < currentSegment.startSec - epsilonSec) {
+ frame.close();
+ continue;
+ }
+
+ if (!heldFrame) {
+ heldFrame = frame;
+ heldFrameSec = frameTimeSec;
+ continue;
+ }
+
+ // Any target timestamp before this midpoint is closer to heldFrame than current frame.
+ const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2;
+ while (!context.cancelled) {
+ const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
+ if (segmentFrameIndex >= segmentFrameCount) {
+ break;
+ }
+
+ const segmentDurationSec = currentSegment.endSec - currentSegment.startSec;
+ const sourceTimeSec =
+ currentSegment.startSec +
+ (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
+ if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
+ break;
+ }
+ if (sourceTimeSec > handoffBoundarySec) {
+ break;
+ }
+
+ const sourceTimestampMs = sourceTimeSec * 1000;
+ await onFrame(
+ heldFrame,
+ exportFrameIndex * frameDurationUs,
+ sourceTimestampMs,
+ sourceTimestampMs,
+ );
+ segmentFrameIndex++;
+ exportFrameIndex++;
+ }
+
+ heldFrame.close();
+ heldFrame = frame;
+ heldFrameSec = frameTimeSec;
+ }
+
+ // Flush remaining output frames for the last decoded frame.
+ if (!decodeError && heldFrame && segmentIdx < segments.length) {
+ while (!context.cancelled && segmentIdx < segments.length) {
+ const segment = segments[segmentIdx];
+ if (heldFrameSec < segment.startSec - epsilonSec) {
+ break;
+ }
+
+ while (!context.cancelled && (await emitHeldFrameForTarget(segment))) {
+ // Keep emitting output frames for the active segment.
+ }
+
+ segmentIdx++;
+ segmentFrameIndex = 0;
+ if (
+ segmentIdx < segments.length &&
+ heldFrameSec < segments[segmentIdx].startSec - epsilonSec
+ ) {
+ break;
+ }
+ }
+ heldFrame.close();
+ heldFrame = null;
+ } else if (heldFrame) {
+ heldFrame.close();
+ heldFrame = null;
+ }
+
+ // Drain leftover decoded frames
+ while (!decodeDone && !decodeError) {
+ const frame = await getNextFrame();
+ if (!frame) break;
+ frame.close();
+ }
+
+ try {
+ reader.cancel();
+ } catch {
+ /* already closed */
+ }
+ await feedPromise;
+ for (const f of pendingFrames) f.close();
+ pendingFrames.length = 0;
+
+ if (context.decoder?.state === "configured") {
+ context.decoder.close();
+ }
+ context.decoder = null;
+
+ if (decodeError) {
+ throw decodeError;
+ }
+
+ const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
+ if (
+ !context.cancelled &&
+ lastDecodedFrameSec !== null &&
+ requiredEndSec - lastDecodedFrameSec > 1 &&
+ exportFrameIndex < expectedOutputFrames
+ ) {
+ throw new Error(
+ `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`,
+ );
+ }
+}
diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts
index ee59a0ac..152fdb19 100644
--- a/src/lib/exporter/streamingDecoder.ts
+++ b/src/lib/exporter/streamingDecoder.ts
@@ -2,120 +2,27 @@ import { WebDemuxer } from "web-demuxer";
import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
import { createFallbackDemuxerSource, resolveMediaResourceUrl } from "./localMediaSource";
+import { decodeVideoStream } from "./streamingDecodePipeline";
+import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments";
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
-const STARTUP_STABILIZATION_SECONDS = 1.25;
-const STARTUP_MAX_DECODE_QUEUE = 12;
-const STARTUP_MAX_PENDING_FRAMES = 28;
-export interface DecodedVideoInfo {
- width: number;
- height: number;
- duration: number; // seconds
- mediaStartTime?: number; // seconds
- streamStartTime?: number; // seconds
- streamDuration?: number; // seconds
- frameRate: number;
- codec: string;
- hasAudio: boolean;
- audioCodec?: string;
- audioSampleRate?: number;
-}
+import type { DecodedVideoInfo } from "./streamingDecoderSupport";
+export {
+ buildVideoDecodeFailure,
+ getDecodedFrameStartupOffsetUs,
+ getDecodedFrameTimelineOffsetUs,
+ getVideoDecodeFailureCode,
+ preserveFirstVideoDecodeFailure,
+ type DecodedVideoInfo,
+ type VideoDecodeFailureContext,
+} from "./streamingDecoderSupport";
interface StreamingVideoDecoderLoadOptions {
useFallbackMediaSource?: boolean;
}
-interface VideoDecodeFailureContext {
- decoderConfig: VideoDecoderConfig;
- sourceMetadata?: DecodedVideoInfo;
- chunkIndex?: number;
- chunk?: EncodedVideoChunk;
- decoderState?: CodecState;
- decodeQueueSize?: number;
-}
-
-/** Maps WebCodecs failures to stable support-facing identifiers. */
-export function getVideoDecodeFailureCode(error: unknown): string {
- const name = error instanceof DOMException ? error.name : "";
- switch (name) {
- case "EncodingError":
- return "VIDEO_DECODE_ENCODING_ERROR";
- case "NotSupportedError":
- return "VIDEO_CODEC_UNSUPPORTED";
- case "QuotaExceededError":
- return "VIDEO_DECODER_RESOURCE_EXHAUSTED";
- case "InvalidStateError":
- return "VIDEO_DECODER_INVALID_STATE";
- default:
- return "VIDEO_DECODE_FAILED";
- }
-}
-
-function describeUnknownError(error: unknown): string {
- if (error instanceof DOMException) {
- return `${error.name}: ${error.message}`;
- }
-
- if (error instanceof Error) {
- return error.message;
- }
-
- return String(error);
-}
-
-/** Builds a decode error with codec, source, chunk, and decoder-state context. */
-export function buildVideoDecodeFailure(error: unknown, context: VideoDecodeFailureContext): Error {
- const details = [`codec=${context.decoderConfig.codec}`];
- const failureCode = getVideoDecodeFailureCode(error);
- const width = context.decoderConfig.codedWidth;
- const height = context.decoderConfig.codedHeight;
- if (width && height) {
- details.push(`codedSize=${width}x${height}`);
- }
- if (context.decoderConfig.hardwareAcceleration) {
- details.push(`hardwareAcceleration=${context.decoderConfig.hardwareAcceleration}`);
- }
- if (context.sourceMetadata) {
- details.push(`sourceFps=${context.sourceMetadata.frameRate}`);
- details.push(`sourceDurationSec=${context.sourceMetadata.duration}`);
- }
- if (context.chunkIndex !== undefined) {
- details.push(`chunkIndex=${context.chunkIndex}`);
- }
- if (context.chunk) {
- details.push(`chunkType=${context.chunk.type}`);
- details.push(`chunkTimestampUs=${context.chunk.timestamp}`);
- details.push(`sourceTimeSec=${(context.chunk.timestamp / 1_000_000).toFixed(3)}`);
- if (typeof context.chunk.duration === "number") {
- details.push(`chunkDurationUs=${context.chunk.duration}`);
- }
- details.push(`chunkBytes=${context.chunk.byteLength}`);
- }
- if (context.decoderState) {
- details.push(`decoderState=${context.decoderState}`);
- }
- if (context.decodeQueueSize !== undefined) {
- details.push(`decodeQueueSize=${context.decodeQueueSize}`);
- }
-
- const failure = new Error(
- `[${failureCode}] VideoDecoder failure: ${describeUnknownError(error)} (${details.join(", ")})`,
- );
- (failure as Error & { cause?: unknown }).cause = error;
- return failure;
-}
-
-/** Keeps the original decoder failure when cleanup triggers secondary errors. */
-export function preserveFirstVideoDecodeFailure(
- existingError: Error | null,
- error: unknown,
- context: VideoDecodeFailureContext,
-): Error {
- return existingError ?? buildVideoDecodeFailure(error, context);
-}
-
/** Decoder retains ownership of the VideoFrame and closes it after use. */
type OnFrameCallback = (
frame: VideoFrame,
@@ -124,32 +31,6 @@ type OnFrameCallback = (
cursorTimestampMs: number,
) => Promise<void>;
-export function getDecodedFrameStartupOffsetUs(
- firstDecodedFrameTimestampUs: number,
- metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
-): number {
- const streamStartTimeUs = Math.round(
- (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
- );
-
- return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs);
-}
-
-export function getDecodedFrameTimelineOffsetUs(
- firstDecodedFrameTimestampUs: number,
- metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
-): number {
- const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000);
- const streamStartTimeUs = Math.round(
- (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
- );
-
- return (
- Math.max(0, streamStartTimeUs - mediaStartTimeUs) +
- getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata)
- );
-}
-
/**
* Decodes video frames via web-demuxer + VideoDecoder in a single forward pass.
* Way faster than seeking an HTMLVideoElement per frame.
@@ -291,498 +172,44 @@ export class StreamingVideoDecoder {
throw new Error("Must call loadMetadata() before decodeAll()");
}
- const decoderConfig = await this.demuxer.getDecoderConfig("video");
- const codec = this.metadata.codec.toLowerCase();
- const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
- const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
- duration: this.metadata.duration,
- streamDuration: this.metadata.streamDuration,
- });
- const segments = this.splitBySpeed(
- this.computeSegments(effectiveVideoDuration, trimRegions),
+ const owner = this;
+ await decodeVideoStream(
+ {
+ demuxer: this.demuxer,
+ metadata: this.metadata,
+ pendingFrames: this.pendingFrames,
+ maxDecodeQueue: this.maxDecodeQueue,
+ maxPendingFrames: this.maxPendingFrames,
+ get cancelled() {
+ return owner.cancelled;
+ },
+ get decoder() {
+ return owner.decoder;
+ },
+ set decoder(value) {
+ owner.decoder = value;
+ },
+ },
+ targetFrameRate,
+ trimRegions,
speedRegions,
+ onFrame,
);
- const segmentOutputFrameCounts = segments.map((segment) =>
- Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate),
- );
- const expectedOutputFrames = segmentOutputFrameCounts.reduce(
- (sum, count) => sum + count,
- 0,
- );
- const frameDurationUs = 1_000_000 / targetFrameRate;
- const epsilonSec = 0.001;
- const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS;
- const startupFrameBudget = Math.max(
- 1,
- Math.round(targetFrameRate * startupStabilizationSeconds),
- );
- let exportFrameIndex = 0;
- let loggedSteadyStateBackpressure = false;
- const backpressureWaiters = new Set<() => void>();
-
- const notifyBackpressureProgress = () => {
- if (backpressureWaiters.size === 0) {
- return;
- }
-
- const waiters = [...backpressureWaiters];
- backpressureWaiters.clear();
- for (const resolve of waiters) {
- resolve();
- }
- };
-
- const waitForBackpressureProgress = () =>
- new Promise<void>((resolve) => {
- backpressureWaiters.add(resolve);
- });
-
- console.log(
- `[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`,
- );
-
- // Async frame queue — decoder pushes, consumer pulls
- this.pendingFrames.length = 0;
- const pendingFrames = this.pendingFrames;
- let frameResolve: ((frame: VideoFrame | null) => void) | null = null;
- let decodeError: Error | null = null;
- let decodeDone = false;
- let firstDecodedFrameTimestampUs: number | null = null;
- let decodedFrameTimelineOffsetUs = 0;
- let submittedChunkCount = 0;
- let lastSubmittedChunk: EncodedVideoChunk | undefined;
- let lastSubmittedChunkIndex: number | undefined;
- const preferredDecoderConfig = shouldPreferSoftwareDecode
- ? {
- ...decoderConfig,
- hardwareAcceleration: "prefer-software" as const,
- }
- : decoderConfig;
- let activeDecoderConfig = preferredDecoderConfig;
- const getDecoderFailureContext = (): VideoDecodeFailureContext => ({
- decoderConfig: activeDecoderConfig,
- sourceMetadata: this.metadata ?? undefined,
- chunkIndex: lastSubmittedChunkIndex,
- chunk: lastSubmittedChunk,
- decoderState: this.decoder?.state,
- decodeQueueSize: this.decoder?.decodeQueueSize,
- });
- const recordFirstDecodeError = (error: unknown) => {
- decodeError = preserveFirstVideoDecodeFailure(
- decodeError,
- error,
- getDecoderFailureContext(),
- );
- };
-
- this.decoder = new VideoDecoder({
- output: (frame: VideoFrame) => {
- if (frameResolve) {
- const resolve = frameResolve;
- frameResolve = null;
- resolve(frame);
- } else {
- pendingFrames.push(frame);
- }
- notifyBackpressureProgress();
- },
- error: (e: DOMException) => {
- recordFirstDecodeError(e);
- if (frameResolve) {
- const resolve = frameResolve;
- frameResolve = null;
- resolve(null);
- }
- notifyBackpressureProgress();
- },
- });
- try {
- this.decoder.configure(preferredDecoderConfig);
- } catch (error) {
- if (!shouldPreferSoftwareDecode) {
- throw buildVideoDecodeFailure(error, getDecoderFailureContext());
- }
- // Fall back to default decoder config if software preference is unsupported.
- activeDecoderConfig = decoderConfig;
- try {
- this.decoder.configure(decoderConfig);
- } catch (fallbackError) {
- throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext());
- }
- }
-
- const getNextFrame = (): Promise<VideoFrame | null> => {
- if (decodeError) return Promise.resolve(null);
- if (pendingFrames.length > 0) {
- const frame = pendingFrames.shift()!;
- notifyBackpressureProgress();
- return Promise.resolve(frame);
- }
- if (decodeDone) return Promise.resolve(null);
- return new Promise((resolve) => {
- frameResolve = resolve;
- });
- };
-
- // One forward stream through the whole file.
- // Pass explicit range because some containers are truncated when no end is provided.
- const readEndSec =
- Math.max(
- this.metadata.duration + (this.metadata.mediaStartTime ?? 0),
- (this.metadata.streamDuration ?? this.metadata.duration) +
- (this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0),
- ) + 0.5;
- const reader = this.demuxer.read("video", 0, readEndSec).getReader();
-
- // Feed chunks to decoder in background with backpressure
- const feedPromise = (async () => {
- try {
- while (!this.cancelled && !decodeError) {
- const { done, value: chunk } = await reader.read();
- if (done || !chunk) break;
-
- if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) {
- loggedSteadyStateBackpressure = true;
- console.log(
- "[StreamingVideoDecoder] Switched to steady-state decode backpressure",
- );
- }
-
- const decodeQueueLimit =
- exportFrameIndex < startupFrameBudget
- ? Math.min(this.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE)
- : this.maxDecodeQueue;
- const pendingFrameLimit =
- exportFrameIndex < startupFrameBudget
- ? Math.min(this.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES)
- : this.maxPendingFrames;
-
- // Backpressure on both decode queue and decoded frame backlog.
- while (
- !decodeError &&
- this.decoder!.state === "configured" &&
- (this.decoder!.decodeQueueSize > decodeQueueLimit ||
- pendingFrames.length > pendingFrameLimit) &&
- !this.cancelled
- ) {
- await waitForBackpressureProgress();
- }
- if (this.cancelled || decodeError) break;
- if (this.decoder!.state !== "configured") {
- recordFirstDecodeError(
- new DOMException(
- "Decoder closed before the next video chunk was submitted.",
- "InvalidStateError",
- ),
- );
- break;
- }
-
- lastSubmittedChunk = chunk;
- lastSubmittedChunkIndex = submittedChunkCount;
- this.decoder!.decode(chunk);
- submittedChunkCount++;
- }
-
- if (!this.cancelled && this.decoder!.state === "configured") {
- await this.decoder!.flush();
- }
- } catch (e) {
- recordFirstDecodeError(e);
- } finally {
- decodeDone = true;
- if (frameResolve) {
- const resolve = frameResolve;
- frameResolve = null;
- resolve(null);
- }
- notifyBackpressureProgress();
- }
- })();
-
- // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling
- let segmentIdx = 0;
- let segmentFrameIndex = 0;
- let lastDecodedFrameSec: number | null = null;
- let heldFrame: VideoFrame | null = null;
- let heldFrameSec = 0;
-
- const emitHeldFrameForTarget = async (segment: {
- startSec: number;
- endSec: number;
- speed: number;
- }) => {
- if (!heldFrame) return false;
- const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
- if (segmentFrameIndex >= segmentFrameCount) return false;
-
- const segmentDurationSec = segment.endSec - segment.startSec;
- const sourceTimeSec =
- segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
- if (sourceTimeSec >= segment.endSec - epsilonSec) return false;
-
- const sourceTimestampMs = sourceTimeSec * 1000;
- await onFrame(
- heldFrame,
- exportFrameIndex * frameDurationUs,
- sourceTimestampMs,
- sourceTimestampMs,
- );
- segmentFrameIndex++;
- exportFrameIndex++;
- return true;
- };
-
- while (!this.cancelled && segmentIdx < segments.length) {
- const frame = await getNextFrame();
- if (!frame) break;
-
- if (firstDecodedFrameTimestampUs === null) {
- firstDecodedFrameTimestampUs = frame.timestamp;
- decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs(
- firstDecodedFrameTimestampUs,
- this.metadata,
- );
- }
-
- const normalizedFrameTimeSec = Math.max(
- 0,
- (frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) /
- 1_000_000,
- );
- const frameTimeSec: number =
- lastDecodedFrameSec === null
- ? normalizedFrameTimeSec
- : Math.max(lastDecodedFrameSec, normalizedFrameTimeSec);
- lastDecodedFrameSec = frameTimeSec;
-
- // Finalize completed segments before handling this frame.
- while (
- segmentIdx < segments.length &&
- frameTimeSec >= segments[segmentIdx].endSec - epsilonSec
- ) {
- const segment = segments[segmentIdx];
- while (!this.cancelled && (await emitHeldFrameForTarget(segment))) {
- // Keep emitting remaining output frames for this segment from the last known frame.
- }
-
- segmentIdx++;
- segmentFrameIndex = 0;
- if (
- heldFrame &&
- segmentIdx < segments.length &&
- heldFrameSec < segments[segmentIdx].startSec - epsilonSec
- ) {
- heldFrame.close();
- heldFrame = null;
- }
- }
-
- if (segmentIdx >= segments.length) {
- frame.close();
- continue;
- }
-
- const currentSegment = segments[segmentIdx];
-
- // Before current segment (trimmed region or pre-roll).
- if (frameTimeSec < currentSegment.startSec - epsilonSec) {
- frame.close();
- continue;
- }
-
- if (!heldFrame) {
- heldFrame = frame;
- heldFrameSec = frameTimeSec;
- continue;
- }
-
- // Any target timestamp before this midpoint is closer to heldFrame than current frame.
- const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2;
- while (!this.cancelled) {
- const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
- if (segmentFrameIndex >= segmentFrameCount) {
- break;
- }
-
- const segmentDurationSec = currentSegment.endSec - currentSegment.startSec;
- const sourceTimeSec =
- currentSegment.startSec +
- (segmentFrameIndex / segmentFrameCount) * segmentDurationSec;
- if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
- break;
- }
- if (sourceTimeSec > handoffBoundarySec) {
- break;
- }
-
- const sourceTimestampMs = sourceTimeSec * 1000;
- await onFrame(
- heldFrame,
- exportFrameIndex * frameDurationUs,
- sourceTimestampMs,
- sourceTimestampMs,
- );
- segmentFrameIndex++;
- exportFrameIndex++;
- }
-
- heldFrame.close();
- heldFrame = frame;
- heldFrameSec = frameTimeSec;
- }
-
- // Flush remaining output frames for the last decoded frame.
- if (!decodeError && heldFrame && segmentIdx < segments.length) {
- while (!this.cancelled && segmentIdx < segments.length) {
- const segment = segments[segmentIdx];
- if (heldFrameSec < segment.startSec - epsilonSec) {
- break;
- }
-
- while (!this.cancelled && (await emitHeldFrameForTarget(segment))) {
- // Keep emitting output frames for the active segment.
- }
-
- segmentIdx++;
- segmentFrameIndex = 0;
- if (
- segmentIdx < segments.length &&
- heldFrameSec < segments[segmentIdx].startSec - epsilonSec
- ) {
- break;
- }
- }
- heldFrame.close();
- heldFrame = null;
- } else if (heldFrame) {
- heldFrame.close();
- heldFrame = null;
- }
-
- // Drain leftover decoded frames
- while (!decodeDone && !decodeError) {
- const frame = await getNextFrame();
- if (!frame) break;
- frame.close();
- }
-
- try {
- reader.cancel();
- } catch {
- /* already closed */
- }
- await feedPromise;
- for (const f of pendingFrames) f.close();
- pendingFrames.length = 0;
-
- if (this.decoder?.state === "configured") {
- this.decoder.close();
- }
- this.decoder = null;
-
- if (decodeError) {
- throw decodeError;
- }
-
- const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0;
- if (
- !this.cancelled &&
- lastDecodedFrameSec !== null &&
- requiredEndSec - lastDecodedFrameSec > 1 &&
- exportFrameIndex < expectedOutputFrames
- ) {
- throw new Error(
- `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`,
- );
- }
- }
-
- private computeSegments(
- totalDuration: number,
- trimRegions?: TrimRegion[],
- ): Array<{ startSec: number; endSec: number }> {
- if (!trimRegions || trimRegions.length === 0) {
- return [{ startSec: 0, endSec: totalDuration }];
- }
-
- const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs);
- const segments: Array<{ startSec: number; endSec: number }> = [];
- let cursor = 0;
-
- for (const trim of sorted) {
- const trimStart = trim.startMs / 1000;
- const trimEnd = trim.endMs / 1000;
- if (cursor < trimStart) {
- segments.push({ startSec: cursor, endSec: trimStart });
- }
- cursor = Math.max(cursor, trimEnd);
- }
-
- if (cursor < totalDuration) {
- segments.push({ startSec: cursor, endSec: totalDuration });
- }
-
- return segments;
}
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
if (!this.metadata) throw new Error("Must call loadMetadata() first");
- const trimSegments = this.computeSegments(
+ const trimSegments = computeVideoSegments(
getEffectiveVideoStreamDurationSeconds({
duration: this.metadata.duration,
streamDuration: this.metadata.streamDuration,
}),
trimRegions,
);
- const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
+ const speedSegments = splitVideoSegmentsBySpeed(trimSegments, speedRegions);
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
}
- private splitBySpeed(
- segments: Array<{ startSec: number; endSec: number }>,
- speedRegions?: SpeedRegion[],
- ): Array<{ startSec: number; endSec: number; speed: number }> {
- if (!speedRegions || speedRegions.length === 0)
- return segments.map((s) => ({ ...s, speed: 1 }));
-
- const result: Array<{ startSec: number; endSec: number; speed: number }> = [];
- for (const segment of segments) {
- const overlapping = speedRegions
- .filter(
- (sr) =>
- sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec,
- )
- .sort((a, b) => a.startMs - b.startMs);
-
- if (overlapping.length === 0) {
- result.push({ ...segment, speed: 1 });
- continue;
- }
-
- let cursor = segment.startSec;
- for (const sr of overlapping) {
- const srStart = Math.max(sr.startMs / 1000, segment.startSec);
- const srEnd = Math.min(sr.endMs / 1000, segment.endSec);
- if (cursor < srStart) {
- result.push({ startSec: cursor, endSec: srStart, speed: 1 });
- }
- const effectiveStart = Math.max(cursor, srStart);
- if (srEnd > effectiveStart) {
- result.push({
- startSec: effectiveStart,
- endSec: srEnd,
- speed: sr.speed,
- });
- }
- cursor = Math.max(cursor, srEnd);
- }
- if (cursor < segment.endSec)
- result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 });
- }
- return result.filter((s) => s.endSec - s.startSec > 0.0001);
- }
-
cancel(): void {
this.cancelled = true;
}
diff --git a/src/lib/exporter/streamingDecoderSupport.ts b/src/lib/exporter/streamingDecoderSupport.ts
new file mode 100644
index 00000000..dd934631
--- /dev/null
+++ b/src/lib/exporter/streamingDecoderSupport.ts
@@ -0,0 +1,128 @@
+export interface DecodedVideoInfo {
+ width: number;
+ height: number;
+ duration: number; // seconds
+ mediaStartTime?: number; // seconds
+ streamStartTime?: number; // seconds
+ streamDuration?: number; // seconds
+ frameRate: number;
+ codec: string;
+ hasAudio: boolean;
+ audioCodec?: string;
+ audioSampleRate?: number;
+}
+
+export interface VideoDecodeFailureContext {
+ decoderConfig: VideoDecoderConfig;
+ sourceMetadata?: DecodedVideoInfo;
+ chunkIndex?: number;
+ chunk?: EncodedVideoChunk;
+ decoderState?: CodecState;
+ decodeQueueSize?: number;
+}
+
+/** Maps WebCodecs failures to stable support-facing identifiers. */
+export function getVideoDecodeFailureCode(error: unknown): string {
+ const name = error instanceof DOMException ? error.name : "";
+ switch (name) {
+ case "EncodingError":
+ return "VIDEO_DECODE_ENCODING_ERROR";
+ case "NotSupportedError":
+ return "VIDEO_CODEC_UNSUPPORTED";
+ case "QuotaExceededError":
+ return "VIDEO_DECODER_RESOURCE_EXHAUSTED";
+ case "InvalidStateError":
+ return "VIDEO_DECODER_INVALID_STATE";
+ default:
+ return "VIDEO_DECODE_FAILED";
+ }
+}
+
+function describeUnknownError(error: unknown): string {
+ if (error instanceof DOMException) {
+ return `${error.name}: ${error.message}`;
+ }
+
+ if (error instanceof Error) {
+ return error.message;
+ }
+
+ return String(error);
+}
+
+/** Builds a decode error with codec, source, chunk, and decoder-state context. */
+export function buildVideoDecodeFailure(error: unknown, context: VideoDecodeFailureContext): Error {
+ const details = [`codec=${context.decoderConfig.codec}`];
+ const failureCode = getVideoDecodeFailureCode(error);
+ const width = context.decoderConfig.codedWidth;
+ const height = context.decoderConfig.codedHeight;
+ if (width && height) {
+ details.push(`codedSize=${width}x${height}`);
+ }
+ if (context.decoderConfig.hardwareAcceleration) {
+ details.push(`hardwareAcceleration=${context.decoderConfig.hardwareAcceleration}`);
+ }
+ if (context.sourceMetadata) {
+ details.push(`sourceFps=${context.sourceMetadata.frameRate}`);
+ details.push(`sourceDurationSec=${context.sourceMetadata.duration}`);
+ }
+ if (context.chunkIndex !== undefined) {
+ details.push(`chunkIndex=${context.chunkIndex}`);
+ }
+ if (context.chunk) {
+ details.push(`chunkType=${context.chunk.type}`);
+ details.push(`chunkTimestampUs=${context.chunk.timestamp}`);
+ details.push(`sourceTimeSec=${(context.chunk.timestamp / 1_000_000).toFixed(3)}`);
+ if (typeof context.chunk.duration === "number") {
+ details.push(`chunkDurationUs=${context.chunk.duration}`);
+ }
+ details.push(`chunkBytes=${context.chunk.byteLength}`);
+ }
+ if (context.decoderState) {
+ details.push(`decoderState=${context.decoderState}`);
+ }
+ if (context.decodeQueueSize !== undefined) {
+ details.push(`decodeQueueSize=${context.decodeQueueSize}`);
+ }
+
+ const failure = new Error(
+ `[${failureCode}] VideoDecoder failure: ${describeUnknownError(error)} (${details.join(", ")})`,
+ );
+ (failure as Error & { cause?: unknown }).cause = error;
+ return failure;
+}
+
+/** Keeps the original decoder failure when cleanup triggers secondary errors. */
+export function preserveFirstVideoDecodeFailure(
+ existingError: Error | null,
+ error: unknown,
+ context: VideoDecodeFailureContext,
+): Error {
+ return existingError ?? buildVideoDecodeFailure(error, context);
+}
+
+export function getDecodedFrameStartupOffsetUs(
+ firstDecodedFrameTimestampUs: number,
+ metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
+): number {
+ const streamStartTimeUs = Math.round(
+ (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
+ );
+
+ return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs);
+}
+
+export function getDecodedFrameTimelineOffsetUs(
+ firstDecodedFrameTimestampUs: number,
+ metadata: Pick<DecodedVideoInfo, "mediaStartTime" | "streamStartTime">,
+): number {
+ const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000);
+ const streamStartTimeUs = Math.round(
+ (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000,
+ );
+
+ return (
+ Math.max(0, streamStartTimeUs - mediaStartTimeUs) +
+ getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata)
+ );
+}
diff --git a/src/lib/exporter/videoTimelineSegments.ts b/src/lib/exporter/videoTimelineSegments.ts
new file mode 100644
index 00000000..9cb19822
--- /dev/null
+++ b/src/lib/exporter/videoTimelineSegments.ts
@@ -0,0 +1,72 @@
+import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types";
+
+export function computeVideoSegments(
+ totalDuration: number,
+ trimRegions?: TrimRegion[],
+): Array<{ startSec: number; endSec: number }> {
+ if (!trimRegions || trimRegions.length === 0) {
+ return [{ startSec: 0, endSec: totalDuration }];
+ }
+
+ const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs);
+ const segments: Array<{ startSec: number; endSec: number }> = [];
+ let cursor = 0;
+
+ for (const trim of sorted) {
+ const trimStart = trim.startMs / 1000;
+ const trimEnd = trim.endMs / 1000;
+ if (cursor < trimStart) {
+ segments.push({ startSec: cursor, endSec: trimStart });
+ }
+ cursor = Math.max(cursor, trimEnd);
+ }
+
+ if (cursor < totalDuration) {
+ segments.push({ startSec: cursor, endSec: totalDuration });
+ }
+
+ return segments;
+}
+
+export function splitVideoSegmentsBySpeed(
+ segments: Array<{ startSec: number; endSec: number }>,
+ speedRegions?: SpeedRegion[],
+): Array<{ startSec: number; endSec: number; speed: number }> {
+ if (!speedRegions || speedRegions.length === 0)
+ return segments.map((s) => ({ ...s, speed: 1 }));
+
+ const result: Array<{ startSec: number; endSec: number; speed: number }> = [];
+ for (const segment of segments) {
+ const overlapping = speedRegions
+ .filter(
+ (sr) => sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec,
+ )
+ .sort((a, b) => a.startMs - b.startMs);
+
+ if (overlapping.length === 0) {
+ result.push({ ...segment, speed: 1 });
+ continue;
+ }
+
+ let cursor = segment.startSec;
+ for (const sr of overlapping) {
+ const srStart = Math.max(sr.startMs / 1000, segment.startSec);
+ const srEnd = Math.min(sr.endMs / 1000, segment.endSec);
+ if (cursor < srStart) {
+ result.push({ startSec: cursor, endSec: srStart, speed: 1 });
+ }
+ const effectiveStart = Math.max(cursor, srStart);
+ if (srEnd > effectiveStart) {
+ result.push({
+ startSec: effectiveStart,
+ endSec: srEnd,
+ speed: sr.speed,
+ });
+ }
+ cursor = Math.max(cursor, srEnd);
+ }
+ if (cursor < segment.endSec)
+ result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 });
+ }
+ return result.filter((s) => s.endSec - s.startSec > 0.0001);
+}