Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

8.4 KB
Raw Blame
import { WebDemuxer } from "web-demuxer";
import { AudioTimelineProcessor } from "./audioTimelineProcessor";
import { DECODE_BACKPRESSURE_LIMIT, OFFLINE_AUDIO_SAMPLE_RATE } from "./audioProcessorShared";
import { resolveMediaElementSource } from "./localMediaSource";

export class AudioMediaProcessor extends AudioTimelineProcessor {
	protected async decodeAudioFromUrl(url: string): Promise<AudioBuffer | null> {
		try {
			const buffer = await this.streamDecodeFromUrl(url);
			if (buffer) return buffer;
		} catch (error) {
			console.warn(
				"[AudioProcessor] Streaming decode failed, falling back to bulk decode:",
				url,
				error,
			);
		}
		return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE);
	}

	// Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk
	// without loading the entire compressed file into a contiguous ArrayBuffer.
	protected async streamDecodeFromUrl(url: string): Promise<AudioBuffer | null> {
		const source = await resolveMediaElementSource(url);
		let demuxer: WebDemuxer | null = null;

		try {
			const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href;
			demuxer = new WebDemuxer({ wasmFilePath: wasmUrl });
			await demuxer.load(source.src);

			let audioConfig: AudioDecoderConfig;
			try {
				audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig;
			} catch {
				return null; // No audio track
			}

			const sampleRate = audioConfig.sampleRate || 48_000;
			const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2);

			// Accumulate decoded PCM per channel
			const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []);
			let totalFrames = 0;
			let decodeError: Error | null = null;
			const decodeCapacityWaiters = new Set<() => void>();

			const notifyDecodeCapacityAvailable = () => {
				if (decodeCapacityWaiters.size === 0) {
					return;
				}

				const waiters = [...decodeCapacityWaiters];
				decodeCapacityWaiters.clear();
				for (const resolve of waiters) {
					resolve();
				}
			};

			const waitForDecodeCapacity = () =>
				new Promise<void>((resolve) => {
					decodeCapacityWaiters.add(resolve);
				});

			const decoder = new AudioDecoder({
				output: (data: AudioData) => {
					try {
						const frames = data.numberOfFrames;
						const dataChannels = Math.min(data.numberOfChannels, numChannels);
						const format = data.format;

						if (format?.includes("planar")) {
							for (let ch = 0; ch < dataChannels; ch++) {
								const size = data.allocationSize({
									planeIndex: ch,
								});
								const bytes = new ArrayBuffer(size);
								data.copyTo(bytes, { planeIndex: ch });
								channelChunks[ch].push(this.rawToFloat32(bytes, format, frames));
							}
						} else if (format) {
							// Interleaved format — deinterleave into per-channel arrays.
							// Use data.numberOfChannels as stride (not capped dataChannels)
							// since the raw buffer contains all source channels.
							const srcChannels = data.numberOfChannels;
							const size = data.allocationSize({ planeIndex: 0 });
							const bytes = new ArrayBuffer(size);
							data.copyTo(bytes, { planeIndex: 0 });
							const interleaved = this.rawToFloat32(
								bytes,
								format,
								frames * srcChannels,
							);
							for (let ch = 0; ch < dataChannels; ch++) {
								const chData = new Float32Array(frames);
								for (let i = 0; i < frames; i++) {
									chData[i] = interleaved[i * srcChannels + ch];
								}
								channelChunks[ch].push(chData);
							}
						}

						// Fill missing channels with silence
						for (let ch = dataChannels; ch < numChannels; ch++) {
							channelChunks[ch].push(new Float32Array(frames));
						}

						totalFrames += frames;
					} finally {
						data.close();
						notifyDecodeCapacityAvailable();
					}
				},
				error: (err: DOMException) => {
					decodeError = new Error(`Streaming audio decode error: ${err.message}`);
					notifyDecodeCapacityAvailable();
				},
			});

			decoder.configure(audioConfig);

			const audioStream = demuxer.read("audio");
			const reader = (audioStream as ReadableStream<EncodedAudioChunk>).getReader();

			try {
				while (!this.cancelled) {
					if (decodeError) throw decodeError;
					const { done, value: chunk } = await reader.read();
					if (done || !chunk) break;

					decoder.decode(chunk);

					while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) {
						if (decodeError) throw decodeError;
						await waitForDecodeCapacity();
					}
				}

				if (decoder.state === "configured") {
					await decoder.flush();
				}
				if (decodeError) throw decodeError;
			} finally {
				notifyDecodeCapacityAvailable();
				try {
					await reader.cancel();
				} catch {
					/* reader already closed */
				}
				if (decoder.state === "configured") {
					decoder.close();
				}
			}

			if (totalFrames === 0) return null;

			// Build AudioBuffer from accumulated chunks
			const audioBuffer = new AudioBuffer({
				length: totalFrames,
				numberOfChannels: numChannels,
				sampleRate,
			});
			for (let ch = 0; ch < numChannels; ch++) {
				const channelData = audioBuffer.getChannelData(ch);
				let writeOffset = 0;
				for (const chunk of channelChunks[ch]) {
					channelData.set(chunk, writeOffset);
					writeOffset += chunk.length;
				}
			}

			return audioBuffer;
		} finally {
			source.revoke();
			try {
				demuxer?.destroy();
			} catch {
				/* cleanup */
			}
		}
	}

	// Convert raw bytes from AudioData to Float32Array based on the sample format.
	protected rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array {
		if (format.startsWith("f32")) {
			return new Float32Array(bytes);
		}
		if (format.startsWith("s16")) {
			const int16 = new Int16Array(bytes);
			const f32 = new Float32Array(sampleCount);
			for (let i = 0; i < sampleCount; i++) {
				f32[i] = int16[i] / 0x8000;
			}
			return f32;
		}
		if (format.startsWith("s32")) {
			const int32 = new Int32Array(bytes);
			const f32 = new Float32Array(sampleCount);
			for (let i = 0; i < sampleCount; i++) {
				f32[i] = int32[i] / 0x80000000;
			}
			return f32;
		}
		if (format.startsWith("u8")) {
			const uint8 = new Uint8Array(bytes);
			const f32 = new Float32Array(sampleCount);
			for (let i = 0; i < sampleCount; i++) {
				f32[i] = (uint8[i] - 128) / 128;
			}
			return f32;
		}
		// Unknown format — attempt float32 interpretation
		return new Float32Array(bytes);
	}

	// Bulk decode fallback: loads entire file into memory and uses decodeAudioData.
	protected async bulkDecodeFromUrl(
		url: string,
		sampleRate: number,
	): Promise<AudioBuffer | null> {
		try {
			const source = await resolveMediaElementSource(url);
			try {
				const response = await fetch(source.src);
				const arrayBuffer = await response.arrayBuffer();
				const tempCtx = new OfflineAudioContext(2, 1, sampleRate);
				return await tempCtx.decodeAudioData(arrayBuffer);
			} finally {
				source.revoke();
			}
		} catch (error) {
			console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error);
			return null;
		}
	}

	// Get the duration of a media file by loading only its metadata.
	protected async getMediaDurationSec(url: string): Promise<number> {
		const source = await resolveMediaElementSource(url);
		try {
			const media = document.createElement("video");
			media.preload = "metadata";
			media.src = source.src;

			return await new Promise<number>((resolve, reject) => {
				const timeout = setTimeout(() => {
					cleanup();
					media.src = "";
					media.load();
					reject(new Error("Timed out getting media duration (30s)"));
				}, 30_000);

				const onLoaded = () => {
					cleanup();
					const duration = media.duration;
					media.src = "";
					media.load();
					resolve(Number.isFinite(duration) ? duration : 0);
				};
				const onError = () => {
					cleanup();
					media.src = "";
					media.load();
					reject(new Error("Failed to get media duration"));
				};
				const cleanup = () => {
					clearTimeout(timeout);
					media.removeEventListener("loadedmetadata", onLoaded);
					media.removeEventListener("error", onError);
				};

				media.addEventListener("loadedmetadata", onLoaded);
				media.addEventListener("error", onError, { once: true });
			});
		} finally {
			source.revoke();
		}
	}

	// Build non-overlapping timeline slices from the source timeline, excluding
	// trimmed regions and tagging each slice with its playback speed.
}