Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

15.4 KB
Raw Blame
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import {
	clampMediaTimeToDuration,
	enablePitchPreservingPlayback,
	estimateCompanionAudioStartDelaySeconds,
	getMediaSyncPlaybackRate,
	resolvePreviewMediaDuration,
} from "@/lib/mediaTiming";
import type { AudioRegion } from "../types";
import { supportsPreviewPlaybackRate } from "../videoPlayback/playbackRate";
import {
	getAudioResourceVersionKey,
	getVersionedAudioResourceUrl,
	isAudioResourceLoadCurrent,
} from "./audioResourceVersion";

const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;

interface UseAudioPreviewSyncParams {
	audioRegions: AudioRegion[];
	previewVolume: number;
	isPlaying: boolean;
	currentTime: number;
	timelineTime: number;
	duration: number;
	sourcePlaybackRate: number;
	previewSourceAudioFallbackPaths: string[];
	sourceAudioFallbackStartDelayMsByPath: Record<string, number>;
	sourceAudioResourceVersion: number;
	isCurrentClipMuted: boolean;
	getSourceTrackPreviewGain: (audioPath: string) => number;
	onSourceFallbackLoadError: (error: unknown) => void;
}

export function useAudioPreviewSync({
	audioRegions,
	previewVolume,
	isPlaying,
	currentTime,
	timelineTime,
	duration,
	sourcePlaybackRate,
	previewSourceAudioFallbackPaths,
	sourceAudioFallbackStartDelayMsByPath,
	sourceAudioResourceVersion,
	isCurrentClipMuted,
	getSourceTrackPreviewGain,
	onSourceFallbackLoadError,
}: UseAudioPreviewSyncParams) {
	const resolvedPlan = useMemo(
		() =>
			buildResolvedAudioPlan({
				videoResource: null,
				sourceAudioFallbackPaths: previewSourceAudioFallbackPaths,
				audioRegions,
			}),
		[audioRegions, previewSourceAudioFallbackPaths],
	);
	const resolvedUserTracks = useMemo(
		() => resolvedPlan.tracks.filter((track) => track.kind === "user"),
		[resolvedPlan],
	);
	const resolvedSourceTracks = useMemo(
		() => resolvedPlan.tracks.filter((track) => track.kind !== "user"),
		[resolvedPlan],
	);

	const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
	const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
	const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
	const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
	const sourceAudioMediaNodesRef = useRef<Map<string, MediaElementAudioSourceNode>>(new Map());
	const sourceAudioGainNodesRef = useRef<Map<string, GainNode>>(new Map());
	const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
	const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
	const sourceAudioContextRef = useRef<AudioContext | null>(null);
	const sourceAudioMasterGainRef = useRef<GainNode | null>(null);
	const sourceAudioResumePromiseRef = useRef<Promise<void> | null>(null);
	const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
	const [sourceLoadVersion, setSourceLoadVersion] = useState(0);

	const ensureSourceAudioContext = useCallback(() => {
		if (!sourceAudioContextRef.current) {
			const context = new AudioContext({ latencyHint: "interactive" });
			const masterGain = context.createGain();
			masterGain.gain.value = 1;
			masterGain.connect(context.destination);
			sourceAudioContextRef.current = context;
			sourceAudioMasterGainRef.current = masterGain;
		}
		return sourceAudioContextRef.current;
	}, []);

	const ensureSourceAudioRunning = useCallback(() => {
		const context = ensureSourceAudioContext();
		if (context.state === "running") {
			return Promise.resolve();
		}
		if (!sourceAudioResumePromiseRef.current) {
			sourceAudioResumePromiseRef.current = context
				.resume()
				.catch(() => undefined)
				.finally(() => {
					sourceAudioResumePromiseRef.current = null;
				});
		}
		return sourceAudioResumePromiseRef.current;
	}, [ensureSourceAudioContext]);

	const playSourceAudioPreview = useCallback(() => {
		void ensureSourceAudioRunning();
	}, [ensureSourceAudioRunning]);

	useEffect(() => {
		let cancelled = false;
		const existing = audioElementsRef.current;
		const currentIds = new Set(resolvedUserTracks.map((track) => track.id));

		for (const [id, audio] of existing) {
			if (!currentIds.has(id)) {
				audio.pause();
				audio.src = "";
				audioElementRevokersRef.current.get(id)?.();
				audioElementRevokersRef.current.delete(id);
				audioElementResourcesRef.current.delete(id);
				existing.delete(id);
			}
		}

		for (const track of resolvedUserTracks) {
			let audio = existing.get(track.id);
			if (!audio) {
				audio = new Audio();
				audio.preload = "auto";
				existing.set(track.id, audio);
			}

			if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) {
				audio.pause();
				audio.src = "";
				audioElementRevokersRef.current.get(track.id)?.();
				audioElementRevokersRef.current.delete(track.id);
				audioElementResourcesRef.current.set(track.id, track.sourceRef.path);

				void (async () => {
					const resolved = await resolveMediaElementSource(track.sourceRef.path);
					const latestAudio = existing.get(track.id);

					if (
						cancelled ||
						latestAudio !== audio ||
						audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path
					) {
						resolved.revoke();
						return;
					}

					audioElementRevokersRef.current.set(track.id, resolved.revoke);
					latestAudio.src = resolved.src;
				})();
			}

			audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume));
		}

		return () => {
			cancelled = true;
		};
	}, [previewVolume, resolvedUserTracks]);

	useEffect(() => {
		const existing = sourceAudioElementsRef.current;
		const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path));

		for (const [id, audio] of existing) {
			if (!currentIds.has(id)) {
				audio.pause();
				audio.src = "";
				sourceAudioMediaNodesRef.current.get(id)?.disconnect();
				sourceAudioMediaNodesRef.current.delete(id);
				sourceAudioGainNodesRef.current.get(id)?.disconnect();
				sourceAudioGainNodesRef.current.delete(id);
				sourceAudioElementRevokersRef.current.get(id)?.();
				sourceAudioElementRevokersRef.current.delete(id);
				sourceAudioElementResourcesRef.current.delete(id);
				existing.delete(id);
			}
		}

		for (const track of resolvedSourceTracks) {
			const audioPath = track.sourceRef.path;
			const resourceKey = getAudioResourceVersionKey(audioPath, sourceAudioResourceVersion);
			let audio = existing.get(audioPath);
			if (!audio) {
				audio = new Audio();
				audio.preload = "auto";
				audio.crossOrigin = "anonymous";
				existing.set(audioPath, audio);
			}
			audio.volume = 1;
			audio.dataset.sourceAudioPath = audioPath;

			// Web Audio API createMediaElementSource breaks preservesPitch on Chromium.
			// We route directly through the HTMLAudioElement to ensure pitch preservation works
			// during speed changes. Note: this limits maximum preview volume to 1.0 (100%).

			if (sourceAudioElementResourcesRef.current.get(audioPath) !== resourceKey) {
				audio.pause();
				audio.src = "";
				sourceAudioElementRevokersRef.current.get(audioPath)?.();
				sourceAudioElementRevokersRef.current.delete(audioPath);
				sourceAudioElementResourcesRef.current.set(audioPath, resourceKey);

				// Resource identity, not effect lifetime, decides staleness. Playback and fallback
				// rerenders must not cancel the only load for the current finalized sidecar version.
				void (async () => {
					try {
						const resolved = await resolveMediaElementSource(audioPath);
						const latestAudio = existing.get(audioPath);

						if (
							latestAudio !== audio ||
							!isAudioResourceLoadCurrent(
								sourceAudioElementResourcesRef.current,
								audioPath,
								resourceKey,
							)
						) {
							resolved.revoke();
							return;
						}

						sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
						latestAudio.src = getVersionedAudioResourceUrl(
							resolved.src,
							sourceAudioResourceVersion,
						);
						latestAudio.load();
						lastSourceAudioSyncTimeRef.current = null;
						setSourceLoadVersion((version) => version + 1);
					} catch (error) {
						const latestAudio = existing.get(audioPath);
						if (
							latestAudio !== audio ||
							!isAudioResourceLoadCurrent(
								sourceAudioElementResourcesRef.current,
								audioPath,
								resourceKey,
							)
						) {
							return;
						}

						sourceAudioElementRevokersRef.current.get(audioPath)?.();
						sourceAudioElementRevokersRef.current.delete(audioPath);
						sourceAudioElementResourcesRef.current.delete(audioPath);
						if (latestAudio === audio) {
							latestAudio.pause();
							latestAudio.src = "";
						}
						onSourceFallbackLoadError(error);
					}
				})();
			}

			audio.volume = Math.max(
				0,
				Math.min(
					1,
					getSourceTrackPreviewGain(audioPath) * (isCurrentClipMuted ? 0 : previewVolume),
				),
			);
		}

		if (sourceAudioMasterGainRef.current) {
			sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
				? 0
				: Math.max(0, Math.min(1, previewVolume));
		}

		if (resolvedSourceTracks.length === 0) {
			lastSourceAudioSyncTimeRef.current = null;
		}
	}, [
		getSourceTrackPreviewGain,
		isCurrentClipMuted,
		onSourceFallbackLoadError,
		resolvedSourceTracks,
		sourceAudioResourceVersion,
		previewVolume,
	]);

	useEffect(() => {
		return () => {
			for (const audio of audioElementsRef.current.values()) {
				audio.pause();
				audio.src = "";
			}
			for (const revoke of audioElementRevokersRef.current.values()) {
				revoke();
			}
			audioElementsRef.current.clear();
			audioElementRevokersRef.current.clear();
			audioElementResourcesRef.current.clear();
			for (const audio of sourceAudioElementsRef.current.values()) {
				audio.pause();
				audio.src = "";
			}
			for (const node of sourceAudioMediaNodesRef.current.values()) {
				node.disconnect();
			}
			for (const node of sourceAudioGainNodesRef.current.values()) {
				node.disconnect();
			}
			for (const revoke of sourceAudioElementRevokersRef.current.values()) {
				revoke();
			}
			sourceAudioElementsRef.current.clear();
			sourceAudioMediaNodesRef.current.clear();
			sourceAudioGainNodesRef.current.clear();
			sourceAudioElementRevokersRef.current.clear();
			sourceAudioElementResourcesRef.current.clear();
			if (sourceAudioMasterGainRef.current) {
				sourceAudioMasterGainRef.current.disconnect();
				sourceAudioMasterGainRef.current = null;
			}
			const context = sourceAudioContextRef.current;
			sourceAudioContextRef.current = null;
			sourceAudioResumePromiseRef.current = null;
			if (context) {
				void context.close();
			}
			lastSourceAudioSyncTimeRef.current = null;
		};
	}, []);

	useEffect(() => {
		const currentTimeMs = timelineTime * 1000;

		for (const track of resolvedUserTracks) {
			const audio = audioElementsRef.current.get(track.id);
			if (!audio) continue;

			const startMs = track.timelineBinding.startMs;
			const endMs = track.timelineBinding.endMs;
			const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs;

			if (isPlaying && isInRegion) {
				enablePitchPreservingPlayback(audio);
				const audioOffset = (currentTimeMs - startMs) / 1000;
				if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
					audio.currentTime = audioOffset;
				}
				const syncedPlaybackRate = getMediaSyncPlaybackRate({
					basePlaybackRate: 1,
					currentTime: audio.currentTime,
					targetTime: audioOffset,
				});
				if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
					audio.playbackRate = syncedPlaybackRate;
				}
				if (audio.paused) {
					audio.play().catch(() => undefined);
				}
			} else if (!audio.paused) {
				audio.pause();
			}
		}
	}, [isPlaying, resolvedUserTracks, timelineTime]);

	useEffect(() => {
		if (resolvedSourceTracks.length === 0) {
			lastSourceAudioSyncTimeRef.current = null;
			return;
		}
		let cancelled = false;
		const seekedListeners: Array<{ audio: HTMLAudioElement; resume: () => void }> = [];
		// A newly resolved source must pass the same playback checks as timeline updates.
		void sourceLoadVersion;

		const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
		const timelineJumped =
			previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
		const driftThreshold = isPlaying
			? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
			: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
		if (sourceAudioMasterGainRef.current) {
			sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
				? 0
				: Math.max(0, Math.min(1, previewVolume));
		}

		for (const audio of sourceAudioElementsRef.current.values()) {
			if (!audio.src) continue;
			if (!supportsPreviewPlaybackRate(sourcePlaybackRate)) {
				audio.pause();
				continue;
			}
			const sourceAudioPath = audio.dataset.sourceAudioPath ?? "";
			audio.volume = Math.max(
				0,
				Math.min(
					1,
					getSourceTrackPreviewGain(sourceAudioPath) *
						(isCurrentClipMuted ? 0 : previewVolume),
				),
			);

			enablePitchPreservingPlayback(audio);
			const audioDuration = resolvePreviewMediaDuration(audio.duration, duration);
			const isMicCompanionTrack = /\.mic\./i.test(sourceAudioPath);
			const rawStartDelaySeconds = estimateCompanionAudioStartDelaySeconds(
				duration,
				audioDuration,
				sourceAudioFallbackStartDelayMsByPath[sourceAudioPath],
			);
			const maxPreviewStartDelaySeconds = isMicCompanionTrack ? 2 : 5;
			const startDelaySeconds = isMicCompanionTrack
				? 0
				: Number.isFinite(duration) &&
						(rawStartDelaySeconds >= Math.max(0, duration - 0.01) ||
							rawStartDelaySeconds >
								Math.max(maxPreviewStartDelaySeconds, duration * 0.9))
					? 0
					: rawStartDelaySeconds;
			const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
			const targetTime = clampMediaTimeToDuration(
				currentTime - startDelaySeconds,
				audioDuration,
			);

			const shouldSeek =
				timelineJumped ||
				(!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) ||
				(isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9);
			if (shouldSeek) {
				try {
					if (!audio.paused) audio.pause();
					audio.currentTime = targetTime;
				} catch {
					// no-op
				}
			}

			// KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter
			// from continuous micro-corrections on system audio.
			const syncedPlaybackRate = sourcePlaybackRate;
			if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
				audio.playbackRate = syncedPlaybackRate;
			}

			const atEnd = audioDuration !== null && targetTime >= audioDuration;
			if (isPlaying && !isCurrentClipMuted && !beforeAudioStart && !atEnd) {
				const resume = () => {
					void ensureSourceAudioRunning().then(() => {
						if (!cancelled && audio.paused && !audio.seeking)
							audio.play().catch(() => undefined);
					});
				};
				audio.addEventListener("seeked", resume);
				seekedListeners.push({ audio, resume });
				resume();
			} else if (!audio.paused) {
				audio.pause();
			}
		}

		lastSourceAudioSyncTimeRef.current = currentTime;
		return () => {
			cancelled = true;
			for (const { audio, resume } of seekedListeners) {
				audio.removeEventListener("seeked", resume);
			}
		};
	}, [
		sourceLoadVersion,
		currentTime,
		duration,
		sourcePlaybackRate,
		getSourceTrackPreviewGain,
		isCurrentClipMuted,
		isPlaying,
		previewVolume,
		resolvedSourceTracks,
		sourceAudioFallbackStartDelayMsByPath,
		ensureSourceAudioRunning,
	]);

	return { playSourceAudioPreview };
}