Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

11.4 KB
Raw Blame
import type { SpeedRegion } from "@/components/video-editor/types";
import { AudioProcessorBase } from "./audioProcessorBase";
import type { TimelineSlice, TrimLikeRegion } from "./audioProcessorShared";

export class AudioTimelineProcessor extends AudioProcessorBase {
	protected buildTimelineSlices(
		sourceDurationMs: number,
		trimRegions: TrimLikeRegion[],
		speedRegions: SpeedRegion[],
	): TimelineSlice[] {
		const boundaries = new Set<number>();
		boundaries.add(0);
		boundaries.add(sourceDurationMs);

		for (const trim of trimRegions) {
			if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs);
			if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs);
		}
		for (const speed of speedRegions) {
			if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs)
				boundaries.add(speed.startMs);
			if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs);
		}

		const sorted = [...boundaries].sort((a, b) => a - b);
		const slices: TimelineSlice[] = [];

		for (let i = 0; i < sorted.length - 1; i++) {
			const start = sorted[i];
			const end = sorted[i + 1];
			if (end - start < 0.001) continue;

			// Skip segments entirely inside a trim region
			const midpoint = (start + end) / 2;
			if (this.isInTrimRegion(midpoint, trimRegions)) continue;

			const speedRegion = speedRegions.find(
				(s) => midpoint >= s.startMs && midpoint < s.endMs,
			);

			slices.push({
				sourceStartMs: start,
				sourceEndMs: end,
				speed: speedRegion?.speed ?? 1,
			});
		}

		return slices;
	}

	// Map a source-timeline timestamp to the corresponding output-timeline timestamp.
	protected sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number {
		let outputMs = 0;

		for (const slice of slices) {
			if (sourceMs <= slice.sourceStartMs) {
				return outputMs;
			}
			const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs;
			if (sourceMs >= slice.sourceEndMs) {
				outputMs += sliceDurationMs / slice.speed;
				continue;
			}
			// Source time falls within this slice
			outputMs += (sourceMs - slice.sourceStartMs) / slice.speed;
			return outputMs;
		}

		return outputMs;
	}

	// Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext.
	// Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate
	// playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are
	// provided, only sources overlapping the chunk window are scheduled.
	protected scheduleBufferThroughTimeline(
		ctx: OfflineAudioContext,
		buffer: AudioBuffer,
		slices: TimelineSlice[],
		sourceStartDelaySec: number,
		gain = 1,
		chunkOutputStartSec = 0,
		chunkDurationSec = Number.POSITIVE_INFINITY,
		mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [],
	): void {
		let outputOffsetSec = 0;

		for (const slice of slices) {
			const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000;
			if (slice.outputStartMs !== undefined) outputOffsetSec = slice.outputStartMs / 1000;
			const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed;

			// Where in the buffer does this slice read from?
			const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec;

			// Skip if slice doesn't overlap with the buffer at all
			if (
				bufferOffsetSec + sliceSourceDurationSec <= 0 ||
				bufferOffsetSec >= buffer.duration
			) {
				outputOffsetSec += sliceOutputDurationSec;
				continue;
			}

			// Clamp to buffer bounds
			let effectiveBufferStartSec = Math.max(0, bufferOffsetSec);
			const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec;
			let effectiveSourceDurationSec = Math.min(
				sliceSourceDurationSec - trimmedFromStartSec,
				buffer.duration - effectiveBufferStartSec,
			);

			if (effectiveSourceDurationSec <= 0.001) {
				outputOffsetSec += sliceOutputDurationSec;
				continue;
			}

			// Calculate output position (global then chunk-local)
			let localOutputStartSec =
				outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec;
			let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed;

			// Skip if entirely outside chunk window
			if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) {
				outputOffsetSec += sliceOutputDurationSec;
				continue;
			}

			// Clip to chunk start
			if (localOutputStartSec < 0) {
				const skipOutputSec = -localOutputStartSec;
				const skipSourceSec = skipOutputSec * slice.speed;
				effectiveBufferStartSec += skipSourceSec;
				effectiveSourceDurationSec -= skipSourceSec;
				localOutputStartSec = 0;
			}

			// Clip to chunk end
			if (localOutputEndSec > chunkDurationSec) {
				const excessOutputSec = localOutputEndSec - chunkDurationSec;
				effectiveSourceDurationSec -= excessOutputSec * slice.speed;
			}

			if (effectiveSourceDurationSec <= 0.001) {
				outputOffsetSec += sliceOutputDurationSec;
				continue;
			}

			const audibleRanges: Array<{ startSec: number; endSec: number }> = [
				{
					startSec: localOutputStartSec + chunkOutputStartSec,
					endSec:
						localOutputStartSec +
						chunkOutputStartSec +
						effectiveSourceDurationSec / slice.speed,
				},
			];
			for (const mutedRange of mutedOutputRangesSec) {
				for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) {
					const current = audibleRanges[rangeIndex];
					const overlapStart = Math.max(current.startSec, mutedRange.startSec);
					const overlapEnd = Math.min(current.endSec, mutedRange.endSec);
					if (overlapEnd <= overlapStart) {
						continue;
					}
					audibleRanges.splice(rangeIndex, 1);
					if (current.startSec < overlapStart) {
						audibleRanges.push({ startSec: current.startSec, endSec: overlapStart });
					}
					if (overlapEnd < current.endSec) {
						audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec });
					}
				}
			}

			for (const audibleRange of audibleRanges) {
				const audibleDurationSec = audibleRange.endSec - audibleRange.startSec;
				if (audibleDurationSec <= 0.001) {
					continue;
				}
				const source = ctx.createBufferSource();
				const gainNode = ctx.createGain();
				gainNode.gain.value = Math.max(0, Math.min(2, gain));

				const sourceOffsetSec =
					effectiveBufferStartSec +
					(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
						slice.speed;
				const localStartSec = audibleRange.startSec - chunkOutputStartSec;
				const sourceDurationSec = audibleDurationSec * slice.speed;

				const stretchedBuffer = this.stretchAudioBuffer(
					buffer,
					slice.speed,
					sourceOffsetSec,
					sourceDurationSec,
					audibleDurationSec,
					ctx,
				);

				source.buffer = stretchedBuffer;
				source.playbackRate.value = 1;
				source.connect(gainNode);
				gainNode.connect(ctx.destination);

				source.start(localStartSec);
			}

			outputOffsetSec += sliceOutputDurationSec;
		}
	}

	protected stretchAudioBuffer(
		originalBuffer: AudioBuffer,
		speed: number,
		sourceOffsetSec: number,
		sourceDurationSec: number,
		audibleDurationSec: number,
		ctx: BaseAudioContext,
	): AudioBuffer {
		const sampleRate = originalBuffer.sampleRate;
		const channels = originalBuffer.numberOfChannels;

		const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate));
		const sourceSamples = Math.floor(sourceDurationSec * sampleRate);
		const endSample = Math.min(originalBuffer.length, startSample + sourceSamples);

		const outSamples = Math.floor(audibleDurationSec * sampleRate);
		if (outSamples <= 0 || startSample >= originalBuffer.length) {
			return ctx.createBuffer(channels, 1, sampleRate);
		}

		const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate);

		if (Math.abs(speed - 1) < 0.001) {
			const copyLength = Math.min(endSample - startSample, outSamples);
			if (copyLength > 0) {
				for (let c = 0; c < channels; c++) {
					outBuffer.copyToChannel(
						originalBuffer
							.getChannelData(c)
							.subarray(startSample, startSample + copyLength),
						c,
					);
				}
			}
			return outBuffer;
		}

		// WSOLA uses windowing which causes fade-in at the start and fade-out at the end.
		// To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it.
		const paddingSec = 0.1;
		const paddingOutSamples = Math.floor(sampleRate * paddingSec);
		const paddingInSamples = Math.floor(paddingOutSamples * speed);

		const workStartIn = Math.max(0, startSample - paddingInSamples);
		const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);

		const actualPaddingInStart = startSample - workStartIn;
		// We expect the output offset for the requested start to be roughly:
		const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);

		const windowSize = Math.floor(sampleRate * 0.04);
		const hopOut = Math.floor(windowSize * 0.5);
		const hopIn = Math.floor(hopOut * speed);
		const searchRange = Math.floor(sampleRate * 0.015);

		const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
		const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);

		const inDataByChannel = Array.from({ length: channels }, (_, c) =>
			originalBuffer.getChannelData(c),
		);
		const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
			workOutBuffer.getChannelData(c),
		);

		const window = new Float32Array(windowSize);
		for (let i = 0; i < windowSize; i++) {
			window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1)));
		}

		let inOffset = workStartIn;
		let outOffset = 0;

		// Initial window
		for (let i = 0; i < windowSize; i++) {
			if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
				for (let c = 0; c < channels; c++) {
					workOutDataByChannel[c][outOffset + i] +=
						inDataByChannel[c][inOffset + i] * window[i];
				}
			}
		}

		outOffset += hopOut;
		inOffset += hopIn;

		while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) {
			let bestOffset = inOffset;
			const minSearch = Math.max(workStartIn, inOffset - searchRange);
			const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange);

			if (maxSearch > minSearch) {
				let maxCorr = -Infinity;
				let bestDelta = 0;

				for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) {
					let corr = 0;
					for (let i = 0; i < hopOut; i += 4) {
						if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
							for (let c = 0; c < channels; c++) {
								corr +=
									workOutDataByChannel[c][outOffset + i] *
									inDataByChannel[c][testOffset + i];
							}
						}
					}
					if (corr > maxCorr) {
						maxCorr = corr;
						bestDelta = testOffset - inOffset;
					}
				}
				bestOffset = inOffset + bestDelta;
			}

			for (let i = 0; i < windowSize; i++) {
				if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
					for (let c = 0; c < channels; c++) {
						workOutDataByChannel[c][outOffset + i] +=
							inDataByChannel[c][bestOffset + i] * window[i];
					}
				}
			}

			outOffset += hopOut;
			inOffset += hopIn;
		}

		// Transfer the stable middle portion to the final buffer
		for (let c = 0; c < channels; c++) {
			const finalData = outBuffer.getChannelData(c);
			const tempData = workOutBuffer.getChannelData(c);
			for (let i = 0; i < outSamples; i++) {
				const srcIdx = actualPaddingOutStart + i;
				if (srcIdx < workOutSamples) {
					finalData[i] = tempData[srcIdx];
				}
			}
		}

		return outBuffer;
	}

	// Create a WAV file header for the given audio parameters.
}