Skip to content

🎥 Make videos programmatically with React

6.7 KB
Raw Blame
import {calculateATempo} from './assets/calculate-atempo';
import {ffmpegVolumeExpression} from './assets/ffmpeg-volume-expression';
import type {AssetVolume, MediaAsset} from './assets/types';
import type {LogLevel} from './log-level';
import {Log} from './logger';
import {getActualTrimLeft} from './seamless-aac-trim';
import {truthy} from './truthy';

export type FilterWithoutPaddingApplied = ProcessedTrack & {
	filter: string | null;
	actualTrimLeft: number;
};

export type ProcessedTrack = {
	pad_start: string | null;
	pad_end: string | null;
};

const cleanUpFloatingPointError = (value: number) => {
	if (value % 1 < 0.0000001) {
		return Math.floor(value);
	}

	if (value % 1 > 0.9999999) {
		return Math.ceil(value);
	}

	return value;
};

const stringifyTrim = (trim: number) => {
	const value = cleanUpFloatingPointError(trim * 1_000_000);

	const asString = `${value}us`;

	// Handle very small values such as `"6e-7us"`, those are essentially rounding errors to 0
	if (asString.includes('e-')) {
		return '0us';
	}

	return asString;
};

// When `preservePitch` is disabled, the speed change must not go through
// `atempo` (which keeps the pitch). Changing the sample rate and then
// resampling back to the target rate speeds up or slows down playback
// while letting the pitch shift along with it (tape / chipmunk effect).
const calculateSpeedChangeFilter = ({
	playbackRate,
	preservePitch,
	sampleRate,
}: {
	playbackRate: number;
	preservePitch: boolean;
	sampleRate: number;
}): string | null => {
	if (playbackRate === 1) {
		return null;
	}

	if (preservePitch) {
		return calculateATempo(playbackRate);
	}

	return `asetrate=${sampleRate}*${playbackRate},aresample=${sampleRate}`;
};

const trimAndSetTempo = ({
	assetDuration,
	asset,
	trimLeftOffset,
	trimRightOffset,
	fps,
	indent,
	logLevel,
	sampleRate,
}: {
	assetDuration: number | null;
	trimLeftOffset: number;
	trimRightOffset: number;
	asset: MediaAsset;
	fps: number;
	indent: boolean;
	logLevel: LogLevel;
	sampleRate: number;
}): {
	actualTrimLeft: number;
	filter: (string | null)[];
	audibleDuration: number;
} => {
	// We need to apply the tempo filter first
	// because the atempo filter is not frame-perfect.
	// It creates a small offset and the offset needs to be the same for all audio tracks, before processing it further.
	// This also affects the trimLeft and trimRight values, as they need to be adjusted.
	const {trimLeft, maxTrim} = getActualTrimLeft({
		trimLeft: asset.trimLeft,
		fps,
		trimLeftOffset,
		seamless: true,
		assetDuration,
		audioStartFrame: asset.audioStartFrame,
		playbackRate: asset.playbackRate,
	});
	const trimRight =
		trimLeft + asset.duration / fps - trimLeftOffset + trimRightOffset;

	let trimRightOrAssetDuration = maxTrim
		? Math.min(trimRight, maxTrim)
		: trimRight;

	if (trimRightOrAssetDuration < trimLeft) {
		Log.warn(
			{indent, logLevel},
			'trimRightOrAssetDuration < trimLeft: ' +
				JSON.stringify({
					trimRight,
					trimLeft,
					assetDuration,
					assetTrimLeft: asset.trimLeft,
				}),
		);
		trimRightOrAssetDuration = trimLeft;
	}

	return {
		filter: [
			calculateSpeedChangeFilter({
				playbackRate: asset.playbackRate,
				preservePitch: asset.preservePitch,
				sampleRate,
			}),
			`atrim=${stringifyTrim(trimLeft)}:${stringifyTrim(trimRightOrAssetDuration)}`,
		],
		actualTrimLeft: trimLeft,
		audibleDuration: trimRightOrAssetDuration - trimLeft,
	};
};

export const stringifyFfmpegFilter = ({
	channels,
	volume,
	fps,
	assetDuration,
	chunkLengthInSeconds,
	forSeamlessAacConcatenation,
	trimLeftOffset,
	trimRightOffset,
	asset,
	indent,
	logLevel,
	presentationTimeOffsetInSeconds,
	sampleRate,
}: {
	channels: number;
	volume: AssetVolume;
	fps: number;
	assetDuration: number | null;
	chunkLengthInSeconds: number;
	forSeamlessAacConcatenation: boolean;
	trimLeftOffset: number;
	trimRightOffset: number;
	asset: MediaAsset;
	indent: boolean;
	logLevel: LogLevel;
	presentationTimeOffsetInSeconds: number;
	sampleRate: number;
}): FilterWithoutPaddingApplied | null => {
	if (channels === 0) {
		return null;
	}

	const {toneFrequency, startInVideo} = asset;

	const startInVideoSeconds = startInVideo / fps;
	// A chunk's leading trim only cuts into assets already playing at its start.
	// For later assets, remove that time from the silence before the asset instead.
	const assetTrimLeftOffset = startInVideo === 0 ? trimLeftOffset : 0;
	const startInChunkSeconds =
		startInVideo === 0 ? 0 : startInVideoSeconds - trimLeftOffset;

	const {trimLeft, maxTrim} = getActualTrimLeft({
		trimLeft: asset.trimLeft,
		fps,
		trimLeftOffset: assetTrimLeftOffset,
		seamless: forSeamlessAacConcatenation,
		assetDuration,
		audioStartFrame: asset.audioStartFrame,
		playbackRate: asset.playbackRate,
	});

	if (maxTrim && trimLeft >= maxTrim) {
		return null;
	}

	if (toneFrequency !== null && (toneFrequency <= 0 || toneFrequency > 2)) {
		throw new Error(
			'toneFrequency must be a positive number between 0.01 and 2',
		);
	}

	const {
		actualTrimLeft,
		audibleDuration,
		filter: trimAndTempoFilter,
	} = trimAndSetTempo({
		assetDuration,
		trimLeftOffset: assetTrimLeftOffset,
		trimRightOffset,
		asset,
		fps,
		indent,
		logLevel,
		sampleRate,
	});

	const volumeFilter = ffmpegVolumeExpression({
		volume,
		fps,
		trimLeft: actualTrimLeft,
	});

	const padAtEnd = chunkLengthInSeconds - audibleDuration - startInChunkSeconds;

	const padStart =
		startInChunkSeconds +
		(asset.trimLeft === 0 ? presentationTimeOffsetInSeconds : 0);

	// Set as few filters as possible, as combining them can create noise
	return {
		filter:
			'[0:a]' +
			[
				`aformat=sample_fmts=s16:sample_rates=${sampleRate}`,
				// The order matters here! For speed and correctness, we first trim the audio
				...trimAndTempoFilter,
				// The timings for volume must include whatever is in atrim, unless the volume
				// filter gets applied before atrim
				volumeFilter.value === '1'
					? null
					: `volume=${volumeFilter.value}:eval=${volumeFilter.eval}`,
				toneFrequency && toneFrequency !== 1
					? `asetrate=${sampleRate}*${toneFrequency},aresample=${sampleRate},atempo=1/${toneFrequency}`
					: null,
			]
				.filter(truthy)
				.join(',') +
			`[a0]`,
		pad_end:
			padAtEnd > 0.0000001
				? 'apad=pad_len=' + Math.round(padAtEnd * sampleRate)
				: null,
		pad_start:
			// For n channels, we delay n + 1 channels.
			// This is because `ffprobe` for some audio files reports the wrong amount
			// of channels.
			// This should be fine because FFMPEG documentation states:
			// "Unused delays will be silently ignored."
			// https://ffmpeg.org/ffmpeg-filters.html#adelay
			padStart === 0
				? null
				: `adelay=${new Array(channels + 1)
						.fill((padStart * 1000).toFixed(0))
						.join('|')}`,
		actualTrimLeft,
	};
};