Skip to content

🎥 Make videos programmatically with React

Keep the web renderer pitch shifter independent of media

JonnyBurgerdfef55d5cc7a
diff --git a/bun.lock b/bun.lock
index bde8af5817..8723c81894 100644
--- a/bun.lock
+++ b/bun.lock
@@ -2791,7 +2791,6 @@
"@mediabunny/flac-encoder": "catalog:",
"@mediabunny/mp3-encoder": "catalog:",
"@remotion/licensing": "workspace:*",
- "@remotion/media": "workspace:*",
"mediabunny": "catalog:",
"remotion": "workspace:*",
},
@@ -2799,6 +2798,7 @@
"@react-three/fiber": "catalog:",
"@remotion/effects": "workspace:*",
"@remotion/eslint-config-internal": "workspace:*",
+ "@remotion/media": "workspace:*",
"@remotion/paths": "workspace:*",
"@remotion/player": "workspace:*",
"@remotion/shapes": "workspace:*",
diff --git a/packages/media/bundle.ts b/packages/media/bundle.ts
index 7fe416b0c7..23a4d1c63d 100644
--- a/packages/media/bundle.ts
+++ b/packages/media/bundle.ts
@@ -6,8 +6,7 @@ if (process.env.NODE_ENV !== 'production') {
}
console.time('Generated.');
const output = await build({
- entrypoints: ['src/index.ts', 'src/internal.ts'],
- splitting: true,
+ entrypoints: ['src/index.ts'],
naming: '[name].mjs',
external: [
'remotion',
diff --git a/packages/media/package.json b/packages/media/package.json
index 544dd7aa02..d67bfd8d88 100644
--- a/packages/media/package.json
+++ b/packages/media/package.json
@@ -51,12 +51,7 @@
"module": "./dist/esm/index.mjs",
"import": "./dist/esm/index.mjs"
},
- "./package.json": "./package.json",
- "./internal": {
- "types": "./dist/internal.d.ts",
- "module": "./dist/esm/internal.mjs",
- "import": "./dist/esm/internal.mjs"
- }
+ "./package.json": "./package.json"
},
"description": "Experimental WebCodecs-based media tags",
"homepage": "https://remotion.dev/docs/media"
diff --git a/packages/media/src/internal.ts b/packages/media/src/internal.ts
deleted file mode 100644
index 1c3243025d..0000000000
--- a/packages/media/src/internal.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-// Internal entry point for other Remotion packages.
-export {StreamingPitchShifter} from './audio/streaming-pitch-shifter';
diff --git a/packages/web-renderer/bundle.ts b/packages/web-renderer/bundle.ts
index a3dbfdf6bd..8446593e25 100644
--- a/packages/web-renderer/bundle.ts
+++ b/packages/web-renderer/bundle.ts
@@ -6,7 +6,6 @@ await buildPackage({
esm: 'build',
},
external: [
- '@remotion/media/internal',
'remotion',
'react',
'react-dom',
diff --git a/packages/web-renderer/package.json b/packages/web-renderer/package.json
index 953509f4c8..59d7609fc7 100644
--- a/packages/web-renderer/package.json
+++ b/packages/web-renderer/package.json
@@ -24,8 +24,7 @@
"@mediabunny/flac-encoder": "catalog:",
"@remotion/licensing": "workspace:*",
"remotion": "workspace:*",
- "mediabunny": "catalog:",
- "@remotion/media": "workspace:*"
+ "mediabunny": "catalog:"
},
"devDependencies": {
"@remotion/effects": "workspace:*",
@@ -33,6 +32,7 @@
"@remotion/eslint-config-internal": "workspace:*",
"@remotion/paths": "workspace:*",
"@remotion/player": "workspace:*",
+ "@remotion/media": "workspace:*",
"@remotion/shapes": "workspace:*",
"@remotion/three": "workspace:*",
"@remotion/transitions": "workspace:*",
diff --git a/packages/web-renderer/src/audio.ts b/packages/web-renderer/src/audio.ts
index 3dabd93a16..3f83bbc99a 100644
--- a/packages/web-renderer/src/audio.ts
+++ b/packages/web-renderer/src/audio.ts
@@ -1,5 +1,5 @@
-import {StreamingPitchShifter} from '@remotion/media/internal';
import type {TRenderAsset} from 'remotion';
+import {StreamingPitchShifter} from './streaming-pitch-shifter';
const TARGET_NUMBER_OF_CHANNELS = 2;
diff --git a/packages/web-renderer/src/streaming-pitch-shifter.ts b/packages/web-renderer/src/streaming-pitch-shifter.ts
new file mode 100644
index 0000000000..fc269421ce
--- /dev/null
+++ b/packages/web-renderer/src/streaming-pitch-shifter.ts
@@ -0,0 +1,491 @@
+// The pitch-shifting algorithm is adapted from Vanilagy's WSOLA audio
+// stretcher: https://gist.github.com/Vanilagy/05f7901f4c4398356657e3a86c7aee05
+const REFERENCE_SAMPLE_RATE = 48_000;
+const REFERENCE_HOP_SIZE = 512;
+
+type PlanarAudio = Float32Array[];
+
+const makePlanarAudio = (numberOfChannels: number, length: number) => {
+ return new Array(numberOfChannels)
+ .fill(null)
+ .map(() => new Float32Array(length));
+};
+
+const ensurePlanarCapacity = ({
+ buffers,
+ requiredLength,
+}: {
+ buffers: PlanarAudio;
+ requiredLength: number;
+}) => {
+ if (buffers[0].length >= requiredLength) {
+ return buffers;
+ }
+
+ let newLength = buffers[0].length;
+ while (newLength < requiredLength) {
+ newLength *= 2;
+ }
+
+ return buffers.map((buffer) => {
+ const expanded = new Float32Array(newLength);
+ expanded.set(buffer);
+ return expanded;
+ });
+};
+
+class PlanarAudioQueue {
+ private chunks: PlanarAudio[] = [];
+ private length = 0;
+
+ public push(audio: PlanarAudio) {
+ if (audio[0].length === 0) {
+ return;
+ }
+
+ this.chunks.push(audio);
+ this.length += audio[0].length;
+ }
+
+ public take(numberOfFrames: number, numberOfChannels: number): PlanarAudio {
+ const framesToTake = Math.min(numberOfFrames, this.length);
+ const result = makePlanarAudio(numberOfChannels, framesToTake);
+ let written = 0;
+
+ while (written < framesToTake) {
+ const first = this.chunks[0];
+ const available = first[0].length;
+ const count = Math.min(available, framesToTake - written);
+
+ for (let channel = 0; channel < numberOfChannels; channel++) {
+ result[channel].set(first[channel].subarray(0, count), written);
+ }
+
+ if (count === available) {
+ this.chunks.shift();
+ } else {
+ this.chunks[0] = first.map((channel) => channel.subarray(count));
+ }
+
+ written += count;
+ this.length -= count;
+ }
+
+ return result;
+ }
+
+ public getLength() {
+ return this.length;
+ }
+}
+
+// A streaming implementation of Waveform Similarity Overlap-Add. It changes
+// duration while retaining pitch. Pitch shifting is achieved by following this
+// stage with a resampler that restores the original duration.
+class StreamingTimeStretcher {
+ private readonly numberOfChannels: number;
+ private readonly factor: number;
+ private readonly hopSize: number;
+ private readonly windowSize: number;
+ private readonly searchRadius: number;
+ private readonly analysisHop: number;
+ private input: PlanarAudio;
+ private inputLength = 0;
+ private output: PlanarAudio;
+ private outputLength = 0;
+ private analysisPosition = 0;
+ private synthesisPosition = 0;
+ private initialized = false;
+ private finalized = false;
+ private totalInputFrames = 0;
+ private totalOutputFrames = 0;
+
+ public constructor({
+ numberOfChannels,
+ sampleRate,
+ factor,
+ }: {
+ numberOfChannels: number;
+ sampleRate: number;
+ factor: number;
+ }) {
+ this.numberOfChannels = numberOfChannels;
+ this.factor = factor;
+ this.hopSize = Math.max(
+ 32,
+ Math.round((REFERENCE_HOP_SIZE * sampleRate) / REFERENCE_SAMPLE_RATE),
+ );
+ this.windowSize = this.hopSize * 2;
+ this.searchRadius = this.hopSize;
+ this.analysisHop = this.hopSize / factor;
+ this.input = makePlanarAudio(numberOfChannels, 65_536);
+ this.output = makePlanarAudio(numberOfChannels, 65_536);
+ }
+
+ public append(audio: PlanarAudio) {
+ if (this.finalized) {
+ throw new Error(
+ 'Cannot append audio after the time stretcher was finalized.',
+ );
+ }
+
+ const {length} = audio[0];
+ this.input = ensurePlanarCapacity({
+ buffers: this.input,
+ requiredLength: this.inputLength + length,
+ });
+
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.input[channel].set(audio[channel], this.inputLength);
+ }
+
+ this.inputLength += length;
+ this.totalInputFrames += length;
+ this.process();
+ return this.drainFinalizedOutput();
+ }
+
+ private findBestAnalysisPosition({
+ expectedPosition,
+ nextSynthesisPosition,
+ }: {
+ expectedPosition: number;
+ nextSynthesisPosition: number;
+ }) {
+ const minimum = Math.max(
+ 0,
+ Math.floor(expectedPosition - this.searchRadius),
+ );
+ const maximum = Math.min(
+ this.inputLength - this.windowSize,
+ Math.ceil(expectedPosition + this.searchRadius),
+ );
+
+ let bestPosition = minimum;
+ let bestCorrelation = -Infinity;
+
+ for (let candidate = minimum; candidate <= maximum; candidate += 4) {
+ let dotProduct = 0;
+ let previousEnergy = 0;
+ let candidateEnergy = 0;
+
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ const previous = this.output[channel];
+ const incoming = this.input[channel];
+ for (let frame = 0; frame < this.hopSize; frame += 2) {
+ const previousValue = previous[nextSynthesisPosition + frame];
+ const candidateValue = incoming[candidate + frame];
+ dotProduct += previousValue * candidateValue;
+ previousEnergy += previousValue * previousValue;
+ candidateEnergy += candidateValue * candidateValue;
+ }
+ }
+
+ const correlation =
+ dotProduct /
+ (Math.sqrt(previousEnergy * candidateEnergy) || Number.EPSILON);
+ if (correlation > bestCorrelation) {
+ bestCorrelation = correlation;
+ bestPosition = candidate;
+ }
+ }
+
+ const fineMinimum = Math.max(minimum, bestPosition - 4);
+ const fineMaximum = Math.min(maximum, bestPosition + 4);
+ for (let candidate = fineMinimum; candidate <= fineMaximum; candidate++) {
+ let dotProduct = 0;
+ let previousEnergy = 0;
+ let candidateEnergy = 0;
+
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ const previous = this.output[channel];
+ const incoming = this.input[channel];
+ for (let frame = 0; frame < this.hopSize; frame++) {
+ const previousValue = previous[nextSynthesisPosition + frame];
+ const candidateValue = incoming[candidate + frame];
+ dotProduct += previousValue * candidateValue;
+ previousEnergy += previousValue * previousValue;
+ candidateEnergy += candidateValue * candidateValue;
+ }
+ }
+
+ const correlation =
+ dotProduct /
+ (Math.sqrt(previousEnergy * candidateEnergy) || Number.EPSILON);
+ if (correlation > bestCorrelation) {
+ bestCorrelation = correlation;
+ bestPosition = candidate;
+ }
+ }
+
+ return bestPosition;
+ }
+
+ private process() {
+ if (!this.initialized) {
+ if (this.inputLength < this.windowSize + this.searchRadius) {
+ return;
+ }
+
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.output[channel].set(
+ this.input[channel].subarray(0, this.windowSize),
+ );
+ }
+
+ this.outputLength = this.windowSize;
+ this.initialized = true;
+ }
+
+ while (true) {
+ const expectedPosition = this.analysisPosition + this.analysisHop;
+ if (
+ expectedPosition + this.searchRadius + this.windowSize >
+ this.inputLength
+ ) {
+ break;
+ }
+
+ const nextSynthesisPosition = this.synthesisPosition + this.hopSize;
+ this.output = ensurePlanarCapacity({
+ buffers: this.output,
+ requiredLength: nextSynthesisPosition + this.windowSize,
+ });
+ const bestPosition = this.findBestAnalysisPosition({
+ expectedPosition,
+ nextSynthesisPosition,
+ });
+
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ for (let frame = 0; frame < this.hopSize; frame++) {
+ const fadeIn =
+ 0.5 - 0.5 * Math.cos((Math.PI * (frame + 1)) / (this.hopSize + 1));
+ const outputIndex = nextSynthesisPosition + frame;
+ this.output[channel][outputIndex] =
+ this.output[channel][outputIndex] * (1 - fadeIn) +
+ this.input[channel][bestPosition + frame] * fadeIn;
+ }
+
+ this.output[channel].set(
+ this.input[channel].subarray(
+ bestPosition + this.hopSize,
+ bestPosition + this.windowSize,
+ ),
+ nextSynthesisPosition + this.hopSize,
+ );
+ }
+
+ // Keep the analysis clock independent from the correlation correction.
+ // Periodic signals can have equally good matches at an earlier period; if
+ // the correction became the next clock position, the iterator could stop
+ // making forward progress.
+ this.analysisPosition = expectedPosition;
+ this.synthesisPosition = nextSynthesisPosition;
+ this.outputLength = nextSynthesisPosition + this.windowSize;
+ }
+ }
+
+ private drainFinalizedOutput() {
+ if (!this.initialized) {
+ return makePlanarAudio(this.numberOfChannels, 0);
+ }
+
+ const finalizedLength = Math.max(0, this.synthesisPosition + this.hopSize);
+ const result = this.output.map((channel) =>
+ channel.slice(0, finalizedLength),
+ );
+ this.totalOutputFrames += finalizedLength;
+
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.output[channel].copyWithin(0, finalizedLength, this.outputLength);
+ }
+
+ this.outputLength -= finalizedLength;
+ this.synthesisPosition -= finalizedLength;
+
+ const inputFramesToDiscard = Math.max(
+ 0,
+ Math.floor(this.analysisPosition) - this.searchRadius,
+ );
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.input[channel].copyWithin(0, inputFramesToDiscard, this.inputLength);
+ }
+
+ this.inputLength -= inputFramesToDiscard;
+ this.analysisPosition -= inputFramesToDiscard;
+ return result;
+ }
+
+ public finalize() {
+ if (this.finalized) {
+ throw new Error('The time stretcher has already been finalized.');
+ }
+
+ this.finalized = true;
+ const targetLength = Math.round(this.totalInputFrames * this.factor);
+ const padding = makePlanarAudio(
+ this.numberOfChannels,
+ this.windowSize + this.searchRadius * 2,
+ );
+ this.input = ensurePlanarCapacity({
+ buffers: this.input,
+ requiredLength: this.inputLength + padding[0].length,
+ });
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.input[channel].set(padding[channel], this.inputLength);
+ }
+
+ this.inputLength += padding[0].length;
+ this.process();
+ const finalized = this.drainFinalizedOutput();
+ const remaining = Math.max(
+ 0,
+ targetLength - this.totalOutputFrames + finalized[0].length,
+ );
+
+ if (finalized[0].length >= remaining) {
+ return finalized.map((channel) => channel.slice(0, remaining));
+ }
+
+ const result = makePlanarAudio(this.numberOfChannels, remaining);
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ result[channel].set(finalized[channel]);
+ }
+
+ return result;
+ }
+}
+
+class StreamingLinearResampler {
+ private readonly numberOfChannels: number;
+ private readonly step: number;
+ private input: PlanarAudio;
+ private inputLength = 0;
+ private position = 0;
+
+ public constructor({
+ numberOfChannels,
+ step,
+ }: {
+ numberOfChannels: number;
+ step: number;
+ }) {
+ this.numberOfChannels = numberOfChannels;
+ this.step = step;
+ this.input = makePlanarAudio(numberOfChannels, 65_536);
+ }
+
+ public append(audio: PlanarAudio) {
+ this.input = ensurePlanarCapacity({
+ buffers: this.input,
+ requiredLength: this.inputLength + audio[0].length,
+ });
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.input[channel].set(audio[channel], this.inputLength);
+ }
+
+ this.inputLength += audio[0].length;
+ return this.process(false);
+ }
+
+ private process(finalizing: boolean) {
+ const outputLength = Math.max(
+ 0,
+ Math.floor(
+ (this.inputLength - (finalizing ? 0 : 1) - this.position) / this.step,
+ ) + 1,
+ );
+ const result = makePlanarAudio(this.numberOfChannels, outputLength);
+
+ for (let outputFrame = 0; outputFrame < outputLength; outputFrame++) {
+ const leftIndex = Math.floor(this.position);
+ const rightIndex = Math.min(leftIndex + 1, this.inputLength - 1);
+ const fraction = this.position - leftIndex;
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ const left = this.input[channel][leftIndex];
+ const right = this.input[channel][rightIndex];
+ result[channel][outputFrame] = left + (right - left) * fraction;
+ }
+
+ this.position += this.step;
+ }
+
+ const discard = Math.min(Math.floor(this.position), this.inputLength);
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ this.input[channel].copyWithin(0, discard, this.inputLength);
+ }
+
+ this.inputLength -= discard;
+ this.position -= discard;
+ return result;
+ }
+
+ public finalize() {
+ return this.process(true);
+ }
+}
+
+export class StreamingPitchShifter {
+ private readonly numberOfChannels: number;
+ private readonly stretcher: StreamingTimeStretcher;
+ private readonly resampler: StreamingLinearResampler;
+ private readonly outputQueue = new PlanarAudioQueue();
+ private totalInputFrames = 0;
+ private totalOutputFrames = 0;
+
+ public constructor({
+ numberOfChannels,
+ sampleRate,
+ toneFrequency,
+ }: {
+ numberOfChannels: number;
+ sampleRate: number;
+ toneFrequency: number;
+ }) {
+ this.numberOfChannels = numberOfChannels;
+ this.stretcher = new StreamingTimeStretcher({
+ numberOfChannels,
+ sampleRate,
+ factor: toneFrequency,
+ });
+ this.resampler = new StreamingLinearResampler({
+ numberOfChannels,
+ step: toneFrequency,
+ });
+ }
+
+ public append(audio: PlanarAudio) {
+ this.totalInputFrames += audio[0].length;
+ const stretched = this.stretcher.append(audio);
+ this.outputQueue.push(this.resampler.append(stretched));
+ return this.takeAvailableOutput();
+ }
+
+ private takeAvailableOutput() {
+ const availableInputFrames = this.totalInputFrames - this.totalOutputFrames;
+ const framesToTake = Math.min(
+ availableInputFrames,
+ this.outputQueue.getLength(),
+ );
+ const result = this.outputQueue.take(framesToTake, this.numberOfChannels);
+ this.totalOutputFrames += framesToTake;
+ return result;
+ }
+
+ public finalize() {
+ this.outputQueue.push(this.resampler.append(this.stretcher.finalize()));
+ this.outputQueue.push(this.resampler.finalize());
+ const remaining = this.totalInputFrames - this.totalOutputFrames;
+ const available = this.outputQueue.take(
+ Math.min(remaining, this.outputQueue.getLength()),
+ this.numberOfChannels,
+ );
+ const result = makePlanarAudio(this.numberOfChannels, remaining);
+ for (let channel = 0; channel < this.numberOfChannels; channel++) {
+ result[channel].set(available[channel]);
+ }
+
+ this.totalOutputFrames += remaining;
+ return result;
+ }
+}