🎥 Make videos programmatically with React
Keep the web renderer pitch shifter independent of media
JonnyBurgerdfef55d5cc7a
diff --git a/bun.lock b/bun.lockindex bde8af5817..8723c81894 100644--- a/bun.lock+++ b/bun.lock@@ -2791,7 +2791,6 @@"@mediabunny/flac-encoder": "catalog:","@mediabunny/mp3-encoder": "catalog:","@remotion/licensing": "workspace:*",- "@remotion/media": "workspace:*","mediabunny": "catalog:","remotion": "workspace:*",},@@ -2799,6 +2798,7 @@"@react-three/fiber": "catalog:","@remotion/effects": "workspace:*","@remotion/eslint-config-internal": "workspace:*",+ "@remotion/media": "workspace:*","@remotion/paths": "workspace:*","@remotion/player": "workspace:*","@remotion/shapes": "workspace:*",diff --git a/packages/media/bundle.ts b/packages/media/bundle.tsindex 7fe416b0c7..23a4d1c63d 100644--- a/packages/media/bundle.ts+++ b/packages/media/bundle.ts@@ -6,8 +6,7 @@ if (process.env.NODE_ENV !== 'production') {}console.time('Generated.');const output = await build({- entrypoints: ['src/index.ts', 'src/internal.ts'],- splitting: true,+ entrypoints: ['src/index.ts'],naming: '[name].mjs',external: ['remotion',diff --git a/packages/media/package.json b/packages/media/package.jsonindex 544dd7aa02..d67bfd8d88 100644--- a/packages/media/package.json+++ b/packages/media/package.json@@ -51,12 +51,7 @@"module": "./dist/esm/index.mjs","import": "./dist/esm/index.mjs"},- "./package.json": "./package.json",- "./internal": {- "types": "./dist/internal.d.ts",- "module": "./dist/esm/internal.mjs",- "import": "./dist/esm/internal.mjs"- }+ "./package.json": "./package.json"},"description": "Experimental WebCodecs-based media tags","homepage": "https://remotion.dev/docs/media"diff --git a/packages/media/src/internal.ts b/packages/media/src/internal.tsdeleted file mode 100644index 1c3243025d..0000000000--- a/packages/media/src/internal.ts+++ /dev/null@@ -1,2 +0,0 @@-// Internal entry point for other Remotion packages.-export {StreamingPitchShifter} from './audio/streaming-pitch-shifter';diff --git a/packages/web-renderer/bundle.ts b/packages/web-renderer/bundle.tsindex a3dbfdf6bd..8446593e25 100644--- a/packages/web-renderer/bundle.ts+++ b/packages/web-renderer/bundle.ts@@ -6,7 +6,6 @@ await buildPackage({esm: 'build',},external: [- '@remotion/media/internal','remotion','react','react-dom',diff --git a/packages/web-renderer/package.json b/packages/web-renderer/package.jsonindex 953509f4c8..59d7609fc7 100644--- a/packages/web-renderer/package.json+++ b/packages/web-renderer/package.json@@ -24,8 +24,7 @@"@mediabunny/flac-encoder": "catalog:","@remotion/licensing": "workspace:*","remotion": "workspace:*",- "mediabunny": "catalog:",- "@remotion/media": "workspace:*"+ "mediabunny": "catalog:"},"devDependencies": {"@remotion/effects": "workspace:*",@@ -33,6 +32,7 @@"@remotion/eslint-config-internal": "workspace:*","@remotion/paths": "workspace:*","@remotion/player": "workspace:*",+ "@remotion/media": "workspace:*","@remotion/shapes": "workspace:*","@remotion/three": "workspace:*","@remotion/transitions": "workspace:*",diff --git a/packages/web-renderer/src/audio.ts b/packages/web-renderer/src/audio.tsindex 3dabd93a16..3f83bbc99a 100644--- a/packages/web-renderer/src/audio.ts+++ b/packages/web-renderer/src/audio.ts@@ -1,5 +1,5 @@-import {StreamingPitchShifter} from '@remotion/media/internal';import type {TRenderAsset} from 'remotion';+import {StreamingPitchShifter} from './streaming-pitch-shifter';const TARGET_NUMBER_OF_CHANNELS = 2;diff --git a/packages/web-renderer/src/streaming-pitch-shifter.ts b/packages/web-renderer/src/streaming-pitch-shifter.tsnew file mode 100644index 0000000000..fc269421ce--- /dev/null+++ b/packages/web-renderer/src/streaming-pitch-shifter.ts@@ -0,0 +1,491 @@+// The pitch-shifting algorithm is adapted from Vanilagy's WSOLA audio+// stretcher: https://gist.github.com/Vanilagy/05f7901f4c4398356657e3a86c7aee05+const REFERENCE_SAMPLE_RATE = 48_000;+const REFERENCE_HOP_SIZE = 512;++type PlanarAudio = Float32Array[];++const makePlanarAudio = (numberOfChannels: number, length: number) => {+ return new Array(numberOfChannels)+ .fill(null)+ .map(() => new Float32Array(length));+};++const ensurePlanarCapacity = ({+ buffers,+ requiredLength,+}: {+ buffers: PlanarAudio;+ requiredLength: number;+}) => {+ if (buffers[0].length >= requiredLength) {+ return buffers;+ }++ let newLength = buffers[0].length;+ while (newLength < requiredLength) {+ newLength *= 2;+ }++ return buffers.map((buffer) => {+ const expanded = new Float32Array(newLength);+ expanded.set(buffer);+ return expanded;+ });+};++class PlanarAudioQueue {+ private chunks: PlanarAudio[] = [];+ private length = 0;++ public push(audio: PlanarAudio) {+ if (audio[0].length === 0) {+ return;+ }++ this.chunks.push(audio);+ this.length += audio[0].length;+ }++ public take(numberOfFrames: number, numberOfChannels: number): PlanarAudio {+ const framesToTake = Math.min(numberOfFrames, this.length);+ const result = makePlanarAudio(numberOfChannels, framesToTake);+ let written = 0;++ while (written < framesToTake) {+ const first = this.chunks[0];+ const available = first[0].length;+ const count = Math.min(available, framesToTake - written);++ for (let channel = 0; channel < numberOfChannels; channel++) {+ result[channel].set(first[channel].subarray(0, count), written);+ }++ if (count === available) {+ this.chunks.shift();+ } else {+ this.chunks[0] = first.map((channel) => channel.subarray(count));+ }++ written += count;+ this.length -= count;+ }++ return result;+ }++ public getLength() {+ return this.length;+ }+}++// A streaming implementation of Waveform Similarity Overlap-Add. It changes+// duration while retaining pitch. Pitch shifting is achieved by following this+// stage with a resampler that restores the original duration.+class StreamingTimeStretcher {+ private readonly numberOfChannels: number;+ private readonly factor: number;+ private readonly hopSize: number;+ private readonly windowSize: number;+ private readonly searchRadius: number;+ private readonly analysisHop: number;+ private input: PlanarAudio;+ private inputLength = 0;+ private output: PlanarAudio;+ private outputLength = 0;+ private analysisPosition = 0;+ private synthesisPosition = 0;+ private initialized = false;+ private finalized = false;+ private totalInputFrames = 0;+ private totalOutputFrames = 0;++ public constructor({+ numberOfChannels,+ sampleRate,+ factor,+ }: {+ numberOfChannels: number;+ sampleRate: number;+ factor: number;+ }) {+ this.numberOfChannels = numberOfChannels;+ this.factor = factor;+ this.hopSize = Math.max(+ 32,+ Math.round((REFERENCE_HOP_SIZE * sampleRate) / REFERENCE_SAMPLE_RATE),+ );+ this.windowSize = this.hopSize * 2;+ this.searchRadius = this.hopSize;+ this.analysisHop = this.hopSize / factor;+ this.input = makePlanarAudio(numberOfChannels, 65_536);+ this.output = makePlanarAudio(numberOfChannels, 65_536);+ }++ public append(audio: PlanarAudio) {+ if (this.finalized) {+ throw new Error(+ 'Cannot append audio after the time stretcher was finalized.',+ );+ }++ const {length} = audio[0];+ this.input = ensurePlanarCapacity({+ buffers: this.input,+ requiredLength: this.inputLength + length,+ });++ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.input[channel].set(audio[channel], this.inputLength);+ }++ this.inputLength += length;+ this.totalInputFrames += length;+ this.process();+ return this.drainFinalizedOutput();+ }++ private findBestAnalysisPosition({+ expectedPosition,+ nextSynthesisPosition,+ }: {+ expectedPosition: number;+ nextSynthesisPosition: number;+ }) {+ const minimum = Math.max(+ 0,+ Math.floor(expectedPosition - this.searchRadius),+ );+ const maximum = Math.min(+ this.inputLength - this.windowSize,+ Math.ceil(expectedPosition + this.searchRadius),+ );++ let bestPosition = minimum;+ let bestCorrelation = -Infinity;++ for (let candidate = minimum; candidate <= maximum; candidate += 4) {+ let dotProduct = 0;+ let previousEnergy = 0;+ let candidateEnergy = 0;++ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ const previous = this.output[channel];+ const incoming = this.input[channel];+ for (let frame = 0; frame < this.hopSize; frame += 2) {+ const previousValue = previous[nextSynthesisPosition + frame];+ const candidateValue = incoming[candidate + frame];+ dotProduct += previousValue * candidateValue;+ previousEnergy += previousValue * previousValue;+ candidateEnergy += candidateValue * candidateValue;+ }+ }++ const correlation =+ dotProduct /+ (Math.sqrt(previousEnergy * candidateEnergy) || Number.EPSILON);+ if (correlation > bestCorrelation) {+ bestCorrelation = correlation;+ bestPosition = candidate;+ }+ }++ const fineMinimum = Math.max(minimum, bestPosition - 4);+ const fineMaximum = Math.min(maximum, bestPosition + 4);+ for (let candidate = fineMinimum; candidate <= fineMaximum; candidate++) {+ let dotProduct = 0;+ let previousEnergy = 0;+ let candidateEnergy = 0;++ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ const previous = this.output[channel];+ const incoming = this.input[channel];+ for (let frame = 0; frame < this.hopSize; frame++) {+ const previousValue = previous[nextSynthesisPosition + frame];+ const candidateValue = incoming[candidate + frame];+ dotProduct += previousValue * candidateValue;+ previousEnergy += previousValue * previousValue;+ candidateEnergy += candidateValue * candidateValue;+ }+ }++ const correlation =+ dotProduct /+ (Math.sqrt(previousEnergy * candidateEnergy) || Number.EPSILON);+ if (correlation > bestCorrelation) {+ bestCorrelation = correlation;+ bestPosition = candidate;+ }+ }++ return bestPosition;+ }++ private process() {+ if (!this.initialized) {+ if (this.inputLength < this.windowSize + this.searchRadius) {+ return;+ }++ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.output[channel].set(+ this.input[channel].subarray(0, this.windowSize),+ );+ }++ this.outputLength = this.windowSize;+ this.initialized = true;+ }++ while (true) {+ const expectedPosition = this.analysisPosition + this.analysisHop;+ if (+ expectedPosition + this.searchRadius + this.windowSize >+ this.inputLength+ ) {+ break;+ }++ const nextSynthesisPosition = this.synthesisPosition + this.hopSize;+ this.output = ensurePlanarCapacity({+ buffers: this.output,+ requiredLength: nextSynthesisPosition + this.windowSize,+ });+ const bestPosition = this.findBestAnalysisPosition({+ expectedPosition,+ nextSynthesisPosition,+ });++ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ for (let frame = 0; frame < this.hopSize; frame++) {+ const fadeIn =+ 0.5 - 0.5 * Math.cos((Math.PI * (frame + 1)) / (this.hopSize + 1));+ const outputIndex = nextSynthesisPosition + frame;+ this.output[channel][outputIndex] =+ this.output[channel][outputIndex] * (1 - fadeIn) ++ this.input[channel][bestPosition + frame] * fadeIn;+ }++ this.output[channel].set(+ this.input[channel].subarray(+ bestPosition + this.hopSize,+ bestPosition + this.windowSize,+ ),+ nextSynthesisPosition + this.hopSize,+ );+ }++ // Keep the analysis clock independent from the correlation correction.+ // Periodic signals can have equally good matches at an earlier period; if+ // the correction became the next clock position, the iterator could stop+ // making forward progress.+ this.analysisPosition = expectedPosition;+ this.synthesisPosition = nextSynthesisPosition;+ this.outputLength = nextSynthesisPosition + this.windowSize;+ }+ }++ private drainFinalizedOutput() {+ if (!this.initialized) {+ return makePlanarAudio(this.numberOfChannels, 0);+ }++ const finalizedLength = Math.max(0, this.synthesisPosition + this.hopSize);+ const result = this.output.map((channel) =>+ channel.slice(0, finalizedLength),+ );+ this.totalOutputFrames += finalizedLength;++ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.output[channel].copyWithin(0, finalizedLength, this.outputLength);+ }++ this.outputLength -= finalizedLength;+ this.synthesisPosition -= finalizedLength;++ const inputFramesToDiscard = Math.max(+ 0,+ Math.floor(this.analysisPosition) - this.searchRadius,+ );+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.input[channel].copyWithin(0, inputFramesToDiscard, this.inputLength);+ }++ this.inputLength -= inputFramesToDiscard;+ this.analysisPosition -= inputFramesToDiscard;+ return result;+ }++ public finalize() {+ if (this.finalized) {+ throw new Error('The time stretcher has already been finalized.');+ }++ this.finalized = true;+ const targetLength = Math.round(this.totalInputFrames * this.factor);+ const padding = makePlanarAudio(+ this.numberOfChannels,+ this.windowSize + this.searchRadius * 2,+ );+ this.input = ensurePlanarCapacity({+ buffers: this.input,+ requiredLength: this.inputLength + padding[0].length,+ });+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.input[channel].set(padding[channel], this.inputLength);+ }++ this.inputLength += padding[0].length;+ this.process();+ const finalized = this.drainFinalizedOutput();+ const remaining = Math.max(+ 0,+ targetLength - this.totalOutputFrames + finalized[0].length,+ );++ if (finalized[0].length >= remaining) {+ return finalized.map((channel) => channel.slice(0, remaining));+ }++ const result = makePlanarAudio(this.numberOfChannels, remaining);+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ result[channel].set(finalized[channel]);+ }++ return result;+ }+}++class StreamingLinearResampler {+ private readonly numberOfChannels: number;+ private readonly step: number;+ private input: PlanarAudio;+ private inputLength = 0;+ private position = 0;++ public constructor({+ numberOfChannels,+ step,+ }: {+ numberOfChannels: number;+ step: number;+ }) {+ this.numberOfChannels = numberOfChannels;+ this.step = step;+ this.input = makePlanarAudio(numberOfChannels, 65_536);+ }++ public append(audio: PlanarAudio) {+ this.input = ensurePlanarCapacity({+ buffers: this.input,+ requiredLength: this.inputLength + audio[0].length,+ });+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.input[channel].set(audio[channel], this.inputLength);+ }++ this.inputLength += audio[0].length;+ return this.process(false);+ }++ private process(finalizing: boolean) {+ const outputLength = Math.max(+ 0,+ Math.floor(+ (this.inputLength - (finalizing ? 0 : 1) - this.position) / this.step,+ ) + 1,+ );+ const result = makePlanarAudio(this.numberOfChannels, outputLength);++ for (let outputFrame = 0; outputFrame < outputLength; outputFrame++) {+ const leftIndex = Math.floor(this.position);+ const rightIndex = Math.min(leftIndex + 1, this.inputLength - 1);+ const fraction = this.position - leftIndex;+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ const left = this.input[channel][leftIndex];+ const right = this.input[channel][rightIndex];+ result[channel][outputFrame] = left + (right - left) * fraction;+ }++ this.position += this.step;+ }++ const discard = Math.min(Math.floor(this.position), this.inputLength);+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ this.input[channel].copyWithin(0, discard, this.inputLength);+ }++ this.inputLength -= discard;+ this.position -= discard;+ return result;+ }++ public finalize() {+ return this.process(true);+ }+}++export class StreamingPitchShifter {+ private readonly numberOfChannels: number;+ private readonly stretcher: StreamingTimeStretcher;+ private readonly resampler: StreamingLinearResampler;+ private readonly outputQueue = new PlanarAudioQueue();+ private totalInputFrames = 0;+ private totalOutputFrames = 0;++ public constructor({+ numberOfChannels,+ sampleRate,+ toneFrequency,+ }: {+ numberOfChannels: number;+ sampleRate: number;+ toneFrequency: number;+ }) {+ this.numberOfChannels = numberOfChannels;+ this.stretcher = new StreamingTimeStretcher({+ numberOfChannels,+ sampleRate,+ factor: toneFrequency,+ });+ this.resampler = new StreamingLinearResampler({+ numberOfChannels,+ step: toneFrequency,+ });+ }++ public append(audio: PlanarAudio) {+ this.totalInputFrames += audio[0].length;+ const stretched = this.stretcher.append(audio);+ this.outputQueue.push(this.resampler.append(stretched));+ return this.takeAvailableOutput();+ }++ private takeAvailableOutput() {+ const availableInputFrames = this.totalInputFrames - this.totalOutputFrames;+ const framesToTake = Math.min(+ availableInputFrames,+ this.outputQueue.getLength(),+ );+ const result = this.outputQueue.take(framesToTake, this.numberOfChannels);+ this.totalOutputFrames += framesToTake;+ return result;+ }++ public finalize() {+ this.outputQueue.push(this.resampler.append(this.stretcher.finalize()));+ this.outputQueue.push(this.resampler.finalize());+ const remaining = this.totalInputFrames - this.totalOutputFrames;+ const available = this.outputQueue.take(+ Math.min(remaining, this.outputQueue.getLength()),+ this.numberOfChannels,+ );+ const result = makePlanarAudio(this.numberOfChannels, remaining);+ for (let channel = 0; channel < this.numberOfChannels; channel++) {+ result[channel].set(available[channel]);+ }++ this.totalOutputFrames += remaining;+ return result;+ }+}