Visualize Your Ideas With Code
feat: programmable sounds (#1082)
Astaviea40d50b87f5a
diff --git a/package-lock.json b/package-lock.jsonindex 8f275e60..36698570 100644--- a/package-lock.json+++ b/package-lock.json@@ -25619,7 +25619,7 @@"packages/ffmpeg": {"name": "@motion-canvas/ffmpeg","version": "3.17.2",- "license": "MIT",+ "license": "GPLv3","dependencies": {"@motion-canvas/core": "^3.17.2","@motion-canvas/vite-plugin": "^3.17.2",diff --git a/packages/core/src/app/Exporter.ts b/packages/core/src/app/Exporter.tsindex 3c141df2..68426f6b 100644--- a/packages/core/src/app/Exporter.ts+++ b/packages/core/src/app/Exporter.ts@@ -1,4 +1,5 @@import type {MetaField} from '../meta';+import type {Sound} from '../scenes';import type {Project} from './Project';import type {RendererResult, RendererSettings} from './Renderer';@@ -63,8 +64,11 @@ export interface Exporter {* is rendered. Once `start()` is called, it is guaranteed that the `stop()`* method will be called as well. Can be used to initialize any resources that* require a clean-up.+ *+ * @param sounds - The sounds to be played during the animation.+ * @param duration - The duration of the animation in frames.*/- start?(): Promise<void>;+ start?(sounds: Sound[], duration: number): Promise<void>;/*** Export a frame.diff --git a/packages/core/src/app/Player.ts b/packages/core/src/app/Player.tsindex ce69bf68..c3377254 100644--- a/packages/core/src/app/Player.ts+++ b/packages/core/src/app/Player.ts@@ -3,8 +3,8 @@ import {EventDispatcher,ValueDispatcher,} from '../events';-import {AudioManager} from '../media';-import {Scene} from '../scenes';+import {AudioManager, AudioManagerPool} from '../media';+import {Scene, Sound} from '../scenes';import {EditableTimeEvents} from '../scenes/timeEvents';import {clamp} from '../tweening';import {Vector2} from '../types';@@ -89,6 +89,7 @@ export class Player {public readonly playback: PlaybackManager;public readonly status: PlaybackStatus;public readonly audio: AudioManager;+ public readonly audioPool: AudioManagerPool;public readonly logger: Logger;private readonly sharedWebGLContext: SharedWebGLContext;@@ -143,6 +144,7 @@ export class Player {this.playback = new PlaybackManager();this.status = new PlaybackStatus(this.playback);this.audio = new AudioManager(this.logger);+ this.audioPool = new AudioManagerPool(this.logger);this.size = settings.size ?? new Vector2(1920, 1080);this.resolutionScale = settings.resolutionScale ?? 1;this.startTime = settings.range?.[0] ?? 0;@@ -378,6 +380,13 @@ export class Player {try {await this.playback.recalculate();this.duration.current = this.playback.frame;++ const sounds: Sound[] = [];+ for (const scene of this.playback.onScenesRecalculated.current) {+ sounds.push(...scene.sounds.getSounds());+ }+ await this.audioPool.setupPool(sounds);+this.recalculated.dispatch();} catch (e) {this.requestSeek(state.seek);@@ -408,6 +417,12 @@ export class Player {state.seek = this.startFrame;}+ // Pause / play sounds.+ this.audioPool.prepare(this.status.time, 1 / this.status.fps);+ await this.audioPool.setPaused(state.paused || this.finished);+ this.audioPool.setMuted(state.muted);+ this.audioPool.setVolume(state.volume);+// Pause / play audio.const audioPaused =state.paused || this.finished || !this.audio.isInRange(this.status.time);@@ -520,8 +535,10 @@ export class Player {}private syncAudio(frameOffset = 0) {- this.audio.setTime(- this.status.framesToSeconds(this.playback.frame + frameOffset),- );+ const time = this.status.framesToSeconds(this.playback.frame + frameOffset);+ this.audio.setTime(time);++ this.audioPool.prepare(time, 1 / this.status.fps);+ this.audioPool.setTime(time);}}diff --git a/packages/core/src/app/Renderer.ts b/packages/core/src/app/Renderer.tsindex 1e774dd5..ff92283a 100644--- a/packages/core/src/app/Renderer.ts+++ b/packages/core/src/app/Renderer.ts@@ -1,5 +1,5 @@import {EventDispatcher, ValueDispatcher} from '../events';-import type {Scene} from '../scenes';+import type {Scene, Sound} from '../scenes';import {ReadOnlyTimeEvents} from '../scenes/timeEvents';import {clampRemap} from '../tweening';import {Vector2} from '../types';@@ -207,6 +207,8 @@ export class Renderer {await this.reloadScenes(settings);await this.playback.recalculate();if (signal.aborted) return RendererResult.Aborted;++ const sounds = this.collectSounds();await this.playback.reset();if (signal.aborted) return RendererResult.Aborted;@@ -217,7 +219,7 @@ export class Renderer {await this.playback.seek(from);if (signal.aborted) return RendererResult.Aborted;- await this.exporter.start?.();+ await this.exporter.start?.(sounds, to - from);let lastRefresh = performance.now();let result = RendererResult.Success;try {@@ -272,6 +274,14 @@ export class Renderer {}}+ private collectSounds() {+ const sounds: Sound[] = [];+ for (const scene of this.playback.onScenesRecalculated.current) {+ sounds.push(...scene.sounds.getSounds());+ }+ return sounds;+ }+private async exportFrame(signal: AbortSignal) {this.frame.current = this.playback.frame;await this.stage.render(diff --git a/packages/core/src/media/AudioManager.ts b/packages/core/src/media/AudioManager.tsindex 80794a73..96cd9166 100644--- a/packages/core/src/media/AudioManager.ts+++ b/packages/core/src/media/AudioManager.ts@@ -1,5 +1,6 @@import {Logger} from '../app';import {ValueDispatcher} from '../events';+import {Sound} from '../scenes';import {useLogger} from '../utils';import {AudioData} from './AudioData';@@ -15,6 +16,9 @@ export class AudioManager {private error = false;private abortController: AbortController | null = null;private offset = 0;+ private start?: number;+ private duration?: number;+ private gainNode?: GainNode;public constructor(private readonly logger: Logger) {if (import.meta.hot) {@@ -25,6 +29,28 @@ export class AudioManager {});}}++ public setSound(sound: Sound) {+ this.setOffset(sound.offset);+ this.setTrim(sound.start, sound.end);+ this.setSource(sound.audio);++ if (this.gainNode === undefined) {+ const sourceNode = this.context.createMediaElementSource(+ this.audioElement,+ );+ this.gainNode = this.context.createGain();+ sourceNode.connect(this.gainNode);+ this.gainNode.connect(this.context.destination);+ }++ this.gainNode.gain.value = Math.pow(10, (sound.gain ?? 0) / 10);+ this.setPlaybackRate(+ Math.pow(2, (sound.detune ?? 0) / 1200) * (sound.playbackRate ?? 1),+ true,+ );+ }+public getTime() {return this.toAbsoluteTime(this.audioElement.currentTime);}@@ -33,10 +59,24 @@ export class AudioManager {this.audioElement.currentTime = this.toRelativeTime(value);}+ public setTrim(start?: number, end?: number) {+ this.start = start;+ if (end !== undefined) {+ this.duration = end - (start ?? 0);+ } else {+ this.duration = undefined;+ }+ }+public setOffset(value: number) {this.offset = value;}+ public setPlaybackRate(value: number, pitchShift: boolean = false) {+ this.audioElement.playbackRate = value;+ this.audioElement.preservesPitch = !pitchShift;+ }+public setMuted(isMuted: boolean) {this.audioElement.muted = isMuted;}@@ -58,18 +98,26 @@ export class AudioManager {}public isInRange(time: number) {- const audioRangeStart = this.offset;- const audioRangeEnd = this.audioElement.duration + this.offset;-- return audioRangeStart <= time && time <= audioRangeEnd;+ return (+ time >= this.offset &&+ time <+ this.offset ++ (this.duration ??+ this.audioElement.duration * this.audioElement.playbackRate)+ );}public toRelativeTime(time: number) {- return Math.max(0, time - this.offset);+ return (+ Math.max(0, time - this.offset + (this.start ?? 0)) *+ this.audioElement.playbackRate+ );}public toAbsoluteTime(time: number) {- return time + this.offset;+ return (+ time / this.audioElement.playbackRate + this.offset - (this.start ?? 0)+ );}public isReady() {@@ -87,7 +135,7 @@ export class AudioManager {if (this.source && this.audioElement.paused !== isPaused) {if (isPaused) {this.audioElement.pause();- } else {+ } else if (this.audioElement.currentTime < this.audioElement.duration) {try {await this.audioElement.play();this.error = false;diff --git a/packages/core/src/media/AudioManagerPool.ts b/packages/core/src/media/AudioManagerPool.tsnew file mode 100644index 00000000..5d46aa70--- /dev/null+++ b/packages/core/src/media/AudioManagerPool.ts@@ -0,0 +1,70 @@+import {Logger} from '../app';+import {Sound} from '../scenes';+import {AudioManager} from './AudioManager';++export class AudioManagerPool {+ private pool: AudioManager[] = [];+ private managers: Map<Sound, AudioManager> = new Map();+ private sounds: Sound[] = [];++ private muted: boolean = true;+ private volume: number = 1;+ private paused: boolean = true;++ public constructor(private readonly logger: Logger) {}++ public async setupPool(sounds: Sound[]) {+ this.pool = [];+ this.managers.clear();+ this.sounds = sounds;+ }++ public setMuted(muted: boolean) {+ this.muted = muted;+ this.managers.forEach(manager => manager.setMuted(muted));+ }++ public setVolume(volume: number) {+ this.volume = volume;+ this.managers.forEach(manager => manager.setVolume(volume));+ }++ public setTime(time: number) {+ this.managers.forEach(manager => manager.setTime(time));+ }++ public async setPaused(paused: boolean) {+ this.paused = paused;+ await Promise.all(+ Array.from(this.managers.values()).map(manager =>+ manager.setPaused(paused),+ ),+ );+ }++ public prepare(time: number, delta: number) {+ for (const sound of this.sounds) {+ if (sound.offset >= time && sound.offset < time + delta) {+ let manager = this.managers.get(sound);+ if (manager) continue;++ // sound is starting+ manager = this.pool.pop() ?? new AudioManager(this.logger);+ manager.setSound(sound);+ manager.setMuted(this.muted);+ manager.setVolume(this.volume);+ manager.setTime(time);+ manager.setPaused(this.paused);+ this.managers.set(sound, manager);+ } else {+ const manager = this.managers.get(sound);+ if (!manager || manager.isInRange(time)) continue;++ // sound has ended+ manager.setPaused(true);+ this.pool.push(manager);+ this.managers.delete(sound);+ }+ }+ }+}diff --git a/packages/core/src/media/index.ts b/packages/core/src/media/index.tsindex 721c5492..32c6cd50 100644--- a/packages/core/src/media/index.ts+++ b/packages/core/src/media/index.ts@@ -6,4 +6,5 @@export * from './AudioData';export * from './AudioManager';+export * from './AudioManagerPool';export * from './loadImage';diff --git a/packages/core/src/scenes/GeneratorScene.ts b/packages/core/src/scenes/GeneratorScene.tsindex 34ba13da..63caf478 100644--- a/packages/core/src/scenes/GeneratorScene.ts+++ b/packages/core/src/scenes/GeneratorScene.ts@@ -24,6 +24,7 @@ import {SceneMetadata} from './SceneMetadata';import {SceneState} from './SceneState';import {Shaders} from './Shaders';import {Slides} from './Slides';+import {Sounds} from './Sounds';import {Threadable} from './Threadable';import {Variables} from './Variables';import {TimeEvents} from './timeEvents';@@ -47,6 +48,7 @@ export abstract class GeneratorScene<T>public readonly timeEvents: TimeEvents;public readonly shaders: Shaders;public readonly slides: Slides;+ public readonly sounds: Sounds;public readonly variables: Variables;public random: Random;public creationStack?: string;@@ -143,6 +145,7 @@ export abstract class GeneratorScene<T>this.variables = new Variables(this);this.shaders = new Shaders(this, description.sharedWebGLContext);this.slides = new Slides(this);+ this.sounds = new Sounds(this);this.random = new Random(this.meta.seed.get());this.previousOnTop = false;diff --git a/packages/core/src/scenes/Scene.ts b/packages/core/src/scenes/Scene.tsindex 9bffed74..7c751d95 100644--- a/packages/core/src/scenes/Scene.ts+++ b/packages/core/src/scenes/Scene.ts@@ -12,6 +12,7 @@ import type {Random} from './Random';import type {SceneMetadata} from './SceneMetadata';import type {Shaders} from './Shaders';import type {Slides} from './Slides';+import type {Sounds} from './Sounds';import type {Variables} from './Variables';import type {TimeEvents} from './timeEvents';@@ -149,6 +150,7 @@ export interface Scene<T = unknown> {*/readonly shaders: Shaders;readonly slides: Slides;+ readonly sounds: Sounds;readonly logger: Logger;readonly variables: Variables;readonly random: Random;diff --git a/packages/core/src/scenes/Sounds.ts b/packages/core/src/scenes/Sounds.tsnew file mode 100644index 00000000..1578348d--- /dev/null+++ b/packages/core/src/scenes/Sounds.ts@@ -0,0 +1,120 @@+import {useScene} from '../utils';+import type {Scene} from './Scene';++export interface SoundSettings {+ audio: string;+ start?: number;+ end?: number;+ gain?: number;+ detune?: number;+ playbackRate?: number;+}++export interface Sound extends SoundSettings {+ offset: number;+}++export class SoundBuilder {+ private settings: SoundSettings;++ /**+ * {@inheritDoc sound}+ */+ public constructor(audio: string | SoundBuilder) {+ if (audio instanceof SoundBuilder) {+ this.settings = {...audio.settings};+ } else {+ this.settings = {audio};+ }+ }++ /**+ * Trim the audio file to a specific portion of it.+ *+ * @param start - The offset in seconds to play from.+ * @param end - The offset in seconds to play to.+ */+ public trim(start?: number, end?: number): this {+ this.settings.start = start;+ this.settings.end = end;+ return this;+ }++ /**+ * Set the amplification of the played sound.+ *+ * @param db - The gain in dB.+ */+ public gain(db: number): this {+ this.settings.gain = db;+ return this;+ }++ /**+ * Pitch shift the played sound.+ *+ * @remarks+ * This also affects the duration of the sound.+ *+ * @param cents - The pitch shift in cents.+ */+ public detune(cents: number): this {+ this.settings.detune = cents;+ return this;+ }++ /**+ * Change the playback rate of the sound.+ *+ * @remarks+ * This also affects the percieved pitch of the sound.+ *+ * @param rate - The new playback rate.+ */+ public playbackRate(rate: number): this {+ this.settings.playbackRate = rate;+ return this;+ }++ /**+ * Play the configured sound at the current frame.+ *+ * @param offset - An offset in seconds from the current frame. Defaults to 0.+ */+ public play(offset?: number) {+ useScene().sounds.add(this.settings, offset);+ }+}++/**+ * Begin configuring a sound to be played back.+ */+export function sound(audio: string | SoundBuilder) {+ return new SoundBuilder(audio);+}++export class Sounds {+ private sounds: Sound[] = [];++ public constructor(private readonly scene: Scene) {+ this.scene.onReset.subscribe(this.handleReset);+ this.scene.onReset.subscribe(this.handleReload);+ }++ public add(settings: SoundSettings, offset?: number) {+ const playbackTime = this.scene.playback.time + (offset ?? 0);+ this.sounds.push({offset: playbackTime, ...settings});+ }++ public getSounds(): readonly Sound[] {+ return this.sounds;+ }++ public handleReset = () => {+ this.sounds = [];+ };++ public handleReload = () => {+ this.sounds = [];+ };+}diff --git a/packages/core/src/scenes/index.ts b/packages/core/src/scenes/index.tsindex 4ceceb80..9138ad13 100644--- a/packages/core/src/scenes/index.ts+++ b/packages/core/src/scenes/index.ts@@ -13,5 +13,6 @@ export * from './SceneMetadata';export * from './SceneState';export * from './Shaders';export * from './Slides';+export * from './Sounds';export * from './Threadable';export * from './Variables';diff --git a/packages/ffmpeg/client/FFmpegExporterClient.ts b/packages/ffmpeg/client/FFmpegExporterClient.tsindex 9378c10a..938a184f 100644--- a/packages/ffmpeg/client/FFmpegExporterClient.ts+++ b/packages/ffmpeg/client/FFmpegExporterClient.ts@@ -4,10 +4,12 @@ import type {Project,RendererResult,RendererSettings,+ Sound,} from '@motion-canvas/core';import {BoolMetaField,EventDispatcher,+ NumberMetaField,ObjectMetaField,ValueOf,} from '@motion-canvas/core';@@ -61,6 +63,7 @@ export class FFmpegExporterClient implements Exporter {includeAudio: new BoolMetaField('include audio', true).disable(!project.audio,),+ audioSampleRate: new NumberMetaField('audio sample rate', 48000),});}@@ -87,7 +90,7 @@ export class FFmpegExporterClient implements Exporter {private readonly settings: RendererSettings,) {}- public async start(): Promise<void> {+ public async start(sounds: Sound[], duration: number): Promise<void> {const options = this.settings.exporter.options as FFmpegExporterOptions;await this.invoke('start', {...this.settings,@@ -95,6 +98,8 @@ export class FFmpegExporterClient implements Exporter {audio: this.project.audio,audioOffset:this.project.meta.shared.audioOffset.get() - this.settings.range[0],+ sounds,+ duration,});}diff --git a/packages/ffmpeg/package.json b/packages/ffmpeg/package.jsonindex 6d68a692..a4dce312 100644--- a/packages/ffmpeg/package.json+++ b/packages/ffmpeg/package.json@@ -6,7 +6,7 @@"author": "motion-canvas","homepage": "https://motioncanvas.io/","bugs": "https://github.com/motion-canvas/motion-canvas/issues",- "license": "MIT",+ "license": "GPLv3","scripts": {"build": "npm run client:build && npm run server:build","client:build": "tsc --project client/tsconfig.json",diff --git a/packages/ffmpeg/server/FFmpegExporterServer.ts b/packages/ffmpeg/server/FFmpegExporterServer.tsindex 37e09771..4d694cc4 100644--- a/packages/ffmpeg/server/FFmpegExporterServer.ts+++ b/packages/ffmpeg/server/FFmpegExporterServer.ts@@ -1,6 +1,11 @@-import type {RendererResult, RendererSettings} from '@motion-canvas/core';+import type {+ RendererResult,+ RendererSettings,+ Sound,+} from '@motion-canvas/core';import type {PluginConfig} from '@motion-canvas/vite-plugin';import {ffmpegPath, ffprobePath} from 'ffmpeg-ffprobe-static';+import type {AudioVideoFilter, FilterSpecification} from 'fluent-ffmpeg';import ffmpeg from 'fluent-ffmpeg';import * as fs from 'fs';import * as path from 'path';@@ -13,8 +18,31 @@ ffmpeg.setFfprobePath(ffprobePath!);export interface FFmpegExporterSettings extends RendererSettings {audio?: string;audioOffset?: number;++ sounds: Sound[];+ duration: number;+fastStart: boolean;includeAudio: boolean;+ audioSampleRate: number;+}++function formatFilters(filters: AudioVideoFilter[]): string {+ return filters+ .map(f => {+ let options: string[] = [];+ if (typeof f.options === 'string') {+ options = [f.options];+ } else if (f.options.constructor === Array) {+ options = f.options;+ } else {+ options = Object.entries(f.options)+ .filter(([, v]) => v !== undefined)+ .map(([k, v]) => `${k}=${v}`);+ }+ return `${f.filter}=${options.join(':')}`;+ })+ .join(',');}/**@@ -43,18 +71,107 @@ export class FFmpegExporterServer {.inputOptions(['-pix_fmt rgba', '-s:v', `${size.x}x${size.y}`]).inputFps(settings.fps);- // Input audio file- if (settings.includeAudio && settings.audio) {- this.command- .input((settings.audio as string).slice(1))- // FIXME Offset only works for negative values.- .inputOptions([`-itsoffset ${settings.audioOffset ?? 0}`]);+ // Input audio+ const sounds = [...settings.sounds];+ if (settings.audio && settings.includeAudio) {+ sounds.push({+ audio: settings.audio,+ offset: settings.audioOffset ?? 0,+ });+ }++ const filterSpec: FilterSpecification[] = [];+ const streams: string[] = [];++ for (let i = 0; i < sounds.length; i++) {+ const sound = sounds[i];+ this.command.input(sound.audio.slice(1));++ const playback =+ Math.pow(2, (sound.detune ?? 0) / 1200) * (sound.playbackRate ?? 1);++ let trimmed = sound.start ?? 0;+ if (sound.offset < 0) {+ trimmed -= sound.offset * playback;+ }++ if (trimmed !== 0) {+ this.command.inputOptions(`-ss ${trimmed}`);+ }++ const filters: AudioVideoFilter[] = [];+ if (sound.end !== undefined) {+ filters.push({+ filter: 'atrim',+ options: {end: sound.end - trimmed},+ });+ }++ filters.push({+ filter: 'aresample',+ options: settings.audioSampleRate.toString(),+ });++ if (sound.gain) {+ filters.push({+ filter: 'volume',+ options: {volume: `${sound.gain}dB`},+ });+ }++ if (playback !== 1) {+ const rate = Math.round(settings.audioSampleRate * playback);+ filters.push({+ filter: 'asetrate',+ options: {r: rate},+ });+ filters.push({+ filter: 'aresample',+ options: settings.audioSampleRate.toString(),+ });+ }++ if (sound.offset > 0) {+ const delay = Math.round(sound.offset * 1000);+ filters.push({+ filter: 'adelay',+ options: {delays: delay, all: 1},+ });+ }++ if (filters.length > 0) {+ filterSpec.push({+ inputs: `${i + 1}:a`,+ filter: formatFilters(filters),+ outputs: `a${i + 1}`,+ });+ streams.push(`a${i + 1}`);+ } else {+ streams.push(`${i + 1}:a`);+ }+ }++ if (sounds.length > 0) {+ this.command.complexFilter([+ ...filterSpec,+ {+ filter: 'amix',+ // eslint-disable-next-line @typescript-eslint/naming-convention+ options: {inputs: sounds.length, dropout_transition: 0, normalize: 0},+ inputs: streams,+ outputs: 'a',+ },+ ]);+ this.command.outputOptions(['-map 0:v', '-map [a]']);}// Output settingsthis.command.output(path.join(this.config.output, `${settings.name}.mp4`))- .outputOptions(['-pix_fmt yuv420p', '-shortest'])+ .outputOptions([+ '-pix_fmt yuv420p',+ `-t ${settings.duration / settings.fps}`,+ ]).outputFps(settings.fps).size(`${size.x}x${size.y}`);if (settings.fastStart) {