Skip to content

Visualize Your Ideas With Code

feat: programmable sounds (#1082)

Astaviea40d50b87f5a
diff --git a/package-lock.json b/package-lock.json
index 8f275e60..36698570 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -25619,7 +25619,7 @@
"packages/ffmpeg": {
"name": "@motion-canvas/ffmpeg",
"version": "3.17.2",
- "license": "MIT",
+ "license": "GPLv3",
"dependencies": {
"@motion-canvas/core": "^3.17.2",
"@motion-canvas/vite-plugin": "^3.17.2",
diff --git a/packages/core/src/app/Exporter.ts b/packages/core/src/app/Exporter.ts
index 3c141df2..68426f6b 100644
--- a/packages/core/src/app/Exporter.ts
+++ b/packages/core/src/app/Exporter.ts
@@ -1,4 +1,5 @@
import type {MetaField} from '../meta';
+import type {Sound} from '../scenes';
import type {Project} from './Project';
import type {RendererResult, RendererSettings} from './Renderer';
@@ -63,8 +64,11 @@ export interface Exporter {
* is rendered. Once `start()` is called, it is guaranteed that the `stop()`
* method will be called as well. Can be used to initialize any resources that
* require a clean-up.
+ *
+ * @param sounds - The sounds to be played during the animation.
+ * @param duration - The duration of the animation in frames.
*/
- start?(): Promise<void>;
+ start?(sounds: Sound[], duration: number): Promise<void>;
/**
* Export a frame.
diff --git a/packages/core/src/app/Player.ts b/packages/core/src/app/Player.ts
index ce69bf68..c3377254 100644
--- a/packages/core/src/app/Player.ts
+++ b/packages/core/src/app/Player.ts
@@ -3,8 +3,8 @@ import {
EventDispatcher,
ValueDispatcher,
} from '../events';
-import {AudioManager} from '../media';
-import {Scene} from '../scenes';
+import {AudioManager, AudioManagerPool} from '../media';
+import {Scene, Sound} from '../scenes';
import {EditableTimeEvents} from '../scenes/timeEvents';
import {clamp} from '../tweening';
import {Vector2} from '../types';
@@ -89,6 +89,7 @@ export class Player {
public readonly playback: PlaybackManager;
public readonly status: PlaybackStatus;
public readonly audio: AudioManager;
+ public readonly audioPool: AudioManagerPool;
public readonly logger: Logger;
private readonly sharedWebGLContext: SharedWebGLContext;
@@ -143,6 +144,7 @@ export class Player {
this.playback = new PlaybackManager();
this.status = new PlaybackStatus(this.playback);
this.audio = new AudioManager(this.logger);
+ this.audioPool = new AudioManagerPool(this.logger);
this.size = settings.size ?? new Vector2(1920, 1080);
this.resolutionScale = settings.resolutionScale ?? 1;
this.startTime = settings.range?.[0] ?? 0;
@@ -378,6 +380,13 @@ export class Player {
try {
await this.playback.recalculate();
this.duration.current = this.playback.frame;
+
+ const sounds: Sound[] = [];
+ for (const scene of this.playback.onScenesRecalculated.current) {
+ sounds.push(...scene.sounds.getSounds());
+ }
+ await this.audioPool.setupPool(sounds);
+
this.recalculated.dispatch();
} catch (e) {
this.requestSeek(state.seek);
@@ -408,6 +417,12 @@ export class Player {
state.seek = this.startFrame;
}
+ // Pause / play sounds.
+ this.audioPool.prepare(this.status.time, 1 / this.status.fps);
+ await this.audioPool.setPaused(state.paused || this.finished);
+ this.audioPool.setMuted(state.muted);
+ this.audioPool.setVolume(state.volume);
+
// Pause / play audio.
const audioPaused =
state.paused || this.finished || !this.audio.isInRange(this.status.time);
@@ -520,8 +535,10 @@ export class Player {
}
private syncAudio(frameOffset = 0) {
- this.audio.setTime(
- this.status.framesToSeconds(this.playback.frame + frameOffset),
- );
+ const time = this.status.framesToSeconds(this.playback.frame + frameOffset);
+ this.audio.setTime(time);
+
+ this.audioPool.prepare(time, 1 / this.status.fps);
+ this.audioPool.setTime(time);
}
}
diff --git a/packages/core/src/app/Renderer.ts b/packages/core/src/app/Renderer.ts
index 1e774dd5..ff92283a 100644
--- a/packages/core/src/app/Renderer.ts
+++ b/packages/core/src/app/Renderer.ts
@@ -1,5 +1,5 @@
import {EventDispatcher, ValueDispatcher} from '../events';
-import type {Scene} from '../scenes';
+import type {Scene, Sound} from '../scenes';
import {ReadOnlyTimeEvents} from '../scenes/timeEvents';
import {clampRemap} from '../tweening';
import {Vector2} from '../types';
@@ -207,6 +207,8 @@ export class Renderer {
await this.reloadScenes(settings);
await this.playback.recalculate();
if (signal.aborted) return RendererResult.Aborted;
+
+ const sounds = this.collectSounds();
await this.playback.reset();
if (signal.aborted) return RendererResult.Aborted;
@@ -217,7 +219,7 @@ export class Renderer {
await this.playback.seek(from);
if (signal.aborted) return RendererResult.Aborted;
- await this.exporter.start?.();
+ await this.exporter.start?.(sounds, to - from);
let lastRefresh = performance.now();
let result = RendererResult.Success;
try {
@@ -272,6 +274,14 @@ export class Renderer {
}
}
+ private collectSounds() {
+ const sounds: Sound[] = [];
+ for (const scene of this.playback.onScenesRecalculated.current) {
+ sounds.push(...scene.sounds.getSounds());
+ }
+ return sounds;
+ }
+
private async exportFrame(signal: AbortSignal) {
this.frame.current = this.playback.frame;
await this.stage.render(
diff --git a/packages/core/src/media/AudioManager.ts b/packages/core/src/media/AudioManager.ts
index 80794a73..96cd9166 100644
--- a/packages/core/src/media/AudioManager.ts
+++ b/packages/core/src/media/AudioManager.ts
@@ -1,5 +1,6 @@
import {Logger} from '../app';
import {ValueDispatcher} from '../events';
+import {Sound} from '../scenes';
import {useLogger} from '../utils';
import {AudioData} from './AudioData';
@@ -15,6 +16,9 @@ export class AudioManager {
private error = false;
private abortController: AbortController | null = null;
private offset = 0;
+ private start?: number;
+ private duration?: number;
+ private gainNode?: GainNode;
public constructor(private readonly logger: Logger) {
if (import.meta.hot) {
@@ -25,6 +29,28 @@ export class AudioManager {
});
}
}
+
+ public setSound(sound: Sound) {
+ this.setOffset(sound.offset);
+ this.setTrim(sound.start, sound.end);
+ this.setSource(sound.audio);
+
+ if (this.gainNode === undefined) {
+ const sourceNode = this.context.createMediaElementSource(
+ this.audioElement,
+ );
+ this.gainNode = this.context.createGain();
+ sourceNode.connect(this.gainNode);
+ this.gainNode.connect(this.context.destination);
+ }
+
+ this.gainNode.gain.value = Math.pow(10, (sound.gain ?? 0) / 10);
+ this.setPlaybackRate(
+ Math.pow(2, (sound.detune ?? 0) / 1200) * (sound.playbackRate ?? 1),
+ true,
+ );
+ }
+
public getTime() {
return this.toAbsoluteTime(this.audioElement.currentTime);
}
@@ -33,10 +59,24 @@ export class AudioManager {
this.audioElement.currentTime = this.toRelativeTime(value);
}
+ public setTrim(start?: number, end?: number) {
+ this.start = start;
+ if (end !== undefined) {
+ this.duration = end - (start ?? 0);
+ } else {
+ this.duration = undefined;
+ }
+ }
+
public setOffset(value: number) {
this.offset = value;
}
+ public setPlaybackRate(value: number, pitchShift: boolean = false) {
+ this.audioElement.playbackRate = value;
+ this.audioElement.preservesPitch = !pitchShift;
+ }
+
public setMuted(isMuted: boolean) {
this.audioElement.muted = isMuted;
}
@@ -58,18 +98,26 @@ export class AudioManager {
}
public isInRange(time: number) {
- const audioRangeStart = this.offset;
- const audioRangeEnd = this.audioElement.duration + this.offset;
-
- return audioRangeStart <= time && time <= audioRangeEnd;
+ return (
+ time >= this.offset &&
+ time <
+ this.offset +
+ (this.duration ??
+ this.audioElement.duration * this.audioElement.playbackRate)
+ );
}
public toRelativeTime(time: number) {
- return Math.max(0, time - this.offset);
+ return (
+ Math.max(0, time - this.offset + (this.start ?? 0)) *
+ this.audioElement.playbackRate
+ );
}
public toAbsoluteTime(time: number) {
- return time + this.offset;
+ return (
+ time / this.audioElement.playbackRate + this.offset - (this.start ?? 0)
+ );
}
public isReady() {
@@ -87,7 +135,7 @@ export class AudioManager {
if (this.source && this.audioElement.paused !== isPaused) {
if (isPaused) {
this.audioElement.pause();
- } else {
+ } else if (this.audioElement.currentTime < this.audioElement.duration) {
try {
await this.audioElement.play();
this.error = false;
diff --git a/packages/core/src/media/AudioManagerPool.ts b/packages/core/src/media/AudioManagerPool.ts
new file mode 100644
index 00000000..5d46aa70
--- /dev/null
+++ b/packages/core/src/media/AudioManagerPool.ts
@@ -0,0 +1,70 @@
+import {Logger} from '../app';
+import {Sound} from '../scenes';
+import {AudioManager} from './AudioManager';
+
+export class AudioManagerPool {
+ private pool: AudioManager[] = [];
+ private managers: Map<Sound, AudioManager> = new Map();
+ private sounds: Sound[] = [];
+
+ private muted: boolean = true;
+ private volume: number = 1;
+ private paused: boolean = true;
+
+ public constructor(private readonly logger: Logger) {}
+
+ public async setupPool(sounds: Sound[]) {
+ this.pool = [];
+ this.managers.clear();
+ this.sounds = sounds;
+ }
+
+ public setMuted(muted: boolean) {
+ this.muted = muted;
+ this.managers.forEach(manager => manager.setMuted(muted));
+ }
+
+ public setVolume(volume: number) {
+ this.volume = volume;
+ this.managers.forEach(manager => manager.setVolume(volume));
+ }
+
+ public setTime(time: number) {
+ this.managers.forEach(manager => manager.setTime(time));
+ }
+
+ public async setPaused(paused: boolean) {
+ this.paused = paused;
+ await Promise.all(
+ Array.from(this.managers.values()).map(manager =>
+ manager.setPaused(paused),
+ ),
+ );
+ }
+
+ public prepare(time: number, delta: number) {
+ for (const sound of this.sounds) {
+ if (sound.offset >= time && sound.offset < time + delta) {
+ let manager = this.managers.get(sound);
+ if (manager) continue;
+
+ // sound is starting
+ manager = this.pool.pop() ?? new AudioManager(this.logger);
+ manager.setSound(sound);
+ manager.setMuted(this.muted);
+ manager.setVolume(this.volume);
+ manager.setTime(time);
+ manager.setPaused(this.paused);
+ this.managers.set(sound, manager);
+ } else {
+ const manager = this.managers.get(sound);
+ if (!manager || manager.isInRange(time)) continue;
+
+ // sound has ended
+ manager.setPaused(true);
+ this.pool.push(manager);
+ this.managers.delete(sound);
+ }
+ }
+ }
+}
diff --git a/packages/core/src/media/index.ts b/packages/core/src/media/index.ts
index 721c5492..32c6cd50 100644
--- a/packages/core/src/media/index.ts
+++ b/packages/core/src/media/index.ts
@@ -6,4 +6,5 @@
export * from './AudioData';
export * from './AudioManager';
+export * from './AudioManagerPool';
export * from './loadImage';
diff --git a/packages/core/src/scenes/GeneratorScene.ts b/packages/core/src/scenes/GeneratorScene.ts
index 34ba13da..63caf478 100644
--- a/packages/core/src/scenes/GeneratorScene.ts
+++ b/packages/core/src/scenes/GeneratorScene.ts
@@ -24,6 +24,7 @@ import {SceneMetadata} from './SceneMetadata';
import {SceneState} from './SceneState';
import {Shaders} from './Shaders';
import {Slides} from './Slides';
+import {Sounds} from './Sounds';
import {Threadable} from './Threadable';
import {Variables} from './Variables';
import {TimeEvents} from './timeEvents';
@@ -47,6 +48,7 @@ export abstract class GeneratorScene<T>
public readonly timeEvents: TimeEvents;
public readonly shaders: Shaders;
public readonly slides: Slides;
+ public readonly sounds: Sounds;
public readonly variables: Variables;
public random: Random;
public creationStack?: string;
@@ -143,6 +145,7 @@ export abstract class GeneratorScene<T>
this.variables = new Variables(this);
this.shaders = new Shaders(this, description.sharedWebGLContext);
this.slides = new Slides(this);
+ this.sounds = new Sounds(this);
this.random = new Random(this.meta.seed.get());
this.previousOnTop = false;
diff --git a/packages/core/src/scenes/Scene.ts b/packages/core/src/scenes/Scene.ts
index 9bffed74..7c751d95 100644
--- a/packages/core/src/scenes/Scene.ts
+++ b/packages/core/src/scenes/Scene.ts
@@ -12,6 +12,7 @@ import type {Random} from './Random';
import type {SceneMetadata} from './SceneMetadata';
import type {Shaders} from './Shaders';
import type {Slides} from './Slides';
+import type {Sounds} from './Sounds';
import type {Variables} from './Variables';
import type {TimeEvents} from './timeEvents';
@@ -149,6 +150,7 @@ export interface Scene<T = unknown> {
*/
readonly shaders: Shaders;
readonly slides: Slides;
+ readonly sounds: Sounds;
readonly logger: Logger;
readonly variables: Variables;
readonly random: Random;
diff --git a/packages/core/src/scenes/Sounds.ts b/packages/core/src/scenes/Sounds.ts
new file mode 100644
index 00000000..1578348d
--- /dev/null
+++ b/packages/core/src/scenes/Sounds.ts
@@ -0,0 +1,120 @@
+import {useScene} from '../utils';
+import type {Scene} from './Scene';
+
+export interface SoundSettings {
+ audio: string;
+ start?: number;
+ end?: number;
+ gain?: number;
+ detune?: number;
+ playbackRate?: number;
+}
+
+export interface Sound extends SoundSettings {
+ offset: number;
+}
+
+export class SoundBuilder {
+ private settings: SoundSettings;
+
+ /**
+ * {@inheritDoc sound}
+ */
+ public constructor(audio: string | SoundBuilder) {
+ if (audio instanceof SoundBuilder) {
+ this.settings = {...audio.settings};
+ } else {
+ this.settings = {audio};
+ }
+ }
+
+ /**
+ * Trim the audio file to a specific portion of it.
+ *
+ * @param start - The offset in seconds to play from.
+ * @param end - The offset in seconds to play to.
+ */
+ public trim(start?: number, end?: number): this {
+ this.settings.start = start;
+ this.settings.end = end;
+ return this;
+ }
+
+ /**
+ * Set the amplification of the played sound.
+ *
+ * @param db - The gain in dB.
+ */
+ public gain(db: number): this {
+ this.settings.gain = db;
+ return this;
+ }
+
+ /**
+ * Pitch shift the played sound.
+ *
+ * @remarks
+ * This also affects the duration of the sound.
+ *
+ * @param cents - The pitch shift in cents.
+ */
+ public detune(cents: number): this {
+ this.settings.detune = cents;
+ return this;
+ }
+
+ /**
+ * Change the playback rate of the sound.
+ *
+ * @remarks
+ * This also affects the percieved pitch of the sound.
+ *
+ * @param rate - The new playback rate.
+ */
+ public playbackRate(rate: number): this {
+ this.settings.playbackRate = rate;
+ return this;
+ }
+
+ /**
+ * Play the configured sound at the current frame.
+ *
+ * @param offset - An offset in seconds from the current frame. Defaults to 0.
+ */
+ public play(offset?: number) {
+ useScene().sounds.add(this.settings, offset);
+ }
+}
+
+/**
+ * Begin configuring a sound to be played back.
+ */
+export function sound(audio: string | SoundBuilder) {
+ return new SoundBuilder(audio);
+}
+
+export class Sounds {
+ private sounds: Sound[] = [];
+
+ public constructor(private readonly scene: Scene) {
+ this.scene.onReset.subscribe(this.handleReset);
+ this.scene.onReset.subscribe(this.handleReload);
+ }
+
+ public add(settings: SoundSettings, offset?: number) {
+ const playbackTime = this.scene.playback.time + (offset ?? 0);
+ this.sounds.push({offset: playbackTime, ...settings});
+ }
+
+ public getSounds(): readonly Sound[] {
+ return this.sounds;
+ }
+
+ public handleReset = () => {
+ this.sounds = [];
+ };
+
+ public handleReload = () => {
+ this.sounds = [];
+ };
+}
diff --git a/packages/core/src/scenes/index.ts b/packages/core/src/scenes/index.ts
index 4ceceb80..9138ad13 100644
--- a/packages/core/src/scenes/index.ts
+++ b/packages/core/src/scenes/index.ts
@@ -13,5 +13,6 @@ export * from './SceneMetadata';
export * from './SceneState';
export * from './Shaders';
export * from './Slides';
+export * from './Sounds';
export * from './Threadable';
export * from './Variables';
diff --git a/packages/ffmpeg/client/FFmpegExporterClient.ts b/packages/ffmpeg/client/FFmpegExporterClient.ts
index 9378c10a..938a184f 100644
--- a/packages/ffmpeg/client/FFmpegExporterClient.ts
+++ b/packages/ffmpeg/client/FFmpegExporterClient.ts
@@ -4,10 +4,12 @@ import type {
Project,
RendererResult,
RendererSettings,
+ Sound,
} from '@motion-canvas/core';
import {
BoolMetaField,
EventDispatcher,
+ NumberMetaField,
ObjectMetaField,
ValueOf,
} from '@motion-canvas/core';
@@ -61,6 +63,7 @@ export class FFmpegExporterClient implements Exporter {
includeAudio: new BoolMetaField('include audio', true).disable(
!project.audio,
),
+ audioSampleRate: new NumberMetaField('audio sample rate', 48000),
});
}
@@ -87,7 +90,7 @@ export class FFmpegExporterClient implements Exporter {
private readonly settings: RendererSettings,
) {}
- public async start(): Promise<void> {
+ public async start(sounds: Sound[], duration: number): Promise<void> {
const options = this.settings.exporter.options as FFmpegExporterOptions;
await this.invoke('start', {
...this.settings,
@@ -95,6 +98,8 @@ export class FFmpegExporterClient implements Exporter {
audio: this.project.audio,
audioOffset:
this.project.meta.shared.audioOffset.get() - this.settings.range[0],
+ sounds,
+ duration,
});
}
diff --git a/packages/ffmpeg/package.json b/packages/ffmpeg/package.json
index 6d68a692..a4dce312 100644
--- a/packages/ffmpeg/package.json
+++ b/packages/ffmpeg/package.json
@@ -6,7 +6,7 @@
"author": "motion-canvas",
"homepage": "https://motioncanvas.io/",
"bugs": "https://github.com/motion-canvas/motion-canvas/issues",
- "license": "MIT",
+ "license": "GPLv3",
"scripts": {
"build": "npm run client:build && npm run server:build",
"client:build": "tsc --project client/tsconfig.json",
diff --git a/packages/ffmpeg/server/FFmpegExporterServer.ts b/packages/ffmpeg/server/FFmpegExporterServer.ts
index 37e09771..4d694cc4 100644
--- a/packages/ffmpeg/server/FFmpegExporterServer.ts
+++ b/packages/ffmpeg/server/FFmpegExporterServer.ts
@@ -1,6 +1,11 @@
-import type {RendererResult, RendererSettings} from '@motion-canvas/core';
+import type {
+ RendererResult,
+ RendererSettings,
+ Sound,
+} from '@motion-canvas/core';
import type {PluginConfig} from '@motion-canvas/vite-plugin';
import {ffmpegPath, ffprobePath} from 'ffmpeg-ffprobe-static';
+import type {AudioVideoFilter, FilterSpecification} from 'fluent-ffmpeg';
import ffmpeg from 'fluent-ffmpeg';
import * as fs from 'fs';
import * as path from 'path';
@@ -13,8 +18,31 @@ ffmpeg.setFfprobePath(ffprobePath!);
export interface FFmpegExporterSettings extends RendererSettings {
audio?: string;
audioOffset?: number;
+
+ sounds: Sound[];
+ duration: number;
+
fastStart: boolean;
includeAudio: boolean;
+ audioSampleRate: number;
+}
+
+function formatFilters(filters: AudioVideoFilter[]): string {
+ return filters
+ .map(f => {
+ let options: string[] = [];
+ if (typeof f.options === 'string') {
+ options = [f.options];
+ } else if (f.options.constructor === Array) {
+ options = f.options;
+ } else {
+ options = Object.entries(f.options)
+ .filter(([, v]) => v !== undefined)
+ .map(([k, v]) => `${k}=${v}`);
+ }
+ return `${f.filter}=${options.join(':')}`;
+ })
+ .join(',');
}
/**
@@ -43,18 +71,107 @@ export class FFmpegExporterServer {
.inputOptions(['-pix_fmt rgba', '-s:v', `${size.x}x${size.y}`])
.inputFps(settings.fps);
- // Input audio file
- if (settings.includeAudio && settings.audio) {
- this.command
- .input((settings.audio as string).slice(1))
- // FIXME Offset only works for negative values.
- .inputOptions([`-itsoffset ${settings.audioOffset ?? 0}`]);
+ // Input audio
+ const sounds = [...settings.sounds];
+ if (settings.audio && settings.includeAudio) {
+ sounds.push({
+ audio: settings.audio,
+ offset: settings.audioOffset ?? 0,
+ });
+ }
+
+ const filterSpec: FilterSpecification[] = [];
+ const streams: string[] = [];
+
+ for (let i = 0; i < sounds.length; i++) {
+ const sound = sounds[i];
+ this.command.input(sound.audio.slice(1));
+
+ const playback =
+ Math.pow(2, (sound.detune ?? 0) / 1200) * (sound.playbackRate ?? 1);
+
+ let trimmed = sound.start ?? 0;
+ if (sound.offset < 0) {
+ trimmed -= sound.offset * playback;
+ }
+
+ if (trimmed !== 0) {
+ this.command.inputOptions(`-ss ${trimmed}`);
+ }
+
+ const filters: AudioVideoFilter[] = [];
+ if (sound.end !== undefined) {
+ filters.push({
+ filter: 'atrim',
+ options: {end: sound.end - trimmed},
+ });
+ }
+
+ filters.push({
+ filter: 'aresample',
+ options: settings.audioSampleRate.toString(),
+ });
+
+ if (sound.gain) {
+ filters.push({
+ filter: 'volume',
+ options: {volume: `${sound.gain}dB`},
+ });
+ }
+
+ if (playback !== 1) {
+ const rate = Math.round(settings.audioSampleRate * playback);
+ filters.push({
+ filter: 'asetrate',
+ options: {r: rate},
+ });
+ filters.push({
+ filter: 'aresample',
+ options: settings.audioSampleRate.toString(),
+ });
+ }
+
+ if (sound.offset > 0) {
+ const delay = Math.round(sound.offset * 1000);
+ filters.push({
+ filter: 'adelay',
+ options: {delays: delay, all: 1},
+ });
+ }
+
+ if (filters.length > 0) {
+ filterSpec.push({
+ inputs: `${i + 1}:a`,
+ filter: formatFilters(filters),
+ outputs: `a${i + 1}`,
+ });
+ streams.push(`a${i + 1}`);
+ } else {
+ streams.push(`${i + 1}:a`);
+ }
+ }
+
+ if (sounds.length > 0) {
+ this.command.complexFilter([
+ ...filterSpec,
+ {
+ filter: 'amix',
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ options: {inputs: sounds.length, dropout_transition: 0, normalize: 0},
+ inputs: streams,
+ outputs: 'a',
+ },
+ ]);
+ this.command.outputOptions(['-map 0:v', '-map [a]']);
}
// Output settings
this.command
.output(path.join(this.config.output, `${settings.name}.mp4`))
- .outputOptions(['-pix_fmt yuv420p', '-shortest'])
+ .outputOptions([
+ '-pix_fmt yuv420p',
+ `-t ${settings.duration / settings.fps}`,
+ ])
.outputFps(settings.fps)
.size(`${size.x}x${size.y}`);
if (settings.fastStart) {