Skip to content

Visualize Your Ideas With Code

feat: use Web Audio API for waveform generation

aarthificial817e244bb218
diff --git a/src/Project.ts b/src/Project.ts
index a6d0dcba..02d6c3de 100644
--- a/src/Project.ts
+++ b/src/Project.ts
@@ -171,9 +171,9 @@ export class Project extends Stage {
this.frame = scene.lastFrame;
} else {
this.currentScene = scene;
- await scene.reset();
scene.firstFrame = this.frame;
scene.transitionDuration = -1;
+ await scene.reset();
while (!scene.canTransitionOut()) {
if (scene.transitionDuration < 0 && scene.isAfterTransitionIn()) {
scene.transitionDuration = this.frame - scene.firstFrame;
diff --git a/src/audio/AudioData.ts b/src/audio/AudioData.ts
new file mode 100644
index 00000000..0507d1ce
--- /dev/null
+++ b/src/audio/AudioData.ts
@@ -0,0 +1,19 @@
+export interface AudioData {
+ /**
+ * An array of minimum and maximum waveform data points, interleaved.
+ * Each value is in range of -1 to 1.
+ */
+ peaks: number[];
+ /**
+ * The amount of samples taken.
+ */
+ length: number;
+ /**
+ * The absolute biggest value from the peaks array.
+ */
+ absoluteMax: number;
+ /**
+ * Samples per seconds.
+ */
+ sampleRate: number;
+}
diff --git a/src/audio/AudioManager.ts b/src/audio/AudioManager.ts
new file mode 100644
index 00000000..ef553226
--- /dev/null
+++ b/src/audio/AudioManager.ts
@@ -0,0 +1,168 @@
+import {AudioData} from './AudioData';
+import {SimpleEventDispatcher} from 'strongly-typed-events';
+
+export class AudioManager {
+ public get DataChanged() {
+ return this.dataChanged.asEvent();
+ }
+
+ public get OffsetChanged() {
+ return this.dataChanged.asEvent();
+ }
+
+ private static readonly context = new AudioContext();
+ private readonly audioElement: HTMLAudioElement = new Audio();
+ private source: string = null;
+ private offset = 0;
+ private error = false;
+ private data: AudioData = null;
+ private dataChanged = new SimpleEventDispatcher<AudioData>();
+ private offsetChanged = new SimpleEventDispatcher<number>();
+ private abortController: AbortController = null;
+
+ public getData() {
+ return this.data;
+ }
+
+ public getTime() {
+ return this.toAbsoluteTime(this.audioElement.currentTime);
+ }
+
+ public setTime(value: number) {
+ this.audioElement.currentTime = this.toRelativeTime(value);
+ }
+
+ public setOffset(value: number) {
+ this.offset = value;
+ this.offsetChanged.dispatch(value);
+ }
+
+ public async setMuted(isMuted: boolean) {
+ this.audioElement.muted = isMuted;
+ }
+
+ public setSource(src: string) {
+ this.source = src;
+ this.audioElement.src = src;
+ this.abortController?.abort();
+ this.abortController = new AbortController();
+ this.loadData(this.abortController.signal).catch(e => {
+ if (e.name !== 'AbortError') {
+ console.error(e);
+ }
+ });
+ }
+
+ public isInRange(time: number) {
+ return time >= this.offset && time < this.audioElement.duration;
+ }
+
+ public toRelativeTime(time: number) {
+ return Math.max(0, time - this.offset);
+ }
+
+ public toAbsoluteTime(time: number) {
+ return time + this.offset;
+ }
+
+ public isReady() {
+ return this.source && !this.error;
+ }
+
+ /**
+ * Pause/resume the audio.
+ *
+ * @param isPaused
+ *
+ * @return `true` if the audio successfully started playing.
+ */
+ public async setPaused(isPaused: boolean): Promise<boolean> {
+ if (this.source && this.audioElement.paused !== isPaused) {
+ if (isPaused) {
+ this.audioElement.pause();
+ } else {
+ try {
+ await this.audioElement.play();
+ return true;
+ this.error = false;
+ } catch (e) {
+ if (!this.error) {
+ console.error(e);
+ }
+ this.error = true;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ private async loadData(signal: AbortSignal) {
+ this.data = null;
+ this.dataChanged.dispatch(this.data);
+
+ const response = await fetch(this.source, {signal});
+ const rawBuffer = await response.arrayBuffer();
+ if (signal.aborted) return;
+ const audioBuffer = await this.decodeAudioData(rawBuffer);
+ if (signal.aborted) return;
+
+ const sampleSize = 256;
+ const samples = ~~(audioBuffer.length / sampleSize);
+
+ const peaks = [];
+ let absoluteMax = 0;
+ for (
+ let channelId = 0;
+ channelId < audioBuffer.numberOfChannels;
+ channelId++
+ ) {
+ const channel = audioBuffer.getChannelData(channelId);
+ for (let i = 0; i < samples; i++) {
+ const start = ~~(i * sampleSize);
+ const end = ~~(start + sampleSize);
+
+ let min = channel[start];
+ let max = min;
+
+ for (let j = start; j < end; j++) {
+ const value = channel[j];
+ if (value > max) {
+ max = value;
+ }
+ if (value < min) {
+ min = value;
+ }
+ }
+
+ if (channelId === 0 || max > peaks[i * 2]) {
+ peaks[i * 2] = max;
+ }
+ if (channelId === 0 || min < peaks[i * 2 + 1]) {
+ peaks[i * 2 + 1] = min;
+ }
+
+ if (max > absoluteMax) {
+ absoluteMax = max;
+ }
+ if (Math.abs(min) > absoluteMax) {
+ absoluteMax = Math.abs(min);
+ }
+ }
+ }
+
+ this.data = {
+ peaks,
+ absoluteMax,
+ length: samples,
+ sampleRate: (audioBuffer.sampleRate / sampleSize) * 2,
+ };
+ this.dataChanged.dispatch(this.data);
+ }
+
+ private decodeAudioData(buffer: ArrayBuffer): Promise<AudioBuffer> {
+ return new Promise<AudioBuffer>((resolve, reject) =>
+ AudioManager.context.decodeAudioData(buffer, resolve, reject),
+ );
+ }
+}
diff --git a/src/bootstrap.ts b/src/bootstrap.ts
index 9c570026..e442e116 100644
--- a/src/bootstrap.ts
+++ b/src/bootstrap.ts
@@ -1,19 +1,17 @@
-import type {Size, Waveform} from './types';
+import type {Size} from './types';
import type {SceneRunner} from './Scene';
import {Project, ProjectSize} from './Project';
import {Player} from './player/Player';
import {hot} from './hot';
+import {AudioManager} from './audio/AudioManager';
interface BootstrapConfig {
name: string;
+ scenes: SceneRunner[];
size?: Size;
background?: string | false;
- scenes: SceneRunner[];
- audio?: {
- meta: Waveform;
- src: string;
- offset?: number;
- };
+ audio?: string;
+ audioOffset?: number;
}
export function bootstrap(config: BootstrapConfig) {
@@ -23,13 +21,14 @@ export function bootstrap(config: BootstrapConfig) {
background: config.background ?? '#141414',
...(config.size ?? ProjectSize.FullHD),
});
- const player = new Player(
- project,
- config.audio && {
- offset: 0,
- ...config.audio,
- },
- );
+ const audio = new AudioManager();
+ if (config.audio) {
+ audio.setSource(config.audio);
+ }
+ if (config.audioOffset) {
+ audio.setOffset(config.audioOffset);
+ }
+ const player = new Player(project, audio);
(<any>window).player = player;
//@ts-ignore
diff --git a/src/hot.ts b/src/hot.ts
index 2cc6bfaa..e5bd1c29 100644
--- a/src/hot.ts
+++ b/src/hot.ts
@@ -1,12 +1,12 @@
-import {Player} from './player/Player';
+import type {Player} from './player/Player';
export function hot(player: Player, root: NodeModule) {
- const update = async (modules: string[]) => {
+ const updateScenes = async (modules: string[]) => {
const runners = [];
for (const module of modules) {
const runner = __webpack_require__(module).default;
if (
- // FIXME Find out a better way to detect runner factories.
+ // FIXME Find a better way to detect runner factories.
runner.name === '__WEBPACK_DEFAULT_EXPORT__' &&
typeof runner === 'function'
) {
@@ -19,20 +19,19 @@ export function hot(player: Player, root: NodeModule) {
player.project.reload(runners);
player.reload();
};
+
const updateAudio = async (modules: string[]) => {
let src = null;
- let meta = null;
for (const module of modules) {
const audio = __webpack_require__(module);
- if (typeof audio === 'object') {
- meta = audio.default;
- } else {
+ if (typeof audio === 'string') {
src = audio;
+ break;
}
}
- if (src && meta) {
- player.reloadAudio(src, meta);
+ if (src) {
+ player.audio.setSource(src);
}
};
@@ -44,8 +43,8 @@ export function hot(player: Player, root: NodeModule) {
(name: string) => name.match(/\.wav/),
);
- root.hot.accept(scenePaths, update);
- module.hot.accept(scenePaths, update);
+ root.hot.accept(scenePaths, updateScenes);
+ module.hot.accept(scenePaths, updateScenes);
root.hot.accept(audioPaths, updateAudio);
module.hot.accept(audioPaths, updateAudio);
}
diff --git a/src/player/Player.ts b/src/player/Player.ts
index 3ea4390f..cf5517a7 100644
--- a/src/player/Player.ts
+++ b/src/player/Player.ts
@@ -1,5 +1,5 @@
import type {Project} from '../Project';
-import type {Waveform} from '../types';
+import {AudioManager} from '../audio/AudioManager';
import {
PromiseSimpleEventDispatcher,
SimpleEventDispatcher,
@@ -41,8 +41,6 @@ export interface PlayerRenderEvent {
data: Blob;
}
-const STORAGE_KEY = 'player-state';
-
export class Player {
public get StateChanged() {
return this.stateChanged.asEvent();
@@ -52,8 +50,8 @@ export class Player {
return this.timeChanged.asEvent();
}
- public get RenderChanged() {
- return this.renderChanged.asEvent();
+ public get FrameRendered() {
+ return this.frameRendered.asEvent();
}
public get Reloaded() {
@@ -62,29 +60,20 @@ export class Player {
private readonly stateChanged = new SimpleEventDispatcher<PlayerState>();
private readonly timeChanged = new SimpleEventDispatcher<PlayerTime>();
- private readonly renderChanged =
+ private readonly frameRendered =
new PromiseSimpleEventDispatcher<PlayerRenderEvent>();
private readonly reloaded = new SimpleEventDispatcher<number>();
- private readonly audioElement: HTMLAudioElement = null;
private startTime: number;
private renderTime: number = 0;
private requestId: number = null;
- private audioError = false;
-
- public getState(): PlayerState {
- return {...this.state};
- }
+ private frame: number = 0;
- public getTime(): PlayerTime {
- return {
- frame: this.frame,
- time: this.project.framesToSeconds(this.frame),
- duration: this.state.duration,
- durationTime: this.project.framesToSeconds(this.state.duration),
- completion: this.frame / this.state.duration,
- };
- }
+ private commands: PlayerCommands = {
+ reset: true,
+ seek: -1,
+ recalculate: true,
+ };
private state: PlayerState = {
duration: Infinity,
@@ -101,13 +90,30 @@ export class Player {
scale: 1,
};
- private frame: number = 0;
+ public constructor(
+ public readonly project: Project,
+ public readonly audio: AudioManager,
+ ) {
+ this.startTime = performance.now();
+ this.project.framerate = this.state.fps;
+ this.project.resolutionScale = this.state.scale;
- private commands: PlayerCommands = {
- reset: true,
- seek: -1,
- recalculate: true,
- };
+ this.request();
+ }
+
+ public getState(): PlayerState {
+ return {...this.state};
+ }
+
+ public getTime(): PlayerTime {
+ return {
+ frame: this.frame,
+ time: this.project.framesToSeconds(this.frame),
+ duration: this.state.duration,
+ durationTime: this.project.framesToSeconds(this.state.duration),
+ completion: this.frame / this.state.duration,
+ };
+ }
public updateState(newState: Partial<PlayerState>) {
let changed = false;
@@ -125,7 +131,6 @@ export class Player {
...newState,
};
this.stateChanged.dispatch(this.state);
- localStorage.setItem(STORAGE_KEY, JSON.stringify(this.state));
}
}
@@ -143,47 +148,46 @@ export class Player {
return commands;
}
- public constructor(
- public readonly project: Project,
- public readonly audio?: {
- src: string;
- meta: Waveform;
- offset: number;
- },
- ) {
- this.startTime = performance.now();
+ public reload() {
+ this.commands.recalculate = true;
+ if (this.requestId === null) {
+ this.request();
+ }
+ }
- const savedState = localStorage.getItem(STORAGE_KEY);
- if (savedState) {
- const state = JSON.parse(savedState) as PlayerState;
- for (const key in this.state) {
- //@ts-ignore
- this.state[key] = state[key] ?? this.state[key];
- }
+ public setRange(
+ startFrame = this.state.startFrame,
+ endFrame = this.state.endFrame,
+ ) {
+ if (startFrame === undefined) {
+ throw new Error('Bruh');
}
- this.project.framerate = this.state.fps;
- this.project.resolutionScale = this.state.scale;
+ if (endFrame >= this.state.duration) {
+ endFrame = Infinity;
+ }
- if (audio) {
- this.audioElement = new Audio(audio.src);
+ if (startFrame < 0) {
+ startFrame = 0;
}
- this.request();
- }
+ if (startFrame > endFrame) {
+ [startFrame, endFrame] = [endFrame, startFrame];
+ }
- public reload() {
- this.commands.recalculate = true;
- if (this.requestId === null) {
- this.request();
+ if (endFrame - startFrame < 1) {
+ if (startFrame !== 0) {
+ startFrame--;
+ } else {
+ endFrame++;
+ }
}
+
+ this.updateState({startFrame, endFrame});
}
- public reloadAudio(src: string, meta: Waveform) {
- this.audioElement.src = src;
- this.audio.src = src;
- // FIXME UI should be notified about this change
- this.audio.meta = meta;
+ public setSpeed(value: number) {
+ this.updateState({speed: value});
}
public setFramerate(fps: number) {
@@ -193,6 +197,7 @@ export class Player {
fps,
startFrame: Math.floor(this.state.startFrame * ratio),
endFrame: Math.floor(this.state.endFrame * ratio),
+ duration: Math.floor(this.state.duration * ratio),
});
this.requestSeek(Math.floor(this.frame * ratio));
this.reload();
@@ -216,22 +221,20 @@ export class Player {
this.commands.seek = this.clampRange(value, this.state);
}
+ public toggleLoop(value: boolean = !this.state.loop) {
+ this.updateState({loop: value});
+ }
+
public togglePlayback(value: boolean = this.state.paused): void {
- this.updateState({
- paused: !value,
- });
+ this.updateState({paused: !value});
}
public toggleRendering(value: boolean = !this.state.render): void {
- this.updateState({
- render: value,
- });
+ this.updateState({render: value});
}
public toggleAudio(value: boolean = this.state.muted): void {
- this.updateState({
- muted: !value,
- });
+ this.updateState({muted: !value});
}
private async run() {
@@ -266,32 +269,21 @@ export class Player {
}
// Pause / play audio.
- const audioPaused = state.paused || state.finished || state.render;
- if (this.audioElement && this.audioElement.paused !== audioPaused) {
- if (audioPaused) {
- this.audioElement.pause();
- } else {
- try {
- await this.audioElement.play();
- this.syncAudio(-3);
- this.audioError = false;
- } catch (e) {
- if (!this.audioError) {
- console.error(e);
- }
- this.audioError = true;
- }
- }
- }
- if (this.audioElement) {
- this.audioElement.muted = state.muted;
+ const audioPaused =
+ state.paused ||
+ state.finished ||
+ state.render ||
+ !this.audio.isInRange(this.project.time);
+ if (await this.audio.setPaused(audioPaused)) {
+ this.syncAudio(-3);
}
+ this.audio.setMuted(state.muted);
// Rendering
if (state.render) {
state.finished = await this.project.next();
this.project.draw();
- await this.renderChanged.dispatchAsync({
+ await this.frameRendered.dispatchAsync({
frame: this.project.frame,
data: await this.project.getBlob(),
});
@@ -320,19 +312,21 @@ export class Player {
else if (
state.paused ||
(state.speed === 1 &&
- this.hasAudio() &&
- this.audioTime() < this.project.time)
+ this.audio.isReady() &&
+ this.audio.isInRange(this.project.time) &&
+ this.audio.getTime() < this.project.time)
) {
this.request();
return;
}
// Seek to synchronize animation with audio.
else if (
- this.hasAudio() &&
+ this.audio.isReady() &&
state.speed === 1 &&
- this.project.time < this.audioTime() - MAX_AUDIO_DESYNC
+ this.audio.isInRange(this.project.time) &&
+ this.project.time < this.audio.getTime() - MAX_AUDIO_DESYNC
) {
- const seekFrame = this.project.secondsToFrames(this.audioTime());
+ const seekFrame = this.project.secondsToFrames(this.audio.getTime());
state.finished = await this.project.seek(seekFrame, state.speed);
}
// Simply move forward one frame
@@ -375,24 +369,12 @@ export class Player {
return frame >= state.startFrame && frame <= state.endFrame;
}
- private hasAudio(): boolean {
- return this.audioElement && !this.audioError;
- }
-
private syncAudio(frameOffset: number = 0) {
- if (!this.audioElement) return;
-
- this.audioElement.currentTime = Math.max(
- 0,
- this.project.framesToSeconds(this.project.frame + frameOffset) +
- this.audio.offset,
+ this.audio.setTime(
+ this.project.framesToSeconds(this.project.frame + frameOffset),
);
}
- private audioTime(): number {
- return (this.audioElement?.currentTime ?? 0) - this.audio.offset;
- }
-
private request() {
this.requestId = requestAnimationFrame(async time => {
if (time - this.renderTime >= 990 / this.state.fps) {
diff --git a/src/types/Waveform.ts b/src/types/Waveform.ts
deleted file mode 100644
index 1352807e..00000000
--- a/src/types/Waveform.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-export interface Waveform {
- version: number;
- channels: number;
- sample_rate: number;
- samples_per_pixel: number;
- bits: 8 | 16;
- length: number;
- data: number[];
-}
diff --git a/src/types/index.ts b/src/types/index.ts
index ca371c74..6aa4cfd5 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -1,4 +1,3 @@
export * from './Size';
export * from './Origin';
-export * from './Spacing';
-export * from './Waveform';
\ No newline at end of file
+export * from './Spacing';
\ No newline at end of file
diff --git a/tools/bin/audiowaveform.exe b/tools/bin/audiowaveform.exe
deleted file mode 100644
index 3a91390b..00000000
Binary files a/tools/bin/audiowaveform.exe and /dev/null differ
diff --git a/tools/loaders/wav-loader.js b/tools/loaders/wav-loader.js
deleted file mode 100644
index 5e56c6b6..00000000
--- a/tools/loaders/wav-loader.js
+++ /dev/null
@@ -1,27 +0,0 @@
-const {exec} = require('child_process');
-const {promises: fs} = require('fs');
-const loaderUtils = require('loader-utils');
-const path = require('path');
-
-function loader() {
- const source = this.resourcePath;
- const destination = source.slice(0, -4) + '.json';
-
- const callback = this.async();
- exec(
- `..\\bin\\audiowaveform.exe -i ${source} -o ${destination}`,
- {cwd: __dirname},
- (error, stdout) => {
- if (error) {
- callback(error);
- return;
- }
-
- fs.readFile(destination, 'utf8')
- .then(data => callback(null, `export default ${data};`))
- .catch(callback);
- },
- );
-}
-
-module.exports = loader;
diff --git a/tools/serve.mjs b/tools/serve.mjs
index 84280063..00627e91 100644
--- a/tools/serve.mjs
+++ b/tools/serve.mjs
@@ -72,6 +72,10 @@ const compiler = webpack({
test: /\.mp4/i,
type: 'asset',
},
+ {
+ test: /\.wav$/i,
+ type: 'asset',
+ },
{
test: /\.csv$/,
loader: 'csv-loader',
@@ -105,23 +109,6 @@ const compiler = webpack({
},
],
},
- {
- oneOf: [
- {
- test: /\.wav$/i,
- resourceQuery: /meta/,
- use: [
- {
- loader: 'wav-loader',
- },
- ],
- },
- {
- test: /\.wav$/i,
- type: 'asset',
- },
- ],
- },
],
},
resolveLoader: {