Visualize Your Ideas With Code
feat: use Web Audio API for waveform generation
aarthificial817e244bb218
diff --git a/src/Project.ts b/src/Project.tsindex a6d0dcba..02d6c3de 100644--- a/src/Project.ts+++ b/src/Project.ts@@ -171,9 +171,9 @@ export class Project extends Stage {this.frame = scene.lastFrame;} else {this.currentScene = scene;- await scene.reset();scene.firstFrame = this.frame;scene.transitionDuration = -1;+ await scene.reset();while (!scene.canTransitionOut()) {if (scene.transitionDuration < 0 && scene.isAfterTransitionIn()) {scene.transitionDuration = this.frame - scene.firstFrame;diff --git a/src/audio/AudioData.ts b/src/audio/AudioData.tsnew file mode 100644index 00000000..0507d1ce--- /dev/null+++ b/src/audio/AudioData.ts@@ -0,0 +1,19 @@+export interface AudioData {+ /**+ * An array of minimum and maximum waveform data points, interleaved.+ * Each value is in range of -1 to 1.+ */+ peaks: number[];+ /**+ * The amount of samples taken.+ */+ length: number;+ /**+ * The absolute biggest value from the peaks array.+ */+ absoluteMax: number;+ /**+ * Samples per seconds.+ */+ sampleRate: number;+}diff --git a/src/audio/AudioManager.ts b/src/audio/AudioManager.tsnew file mode 100644index 00000000..ef553226--- /dev/null+++ b/src/audio/AudioManager.ts@@ -0,0 +1,168 @@+import {AudioData} from './AudioData';+import {SimpleEventDispatcher} from 'strongly-typed-events';++export class AudioManager {+ public get DataChanged() {+ return this.dataChanged.asEvent();+ }++ public get OffsetChanged() {+ return this.dataChanged.asEvent();+ }++ private static readonly context = new AudioContext();+ private readonly audioElement: HTMLAudioElement = new Audio();+ private source: string = null;+ private offset = 0;+ private error = false;+ private data: AudioData = null;+ private dataChanged = new SimpleEventDispatcher<AudioData>();+ private offsetChanged = new SimpleEventDispatcher<number>();+ private abortController: AbortController = null;++ public getData() {+ return this.data;+ }++ public getTime() {+ return this.toAbsoluteTime(this.audioElement.currentTime);+ }++ public setTime(value: number) {+ this.audioElement.currentTime = this.toRelativeTime(value);+ }++ public setOffset(value: number) {+ this.offset = value;+ this.offsetChanged.dispatch(value);+ }++ public async setMuted(isMuted: boolean) {+ this.audioElement.muted = isMuted;+ }++ public setSource(src: string) {+ this.source = src;+ this.audioElement.src = src;+ this.abortController?.abort();+ this.abortController = new AbortController();+ this.loadData(this.abortController.signal).catch(e => {+ if (e.name !== 'AbortError') {+ console.error(e);+ }+ });+ }++ public isInRange(time: number) {+ return time >= this.offset && time < this.audioElement.duration;+ }++ public toRelativeTime(time: number) {+ return Math.max(0, time - this.offset);+ }++ public toAbsoluteTime(time: number) {+ return time + this.offset;+ }++ public isReady() {+ return this.source && !this.error;+ }++ /**+ * Pause/resume the audio.+ *+ * @param isPaused+ *+ * @return `true` if the audio successfully started playing.+ */+ public async setPaused(isPaused: boolean): Promise<boolean> {+ if (this.source && this.audioElement.paused !== isPaused) {+ if (isPaused) {+ this.audioElement.pause();+ } else {+ try {+ await this.audioElement.play();+ return true;+ this.error = false;+ } catch (e) {+ if (!this.error) {+ console.error(e);+ }+ this.error = true;+ }+ }+ }++ return false;+ }++ private async loadData(signal: AbortSignal) {+ this.data = null;+ this.dataChanged.dispatch(this.data);++ const response = await fetch(this.source, {signal});+ const rawBuffer = await response.arrayBuffer();+ if (signal.aborted) return;+ const audioBuffer = await this.decodeAudioData(rawBuffer);+ if (signal.aborted) return;++ const sampleSize = 256;+ const samples = ~~(audioBuffer.length / sampleSize);++ const peaks = [];+ let absoluteMax = 0;+ for (+ let channelId = 0;+ channelId < audioBuffer.numberOfChannels;+ channelId+++ ) {+ const channel = audioBuffer.getChannelData(channelId);+ for (let i = 0; i < samples; i++) {+ const start = ~~(i * sampleSize);+ const end = ~~(start + sampleSize);++ let min = channel[start];+ let max = min;++ for (let j = start; j < end; j++) {+ const value = channel[j];+ if (value > max) {+ max = value;+ }+ if (value < min) {+ min = value;+ }+ }++ if (channelId === 0 || max > peaks[i * 2]) {+ peaks[i * 2] = max;+ }+ if (channelId === 0 || min < peaks[i * 2 + 1]) {+ peaks[i * 2 + 1] = min;+ }++ if (max > absoluteMax) {+ absoluteMax = max;+ }+ if (Math.abs(min) > absoluteMax) {+ absoluteMax = Math.abs(min);+ }+ }+ }++ this.data = {+ peaks,+ absoluteMax,+ length: samples,+ sampleRate: (audioBuffer.sampleRate / sampleSize) * 2,+ };+ this.dataChanged.dispatch(this.data);+ }++ private decodeAudioData(buffer: ArrayBuffer): Promise<AudioBuffer> {+ return new Promise<AudioBuffer>((resolve, reject) =>+ AudioManager.context.decodeAudioData(buffer, resolve, reject),+ );+ }+}diff --git a/src/bootstrap.ts b/src/bootstrap.tsindex 9c570026..e442e116 100644--- a/src/bootstrap.ts+++ b/src/bootstrap.ts@@ -1,19 +1,17 @@-import type {Size, Waveform} from './types';+import type {Size} from './types';import type {SceneRunner} from './Scene';import {Project, ProjectSize} from './Project';import {Player} from './player/Player';import {hot} from './hot';+import {AudioManager} from './audio/AudioManager';interface BootstrapConfig {name: string;+ scenes: SceneRunner[];size?: Size;background?: string | false;- scenes: SceneRunner[];- audio?: {- meta: Waveform;- src: string;- offset?: number;- };+ audio?: string;+ audioOffset?: number;}export function bootstrap(config: BootstrapConfig) {@@ -23,13 +21,14 @@ export function bootstrap(config: BootstrapConfig) {background: config.background ?? '#141414',...(config.size ?? ProjectSize.FullHD),});- const player = new Player(- project,- config.audio && {- offset: 0,- ...config.audio,- },- );+ const audio = new AudioManager();+ if (config.audio) {+ audio.setSource(config.audio);+ }+ if (config.audioOffset) {+ audio.setOffset(config.audioOffset);+ }+ const player = new Player(project, audio);(<any>window).player = player;//@ts-ignorediff --git a/src/hot.ts b/src/hot.tsindex 2cc6bfaa..e5bd1c29 100644--- a/src/hot.ts+++ b/src/hot.ts@@ -1,12 +1,12 @@-import {Player} from './player/Player';+import type {Player} from './player/Player';export function hot(player: Player, root: NodeModule) {- const update = async (modules: string[]) => {+ const updateScenes = async (modules: string[]) => {const runners = [];for (const module of modules) {const runner = __webpack_require__(module).default;if (- // FIXME Find out a better way to detect runner factories.+ // FIXME Find a better way to detect runner factories.runner.name === '__WEBPACK_DEFAULT_EXPORT__' &&typeof runner === 'function') {@@ -19,20 +19,19 @@ export function hot(player: Player, root: NodeModule) {player.project.reload(runners);player.reload();};+const updateAudio = async (modules: string[]) => {let src = null;- let meta = null;for (const module of modules) {const audio = __webpack_require__(module);- if (typeof audio === 'object') {- meta = audio.default;- } else {+ if (typeof audio === 'string') {src = audio;+ break;}}- if (src && meta) {- player.reloadAudio(src, meta);+ if (src) {+ player.audio.setSource(src);}};@@ -44,8 +43,8 @@ export function hot(player: Player, root: NodeModule) {(name: string) => name.match(/\.wav/),);- root.hot.accept(scenePaths, update);- module.hot.accept(scenePaths, update);+ root.hot.accept(scenePaths, updateScenes);+ module.hot.accept(scenePaths, updateScenes);root.hot.accept(audioPaths, updateAudio);module.hot.accept(audioPaths, updateAudio);}diff --git a/src/player/Player.ts b/src/player/Player.tsindex 3ea4390f..cf5517a7 100644--- a/src/player/Player.ts+++ b/src/player/Player.ts@@ -1,5 +1,5 @@import type {Project} from '../Project';-import type {Waveform} from '../types';+import {AudioManager} from '../audio/AudioManager';import {PromiseSimpleEventDispatcher,SimpleEventDispatcher,@@ -41,8 +41,6 @@ export interface PlayerRenderEvent {data: Blob;}-const STORAGE_KEY = 'player-state';-export class Player {public get StateChanged() {return this.stateChanged.asEvent();@@ -52,8 +50,8 @@ export class Player {return this.timeChanged.asEvent();}- public get RenderChanged() {- return this.renderChanged.asEvent();+ public get FrameRendered() {+ return this.frameRendered.asEvent();}public get Reloaded() {@@ -62,29 +60,20 @@ export class Player {private readonly stateChanged = new SimpleEventDispatcher<PlayerState>();private readonly timeChanged = new SimpleEventDispatcher<PlayerTime>();- private readonly renderChanged =+ private readonly frameRendered =new PromiseSimpleEventDispatcher<PlayerRenderEvent>();private readonly reloaded = new SimpleEventDispatcher<number>();- private readonly audioElement: HTMLAudioElement = null;private startTime: number;private renderTime: number = 0;private requestId: number = null;- private audioError = false;-- public getState(): PlayerState {- return {...this.state};- }+ private frame: number = 0;- public getTime(): PlayerTime {- return {- frame: this.frame,- time: this.project.framesToSeconds(this.frame),- duration: this.state.duration,- durationTime: this.project.framesToSeconds(this.state.duration),- completion: this.frame / this.state.duration,- };- }+ private commands: PlayerCommands = {+ reset: true,+ seek: -1,+ recalculate: true,+ };private state: PlayerState = {duration: Infinity,@@ -101,13 +90,30 @@ export class Player {scale: 1,};- private frame: number = 0;+ public constructor(+ public readonly project: Project,+ public readonly audio: AudioManager,+ ) {+ this.startTime = performance.now();+ this.project.framerate = this.state.fps;+ this.project.resolutionScale = this.state.scale;- private commands: PlayerCommands = {- reset: true,- seek: -1,- recalculate: true,- };+ this.request();+ }++ public getState(): PlayerState {+ return {...this.state};+ }++ public getTime(): PlayerTime {+ return {+ frame: this.frame,+ time: this.project.framesToSeconds(this.frame),+ duration: this.state.duration,+ durationTime: this.project.framesToSeconds(this.state.duration),+ completion: this.frame / this.state.duration,+ };+ }public updateState(newState: Partial<PlayerState>) {let changed = false;@@ -125,7 +131,6 @@ export class Player {...newState,};this.stateChanged.dispatch(this.state);- localStorage.setItem(STORAGE_KEY, JSON.stringify(this.state));}}@@ -143,47 +148,46 @@ export class Player {return commands;}- public constructor(- public readonly project: Project,- public readonly audio?: {- src: string;- meta: Waveform;- offset: number;- },- ) {- this.startTime = performance.now();+ public reload() {+ this.commands.recalculate = true;+ if (this.requestId === null) {+ this.request();+ }+ }- const savedState = localStorage.getItem(STORAGE_KEY);- if (savedState) {- const state = JSON.parse(savedState) as PlayerState;- for (const key in this.state) {- //@ts-ignore- this.state[key] = state[key] ?? this.state[key];- }+ public setRange(+ startFrame = this.state.startFrame,+ endFrame = this.state.endFrame,+ ) {+ if (startFrame === undefined) {+ throw new Error('Bruh');}- this.project.framerate = this.state.fps;- this.project.resolutionScale = this.state.scale;+ if (endFrame >= this.state.duration) {+ endFrame = Infinity;+ }- if (audio) {- this.audioElement = new Audio(audio.src);+ if (startFrame < 0) {+ startFrame = 0;}- this.request();- }+ if (startFrame > endFrame) {+ [startFrame, endFrame] = [endFrame, startFrame];+ }- public reload() {- this.commands.recalculate = true;- if (this.requestId === null) {- this.request();+ if (endFrame - startFrame < 1) {+ if (startFrame !== 0) {+ startFrame--;+ } else {+ endFrame++;+ }}++ this.updateState({startFrame, endFrame});}- public reloadAudio(src: string, meta: Waveform) {- this.audioElement.src = src;- this.audio.src = src;- // FIXME UI should be notified about this change- this.audio.meta = meta;+ public setSpeed(value: number) {+ this.updateState({speed: value});}public setFramerate(fps: number) {@@ -193,6 +197,7 @@ export class Player {fps,startFrame: Math.floor(this.state.startFrame * ratio),endFrame: Math.floor(this.state.endFrame * ratio),+ duration: Math.floor(this.state.duration * ratio),});this.requestSeek(Math.floor(this.frame * ratio));this.reload();@@ -216,22 +221,20 @@ export class Player {this.commands.seek = this.clampRange(value, this.state);}+ public toggleLoop(value: boolean = !this.state.loop) {+ this.updateState({loop: value});+ }+public togglePlayback(value: boolean = this.state.paused): void {- this.updateState({- paused: !value,- });+ this.updateState({paused: !value});}public toggleRendering(value: boolean = !this.state.render): void {- this.updateState({- render: value,- });+ this.updateState({render: value});}public toggleAudio(value: boolean = this.state.muted): void {- this.updateState({- muted: !value,- });+ this.updateState({muted: !value});}private async run() {@@ -266,32 +269,21 @@ export class Player {}// Pause / play audio.- const audioPaused = state.paused || state.finished || state.render;- if (this.audioElement && this.audioElement.paused !== audioPaused) {- if (audioPaused) {- this.audioElement.pause();- } else {- try {- await this.audioElement.play();- this.syncAudio(-3);- this.audioError = false;- } catch (e) {- if (!this.audioError) {- console.error(e);- }- this.audioError = true;- }- }- }- if (this.audioElement) {- this.audioElement.muted = state.muted;+ const audioPaused =+ state.paused ||+ state.finished ||+ state.render ||+ !this.audio.isInRange(this.project.time);+ if (await this.audio.setPaused(audioPaused)) {+ this.syncAudio(-3);}+ this.audio.setMuted(state.muted);// Renderingif (state.render) {state.finished = await this.project.next();this.project.draw();- await this.renderChanged.dispatchAsync({+ await this.frameRendered.dispatchAsync({frame: this.project.frame,data: await this.project.getBlob(),});@@ -320,19 +312,21 @@ export class Player {else if (state.paused ||(state.speed === 1 &&- this.hasAudio() &&- this.audioTime() < this.project.time)+ this.audio.isReady() &&+ this.audio.isInRange(this.project.time) &&+ this.audio.getTime() < this.project.time)) {this.request();return;}// Seek to synchronize animation with audio.else if (- this.hasAudio() &&+ this.audio.isReady() &&state.speed === 1 &&- this.project.time < this.audioTime() - MAX_AUDIO_DESYNC+ this.audio.isInRange(this.project.time) &&+ this.project.time < this.audio.getTime() - MAX_AUDIO_DESYNC) {- const seekFrame = this.project.secondsToFrames(this.audioTime());+ const seekFrame = this.project.secondsToFrames(this.audio.getTime());state.finished = await this.project.seek(seekFrame, state.speed);}// Simply move forward one frame@@ -375,24 +369,12 @@ export class Player {return frame >= state.startFrame && frame <= state.endFrame;}- private hasAudio(): boolean {- return this.audioElement && !this.audioError;- }-private syncAudio(frameOffset: number = 0) {- if (!this.audioElement) return;-- this.audioElement.currentTime = Math.max(- 0,- this.project.framesToSeconds(this.project.frame + frameOffset) +- this.audio.offset,+ this.audio.setTime(+ this.project.framesToSeconds(this.project.frame + frameOffset),);}- private audioTime(): number {- return (this.audioElement?.currentTime ?? 0) - this.audio.offset;- }-private request() {this.requestId = requestAnimationFrame(async time => {if (time - this.renderTime >= 990 / this.state.fps) {diff --git a/src/types/Waveform.ts b/src/types/Waveform.tsdeleted file mode 100644index 1352807e..00000000--- a/src/types/Waveform.ts+++ /dev/null@@ -1,9 +0,0 @@-export interface Waveform {- version: number;- channels: number;- sample_rate: number;- samples_per_pixel: number;- bits: 8 | 16;- length: number;- data: number[];-}diff --git a/src/types/index.ts b/src/types/index.tsindex ca371c74..6aa4cfd5 100644--- a/src/types/index.ts+++ b/src/types/index.ts@@ -1,4 +1,3 @@export * from './Size';export * from './Origin';-export * from './Spacing';-export * from './Waveform';\ No newline at end of file+export * from './Spacing';\ No newline at end of filediff --git a/tools/bin/audiowaveform.exe b/tools/bin/audiowaveform.exedeleted file mode 100644index 3a91390b..00000000Binary files a/tools/bin/audiowaveform.exe and /dev/null differdiff --git a/tools/loaders/wav-loader.js b/tools/loaders/wav-loader.jsdeleted file mode 100644index 5e56c6b6..00000000--- a/tools/loaders/wav-loader.js+++ /dev/null@@ -1,27 +0,0 @@-const {exec} = require('child_process');-const {promises: fs} = require('fs');-const loaderUtils = require('loader-utils');-const path = require('path');--function loader() {- const source = this.resourcePath;- const destination = source.slice(0, -4) + '.json';-- const callback = this.async();- exec(- `..\\bin\\audiowaveform.exe -i ${source} -o ${destination}`,- {cwd: __dirname},- (error, stdout) => {- if (error) {- callback(error);- return;- }-- fs.readFile(destination, 'utf8')- .then(data => callback(null, `export default ${data};`))- .catch(callback);- },- );-}--module.exports = loader;diff --git a/tools/serve.mjs b/tools/serve.mjsindex 84280063..00627e91 100644--- a/tools/serve.mjs+++ b/tools/serve.mjs@@ -72,6 +72,10 @@ const compiler = webpack({test: /\.mp4/i,type: 'asset',},+ {+ test: /\.wav$/i,+ type: 'asset',+ },{test: /\.csv$/,loader: 'csv-loader',@@ -105,23 +109,6 @@ const compiler = webpack({},],},- {- oneOf: [- {- test: /\.wav$/i,- resourceQuery: /meta/,- use: [- {- loader: 'wav-loader',- },- ],- },- {- test: /\.wav$/i,- type: 'asset',- },- ],- },],},resolveLoader: {