the ultimate vocal synthesizer - sing, speak, beatbox, and more
Rust
/**
* Main-thread side of the audio engine: AudioContext + worklet node + analyser, coalesced
* parameter messages (at most one per animation frame per kind) and telemetry fan-out.
*/
import workletUrl from "../worklet/processor.ts?worker&url";
import { chimeraModule } from "../wasm/chimera.ts";
import { preferPlaybackSession } from "./session.ts";
import type { ToWorklet, FromWorklet, TelemetryMessage, PlannedMessage, CreatureTelemetryMessage } from "../worklet/protocol.ts";
import type { BodyMix } from "../wasm/types.ts";
import type { HumanArtic } from "../wasm/types.ts";
import type { VoiceSource } from "../wasm/types.ts";
type Listener = (t: TelemetryMessage) => void;
// 48 kHz whatever the device runs at (the browser resamples): the airway's cost grows with the
// square of its rate, and a 192 kHz device took ~3x the CPU of 48 kHz and starved the worklet
// (load 120-150%, no sound; issue #33). 32 kHz is the fallback for a device too slow for 48: its
// airway runs at 64 kHz instead of 96 (about 40 % less CPU), a little duller (output up to 16 kHz).
const FULL_RATE = 48000;
const LIGHT_RATE = 32000;
/** the averaged load (see watchLoad) past which the device can't keep up, and back under which it can */
const OVERLOADED = 0.9;
const KEEPING_UP = 0.6;
const AVERAGE_S = 2;
/** a reading counts as no more than this: a stall (a hidden tab, a laptop asleep, a busy moment)
* mustn't alone make a fast device look slow, so it takes ~1 s of overload at least to fall back */
const MOST = 1.5;
/** how long a start or rebuild waits for the context to run */
const RESUME_WAIT_MS = 3000;
/** the first moments' load is the wasm compiling, not the device's speed */
const WARM_UP_MS = 2000;
export class AudioEngine {
ctx: AudioContext | null = null;
private node: AudioWorkletNode | null = null;
analyser: AnalyserNode | null = null;
/** unsmoothed, short-window analyser for the spectrogram */
analyserFast: AnalyserNode | null = null;
/** long time-domain window for the waveform scope */
analyserScope: AnalyserNode | null = null;
private master: GainNode | null = null;
/** "close mic" chain: proximity bass, presence, compression (beatboxers cup the mic) */
private micIn: AudioNode | null = null;
private micOn = false;
latest: TelemetryMessage | null = null;
/** performance.now() when `latest` arrived */
private latestAt = 0;
latestCreature: CreatureTelemetryMessage | null = null;
/** load of the most recent telemetry of either kind */
load = 0;
sampleRate = FULL_RATE;
/** the device couldn't keep up at the full rate: the voice runs at the lighter one */
light = false;
/** the device can't keep up even so: the voice breaks up or goes silent */
overloaded = false;
private slowLoad = 0;
private loadAt = 0;
private builtAt = 0;
private rebuilding = false;
private triedLighter = false;
private statusListeners = new Set<() => void>();
private rebuiltListeners = new Set<() => void>();
private listeners = new Set<Listener>();
private pendingArtic: Partial<HumanArtic> | null = null;
private pendingTipFree: boolean | undefined;
private pendingSource: Partial<VoiceSource> | null = null;
private pendingCParams: Record<string, number> | null = null;
private flushScheduled = false;
private readyListeners = new Set<() => void>();
private plannedListeners = new Set<(p: PlannedMessage) => void>();
/** most recent planned utterance (for timelines) */
lastPlan: PlannedMessage | null = null;
private speakId = 0;
error: string | null = null;
/** The human voice's clock now (s), extrapolated from the latest telemetry; null if none. */
voiceClock(): number | null {
if (!this.latest) return null;
return this.latest.t.clock + Math.min(0.25, (performance.now() - this.latestAt) / 1000);
}
private tap: MediaStreamAudioDestinationNode | null = null;
/** What the speakers play, as a media stream (to record a clip); null before start(). */
recordStream(): MediaStream | null {
if (!this.ctx || !this.master) return null;
if (!this.tap) {
this.tap = this.ctx.createMediaStreamDestination();
this.master.connect(this.tap);
}
return this.tap.stream;
}
get running(): boolean {
return !!this.ctx && this.ctx.state === "running";
}
async start(): Promise<void> {
if (this.ctx) {
await this.ctx.resume();
return;
}
await this.build(FULL_RATE);
}
/**
* The context, the worklet and the graph, at this rate; they replace the current ones only once
* built (a failure leaves things as they were).
*/
private async build(rate: number): Promise<void> {
preferPlaybackSession();
const ctx = new AudioContext({ latencyHint: "interactive", sampleRate: rate });
// resume now, while the tap that called start() is still the current gesture: iOS Safari
// refuses a resume() that comes after the awaits below (the context stays suspended, silent)
const resumed = ctx.resume().catch(() => {});
let node: AudioWorkletNode;
try {
await ctx.audioWorklet.addModule(workletUrl);
// the worklet instantiates the engine from the module compiled at startup (main.tsx)
node = new AudioWorkletNode(ctx, "chimera", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [2], processorOptions: { module: chimeraModule() } });
} catch (e) {
// without its worklet the context is empty: forget it, so a retry builds the graph again
// instead of resuming a context with nothing in it (and reporting that the audio started)
void ctx.close().catch(() => {});
throw e;
}
const old = this.ctx;
// the old engine's last telemetry mustn't read as the new one's
if (this.node) this.node.port.onmessage = null;
this.ctx = ctx;
this.sampleRate = ctx.sampleRate;
this.node = node;
this.tap = null;
this.latest = null;
this.latestCreature = null;
this.lastPlan = null;
this.builtAt = this.loadAt = performance.now();
this.slowLoad = 0;
if (old) void old.close().catch(() => {});
const master = ctx.createGain();
master.gain.value = 0.8;
this.master = master;
const analyser = ctx.createAnalyser();
analyser.fftSize = 4096;
analyser.smoothingTimeConstant = 0.6;
this.analyser = analyser;
const fast = ctx.createAnalyser();
fast.fftSize = 2048;
fast.smoothingTimeConstant = 0;
this.analyserFast = fast;
const scope = ctx.createAnalyser();
scope.fftSize = 16384;
this.analyserScope = scope;
// close-mic chain: proximity effect (low shelf), presence peak, fast compressor, make-up
const shelf = ctx.createBiquadFilter();
shelf.type = "lowshelf";
shelf.frequency.value = 120;
shelf.gain.value = 9;
const presence = ctx.createBiquadFilter();
presence.type = "peaking";
presence.frequency.value = 3500;
presence.Q.value = 0.8;
presence.gain.value = 3;
const comp = ctx.createDynamicsCompressor();
comp.threshold.value = -28;
comp.knee.value = 8;
comp.ratio.value = 5;
comp.attack.value = 0.002;
comp.release.value = 0.12;
const makeup = ctx.createGain();
makeup.gain.value = 2.2;
shelf.connect(presence).connect(comp).connect(makeup).connect(master);
this.micIn = shelf;
node.connect(this.micOn ? shelf : master);
master.connect(ctx.destination);
master.connect(analyser);
master.connect(fast);
master.connect(scope);
node.port.onmessage = (e: MessageEvent<FromWorklet>) => {
const m = e.data;
if (m.type === "telemetry") {
this.latest = m;
this.latestAt = performance.now();
this.latestCreature = null;
this.watchLoad(m.load);
for (const l of this.listeners) l(m);
} else if (m.type === "ctelemetry") {
this.latestCreature = m;
this.latest = null;
this.watchLoad(m.load);
} else if (m.type === "planned") {
this.lastPlan = m;
for (const l of this.plannedListeners) l(m);
} else if (m.type === "ready") {
for (const l of this.readyListeners) l();
} else if (m.type === "error") {
this.error = m.message;
console.error("worklet:", m.message);
}
};
// a rebuild comes from no gesture, and a browser may hold the new context until there is one
// (its resume() then doesn't settle until then): the next tap or key resumes it, and nothing
// waits on it for long, so the rebuild still finishes (the state is sent, the old song stops)
const resume = () => void ctx.resume().catch(() => {});
for (const ev of ["pointerdown", "keydown"]) window.addEventListener(ev, resume, { once: true, capture: true });
// a second chance where the first resume was refused (the result is what callers see)
await Promise.race([resumed.then(() => ctx.resume()), new Promise((r) => setTimeout(r, RESUME_WAIT_MS))]);
}
/**
* Whether the audio thread keeps up, from the worklet's load (render time over the block's
* time). Averaged over ~2 s, past the start's warm-up: over 90 %, the device can't keep up,
* and the voice breaks up or goes silent. At 48 kHz it then rebuilds at 32 kHz, once; at 32 kHz
* it says so. A fast device never comes near (about 30-40 %), so its sound doesn't change.
*/
private watchLoad(load: number): void {
this.load = load;
const t = performance.now();
// a gap between readings counts as no more than a moment
const dt = Math.min(0.1, (t - this.loadAt) / 1000);
this.loadAt = t;
if (t - this.builtAt < WARM_UP_MS || this.rebuilding) return;
this.slowLoad += ((Math.min(MOST, load) - this.slowLoad) * dt) / AVERAGE_S;
if (this.slowLoad > OVERLOADED && this.sampleRate > LIGHT_RATE && !this.triedLighter) void this.rebuildLighter();
else if (this.slowLoad > OVERLOADED) this.setOverloaded(true);
else if (this.slowLoad < KEEPING_UP) this.setOverloaded(false);
}
private setOverloaded(on: boolean): void {
if (on === this.overloaded) return;
this.overloaded = on;
for (const l of this.statusListeners) l();
}
/** The lighter rate for a device that can't keep up: ~40 % less work, output up to 16 kHz. */
private async rebuildLighter(): Promise<void> {
this.rebuilding = this.triedLighter = true;
const load = this.load;
try {
await this.build(LIGHT_RATE);
} catch (e) {
// the full-rate voice goes on (and says it can't keep up)
console.error("audio: couldn't rebuild at the lighter rate:", e);
return;
} finally {
this.rebuilding = false;
}
console.warn(`audio: this device can't keep up at ${FULL_RATE} Hz (engine load ${Math.round(load * 100)} %): running at ${this.sampleRate} Hz`);
this.light = true;
for (const l of this.statusListeners) l();
// the new engine starts from scratch: what it plays is sent again
for (const l of this.rebuiltListeners) l();
}
/** Called when `light` or `overloaded` changes. */
onStatus(l: () => void): () => void {
this.statusListeners.add(l);
return () => this.statusListeners.delete(l);
}
/** Called after the audio was rebuilt lighter, with a fresh engine that knows nothing yet. */
onRebuilt(l: () => void): () => void {
this.rebuiltListeners.add(l);
return () => this.rebuiltListeners.delete(l);
}
onTelemetry(l: Listener): () => void {
this.listeners.add(l);
return () => this.listeners.delete(l);
}
onPlanned(l: (p: PlannedMessage) => void): () => void {
this.plannedListeners.add(l);
return () => this.plannedListeners.delete(l);
}
/** Send a gestural plan (times relative to now). Returns the request id. */
schedule(plan: { gestures: import("../wasm/types.ts").Gesture[]; segments: import("../wasm/types.ts").PlannedSegment[]; end: number }, replace = true): number {
const id = ++this.speakId;
this.send({ type: "schedule", gestures: plan.gestures, segments: plan.segments, end: plan.end, id, replace });
return id;
}
/** Switch voice ("human" or a creature id), starting the creature from the given settings. */
setCreature(id: string, params?: Record<string, number>): void {
// settings still waiting for the next frame belong to the previous creature
this.pendingCParams = null;
this.send({ type: "creature", id, params });
}
setCreatureParams(values: Record<string, number>): void {
this.pendingCParams = { ...(this.pendingCParams ?? {}), ...values };
this.scheduleFlush();
}
playCall(id: string, pitch?: number, amp?: number): void {
this.send({ type: "call", id, pitch, amp });
}
stopCall(): void {
this.send({ type: "stopCall" });
}
stopSpeech(): void {
this.send({ type: "stopSpeech" });
}
/** The output's RMS over the fast analyser's window (~40 ms); 0 before start(). */
level(): number {
const a = this.analyserFast;
if (!a) return 0;
const buf = new Float32Array(a.fftSize);
a.getFloatTimeDomainData(buf);
let s = 0;
for (const x of buf) s += x * x;
return Math.sqrt(s / buf.length);
}
/**
* Resolves once the output has stayed below `floor` for `hold` s, or after `max` s: a clip
* mustn't open on the tail of what was sounding (a stopped song keeps ringing for a moment).
*/
async quiet(floor = 0.003, hold = 0.15, max = 2): Promise<void> {
const t0 = performance.now();
let since = t0;
while (performance.now() - t0 < max * 1000) {
const t = performance.now();
if (this.level() >= floor) since = t;
else if (t - since >= hold * 1000) return;
await new Promise((r) => setTimeout(r, 20));
}
}
onReady(l: () => void): () => void {
this.readyListeners.add(l);
return () => this.readyListeners.delete(l);
}
private send(m: ToWorklet): void {
this.node?.port.postMessage(m);
}
/** Route the voice through the close-mic chain (on) or straight to the output (off). */
setMic(on: boolean): void {
if (on === this.micOn) return;
this.micOn = on;
if (!this.node || !this.master || !this.micIn) return;
this.node.disconnect();
this.node.connect(on ? this.micIn : this.master);
}
setBeat(m: { steps?: import("../wasm/types.ts").BeatStep[] | (import("../wasm/types.ts").BeatStep | null)[]; bpm?: number; swing?: number; playing?: boolean }): void {
this.send({ type: "beat", ...m });
}
/** End the sung note (open-ended gestures) with this tag. */
release(tag: number): void {
this.send({ type: "release", tag });
}
/** Gestures planned on the main thread, times relative to arrival + lead (live playing). */
scheduleGestures(gestures: import("../wasm/types.ts").Gesture[], lead = 0.006): void {
this.send({ type: "schedule", gestures, segments: [], end: 0, id: ++this.speakId, replace: false, lead });
}
beatHit(id: string, note?: number, accent?: number): void {
this.send({ type: "beatHit", id, note, accent });
}
setVolume(v: number): void {
if (this.master && this.ctx) this.master.gain.setTargetAtTime(v, this.ctx.currentTime, 0.02);
}
setBody(body: BodyMix): void {
this.send({ type: "body", body });
}
gate(level: number): void {
this.send({ type: "gate", level });
}
/** Coalesced: many calls within a frame become one message. */
setArtic(values: Partial<HumanArtic>, tipFree?: boolean): void {
this.pendingArtic = { ...(this.pendingArtic ?? {}), ...values };
if (tipFree !== undefined) this.pendingTipFree = tipFree;
this.scheduleFlush();
}
setSource(values: Partial<VoiceSource>): void {
this.pendingSource = { ...(this.pendingSource ?? {}), ...values };
this.scheduleFlush();
}
private scheduleFlush(): void {
if (this.flushScheduled) return;
this.flushScheduled = true;
const flush = () => {
this.flushScheduled = false;
if (this.pendingArtic || this.pendingTipFree !== undefined) {
this.send({ type: "artic", values: this.pendingArtic ?? {}, tipFree: this.pendingTipFree });
this.pendingArtic = null;
this.pendingTipFree = undefined;
}
if (this.pendingSource) {
this.send({ type: "source", values: this.pendingSource });
this.pendingSource = null;
}
if (this.pendingCParams) {
this.send({ type: "cparams", values: this.pendingCParams });
this.pendingCParams = null;
}
};
if (typeof requestAnimationFrame === "function") requestAnimationFrame(flush);
else setTimeout(flush, 16);
}
}
export const engine = new AudioEngine();