the ultimate vocal synthesizer - sing, speak, beatbox, and more
Rust
/** Main-thread speech planning: text/IPA → phrases → gestural plan → worklet. */
import type { BodyMix } from "../wasm/types.ts";
import { EnglishG2P, MandarinLexicon, type Phrase } from "../wasm/engine.ts";
import { tokenizeIpa } from "../wasm/engine.ts";
import { planPhrases, type Plan, type Segment } from "../wasm/engine.ts";
import { engine } from "../audio/AudioEngine.ts";
import type { Gesture } from "../wasm/types.ts";
import { ipaPhraseTexts } from "./skipped.ts";
import { wordSpans, type WordSpan } from "./words.ts";
export const g2p = new EnglishG2P();
/** the Mandarin front end's lexicon, loaded when lyrics first have Han characters */
export const zh = new MandarinLexicon();
export interface SpokenPlan {
plan: Plan;
/** worklet-clock offset of the plan's time 0 (set when the worklet confirms) */
offset: number | null;
id: number;
transcription: string;
/** each word's span in the plan */
words: WordSpan[];
}
let current: SpokenPlan | null = null;
/** the plan a stop cut short: its words are no longer being said */
let cutId = -1;
const listeners = new Set<(p: SpokenPlan | null) => void>();
engine.onPlanned((m) => {
if (current && m.id === current.id) {
current = { ...current, offset: m.t0 };
for (const l of listeners) l(current);
}
});
export function onPlan(l: (p: SpokenPlan | null) => void): () => void {
listeners.add(l);
return () => listeners.delete(l);
}
export function currentPlan(): SpokenPlan | null {
return current;
}
export function ipaToPhrases(text: string): Phrase[] {
return ipaPhraseTexts(text).map((p) => ({ segments: tokenizeIpa(p), intonation: "statement" as const, pause: 0.3, words: [] }));
}
export function transcribe(phrases: { segments: Segment[]; intonation?: string }[]): string {
return phrases.map((p) => p.segments.map((s) => (s.stress === 1 ? "ˈ" : s.stress === 2 ? "ˌ" : "") + s.ipa + (s.wordEnd ? " " : "")).join("").trim() + (p.intonation === "question" ? "?" : "")).join(" | ");
}
export function speakPhrases(phrases: { segments: Segment[]; intonation?: "statement" | "question" | "flat"; pause?: number; words?: string[] }[], o: { body: BodyMix; rate: number; f0: number }): SpokenPlan {
const plan = planPhrases(phrases, { body: o.body, t0: 0, rate: o.rate, f0: o.f0 });
const id = engine.schedule({ gestures: plan.gestures.map((g: Gesture) => ({ ...g, targets: { ...g.targets } })), segments: plan.segments, end: plan.end });
current = { plan, offset: null, id, transcription: transcribe(phrases), words: wordSpans(phrases, plan) };
for (const l of listeners) l(current);
return current;
}
export async function speakText(text: string, o: { ipa: boolean; body: BodyMix; rate: number; f0: number }): Promise<SpokenPlan | null> {
if (!text.trim()) return null;
let phrases: Phrase[];
if (o.ipa) phrases = ipaToPhrases(text);
else {
try {
await g2p.load();
} catch {
// dictionary unavailable: the letter-to-sound rules still work
}
phrases = g2p.toPhrases(text);
}
if (!phrases.length) return null;
return speakPhrases(phrases, o);
}
export function stopSpeaking(): void {
engine.stopSpeech();
if (current) cutId = current.id;
}
/** Whether an utterance is sounding: scheduled (or waiting for the worklet) and not over or stopped. */
export function speakingNow(): boolean {
const p = current;
if (!p || p.id === cutId) return false;
const t = engine.voiceClock();
if (p.offset === null || t === null) return true;
return t - p.offset < p.plan.end;
}
/** The word being spoken now (null between words and once the speech is over or stopped). */
export function spokenWordNow(): { key: string; text: string } | null {
const p = current;
const t = engine.voiceClock();
if (!p || p.offset === null || t === null || p.id === cutId) return null;
const pt = t - p.offset;
const i = p.words.findIndex((w) => pt >= w.t0 && pt < w.t1 + 0.05);
return i < 0 ? null : { key: `${p.id}:${i}`, text: p.words[i].text };
}