Skip to content

the ultimate vocal synthesizer - sing, speak, beatbox, and more

Rust
8.7 KB
Raw Blame
/** Engine types as the UI sees them (the JSON shapes of the Rust engine's serde types). */
export type ArticKey = "jaw" | "tbX" | "tbY" | "ttX" | "ttY" | "ttCurl" | "root" | "lipAp" | "lipProt" | "lipWidth" | "labiodental" | "velum" | "larynx" | "epilarynx" | "ventricular" | "lateral" | "dorsum";
export type SourceKey = "abduction" | "closure" | "breath" | "f0" | "vibrato";
export type ControlKey = ArticKey | SourceKey;

export interface Gesture {
  t0: number;
  t1: number;
  rampIn: number;
  rampOut: number;
  strength: number;
  targets: Partial<Record<ControlKey, number>>;
  /** articulator speed multiplier while active and during `releaseHold` seconds after t1 */
  speed?: number;
  releaseHold?: number;
  /** label for visualisation (phoneme / gesture name) */
  label?: string;
  /** open-ended gestures (a sung note held until its key is released) end via release(tag) */
  tag?: number;
}

export interface Vec {
  x: number;
  y: number;
}

export interface HumanArtic {
  jaw: number;
  tbX: number;
  tbY: number;
  ttX: number;
  ttY: number;
  ttCurl: number;
  root: number;
  lipAp: number;
  lipProt: number;
  lipWidth: number;
  labiodental: number;
  velum: number;
  larynx: number;
  epilarynx: number;
  ventricular: number;
  lateral: number;
  dorsum: number;
}

/**
 * A body between the three calibrated ones (mirrors the engine's BodyMix): `female` from the adult
 * male (0) to the adult female (1), `child` from adult (0) to child (1).
 */
export interface BodyMix {
  female: number;
  child: number;
}

export interface HumanBody {
  name: string;
  /** overall size scale (1 = adult male) */
  scale: number;
  /** extra pharynx length scale (female/child have proportionally shorter pharynx) */
  pharynx: number;
  /** tongue body radius (cm, before scale) */
  tongueRadius: number;
  /** max lip width (cm, before scale) */
  lipWidth: number;
  /** length of the membranous vocal fold, the part that vibrates (1 = adult male) */
  foldLength: number;
  /** how much more of the cycle the folds spend apart than an adult man's (0.13 = 13 points of open quotient) */
  openPhase: number;
}

export interface ArticSpec {
  min: number;
  max: number;
  /** natural frequency (Hz) of the critically damped tracking dynamics */
  hz: number;
}

export interface GridSample {
  inner: Vec;
  outer: Vec;
  mid: Vec;
  d: number;
  area: number;
  s: number;
  region: 0 | 1 | 2 | 3;
  ray: number;
}

export interface HumanFrame {
  outer: Vec[];
  inner: Vec[];
  tongue: Vec[];
  velum: Vec[];
  upperLip: Vec[];
  lowerLip: Vec[];
  upperTeeth: Vec[];
  lowerTeeth: Vec[];
  jawBone: Vec[];
  epiglottis: Vec[];
  nasalCavity: Vec[];
  glottis: Vec;
  grid: GridSample[];
  length: number;
  pharynxLength: number;
  teethS: number;
  lipS: number;
  velumPort: number;
  lipArea: number;
  tongueCenter: Vec;
  tongueRadius: number;
  tip: Vec;
  splitIndex: number;
}

export interface VoiceSource {
  f0: number;
  glide: number;
  pressure: number;
  breathiness: number;
  tension: number;
  fry: number;
  roughness: number;
  vibratoDepth: number;
  vibratoRate: number;
  drift: number;
  voiced: boolean;
  /** 0 dark … 1 bright (default 0.25: as tension sets it) */
  brightness: number;
}

export interface VoiceTelemetry {
  artic: Float32Array;
  phAreas: Float32Array;
  orAreas: Float32Array;
  phLen: number;
  orLen: number;
  pressureRms: Float32Array;
  flow: Float32Array;
  velumPort: number;
  glottalArea: Float32Array;
  glottalFlow: Float32Array;
  f0: number;
  voicing: number;
  pLung: number;
  outRms: number;
  noiseLevel: number;
  noiseNode: number;
  gate: number;
  /** voice clock (s) for score playheads */
  clock: number;
  /** labels of gestures active now (most recent first) */
  active: string[];
}

export interface Segment {
  ipa: string;
  dur?: number;
  stress?: number;
  f0?: number;
  wordEnd?: boolean;
  /** a voiceless stop or affricate's aspiration as its front end decides it (unset: English's rule) */
  aspirated?: boolean;
  /** how long an aspirated stop aspirates (s); unset: 50 ms */
  vot?: number;
  /** a vowel said with the velum lowered (IPA combining tilde) */
  nasalized?: boolean;
  /** a consonant with a secondary palatal gesture, the tongue body raised and fronted (IPA ʲ) */
  palatalized?: boolean;
  /** breathy voice: a voiced stop or affricate's release (IPA ʱ), or a vowel (combining diaeresis below) */
  breathy?: boolean;
  /** a vowel that is a glide off the vowel before it, reached over this long (s): one movement, not a second held vowel */
  glide?: number;
}

/** What the engine will not say as written in a piece of IPA text (see `skippedIpa`). */
export interface SkippedIpa {
  /** symbols it does not know (a tone letter, a click, a capital, a typo) */
  unknown: string[];
  /** symbols it knows but has no sound for (a vowel letter without a calibrated vowel) */
  noSound: string[];
  /** letters written with a mark it does not model or cannot apply there: the letter is said, the mark is not */
  ignoredMarks: string[];
}

export interface PlannedSegment {
  ipa: string;
  t0: number;
  t1: number;
  stress?: number;
  kind: string;
}

export interface Plan {
  gestures: Gesture[];
  segments: PlannedSegment[];
  end: number;
}

export interface Phrase {
  segments: Segment[];
  intonation: "statement" | "question";
  /** pause after the phrase (s) */
  pause: number;
  words: string[];
}

export interface Syllable {
  text: string;
  onset: Segment[];
  nucleus: Segment;
  coda: Segment[];
  melisma?: boolean;
  wordEnd: boolean;
  phraseEnd: boolean;
  /** pitch bends inside the sung note, written by the syllable's front end (a tone, an accent) */
  pitch?: Bend[];
  /** nasalise this fraction of the vowel before a closing rhyme's nasal coda (see `closing`; unset: none) */
  nasalLead?: number;
  /** the rhyme's ending (an off-glide, a nasal coda) is one closing movement at the note's end, this long at most (s); unset: a held part */
  closing?: number;
}

/** A bend of the pitch inside a sung note: `st` semitones, at its vowel's start or end, over `len` of the vowel (at most `max` s). */
export interface Bend {
  at: "start" | "middle" | "end";
  len: number;
  max: number;
  st: number;
}

export interface SungNote {
  midi: number;
  /** length in beats */
  beats: number;
  syl: Syllable;
}

export type BeatRole = "kick" | "snare" | "hat" | "perc" | "bass";
export type Airstream = "pulmonic" | "ejective" | "click" | "ingressive" | "voiced";

export interface BeatStep {
  id: string;
  note?: number;
  accent?: number;
}

/** A beatbox sound as the UI lists it (its gestures are built in the engine). */
export interface BeatSound {
  id: string;
  name: string;
  role: BeatRole;
  airstream: Airstream;
  hint: string;
  latency: number;
}

export interface BeatPreset {
  name: string;
  bpm: number;
  swing: number;
  text: string;
}

export interface DemoSong {
  name: string;
  bpm: number;
  lyrics: string;
  notes: [number, number][];
  octave: { male: number; female: number; child: number };
}

export interface VowelRef {
  ipa: string;
  /** example word / description */
  example: string;
  male: [number, number, number];
  female: [number, number, number];
  child: [number, number, number];
  round: boolean;
  /** 0 close … 1 open */
  height: number;
  /** 0 front … 1 back */
  back: number;
}

export type CreatureGroup = "human" | "bird" | "mammal" | "marine" | "reptile" | "amphibian" | "insect" | "crustacean" | "fantasy";

export interface ParamDef {
  key: string;
  label: string;
  min: number;
  max: number;
  default: number;
  unit?: string;
  log?: boolean;
  /** smoothing time constant (s) */
  smooth?: number;
  /** "pitch" params follow the keyboard */
  role?: "pitch";
  /** decimals for display */
  digits?: number;
}

export interface CallKey {
  t: number;
  v: Record<string, number>;
  curve?: "lin" | "exp" | "step";
}

export interface CallDef {
  id: string;
  name: string;
  hint: string;
  keys: CallKey[];
  pitchRef?: number;
}

/** A creature: the engine's definition joined with its drawing data (src/ui/creatureVisuals.ts). */
export interface CreatureDef {
  id: string;
  name: string;
  latin: string;
  group: CreatureGroup;
  blurb: string;
  role: string;
  oversample: number;
  gateTimes?: { attack: number; release: number };
  params: ParamDef[];
  calls: CallDef[];
  visual: import("../ui/creatureVisuals.ts").CreatureVisual;
}

export interface CreatureTelemetry {
  kind: "creature";
  id: string;
  params: Record<string, number>;
  gate: number;
  outRms: number;
  clock: number;
  /** per tube: areas (cm²) and acoustic pressure RMS (dyn/cm², static pressure removed) per node */
  tubes: Record<string, { areas: Float32Array | number[]; rms: Float32Array | number[] }>;
  source: Record<string, number>;
  call: string | null;
}