Skip to content

Visual UI for Explainroo. Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

12.8 KB
Raw Blame
// Audio Engine with real-time generative background music, Web Speech TTS, and auto-ducking

export interface AudioPlaybackState {
  isPlaying: boolean;
  currentTime: number;
  currentWordIndex: number;
  duckingActive: boolean;
}

export type WordBoundaryListener = (word: string, charIndex: number, wordIndex: number) => void;

class AudioEngine {
  private ctx: AudioContext | null = null;
  private musicGain: GainNode | null = null;
  private sfxGain: GainNode | null = null;
  private masterGain: GainNode | null = null;
  private isMusicPlaying = false;
  private musicInterval: number | null = null;
  private currentTrack = 'lofi';
  private speechUtterance: SpeechSynthesisUtterance | null = null;
  private wordListeners: Set<WordBoundaryListener> = new Set();
  private currentWordIndex: number = -1;
  private currentSpokenWord: string = '';
  private fallbackWordTimer: number | null = null;
  private hasReceivedNativeBoundary: boolean = false;

  public getCurrentTrack(): string {
    return this.currentTrack;
  }

  public isSpeaking(): boolean {
    return Boolean(this.speechUtterance);
  }

  public addWordListener(listener: WordBoundaryListener): () => void {
    this.wordListeners.add(listener);
    if (this.currentWordIndex >= 0) {
      try {
        listener(this.currentSpokenWord, 0, this.currentWordIndex);
      } catch {}
    }
    return () => {
      this.wordListeners.delete(listener);
    };
  }

  public notifyWord(word: string, charIndex: number, wordIndex: number) {
    this.currentSpokenWord = word;
    this.currentWordIndex = wordIndex;
    this.wordListeners.forEach((listener) => {
      try {
        listener(word, charIndex, wordIndex);
      } catch (err) {
        console.error('Error in word listener:', err);
      }
    });
  }

  public getCurrentWord(): { word: string; index: number } {
    return { word: this.currentSpokenWord, index: this.currentWordIndex };
  }

  private initContext() {
    if (!this.ctx) {
      const AudioCtx = window.AudioContext || (window as any).webkitAudioContext;
      this.ctx = new AudioCtx();

      this.masterGain = this.ctx.createGain();
      this.masterGain.gain.setValueAtTime(0.9, this.ctx.currentTime);
      this.masterGain.connect(this.ctx.destination);

      this.musicGain = this.ctx.createGain();
      this.musicGain.gain.setValueAtTime(0.25, this.ctx.currentTime);
      this.musicGain.connect(this.masterGain);

      this.sfxGain = this.ctx.createGain();
      this.sfxGain.gain.setValueAtTime(0.4, this.ctx.currentTime);
      this.sfxGain.connect(this.masterGain);
    }

    if (this.ctx.state === 'suspended') {
      this.ctx.resume().catch(() => {});
    }
  }

  // Play Sound Effects
  public playPop() {
    try {
      this.initContext();
      if (!this.ctx || !this.sfxGain) return;
      const t = this.ctx.currentTime;
      const osc = this.ctx.createOscillator();
      const gain = this.ctx.createGain();

      osc.type = 'sine';
      osc.frequency.setValueAtTime(320, t);
      osc.frequency.exponentialRampToValueAtTime(740, t + 0.08);

      gain.gain.setValueAtTime(0.3, t);
      gain.gain.exponentialRampToValueAtTime(0.001, t + 0.09);

      osc.connect(gain);
      gain.connect(this.sfxGain);

      osc.start(t);
      osc.stop(t + 0.1);
    } catch {
      // Audio fallback
    }
  }

  public playWhoosh() {
    try {
      this.initContext();
      if (!this.ctx || !this.sfxGain) return;
      const t = this.ctx.currentTime;

      // Filtered noise swoosh
      const bufferSize = this.ctx.sampleRate * 0.2;
      const buffer = this.ctx.createBuffer(1, bufferSize, this.ctx.sampleRate);
      const output = buffer.getChannelData(0);
      for (let i = 0; i < bufferSize; i++) {
        output[i] = (Math.random() * 2 - 1) * 0.3;
      }

      const noise = this.ctx.createBufferSource();
      noise.buffer = buffer;

      const filter = this.ctx.createBiquadFilter();
      filter.type = 'bandpass';
      filter.frequency.setValueAtTime(400, t);
      filter.frequency.exponentialRampToValueAtTime(1400, t + 0.1);
      filter.frequency.exponentialRampToValueAtTime(300, t + 0.2);
      filter.Q.value = 3.0;

      const gain = this.ctx.createGain();
      gain.gain.setValueAtTime(0.01, t);
      gain.gain.linearRampToValueAtTime(0.2, t + 0.08);
      gain.gain.exponentialRampToValueAtTime(0.001, t + 0.2);

      noise.connect(filter);
      filter.connect(gain);
      gain.connect(this.sfxGain);

      noise.start(t);
      noise.stop(t + 0.2);
    } catch {
      // Audio fallback
    }
  }

  public playDrawChime() {
    try {
      this.initContext();
      if (!this.ctx || !this.sfxGain) return;
      const t = this.ctx.currentTime;
      const notes = [523.25, 659.25, 783.99]; // C5, E5, G5
      notes.forEach((freq, idx) => {
        const osc = this.ctx!.createOscillator();
        const gain = this.ctx!.createGain();
        osc.type = 'triangle';
        osc.frequency.setValueAtTime(freq, t + idx * 0.04);
        gain.gain.setValueAtTime(0.12, t + idx * 0.04);
        gain.gain.exponentialRampToValueAtTime(0.001, t + idx * 0.04 + 0.25);
        osc.connect(gain);
        gain.connect(this.sfxGain!);
        osc.start(t + idx * 0.04);
        osc.stop(t + idx * 0.04 + 0.28);
      });
    } catch {
      // Audio fallback
    }
  }

  // Audio Ducking: lowers music when narrator is speaking
  public setDucking(active: boolean) {
    if (!this.ctx || !this.musicGain) return;
    const t = this.ctx.currentTime;
    const target = active ? 0.08 : 0.25;
    this.musicGain.gain.cancelScheduledValues(t);
    this.musicGain.gain.linearRampToValueAtTime(target, t + 0.25);
  }

  // Generative Background Music Tracks
  public startMusic(track: string = 'lofi') {
    this.initContext();
    this.stopMusic();
    this.currentTrack = track;
    this.isMusicPlaying = true;

    // Generative chord progression based on track
    const chordProgressions: Record<string, number[][]> = {
      lofi: [
        [261.63, 329.63, 392.0, 493.88], // Cmaj7
        [220.0, 261.63, 329.63, 392.0],  // Am7
        [174.61, 220.0, 261.63, 329.63], // Fmaj7
        [196.0, 246.94, 293.66, 349.23], // G7
      ],
      tech: [
        [130.81, 196.0, 261.63, 392.0], // C power
        [110.0, 164.81, 220.0, 329.63], // A power
        [146.83, 220.0, 293.66, 440.0], // D power
        [123.47, 185.0, 246.94, 370.0], // B power
      ],
      acoustic: [
        [261.63, 329.63, 392.0],        // C
        [196.0, 246.94, 293.66],        // G
        [220.0, 261.63, 329.63],        // Am
        [174.61, 220.0, 261.63],        // F
      ],
    };

    const chords = chordProgressions[track] || chordProgressions.lofi;
    let chordIdx = 0;

    const playChord = () => {
      if (!this.isMusicPlaying || !this.ctx || !this.musicGain) return;
      const notes = chords[chordIdx % chords.length];
      chordIdx++;

      const t = this.ctx.currentTime;
      notes.forEach((freq, i) => {
        const osc = this.ctx!.createOscillator();
        const gain = this.ctx!.createGain();
        osc.type = track === 'tech' ? 'sawtooth' : 'sine';
        osc.frequency.setValueAtTime(freq, t);

        const filter = this.ctx!.createBiquadFilter();
        filter.type = 'lowpass';
        filter.frequency.setValueAtTime(track === 'tech' ? 1200 : 700, t);

        gain.gain.setValueAtTime(0.02, t);
        gain.gain.linearRampToValueAtTime(0.08, t + 0.6 + i * 0.05);
        gain.gain.exponentialRampToValueAtTime(0.005, t + 3.2);

        osc.connect(filter);
        filter.connect(gain);
        gain.connect(this.musicGain!);

        osc.start(t);
        osc.stop(t + 3.3);
      });
    };

    playChord();
    this.musicInterval = window.setInterval(playChord, 3200);
  }

  public stopMusic() {
    this.isMusicPlaying = false;
    if (this.musicInterval) {
      clearInterval(this.musicInterval);
      this.musicInterval = null;
    }
  }

  // Web Speech API Voiceover with Word Boundary Synchronization
  public speakNarration(
    text: string,
    onWordSpoken?: (word: string, charIndex: number, wordIndex: number) => void,
    onEnd?: () => void,
    rate: number = 1.0,
    pitch: number = 1.0
  ) {
    if (!('speechSynthesis' in window)) {
      if (onEnd) onEnd();
      return;
    }

    this.stopSpeaking();
    this.setDucking(true);

    if (!text || !text.trim()) {
      if (onEnd) onEnd();
      return;
    }

    const trimmedText = text.trim();
    const wordsList = trimmedText.split(/\s+/).filter(Boolean);

    const utterance = new SpeechSynthesisUtterance(trimmedText);
    utterance.rate = Math.max(0.7, Math.min(1.5, rate));
    utterance.pitch = Math.max(0.8, Math.min(1.3, pitch));

    // Try to pick a natural English voice
    const voices = window.speechSynthesis.getVoices();
    const naturalVoice = voices.find(
      (v) => (v.name.includes('Natural') || v.name.includes('Google') || v.name.includes('Siri') || v.name.includes('Samantha') || v.name.includes('Daniel')) && v.lang.startsWith('en')
    ) || voices.find((v) => v.lang.startsWith('en'));

    if (naturalVoice) {
      utterance.voice = naturalVoice;
    }

    this.hasReceivedNativeBoundary = false;

    // Fallback timer: in case SpeechSynthesis onboundary is not fired by the OS/voice
    // Estimates speech progress (~2.6 words per second scaled by rate)
    let fallbackIdx = 0;
    const intervalMs = Math.max(150, Math.round(1000 / (2.6 * utterance.rate)));
    this.fallbackWordTimer = window.setInterval(() => {
      if (this.hasReceivedNativeBoundary) {
        if (this.fallbackWordTimer) {
          clearInterval(this.fallbackWordTimer);
          this.fallbackWordTimer = null;
        }
        return;
      }
      if (fallbackIdx < wordsList.length) {
        const spokenWord = wordsList[fallbackIdx];
        if (onWordSpoken) {
          onWordSpoken(spokenWord, 0, fallbackIdx);
        }
        this.notifyWord(spokenWord, 0, fallbackIdx);
        fallbackIdx++;
      }
    }, intervalMs);

    utterance.onboundary = (event) => {
      if (event.name === 'word') {
        this.hasReceivedNativeBoundary = true;
        if (this.fallbackWordTimer) {
          clearInterval(this.fallbackWordTimer);
          this.fallbackWordTimer = null;
        }

        const charIdx = event.charIndex;
        const slice = trimmedText.slice(charIdx);
        const match = slice.match(/^[\w'-]+/);
        const spokenWord = match
          ? match[0]
          : event.charLength
          ? trimmedText.substring(charIdx, charIdx + event.charLength)
          : '';

        // Calculate 0-based word index
        const textBefore = trimmedText.slice(0, charIdx);
        const wordsBefore = textBefore.trim() ? textBefore.trim().split(/\s+/).filter(Boolean) : [];
        const wordIndex = wordsBefore.length;

        if (onWordSpoken) {
          onWordSpoken(spokenWord, charIdx, wordIndex);
        }
        this.notifyWord(spokenWord, charIdx, wordIndex);
      }
    };

    utterance.onend = () => {
      if (this.fallbackWordTimer) {
        clearInterval(this.fallbackWordTimer);
        this.fallbackWordTimer = null;
      }
      this.setDucking(false);
      this.speechUtterance = null;
      this.notifyWord('', -1, -1);
      if (onEnd) onEnd();
    };

    utterance.onerror = () => {
      if (this.fallbackWordTimer) {
        clearInterval(this.fallbackWordTimer);
        this.fallbackWordTimer = null;
      }
      this.setDucking(false);
      this.speechUtterance = null;
      this.notifyWord('', -1, -1);
      if (onEnd) onEnd();
    };

    this.speechUtterance = utterance;
    window.speechSynthesis.speak(utterance);
  }

  private customAudio: HTMLAudioElement | null = null;

  public playCustomAudio(url: string, onEnd?: () => void) {
    this.stopSpeaking();
    this.setDucking(true);
    if (this.customAudio) {
      this.customAudio.pause();
      this.customAudio = null;
    }
    const audio = new Audio(url);
    audio.onended = () => {
      this.setDucking(false);
      this.customAudio = null;
      this.notifyWord('', -1, -1);
      if (onEnd) onEnd();
    };
    audio.onerror = () => {
      this.setDucking(false);
      this.customAudio = null;
      this.notifyWord('', -1, -1);
      if (onEnd) onEnd();
    };
    this.customAudio = audio;
    audio.play().catch(() => {});
  }

  public stopCustomAudio() {
    if (this.customAudio) {
      this.customAudio.pause();
      this.customAudio.currentTime = 0;
      this.customAudio = null;
    }
    this.setDucking(false);
    this.notifyWord('', -1, -1);
  }

  public stopSpeaking() {
    if (this.fallbackWordTimer) {
      clearInterval(this.fallbackWordTimer);
      this.fallbackWordTimer = null;
    }
    if ('speechSynthesis' in window) {
      window.speechSynthesis.cancel();
    }
    this.speechUtterance = null;
    this.notifyWord('', -1, -1);
    this.stopCustomAudio();
    this.setDucking(false);
  }

  public stopAll() {
    this.stopSpeaking();
    this.stopCustomAudio();
    this.stopMusic();
    this.notifyWord('', -1, -1);
  }
}

export const audioEngine = new AudioEngine();