Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

7.9 KB
Raw Blame
import { unwrapFinalAnswerContent } from "./final-answer"

export type VoicePhase = "idle" | "connecting" | "listening" | "thinking" | "speaking" | "muted" | "error"

export function speechText(value: string): string {
  return unwrapFinalAnswerContent(value)
    .replace(/```[\s\S]*?(?:```|$)/g, " ")
    .replace(/!\[[^\]]*\]\([^)]*\)/g, " ")
    .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
    .replace(/https?:\/\/\S+/g, " ")
    .replace(/[*_`#>|]/g, "")
    // Unpunctuated list items still need boundaries for streaming synthesis.
    .replace(/(^[ \t]*\d+[.)][ \t]+[^\r\n]*\S)[ \t]*\r?\n(?:[ \t]*\r?\n)*(?=[ \t]*\d+[.)][ \t]+)/gm,
      (_match, line: string) => `${line}${/[.!?。!?]$/.test(line) ? "" : "."}\n`)
    .replace(/^[ \t]*\d+[.)][ \t]+/gm, "")
    .replace(/\[|\]/g, "")
    .replace(/\s+/g, " ").trim()
}

/** Encode resampled 16 kHz mono frames as raw PCM16 LE for ASR. */
export function encodePcm16(input: Float32Array): ArrayBuffer {
  const bytes = new ArrayBuffer(input.length * 2)
  const view = new DataView(bytes)
  input.forEach((sample, index) => {
    const value = Math.max(-1, Math.min(1, sample))
    view.setInt16(index * 2, Math.round(value * (value < 0 ? 32768 : 32767)), true)
  })
  return bytes
}

/** Preserve device-rate frame boundaries while downsampling to 16 kHz. */
export class VoiceResampler {
  private sum = 0
  private weight = 0
  constructor(private inputRate: number) {
    if (inputRate < 16000) throw new Error("Unsupported microphone sample rate")
  }
  process(input: Float32Array): Float32Array {
    const ratio = this.inputRate / 16000
    const output: number[] = []
    for (const value of input) {
      let left = 1
      while (left > 1e-8) {
        const take = Math.min(left, ratio - this.weight)
        this.sum += value * take; this.weight += take; left -= take
        if (this.weight >= ratio - 1e-8) {
          output.push(this.sum / this.weight)
          this.sum = this.weight = 0
        }
      }
    }
    return Float32Array.from(output)
  }
}

export interface VoiceReply {
  id: string
  text: string
  complete: boolean
}

/** Own a voice turn independently of React renders and late network callbacks. */
export class VoiceTurns {
  private segments: string[] = []
  private segmentIds = new Set<number>()
  private timer: ReturnType<typeof setTimeout> | undefined
  private pending = ""
  private sending = false
  private replyAllowed = false
  private replyOffsets = new Map<string, string>()
  private epoch = 0
  private speaking = false
  private interrupted = false
  private interruptWanted = false
  private userSpeaking = false
  private speechAccepted = false
  private transcriptPending = false
  private stopped = false
  private queue: string[] = []
  private draining = false
  muted = false
  busy = false

  constructor(private actions: {
    send: (text: string) => Promise<void>
    interrupt: () => boolean | void
    stopAudio: () => void
    speak: (text: string) => Promise<void>
    phase: (phase: VoicePhase) => void
    caption: (text: string) => void
    error: (error: unknown) => void
    playbackError: (error: unknown) => void
  }) {}

  activity(active: boolean) {
    if (this.stopped || this.muted) return
    this.userSpeaking = active
    if (active) {
      clearTimeout(this.timer)
    } else this.schedule()
  }

  private cutOff() {
    this.actions.stopAudio()
    this.queue = []
    this.epoch += 1
    this.replyAllowed = false
    this.speaking = false
    this.actions.phase("listening")
  }

  transcript(type: "partial" | "final", text: string, segmentId: number) {
    if (this.stopped || this.muted || this.segmentIds.has(segmentId)) return
    if (!text.trim()) {
      if (type === "partial") {
        this.transcriptPending = false
        this.actions.caption([this.pending, ...this.segments].filter(Boolean).join(" "))
        this.schedule()
      }
      return
    }
    if (!this.speechAccepted) {
      this.speechAccepted = true
      this.cutOff()
    }
    this.transcriptPending = type === "partial"
    // A transcript is speech confirmation: pause generation only after this,
    // rather than cancelling an agent because of a short microphone noise.
    if ((this.busy || this.sending) && !this.interrupted) {
      this.interruptWanted = true
      this.interrupted = this.actions.interrupt() !== false
    }
    if (type === "final") {
      this.segmentIds.add(segmentId)
      this.segments.push(text.trim())
      this.actions.caption(this.segments.join(" "))
      this.schedule()
    } else {
      clearTimeout(this.timer)
      this.actions.caption([...this.segments, text.trim()].join(" "))
    }
  }

  private schedule() {
    clearTimeout(this.timer)
    if (this.userSpeaking || (!this.segments.length && !this.pending) || this.muted || this.stopped) return
    this.timer = setTimeout(() => {
      this.pending = [this.pending, ...this.segments].filter(Boolean).join(" ")
      this.segments = []
      void this.flush()
    }, 300)
  }

  updateBusy(busy: boolean) {
    this.busy = busy
    if (busy && this.interruptWanted && !this.interrupted) {
      this.interrupted = this.actions.interrupt() !== false
    }
    if (!busy) void this.flush()
  }

  private async flush() {
    if (!this.pending || this.busy || this.sending || this.stopped || this.muted || this.userSpeaking || this.transcriptPending || this.segments.length) return
    const text = this.pending
    this.pending = ""
    this.speechAccepted = false
    this.sending = true
    this.replyOffsets.clear()
    this.replyAllowed = true
    this.interrupted = false
    this.interruptWanted = false
    this.actions.phase("thinking")
    try { await this.actions.send(text) }
    catch (error) { this.replyAllowed = false; this.actions.error(error) }
    finally { this.sending = false }
  }

  reply(reply: VoiceReply) {
    if (!this.replyAllowed || this.stopped) return
    const text = speechText(reply.text)
    if (!text || text.startsWith("{")) return
    const previous = this.replyOffsets.get(reply.id) || ""
    if (text.length < previous.length) return
    // Streaming markdown can revise an earlier prefix. Keep the spoken
    // position without replaying it or blocking all subsequent sentences.
    let consumed = text.slice(0, previous.length)
    let rest = text.slice(consumed.length)
    while (rest) {
      const sentence = rest.match(/^[\s\S]*?(?:[.!?]\s+|[。!?]\s*)/)?.[0]
      const chunk = sentence || (reply.complete ? rest : "")
      if (!chunk) break
      // Each synthesis request stays within the transport's text limit.
      for (let index = 0; index < chunk.length; index += 4000) this.queue.push(chunk.slice(index, index + 4000))
      consumed += chunk
      rest = rest.slice(chunk.length)
    }
    this.replyOffsets.set(reply.id, consumed)
    void this.drain()
  }

  private async drain() {
    if (this.draining || this.stopped) return
    this.draining = true
    try {
      while (this.queue.length && !this.stopped) {
        const epoch = this.epoch
        const text = this.queue.shift()!
        this.speaking = true
        this.actions.phase("speaking")
        try { await this.actions.speak(text) }
        catch (error) {
          if (epoch === this.epoch && !this.stopped && !(error instanceof Error && error.name === "AbortError")) this.actions.playbackError(error)
        }
        if (epoch === this.epoch) this.speaking = false
      }
    } finally {
      this.draining = false
      if (!this.stopped && !this.speaking) this.actions.phase(this.muted ? "muted" : this.userSpeaking ? "listening" : this.busy ? "thinking" : "listening")
    }
  }

  mute(muted: boolean) {
    this.muted = muted
    this.segments = []
    this.pending = ""
    this.userSpeaking = false
    this.speechAccepted = false
    this.transcriptPending = false
    clearTimeout(this.timer)
    this.actions.phase(muted ? "muted" : this.busy ? "thinking" : "listening")
  }

  stop() {
    this.stopped = true
    this.epoch += 1
    this.queue = []
    this.pending = ""
    clearTimeout(this.timer)
    this.actions.stopAudio()
  }
}