Enterprise agent platform for building, deploying, and operating AI teammates.
import { readAuthSessionSnapshot } from "./auth-cache"
import { getApiUrl, getWsUrl } from "./utils"
import { websocketAuthProtocols } from "./websocket-auth"
import { encodePcm16, VoiceResampler } from "./voice-conversation"
import { createVoiceVad, type VoiceVad } from "./voice-vad"
export type VoiceAudioFailure = "session_timeout" | "authentication_required" | "connection_failed"
export class VoiceAudioError extends Error {
constructor(public code: VoiceAudioFailure) { super(code) }
}
/** Which side of the call a level reading is for. */
export type VoiceLevelSource = "mic" | "output"
/** One call owns its microphone, socket, playback sources and fetches. */
export class VoiceAudio {
private closed = false
private context: AudioContext | null = null
private microphone: MediaStream | null = null
private capture: AudioWorkletNode | null = null
private source: MediaStreamAudioSourceNode | null = null
// Taps for the waveform. The microphone tap is read-only; playback goes
// through the output tap on its way to the speakers.
private micTap: AnalyserNode | null = null
private outputTap: AnalyserNode | null = null
private levelSamples: Float32Array<ArrayBuffer> | null = null
private sink: GainNode | null = null
private socket: WebSocket | null = null
private playback = new Set<AudioBufferSourceNode>()
private synthesis: AbortController | null = null
private vad: VoiceVad | null = null
private serverSpeaking = false
private vadLoading = new AbortController()
private rejectReady: ((error: Error) => void) | null = null
private readyTimeout: ReturnType<typeof setTimeout> | undefined
private _muted = false
get muted() { return this._muted }
set muted(value: boolean) {
this._muted = value
this.microphone?.getTracks().forEach(track => { track.enabled = !value })
}
constructor(private options: {
/**
* An audio context already created and resumed inside the user's click.
* Browsers (Safari above all) only let a context start from a user gesture,
* and a call can only begin after its models and settings have loaded, so
* the caller unlocks one up front and hands it over here. The call owns
* it from then on and closes it.
*/
context?: AudioContext
asr: string
tts: string
voice?: string
vadSilenceThreshold?: number
token: string
sessionId: string | null
activity: (active: boolean) => void
transcript: (type: "partial" | "final", text: string, segmentId: number) => void
error: (reason?: VoiceAudioFailure) => void
vadFallback?: () => void
}) {}
async start(): Promise<void> {
this.context = this.options.context ?? new AudioContext()
// Usually this is the context unlocked by the click that opened the call
// (see `useVoiceCall`); resume is a no-op then. A context made here still
// needs the user activation to be live, so resume before any network await.
await this.context.resume()
if (this.closed) return
const context = this.context
this.outputTap = context.createAnalyser()
this.outputTap.fftSize = 1024
this.outputTap.connect(context.destination)
const media = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
})
if (this.closed) { media.getTracks().forEach(track => track.stop()); return }
this.microphone = media
await context.audioWorklet.addModule("/voice-capture-worklet.js")
if (this.closed) return
// Backend ASR/VAD owns the baseline protocol. Local activity refines turn
// boundaries and must not block a call or SDK parity.
void createVoiceVad({
signal: this.vadLoading.signal,
speech: speaking => { if (!this.closed && !this.muted) this.options.activity(speaking) },
error: () => this.useServerVad(),
}).then(async vad => {
if (this.closed) { await vad.destroy(); return }
if (this.serverSpeaking) { this.serverSpeaking = false; this.options.activity(false) }
this.vad = vad
}).catch(() => this.useServerVad())
const socket = new WebSocket(`${getWsUrl().replace(/\/$/, "")}/api/models/speech/asr/stream`, websocketAuthProtocols(this.options.token))
this.socket = socket
await new Promise<void>((resolve, reject) => {
this.rejectReady = reject
this.readyTimeout = setTimeout(() => this.fail("connection_failed"), 30000)
socket.onopen = () => socket.send(JSON.stringify({
type: "start", model_id: this.options.asr, format: "pcm", sample_rate: 16000,
vad: true, vad_silence_threshold_secs: this.options.vadSilenceThreshold,
}))
socket.onmessage = ({ data }) => {
if (this.closed) return
try {
const event = JSON.parse(data)
if (event.type === "session_started") {
clearTimeout(this.readyTimeout)
this.rejectReady = null
resolve()
} else if ((event.type === "partial" || event.type === "final") && typeof event.text === "string") {
if (!this.muted) {
if (!this.vad && event.text.trim() && !this.serverSpeaking) {
this.serverSpeaking = true
this.options.activity(true)
}
this.options.transcript(event.type, event.text, event.segment_id)
if (!this.vad && event.type === "final") {
this.serverSpeaking = false
this.options.activity(false)
}
}
} else if (event.type === "error") this.fail(event.code === "session_timeout" ? "session_timeout" : "connection_failed")
} catch { this.fail("connection_failed") }
}
socket.onerror = () => this.fail("connection_failed")
socket.onclose = event => this.fail(event.code === 4001 ? "authentication_required" : event.reason === "Speech session timed out" ? "session_timeout" : "connection_failed")
})
if (this.closed) return
const resampler = new VoiceResampler(context.sampleRate)
this.source = context.createMediaStreamSource(media)
this.micTap = context.createAnalyser()
this.micTap.fftSize = 1024
this.source.connect(this.micTap)
this.capture = new AudioWorkletNode(context, "xagent-voice-capture", { channelCount: 1, channelCountMode: "explicit" })
this.sink = context.createGain()
this.sink.gain.value = 0
this.capture.port.onmessage = ({ data }: MessageEvent<Float32Array>) => {
if (this.closed || socket.readyState !== WebSocket.OPEN) return
if (socket.bufferedAmount > 256 * 1024) { this.fail("connection_failed"); return }
const samples = this.muted ? new Float32Array(data.length) : data
const frame = resampler.process(samples)
this.vad?.process(frame)
socket.send(encodePcm16(frame))
}
this.source.connect(this.capture)
this.capture.connect(this.sink).connect(context.destination)
}
private fail(reason: VoiceAudioFailure) {
if (this.closed) return
const reject = this.rejectReady
this.rejectReady = null
reject?.(new VoiceAudioError(reason))
this.close()
this.options.error(reason)
}
private useServerVad() {
const vad = this.vad
this.vad = null
if (vad) {
void vad.destroy().catch(() => {})
if (!this.closed) this.options.activity(false)
}
if (!this.closed) this.options.vadFallback?.()
}
/**
* Current loudness of the microphone or of the assistant's voice, 0..1.
* Cheap enough to read every animation frame; 0 before the call is running
* and while the microphone is muted (a muted track reads as silence).
*/
level(source: VoiceLevelSource): number {
const tap = source === "mic" ? this.micTap : this.outputTap
if (!tap || this.closed) return 0
const samples = (this.levelSamples ??= new Float32Array(new ArrayBuffer(tap.fftSize * 4)))
tap.getFloatTimeDomainData(samples)
let sum = 0
for (let index = 0; index < samples.length; index++) sum += samples[index] * samples[index]
return Math.min(1, Math.sqrt(sum / samples.length) * 5)
}
stopAudio() {
this.synthesis?.abort()
this.synthesis = null
for (const source of this.playback) { try { source.stop() } catch { /* ended */ } source.disconnect() }
this.playback.clear()
}
async speak(text: string): Promise<void> {
const context = this.context
if (!context || this.closed) return
const controller = new AbortController()
this.synthesis = controller
const signal = controller.signal
const login = readAuthSessionSnapshot()
const token = login.accessToken
if (!token || login.sessionId !== this.options.sessionId) throw new VoiceAudioError("authentication_required")
// A synthesis request is billable; never automatically replay a failed POST.
const response = await fetch(`${getApiUrl()}/api/models/speech/tts/stream`, {
method: "POST", signal,
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
body: JSON.stringify({ model_id: this.options.tts, voice: this.options.voice, text, format: "pcm", sample_rate: 24000 }),
})
if (!response.ok || !response.body) {
await response.body?.cancel()
if (response.status === 401 || response.status === 403) throw new VoiceAudioError("authentication_required")
throw new Error("Speech synthesis failed")
}
const rate = Number(response.headers.get("X-Audio-Sample-Rate"))
if (response.headers.get("X-Audio-Format") !== "pcm" || !Number.isFinite(rate) || rate < 8000 || rate > 48000) {
await response.body.cancel()
throw new Error("Invalid speech audio format")
}
const reader = response.body.getReader()
let pending = new Uint8Array(0)
let next = context.currentTime + 0.025
const wait = () => new Promise<void>((resolve, reject) => {
if (signal.aborted) { reject(new DOMException("Aborted", "AbortError")); return }
const abort = () => { clearTimeout(timer); reject(new DOMException("Aborted", "AbortError")) }
const timer = setTimeout(() => { signal.removeEventListener("abort", abort); resolve() }, 20)
signal.addEventListener("abort", abort, { once: true })
})
try {
while (!signal.aborted && !this.closed) {
const { done, value } = await reader.read()
if (done) break
const bytes = new Uint8Array(pending.length + value.length)
bytes.set(pending); bytes.set(value, pending.length)
const aligned = bytes.length - bytes.length % 2
const view = new DataView(bytes.buffer)
for (let offset = 0; offset < aligned; offset += 24000) {
while (next - context.currentTime > 1) await wait()
if (signal.aborted || this.closed) return
const length = Math.min(24000, aligned - offset) / 2
const buffer = context.createBuffer(1, length, rate)
const samples = buffer.getChannelData(0)
for (let index = 0; index < length; index++) samples[index] = view.getInt16(offset + index * 2, true) / 32768
const source = context.createBufferSource()
source.buffer = buffer
source.connect(this.outputTap ?? context.destination)
source.onended = () => { source.disconnect(); this.playback.delete(source) }
this.playback.add(source)
next = Math.max(next, context.currentTime + 0.01)
source.start(next)
next += length / rate
}
pending = bytes.slice(aligned)
}
if (pending.length && !signal.aborted) throw new Error("Truncated PCM sample")
while (!signal.aborted && !this.closed && this.playback.size) await wait()
} catch (error) {
if (this.synthesis === controller) this.stopAudio()
throw error
} finally {
await reader.cancel().catch(() => {})
reader.releaseLock()
if (this.synthesis === controller) this.synthesis = null
}
}
close() {
if (this.closed) return
this.closed = true
this.vadLoading.abort()
void this.vad?.destroy().catch(() => {})
clearTimeout(this.readyTimeout)
this.rejectReady?.(new Error("Call ended"))
this.rejectReady = null
this.stopAudio()
if (this.socket) { this.socket.onclose = this.socket.onerror = this.socket.onmessage = this.socket.onopen = null; this.socket.close() }
this.capture?.disconnect(); this.source?.disconnect(); this.sink?.disconnect()
this.micTap?.disconnect(); this.outputTap?.disconnect()
this.microphone?.getTracks().forEach(track => track.stop())
void this.context?.close().catch(() => {})
}
}