Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

14.9 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
import { VoiceAudio, VoiceAudioError } from "./voice-audio"
import { createVoiceVad } from "./voice-vad"

const detectors = vi.hoisted(() => [] as Array<{ options: { speech: (active: boolean) => void; error: () => void }; process: ReturnType<typeof vi.fn>; destroy: ReturnType<typeof vi.fn> }>)
vi.mock("./voice-vad", () => ({ createVoiceVad: vi.fn(async options => {
  const vad = { options, process: vi.fn(), destroy: vi.fn().mockResolvedValue(undefined) }
  detectors.push(vad)
  return vad
}) }))

const login = vi.hoisted(() => ({ accessToken: "current-login-token", sessionId: "login" }))
vi.mock("./auth-cache", () => ({ readAuthSessionSnapshot: () => login }))
vi.mock("./utils", () => ({ getWsUrl: () => "wss://example.com", getApiUrl: () => "https://example.com" }))

class Socket {
  static OPEN = 1
  static instances: Socket[] = []
  readyState = 1
  bufferedAmount = 0
  onopen: (() => void) | null = null
  onmessage: ((event: { data: string }) => void) | null = null
  onclose: ((event: { code: number; reason: string }) => void) | null = null
  onerror: (() => void) | null = null
  send = vi.fn()
  close = vi.fn()
  constructor(public url: string, public protocols: string[]) { Socket.instances.push(this) }
}
const captures: Array<{ port: { onmessage: ((event: { data: Float32Array }) => void) | null } }> = []
const playback: Array<{ start: ReturnType<typeof vi.fn>; stop: ReturnType<typeof vi.fn>; onended: (() => void) | null }> = []
const analyserLevel = { value: 0 }
const stopTrack = vi.fn()
const closeContext = vi.fn().mockResolvedValue(undefined)
class Context {
  static created = 0
  constructor() { Context.created += 1 }
  sampleRate = 48000
  currentTime = 0
  destination = {}
  resume = vi.fn().mockResolvedValue(undefined)
  close = closeContext
  audioWorklet = { addModule: vi.fn().mockResolvedValue(undefined) }
  createMediaStreamSource = () => ({ connect: vi.fn(), disconnect: vi.fn() })
  createAnalyser = () => ({ fftSize: 0, connect: vi.fn(), disconnect: vi.fn(), getFloatTimeDomainData: (samples: Float32Array) => samples.fill(analyserLevel.value) })
  createGain = () => ({ gain: { value: 0 }, connect: vi.fn(), disconnect: vi.fn() })
  createBuffer = (_channels: number, length: number) => ({ getChannelData: () => new Float32Array(length) })
  createBufferSource = () => {
    const source = { buffer: null, connect: vi.fn(), disconnect: vi.fn(), start: vi.fn(), stop: vi.fn(), onended: null as (() => void) | null }
    playback.push(source)
    return source
  }
}

const options = () => ({ asr: "live", tts: "speaker", token: "login-token", sessionId: "login", activity: vi.fn(), transcript: vi.fn(), error: vi.fn(), vadFallback: vi.fn() })

beforeEach(() => {
  Socket.instances = []; captures.length = 0; playback.length = 0
  detectors.length = 0
  login.sessionId = "login"
  vi.stubGlobal("WebSocket", Socket)
  vi.stubGlobal("AudioContext", Context)
  vi.stubGlobal("AudioWorkletNode", class {
    port = { onmessage: null }
    constructor() { captures.push(this) }
    connect = () => ({ connect: vi.fn() })
    disconnect = vi.fn()
  })
  Object.defineProperty(navigator, "mediaDevices", { configurable: true, value: {
    getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [{ stop: stopTrack }] }),
  } })
})
afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); vi.clearAllMocks() })

async function start(call: VoiceAudio) {
  const opening = call.start()
  for (let step = 0; step < 20 && !Socket.instances.length; step++) await Promise.resolve()
  const socket = Socket.instances[0]
  socket.onopen!()
  socket.onmessage!({ data: JSON.stringify({ type: "session_started" }) })
  await opening
  return socket
}

describe("browser voice resources", () => {
  it.each(["error", "close", "timeout"])("cleans up a failed ASR handshake (%s)", async failure => {
    if (failure === "timeout") vi.useFakeTimers()
    const events = options()
    const call = new VoiceAudio(events)
    const opening = call.start().catch(error => error)
    for (let step = 0; step < 20 && !Socket.instances.length; step++) await Promise.resolve()
    const socket = Socket.instances[0]
    if (failure === "error") socket.onerror!()
    if (failure === "close") socket.onclose!({ code: 1006, reason: "" })
    if (failure === "timeout") await vi.advanceTimersByTimeAsync(30000)
    expect(await opening).toBeInstanceOf(VoiceAudioError)
    expect(events.error).toHaveBeenCalledExactlyOnceWith("connection_failed")
    expect(socket.close).toHaveBeenCalledOnce()
    expect(stopTrack).toHaveBeenCalledOnce()
    expect(closeContext).toHaveBeenCalledOnce()
  })

  it("uses an audio context unlocked earlier instead of creating its own", async () => {
    const unlocked = new Context()
    Context.created = 0
    const call = new VoiceAudio({ ...options(), context: unlocked as unknown as AudioContext })
    await start(call)
    expect(Context.created).toBe(0)
    expect(unlocked.resume).toHaveBeenCalled()
    // The call owns it from here on.
    call.close()
    expect(closeContext).toHaveBeenCalledOnce()
  })

  it("creates its own audio context when none was handed in", async () => {
    Context.created = 0
    await start(new VoiceAudio(options()))
    expect(Context.created).toBe(1)
  })

  it("reads microphone and playback loudness for the waveform", async () => {
    const call = new VoiceAudio(options())
    expect(call.level("mic")).toBe(0)
    await start(call)
    analyserLevel.value = 0
    expect(call.level("mic")).toBe(0)
    analyserLevel.value = 0.1
    expect(call.level("mic")).toBeCloseTo(0.5)
    expect(call.level("output")).toBeCloseTo(0.5)
    analyserLevel.value = 0.9
    expect(call.level("mic")).toBe(1)
    call.close()
    expect(call.level("mic")).toBe(0)
    analyserLevel.value = 0
  })

  it("reports a server session timeout distinctly and releases microphone resources", async () => {
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    socket.onmessage!({ data: JSON.stringify({ type: "error", code: "session_timeout" }) })
    expect(events.error).toHaveBeenCalledExactlyOnceWith("session_timeout")
    expect(stopTrack).toHaveBeenCalledOnce()
    expect(socket.close).toHaveBeenCalledOnce()
  })

  it("reports session revocation distinctly", async () => {
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    socket.onclose!({ code: 4001, reason: "Session expired" })
    expect(events.error).toHaveBeenCalledExactlyOnceWith("authentication_required")
    expect(stopTrack).toHaveBeenCalledOnce()
  })

  it.each(["0", "NaN", "96000"])("rejects invalid sample-rate metadata (%s)", async rate => {
    const call = new VoiceAudio(options())
    await start(call)
    vi.stubGlobal("fetch", vi.fn(async () => new Response(new Uint8Array([0, 0]), { headers: { "X-Audio-Format": "pcm", "X-Audio-Sample-Rate": rate } })))
    await expect(call.speak("Hello")).rejects.toThrow("Invalid speech audio format")
    expect(playback).toHaveLength(0)
    call.close()
  })

  it("does not synthesize with a replacement login's credentials", async () => {
    const call = new VoiceAudio(options())
    await start(call)
    login.sessionId = "replacement-login"
    const request = vi.fn()
    vi.stubGlobal("fetch", request)
    await expect(call.speak("Hello")).rejects.toMatchObject({ code: "authentication_required" })
    expect(request).not.toHaveBeenCalled()
    call.close()
  })

  it("classifies synthesis authentication errors without replaying the request", async () => {
    const call = new VoiceAudio(options())
    await start(call)
    vi.stubGlobal("fetch", vi.fn(async () => new Response(null, { status: 401 })))
    await expect(call.speak("Hello")).rejects.toMatchObject({ code: "authentication_required" })
    expect(fetch).toHaveBeenCalledOnce()
    call.close()
  })

  it("relays quiet server transcripts while local VAD reports no activity", async () => {
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    socket.onmessage!({ data: JSON.stringify({ type: "partial", text: "Quiet words", segment_id: 1 }) })
    socket.onmessage!({ data: JSON.stringify({ type: "final", text: "Quiet words", segment_id: 1 }) })
    expect(events.transcript).toHaveBeenLastCalledWith("final", "Quiet words", 1)
    expect(events.activity).not.toHaveBeenCalled()
    call.close()
  })

  it("uses the backend ASR/VAD protocol when the local detector cannot load", async () => {
    vi.mocked(createVoiceVad).mockRejectedValueOnce(new Error("WASM unavailable"))
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    expect(JSON.parse(socket.send.mock.calls[0][0])).toMatchObject({ vad: true })
    expect(events.vadFallback).toHaveBeenCalledOnce()
    captures[0].port.onmessage!({ data: new Float32Array(1024).fill(0.9) })
    expect(events.activity).not.toHaveBeenCalled()
    socket.onmessage!({ data: JSON.stringify({ type: "partial", text: "Hello", segment_id: 1 }) })
    socket.onmessage!({ data: JSON.stringify({ type: "final", text: "Hello there.", segment_id: 1 }) })
    expect(events.activity.mock.calls).toEqual([[true], [false]])
    expect(events.transcript).toHaveBeenLastCalledWith("final", "Hello there.", 1)
    expect(events.error).not.toHaveBeenCalled()
    call.close()
  })

  it("starts without waiting for local VAD and releases a detector arriving after close", async () => {
    let release!: (vad: Awaited<ReturnType<typeof createVoiceVad>>) => void
    vi.mocked(createVoiceVad).mockReturnValueOnce(new Promise(resolve => { release = resolve }))
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    socket.onmessage!({ data: JSON.stringify({ type: "final", text: "Server VAD works.", segment_id: 1 }) })
    expect(events.activity.mock.calls).toEqual([[true], [false]])
    call.close()
    const detector = { process: vi.fn(), destroy: vi.fn().mockResolvedValue(undefined) }
    release(detector)
    await Promise.resolve()
    expect(detector.destroy).toHaveBeenCalledOnce()
  })

  it("keeps streaming after a running local detector fails", async () => {
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    detectors[0].options.error()
    expect(detectors[0].destroy).toHaveBeenCalledOnce()
    socket.onmessage!({ data: JSON.stringify({ type: "final", text: "Continue.", segment_id: 1 }) })
    expect(events.transcript).toHaveBeenCalledWith("final", "Continue.", 1)
    expect(events.vadFallback).toHaveBeenCalledOnce()
    expect(events.error).not.toHaveBeenCalled()
    call.close()
  })

  it("finishes server speech activity when a local detector joins mid-utterance", async () => {
    let release!: (vad: Awaited<ReturnType<typeof createVoiceVad>>) => void
    vi.mocked(createVoiceVad).mockReturnValueOnce(new Promise(resolve => { release = resolve }))
    const events = options()
    const call = new VoiceAudio(events)
    const socket = await start(call)
    socket.onmessage!({ data: JSON.stringify({ type: "partial", text: "Hello", segment_id: 1 }) })
    release({ process: vi.fn(), destroy: vi.fn().mockResolvedValue(undefined) })
    await Promise.resolve()
    socket.onmessage!({ data: JSON.stringify({ type: "final", text: "Hello there.", segment_id: 1 }) })
    expect(events.activity.mock.calls).toEqual([[true], [false]])
    expect(events.transcript).toHaveBeenLastCalledWith("final", "Hello there.", 1)
    call.close()
  })
  it("uses header login, mono PCM capture, local activity, and releases every resource", async () => {
    const callbacks = options()
    const call = new VoiceAudio(callbacks)
    const socket = await start(call)
    expect(socket.url).not.toContain("token")
    expect(socket.protocols).toEqual(["xagent-auth-v1", "xagent-auth-token.login-token"])
    expect(JSON.parse(socket.send.mock.calls[0][0])).toMatchObject({ model_id: "live", sample_rate: 16000, vad: true })
    for (let frame = 0; frame < 4; frame++) captures[0].port.onmessage!({ data: new Float32Array(1024).fill(0.1) })
    expect(callbacks.activity).not.toHaveBeenCalled()
    detectors[0].options.speech(true)
    expect(callbacks.activity).toHaveBeenCalledWith(true)
    expect(socket.send.mock.calls[1][0]).toBeInstanceOf(ArrayBuffer)
    call.close(); call.close()
    expect(socket.close).toHaveBeenCalledOnce()
    expect(stopTrack).toHaveBeenCalledOnce()
    expect(closeContext).toHaveBeenCalledOnce()
    expect(detectors[0].destroy).toHaveBeenCalledOnce()
  })

  it("stops a microphone granted after the user has already left", async () => {
    let grant!: (media: MediaStream) => void
    vi.mocked(navigator.mediaDevices.getUserMedia).mockImplementation(() => new Promise<MediaStream>(resolve => { grant = resolve }))
    const call = new VoiceAudio(options())
    const opening = call.start()
    await Promise.resolve()
    call.close()
    grant({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream)
    await opening
    expect(stopTrack).toHaveBeenCalledOnce()
    expect(Socket.instances).toHaveLength(0)
  })

  it("plays the first PCM chunk before completion and aborts the fetch and scheduled audio on interruption", async () => {
    const call = new VoiceAudio({ ...options(), voice: "selected-voice" })
    await start(call)
    let stream!: ReadableStreamDefaultController<Uint8Array>
    let signal!: AbortSignal
    const body = new ReadableStream<Uint8Array>({ start(controller) { stream = controller } })
    vi.stubGlobal("fetch", vi.fn((_url: string, request: RequestInit) => {
      signal = request.signal as AbortSignal
      signal.addEventListener("abort", () => stream.error(new DOMException("Aborted", "AbortError")), { once: true })
      return Promise.resolve(new Response(body, { headers: { "X-Audio-Format": "pcm", "X-Audio-Sample-Rate": "24000" } }))
    }))
    const playing = call.speak("Hello.")
    const settled = playing.catch(error => error)
    stream.enqueue(new Uint8Array([0, 0, 1, 0]))
    for (let step = 0; step < 30 && !playback.length; step++) await Promise.resolve()
    expect(playback[0].start).toHaveBeenCalledOnce()
    expect(signal.aborted).toBe(false)
    call.stopAudio()
    expect(signal.aborted).toBe(true)
    expect(playback[0].stop).toHaveBeenCalledOnce()
    expect((await settled).name).toBe("AbortError")
    expect(vi.mocked(fetch)).toHaveBeenCalledOnce()
    expect(JSON.parse(vi.mocked(fetch).mock.calls[0][1]!.body as string)).toMatchObject({ model_id: "speaker", voice: "selected-voice" })
    call.close()
  })
})

it.each([undefined, 0.5])("only submits an explicitly supported VAD timing override (%s)", async threshold => {
  const call = new VoiceAudio({ ...options(), vadSilenceThreshold: threshold })
  const socket = await start(call)
  const payload = JSON.parse(socket.send.mock.calls[0][0])
  expect(payload.vad_silence_threshold_secs).toBe(threshold)
  if (threshold === undefined) expect(payload).not.toHaveProperty("vad_silence_threshold_secs")
  call.close()
})