Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

14.0 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from "vitest"
import { encodePcm16, VoiceResampler, VoiceTurns, speechText } from "./voice-conversation"

afterEach(() => vi.useRealTimers())

function coordinator() {
  const actions = {
    send: vi.fn().mockResolvedValue(undefined), interrupt: vi.fn(), stopAudio: vi.fn(),
    speak: vi.fn().mockResolvedValue(undefined), phase: vi.fn(), caption: vi.fn(), error: vi.fn(), playbackError: vi.fn(),
  }
  return { actions, turns: new VoiceTurns(actions) }
}

describe("voice turn ownership", () => {
  it("keeps the answer audible through local VAD activity without a transcript", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.updateBusy(true)
    turns.reply({ id: "answer", text: "First sentence. More", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    const stops = actions.stopAudio.mock.calls.length
    turns.activity(true); turns.activity(false)
    turns.reply({ id: "answer", text: "First sentence. More words. Tail", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.stopAudio).toHaveBeenCalledTimes(stops)
    expect(actions.interrupt).not.toHaveBeenCalled()
    expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First sentence. ", "More words. "])
    turns.stop()
  })

  it("accepts server-confirmed speech even when local VAD misses it", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.transcript("partial", "Quiet speech", 0)
    turns.transcript("final", "Quiet speech", 0)
    await vi.advanceTimersByTimeAsync(300)
    expect(actions.send).toHaveBeenCalledExactlyOnceWith("Quiet speech")
    turns.stop()
  })

  it("waits for an in-progress utterance before flushing a previously queued segment", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.updateBusy(true)
    turns.activity(true); turns.transcript("final", "First", 0); turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.activity(true)
    turns.updateBusy(false)
    expect(actions.send).not.toHaveBeenCalled()
    turns.transcript("partial", "Second", 1)
    turns.activity(false)
    turns.updateBusy(false)
    expect(actions.send).not.toHaveBeenCalled()
    turns.transcript("final", "Second", 1)
    await vi.advanceTimersByTimeAsync(300)
    expect(actions.send).toHaveBeenCalledExactlyOnceWith("First Second")
    turns.stop()
  })

  it("holds incomplete ASCII sentence ends and streams CJK punctuation", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.reply({ id: "answer", text: "3.", complete: false })
    expect(actions.speak).not.toHaveBeenCalled()
    turns.reply({ id: "answer", text: "3.14 is pi. More", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak).toHaveBeenLastCalledWith("3.14 is pi. ")
    turns.reply({ id: "cjk", text: "你好。还有", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak).toHaveBeenLastCalledWith("你好。")
    turns.stop()
  })

  it.each([false, true])("submits committed speech after a false-start partial clears (queued=%s)", async queued => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.updateBusy(queued)
    turns.transcript("final", "Open settings", 0)
    if (queued) await vi.advanceTimersByTimeAsync(300)
    turns.transcript("partial", "uh", 1)
    turns.updateBusy(false)
    await vi.advanceTimersByTimeAsync(500)
    expect(actions.send).not.toHaveBeenCalled()
    turns.transcript("partial", "", 1)
    expect(actions.caption).toHaveBeenLastCalledWith("Open settings")
    await vi.advanceTimersByTimeAsync(300)
    expect(actions.send).toHaveBeenCalledExactlyOnceWith("Open settings")
    turns.stop()
  })

  it("continues after a consumed markdown prefix changes without replaying it", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.reply({ id: "answer", text: "[Read this. More", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    turns.reply({ id: "answer", text: "Read this. More words. Tail", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["Read this. ", "More words. "])
    turns.stop()
  })

  it.each(["plain", "bold", "quote"])("speaks streamed %s list items without synthesizing their numbers separately", async style => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.transcript("final", "How do I add a vehicle?", 0)
    await vi.advanceTimersByTimeAsync(300)
    const marker = (number: number) => style === "bold" ? `**${number}.** ` : style === "quote" ? `> ${number}. ` : `${number}. `
    const prefixes = [
      `To add a vehicle in Detrack:\n\n${marker(1)}`,
      `In your admin dashboard, open the **Vehicles** tab.\n${marker(2)}`,
      `Choose the option to add a vehicle and fill in the vehicle details.\n${marker(3)}`,
      "Click **Save** to add it to your account.",
    ]
    let text = ""
    for (const [index, prefix] of prefixes.entries()) {
      text += prefix
      turns.reply({ id: "answer", text, complete: index === prefixes.length - 1 })
      await vi.advanceTimersByTimeAsync(0)
    }
    expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual([
      "To add a vehicle in Detrack: In your admin dashboard, open the Vehicles tab.",
      "Choose the option to add a vehicle and fill in the vehicle details.",
      "Click Save to add it to your account.",
    ])
    turns.stop()
  })

  it.each(["plain", "bold", "quote"])("streams unpunctuated %s list items before completion", async style => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.transcript("final", "What are the steps?", 0)
    await vi.advanceTimersByTimeAsync(300)
    const marker = (number: number) => style === "bold" ? `**${number}.** ` : style === "quote" ? `> ${number}. ` : `${number}. `
    let text = `Steps:\n${marker(1)}Open settings\n${marker(2)}Click Save`
    turns.reply({ id: "answer", text, complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual(["Steps: Open settings."])
    text += `\n${marker(3)}Done`
    turns.reply({ id: "answer", text, complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual(["Steps: Open settings.", "Click Save."])
    turns.reply({ id: "answer", text, complete: true })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual(["Steps: Open settings.", "Click Save.", "Done"])
    turns.stop()
  })

  it("reports one failed sentence and continues with the next without replay", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    actions.speak.mockRejectedValueOnce(new Error("temporary synthesis failure"))
    turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.reply({ id: "answer", text: "First. Second.", complete: true })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First. ", "Second."])
    expect(actions.playbackError).toHaveBeenCalledOnce()
    expect(actions.error).not.toHaveBeenCalled()
    turns.stop()
  })

  it("coalesces VAD segments, never sends partials, and ignores repeated finals", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.activity(true)
    turns.transcript("partial", "Hello", 0)
    await vi.advanceTimersByTimeAsync(500)
    expect(actions.send).not.toHaveBeenCalled()
    turns.transcript("final", "Hello.", 0)
    turns.transcript("final", "How are you?", 1)
    turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    expect(actions.send).toHaveBeenCalledExactlyOnceWith("Hello. How are you?")
    turns.updateBusy(true)
    turns.transcript("final", "How are you?", 1)
    expect(actions.interrupt).not.toHaveBeenCalled()
    turns.stop()
  })

  it("automatically cuts off speech, interrupts confirmed generation, and rejects late old replies", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    let abortAudio: (() => void) | undefined
    actions.speak.mockImplementationOnce(() => new Promise<void>((_, reject) => {
      abortAudio = () => reject(new DOMException("Aborted", "AbortError"))
    }))
    actions.stopAudio.mockImplementation(() => abortAudio?.())
    turns.activity(true)
    turns.transcript("final", "First question.", 0)
    turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.updateBusy(true)
    turns.reply({ id: "old", text: "First sentence. More old speech.", complete: true })
    expect(actions.speak).toHaveBeenCalledExactlyOnceWith("First sentence. ")
    const previousStops = actions.stopAudio.mock.calls.length
    turns.activity(true)
    expect(actions.stopAudio).toHaveBeenCalledTimes(previousStops)
    expect(actions.interrupt).not.toHaveBeenCalled()
    turns.transcript("partial", "Actually", 1)
    expect(actions.stopAudio).toHaveBeenCalledTimes(previousStops + 1)
    turns.transcript("partial", "Actually change that", 1)
    expect(actions.interrupt).toHaveBeenCalledOnce()
    turns.reply({ id: "old", text: "First sentence. More old speech. Too late.", complete: true })
    turns.transcript("final", "Actually change that.", 1)
    turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    expect(actions.send).toHaveBeenCalledTimes(1)
    turns.updateBusy(false) // The old run has acknowledged its interruption.
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.send).toHaveBeenLastCalledWith("Actually change that.")
    turns.reply({ id: "new", text: "New answer.", complete: true })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First sentence. ", "New answer."])
    expect(actions.error).not.toHaveBeenCalled()
    turns.stop()
  })

  it("begins speaking completed sentences before the whole answer ends, without replaying them", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.activity(true)
    turns.transcript("final", "Hi.", 0)
    turns.activity(false)
    await vi.advanceTimersByTimeAsync(300)
    turns.reply({ id: "answer", text: "First sentence. Sec", complete: false })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak).toHaveBeenCalledExactlyOnceWith("First sentence. ")
    turns.reply({ id: "answer", text: "First sentence. Second sentence.", complete: true })
    turns.reply({ id: "answer", text: "First sentence. Second sentence.", complete: true })
    await vi.advanceTimersByTimeAsync(0)
    expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First sentence. ", "Second sentence."])
    turns.stop()
  })

  it("does not send muted, unfinished, or ended-call transcripts", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.activity(true)
    turns.transcript("final", "Unfinished", 0)
    turns.activity(false)
    turns.mute(true)
    turns.transcript("final", "Muted", 1)
    await vi.advanceTimersByTimeAsync(500)
    expect(actions.send).not.toHaveBeenCalled()
    turns.mute(false)
    turns.activity(true)
    turns.transcript("final", "After unmute", 2)
    turns.stop()
    await vi.advanceTimersByTimeAsync(500)
    turns.transcript("final", "After ending", 3)
    turns.reply({ id: "late", text: "Old reply.", complete: true })
    expect(actions.send).not.toHaveBeenCalled()
    expect(actions.speak).not.toHaveBeenCalled()
  })

  it("retains an interruption that arrives before the submitted task starts", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    actions.interrupt.mockReturnValue(false)
    turns.updateBusy(true)
    turns.activity(true)
    turns.transcript("partial", "Wait", 0)
    expect(actions.interrupt).toHaveBeenCalledTimes(1)
    actions.interrupt.mockReturnValue(true)
    turns.updateBusy(true)
    expect(actions.interrupt).toHaveBeenCalledTimes(2)
    turns.updateBusy(true)
    expect(actions.interrupt).toHaveBeenCalledTimes(2)
    turns.stop()
  })
})

describe("audio format and activity", () => {
  it.each([44100, 48000, 16000])("preserves duration across %i Hz worklet block boundaries", rate => {
    const input = new Float32Array(rate).fill(0.5)
    const encoder = new VoiceResampler(rate)
    const bytes: number[] = []
    for (let offset = 0; offset < input.length; offset += 1024) bytes.push(...new Uint8Array(encodePcm16(encoder.process(input.slice(offset, offset + 1024)))))
    expect(bytes).toHaveLength(32000)
    const view = new DataView(Uint8Array.from(bytes).buffer)
    expect(view.getInt16(0, true)).toBe(16384)
    expect(view.getInt16(31998, true)).toBe(16384)
  })

  it("does not interrupt or submit empty server transcripts", async () => {
    vi.useFakeTimers()
    const { turns, actions } = coordinator()
    turns.updateBusy(true)
    turns.transcript("partial", "  ", 0)
    turns.transcript("final", "", 0)
    await vi.advanceTimersByTimeAsync(500)
    expect(actions.interrupt).not.toHaveBeenCalled()
    expect(actions.stopAudio).not.toHaveBeenCalled()
    expect(actions.send).not.toHaveBeenCalled()
    turns.stop()
  })

  it("reads final answer text while omitting code blocks, image URLs and markdown", () => {
    expect(speechText('**Hello** [there](https://example.com). ```python\nsecret()\n```')).toBe("Hello there.")
    expect(speechText('{"action":"final_answer","action_input":"Hello."}')).toBe("Hello.")
  })
})