Enterprise agent platform for building, deploying, and operating AI teammates.
import { afterEach, describe, expect, it, vi } from "vitest"
import { encodePcm16, VoiceResampler, VoiceTurns, speechText } from "./voice-conversation"
afterEach(() => vi.useRealTimers())
function coordinator() {
const actions = {
send: vi.fn().mockResolvedValue(undefined), interrupt: vi.fn(), stopAudio: vi.fn(),
speak: vi.fn().mockResolvedValue(undefined), phase: vi.fn(), caption: vi.fn(), error: vi.fn(), playbackError: vi.fn(),
}
return { actions, turns: new VoiceTurns(actions) }
}
describe("voice turn ownership", () => {
it("keeps the answer audible through local VAD activity without a transcript", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.updateBusy(true)
turns.reply({ id: "answer", text: "First sentence. More", complete: false })
await vi.advanceTimersByTimeAsync(0)
const stops = actions.stopAudio.mock.calls.length
turns.activity(true); turns.activity(false)
turns.reply({ id: "answer", text: "First sentence. More words. Tail", complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.stopAudio).toHaveBeenCalledTimes(stops)
expect(actions.interrupt).not.toHaveBeenCalled()
expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First sentence. ", "More words. "])
turns.stop()
})
it("accepts server-confirmed speech even when local VAD misses it", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.transcript("partial", "Quiet speech", 0)
turns.transcript("final", "Quiet speech", 0)
await vi.advanceTimersByTimeAsync(300)
expect(actions.send).toHaveBeenCalledExactlyOnceWith("Quiet speech")
turns.stop()
})
it("waits for an in-progress utterance before flushing a previously queued segment", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.updateBusy(true)
turns.activity(true); turns.transcript("final", "First", 0); turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.activity(true)
turns.updateBusy(false)
expect(actions.send).not.toHaveBeenCalled()
turns.transcript("partial", "Second", 1)
turns.activity(false)
turns.updateBusy(false)
expect(actions.send).not.toHaveBeenCalled()
turns.transcript("final", "Second", 1)
await vi.advanceTimersByTimeAsync(300)
expect(actions.send).toHaveBeenCalledExactlyOnceWith("First Second")
turns.stop()
})
it("holds incomplete ASCII sentence ends and streams CJK punctuation", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.reply({ id: "answer", text: "3.", complete: false })
expect(actions.speak).not.toHaveBeenCalled()
turns.reply({ id: "answer", text: "3.14 is pi. More", complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak).toHaveBeenLastCalledWith("3.14 is pi. ")
turns.reply({ id: "cjk", text: "你好。还有", complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak).toHaveBeenLastCalledWith("你好。")
turns.stop()
})
it.each([false, true])("submits committed speech after a false-start partial clears (queued=%s)", async queued => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.updateBusy(queued)
turns.transcript("final", "Open settings", 0)
if (queued) await vi.advanceTimersByTimeAsync(300)
turns.transcript("partial", "uh", 1)
turns.updateBusy(false)
await vi.advanceTimersByTimeAsync(500)
expect(actions.send).not.toHaveBeenCalled()
turns.transcript("partial", "", 1)
expect(actions.caption).toHaveBeenLastCalledWith("Open settings")
await vi.advanceTimersByTimeAsync(300)
expect(actions.send).toHaveBeenCalledExactlyOnceWith("Open settings")
turns.stop()
})
it("continues after a consumed markdown prefix changes without replaying it", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.reply({ id: "answer", text: "[Read this. More", complete: false })
await vi.advanceTimersByTimeAsync(0)
turns.reply({ id: "answer", text: "Read this. More words. Tail", complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["Read this. ", "More words. "])
turns.stop()
})
it.each(["plain", "bold", "quote"])("speaks streamed %s list items without synthesizing their numbers separately", async style => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.transcript("final", "How do I add a vehicle?", 0)
await vi.advanceTimersByTimeAsync(300)
const marker = (number: number) => style === "bold" ? `**${number}.** ` : style === "quote" ? `> ${number}. ` : `${number}. `
const prefixes = [
`To add a vehicle in Detrack:\n\n${marker(1)}`,
`In your admin dashboard, open the **Vehicles** tab.\n${marker(2)}`,
`Choose the option to add a vehicle and fill in the vehicle details.\n${marker(3)}`,
"Click **Save** to add it to your account.",
]
let text = ""
for (const [index, prefix] of prefixes.entries()) {
text += prefix
turns.reply({ id: "answer", text, complete: index === prefixes.length - 1 })
await vi.advanceTimersByTimeAsync(0)
}
expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual([
"To add a vehicle in Detrack: In your admin dashboard, open the Vehicles tab.",
"Choose the option to add a vehicle and fill in the vehicle details.",
"Click Save to add it to your account.",
])
turns.stop()
})
it.each(["plain", "bold", "quote"])("streams unpunctuated %s list items before completion", async style => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.transcript("final", "What are the steps?", 0)
await vi.advanceTimersByTimeAsync(300)
const marker = (number: number) => style === "bold" ? `**${number}.** ` : style === "quote" ? `> ${number}. ` : `${number}. `
let text = `Steps:\n${marker(1)}Open settings\n${marker(2)}Click Save`
turns.reply({ id: "answer", text, complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual(["Steps: Open settings."])
text += `\n${marker(3)}Done`
turns.reply({ id: "answer", text, complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual(["Steps: Open settings.", "Click Save."])
turns.reply({ id: "answer", text, complete: true })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([value]) => value.trim())).toEqual(["Steps: Open settings.", "Click Save.", "Done"])
turns.stop()
})
it("reports one failed sentence and continues with the next without replay", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
actions.speak.mockRejectedValueOnce(new Error("temporary synthesis failure"))
turns.activity(true); turns.transcript("final", "Hello", 0); turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.reply({ id: "answer", text: "First. Second.", complete: true })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First. ", "Second."])
expect(actions.playbackError).toHaveBeenCalledOnce()
expect(actions.error).not.toHaveBeenCalled()
turns.stop()
})
it("coalesces VAD segments, never sends partials, and ignores repeated finals", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.activity(true)
turns.transcript("partial", "Hello", 0)
await vi.advanceTimersByTimeAsync(500)
expect(actions.send).not.toHaveBeenCalled()
turns.transcript("final", "Hello.", 0)
turns.transcript("final", "How are you?", 1)
turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
expect(actions.send).toHaveBeenCalledExactlyOnceWith("Hello. How are you?")
turns.updateBusy(true)
turns.transcript("final", "How are you?", 1)
expect(actions.interrupt).not.toHaveBeenCalled()
turns.stop()
})
it("automatically cuts off speech, interrupts confirmed generation, and rejects late old replies", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
let abortAudio: (() => void) | undefined
actions.speak.mockImplementationOnce(() => new Promise<void>((_, reject) => {
abortAudio = () => reject(new DOMException("Aborted", "AbortError"))
}))
actions.stopAudio.mockImplementation(() => abortAudio?.())
turns.activity(true)
turns.transcript("final", "First question.", 0)
turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.updateBusy(true)
turns.reply({ id: "old", text: "First sentence. More old speech.", complete: true })
expect(actions.speak).toHaveBeenCalledExactlyOnceWith("First sentence. ")
const previousStops = actions.stopAudio.mock.calls.length
turns.activity(true)
expect(actions.stopAudio).toHaveBeenCalledTimes(previousStops)
expect(actions.interrupt).not.toHaveBeenCalled()
turns.transcript("partial", "Actually", 1)
expect(actions.stopAudio).toHaveBeenCalledTimes(previousStops + 1)
turns.transcript("partial", "Actually change that", 1)
expect(actions.interrupt).toHaveBeenCalledOnce()
turns.reply({ id: "old", text: "First sentence. More old speech. Too late.", complete: true })
turns.transcript("final", "Actually change that.", 1)
turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
expect(actions.send).toHaveBeenCalledTimes(1)
turns.updateBusy(false) // The old run has acknowledged its interruption.
await vi.advanceTimersByTimeAsync(0)
expect(actions.send).toHaveBeenLastCalledWith("Actually change that.")
turns.reply({ id: "new", text: "New answer.", complete: true })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First sentence. ", "New answer."])
expect(actions.error).not.toHaveBeenCalled()
turns.stop()
})
it("begins speaking completed sentences before the whole answer ends, without replaying them", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.activity(true)
turns.transcript("final", "Hi.", 0)
turns.activity(false)
await vi.advanceTimersByTimeAsync(300)
turns.reply({ id: "answer", text: "First sentence. Sec", complete: false })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak).toHaveBeenCalledExactlyOnceWith("First sentence. ")
turns.reply({ id: "answer", text: "First sentence. Second sentence.", complete: true })
turns.reply({ id: "answer", text: "First sentence. Second sentence.", complete: true })
await vi.advanceTimersByTimeAsync(0)
expect(actions.speak.mock.calls.map(([text]) => text)).toEqual(["First sentence. ", "Second sentence."])
turns.stop()
})
it("does not send muted, unfinished, or ended-call transcripts", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.activity(true)
turns.transcript("final", "Unfinished", 0)
turns.activity(false)
turns.mute(true)
turns.transcript("final", "Muted", 1)
await vi.advanceTimersByTimeAsync(500)
expect(actions.send).not.toHaveBeenCalled()
turns.mute(false)
turns.activity(true)
turns.transcript("final", "After unmute", 2)
turns.stop()
await vi.advanceTimersByTimeAsync(500)
turns.transcript("final", "After ending", 3)
turns.reply({ id: "late", text: "Old reply.", complete: true })
expect(actions.send).not.toHaveBeenCalled()
expect(actions.speak).not.toHaveBeenCalled()
})
it("retains an interruption that arrives before the submitted task starts", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
actions.interrupt.mockReturnValue(false)
turns.updateBusy(true)
turns.activity(true)
turns.transcript("partial", "Wait", 0)
expect(actions.interrupt).toHaveBeenCalledTimes(1)
actions.interrupt.mockReturnValue(true)
turns.updateBusy(true)
expect(actions.interrupt).toHaveBeenCalledTimes(2)
turns.updateBusy(true)
expect(actions.interrupt).toHaveBeenCalledTimes(2)
turns.stop()
})
})
describe("audio format and activity", () => {
it.each([44100, 48000, 16000])("preserves duration across %i Hz worklet block boundaries", rate => {
const input = new Float32Array(rate).fill(0.5)
const encoder = new VoiceResampler(rate)
const bytes: number[] = []
for (let offset = 0; offset < input.length; offset += 1024) bytes.push(...new Uint8Array(encodePcm16(encoder.process(input.slice(offset, offset + 1024)))))
expect(bytes).toHaveLength(32000)
const view = new DataView(Uint8Array.from(bytes).buffer)
expect(view.getInt16(0, true)).toBe(16384)
expect(view.getInt16(31998, true)).toBe(16384)
})
it("does not interrupt or submit empty server transcripts", async () => {
vi.useFakeTimers()
const { turns, actions } = coordinator()
turns.updateBusy(true)
turns.transcript("partial", " ", 0)
turns.transcript("final", "", 0)
await vi.advanceTimersByTimeAsync(500)
expect(actions.interrupt).not.toHaveBeenCalled()
expect(actions.stopAudio).not.toHaveBeenCalled()
expect(actions.send).not.toHaveBeenCalled()
turns.stop()
})
it("reads final answer text while omitting code blocks, image URLs and markdown", () => {
expect(speechText('**Hello** [there](https://example.com). ```python\nsecret()\n```')).toBe("Hello there.")
expect(speechText('{"action":"final_answer","action_input":"Hello."}')).toBe("Hello.")
})
})