The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { setApiRequestConnection, setApiRequestProfile } from '@/hermes'
import {
clearVoiceClientConfigCache,
type DirectTtsConfig,
fetchVoiceClientConfig,
isSttSilenceHallucination,
synthesizeSpeechClientDirect,
transcribeAudioClientDirect,
transcriptFromOpenAiMultipartBody
} from './voice-client-direct'
const directStt = {
mode: 'direct',
wire: 'openai-multipart',
provider: 'groq',
base_url: 'https://api.groq.com/openai/v1',
api_key: 'gsk_test',
model: 'whisper-large-v3-turbo',
language: 'en'
} as const
const relay = { mode: 'relay', reason: 'local provider' } as const
function mockDesktopApi(response: unknown) {
const api = vi.fn(async (_request: unknown) => response)
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: { api }
})
return api
}
describe('fetchVoiceClientConfig', () => {
beforeEach(() => clearVoiceClientConfigCache())
afterEach(() => {
setApiRequestConnection(null)
setApiRequestProfile(null)
Reflect.deleteProperty(window, 'hermesDesktop')
vi.restoreAllMocks()
})
it('fetches from /api/audio/voice-config with the ambient scope and caches per scope', async () => {
const api = mockDesktopApi({ ok: true, stt: directStt, tts: relay })
setApiRequestConnection('gw-remote')
setApiRequestProfile('research')
const first = await fetchVoiceClientConfig()
const second = await fetchVoiceClientConfig()
expect(first?.stt).toEqual(directStt)
expect(second).toBe(first)
// One fetch — the second call served from the scope cache.
expect(api).toHaveBeenCalledTimes(1)
const request = api.mock.calls[0][0] as { connectionId?: string; path: string; profile?: string }
expect(request.path).toBe('/api/audio/voice-config')
expect(request.profile).toBe('research')
// hermesApi carries the ambient registry connection tag — the config
// must come from the backend the user is talking to.
expect(request.connectionId).toBe('gw-remote')
})
it("a scope switch never reuses another scope's credentials", async () => {
const api = mockDesktopApi({ ok: true, stt: directStt, tts: relay })
setApiRequestProfile('alpha')
await fetchVoiceClientConfig()
setApiRequestProfile('beta')
await fetchVoiceClientConfig()
expect(api).toHaveBeenCalledTimes(2)
})
it('resolves null on an older backend without the endpoint', async () => {
Object.defineProperty(window, 'hermesDesktop', {
configurable: true,
value: { api: vi.fn(async () => Promise.reject(new Error('404'))) }
})
expect(await fetchVoiceClientConfig()).toBeNull()
})
})
describe('transcribeAudioClientDirect', () => {
beforeEach(() => clearVoiceClientConfigCache())
afterEach(() => {
setApiRequestConnection(null)
setApiRequestProfile(null)
Reflect.deleteProperty(window, 'hermesDesktop')
vi.restoreAllMocks()
vi.unstubAllGlobals()
})
it('POSTs multipart to the provider and returns the transcript', async () => {
mockDesktopApi({ ok: true, stt: directStt, tts: relay })
const fetchMock = vi.fn(async () => new Response(' hello world ', { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
const transcript = await transcribeAudioClientDirect(new Blob(['x'], { type: 'audio/webm' }))
expect(transcript).toBe('hello world')
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe('https://api.groq.com/openai/v1/audio/transcriptions')
expect((init.headers as Record<string, string>).Authorization).toBe('Bearer gsk_test')
// A hung provider must not stall dictation forever: every STT upload carries a timeout signal.
expect(init.signal).toBeInstanceOf(AbortSignal)
const form = init.body as FormData
expect(form.get('model')).toBe('whisper-large-v3-turbo')
expect(form.get('language')).toBe('en')
expect(form.get('response_format')).toBe('text')
})
it('unwraps Mistral Voxtral JSON instead of dumping it into the composer', async () => {
mockDesktopApi({
ok: true,
stt: {
...directStt,
provider: 'mistral',
base_url: 'https://api.mistral.ai/v1',
model: 'voxtral-mini-latest',
language: null
},
tts: relay
})
const voxtral = {
model: 'voxtral-mini-latest',
text: 'Hallo, bist du da?',
language: null,
segments: [],
usage: { prompt_audio_seconds: 4, total_tokens: 388 },
finish_reason: null
}
const fetchMock = vi.fn(async () => new Response(JSON.stringify(voxtral), { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
expect(await transcribeAudioClientDirect(new Blob(['x'], { type: 'audio/webm' }))).toBe('Hallo, bist du da?')
})
it('returns null (relay) when the provider is not client-callable', async () => {
mockDesktopApi({ ok: true, stt: relay, tts: relay })
const fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBeNull()
expect(fetchMock).not.toHaveBeenCalled()
})
it('treats a Whisper silence hallucination like the relay path: silence, not a turn (#126708)', async () => {
const filter = {
phrases: ['thank you', 'bye', 'you', 'the end'],
repeat_regex: '^(?:thank you|thanks|bye|you|ok|okay|the end|[.,!\\s])+$'
}
mockDesktopApi({ ok: true, stt: { ...directStt, hallucination_filter: filter }, tts: relay })
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('Thank you.', { status: 200 }))
)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('')
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('OK. OK. OK.', { status: 200 }))
)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('')
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('The end', { status: 200 }))
)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('')
// A real utterance passes through untouched, and an older backend without
// the filter never drops a transcript.
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('Thanks, that fixed it', { status: 200 }))
)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('Thanks, that fixed it')
// Older backend: no hallucination_filter on the config → pass-through.
clearVoiceClientConfigCache()
mockDesktopApi({ ok: true, stt: directStt, tts: relay })
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('Thank you.', { status: 200 }))
)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('Thank you.')
})
it('isSttSilenceHallucination mirrors the relay contract', () => {
const filter = {
phrases: ['thank you', 'bye', 'you'],
repeat_regex: '^(?:thank you|thanks|bye|you|ok|okay|the end|[.,!\\s])+$'
}
// Known hallucination, case/punctuation-insensitive.
expect(isSttSilenceHallucination('Thank you!', filter)).toBe(true)
// Repetitive filler.
expect(isSttSilenceHallucination('ok ok ok', filter)).toBe(true)
// Empty = silence.
expect(isSttSilenceHallucination(' ', filter)).toBe(true)
// A genuine short utterance is NOT a hallucination.
expect(isSttSilenceHallucination('OK, do it', filter)).toBe(false)
// No filter (older backend) → never drop.
expect(isSttSilenceHallucination('Thank you.', null)).toBe(false)
})
it('strips only trailing .! like the relay rstrip — internal punctuation is a real turn', () => {
const filter = {
phrases: ['thank you', 'bye', 'you', 'the end'],
repeat_regex: '^(?:thank you|thanks|bye|you|ok|okay|the end|[.,!\\\\s])+$'
}
// Internal punctuation survives the strip, so `thank. you` is not the
// phrase `thank you` — the relay keeps it as a real turn, and the
// client-direct path must agree (wire parity).
expect(isSttSilenceHallucination('thank. you', filter)).toBe(false)
expect(isSttSilenceHallucination('Than-k you. thank! you', filter)).toBe(false)
// Trailing punctuation is still stripped the way `rstrip('.!')` does.
expect(isSttSilenceHallucination('Thank you.!', filter)).toBe(true)
expect(isSttSilenceHallucination('The end...', filter)).toBe(true)
})
it('surfaces provider rejections instead of silently relaying', async () => {
mockDesktopApi({ ok: true, stt: directStt, tts: relay })
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(JSON.stringify({ error: { message: 'invalid api key' } }), { status: 401 }))
)
await expect(transcribeAudioClientDirect(new Blob(['x']))).rejects.toThrow(/groq STT error.*invalid api key/)
})
it('speaks the xai wire shape', async () => {
mockDesktopApi({
ok: true,
stt: {
...directStt,
wire: 'xai-stt',
provider: 'xai',
base_url: 'https://api.x.ai/v1',
model: 'grok-voice-transcribe-2.0'
},
tts: relay
})
const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'grok heard this' }), { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('grok heard this')
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe('https://api.x.ai/v1/stt')
expect((init.body as FormData).get('format')).toBe('true')
expect((init.body as FormData).get('model')).toBe('grok-voice-transcribe-2.0')
})
it('drops format=true on the xai wire when no language is set (xAI answers HTTP 400 otherwise)', async () => {
mockDesktopApi({
ok: true,
stt: {
...directStt,
wire: 'xai-stt',
provider: 'xai',
base_url: 'https://api.x.ai/v1',
model: 'grok-voice-transcribe-2.0',
language: ''
},
tts: relay
})
const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'auto detected' }), { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('auto detected')
const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect((init.body as FormData).has('format')).toBe(false)
expect((init.body as FormData).has('language')).toBe(false)
})
it('speaks the elevenlabs wire shape with xi-api-key auth', async () => {
mockDesktopApi({
ok: true,
stt: {
...directStt,
wire: 'elevenlabs-stt',
provider: 'elevenlabs',
base_url: 'https://api.elevenlabs.io/v1',
model: 'scribe_v2'
},
tts: relay
})
const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'scribe text' }), { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('scribe text')
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe('https://api.elevenlabs.io/v1/speech-to-text')
expect((init.headers as Record<string, string>)['xi-api-key']).toBe('gsk_test')
expect((init.body as FormData).get('model_id')).toBe('scribe_v2')
})
/** A fetch that only settles when its AbortSignal fires — a wedged STT endpoint. */
function hangingFetch() {
return vi.fn(
(_url: string, init?: RequestInit) =>
new Promise<Response>((_resolve, reject) => {
init?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')))
})
)
}
it('aborts a hanging transcription at the default 60 s instead of transcribing forever', async () => {
vi.useFakeTimers()
try {
mockDesktopApi({ ok: true, stt: directStt, tts: relay })
const fetchMock = hangingFetch()
vi.stubGlobal('fetch', fetchMock)
const pending = transcribeAudioClientDirect(new Blob(['x'], { type: 'audio/webm' }))
const settled = vi.fn()
pending.then(settled, settled)
await vi.advanceTimersByTimeAsync(0)
const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(init.signal).toBeInstanceOf(AbortSignal)
await vi.advanceTimersByTimeAsync(59_000)
expect(settled).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1_000)
await expect(pending).rejects.toThrow(/Transcription timed out after 60s/)
} finally {
vi.useRealTimers()
}
})
it('honours the gateway-resolved stt.openai.timeout for the direct request', async () => {
vi.useFakeTimers()
try {
mockDesktopApi({ ok: true, stt: { ...directStt, timeout_s: 5 }, tts: relay })
vi.stubGlobal('fetch', hangingFetch())
const pending = transcribeAudioClientDirect(new Blob(['x'], { type: 'audio/webm' }))
const settled = vi.fn()
pending.then(settled, settled)
await vi.advanceTimersByTimeAsync(4_900)
expect(settled).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(200)
await expect(pending).rejects.toThrow(/Transcription timed out after 5s/)
} finally {
vi.useRealTimers()
}
})
})
describe('synthesizeSpeechClientDirect', () => {
afterEach(() => {
vi.restoreAllMocks()
vi.unstubAllGlobals()
})
const openaiTts: DirectTtsConfig = {
mode: 'direct',
wire: 'openai-speech',
provider: 'openai',
base_url: 'https://api.openai.com/v1',
api_key: 'sk_tts',
model: 'gpt-4o-mini-tts',
voice: 'nova',
speed: null
}
it('POSTs the openai speech shape and returns audio bytes', async () => {
const bytes = new Uint8Array([1, 2, 3]).buffer
const fetchMock = vi.fn(async () => new Response(bytes, { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
const audio = await synthesizeSpeechClientDirect(
{ ...openaiTts, extra_body: { consent_attestation: 'I own this voice' } },
'Hello there.'
)
expect(new Uint8Array(audio)).toEqual(new Uint8Array([1, 2, 3]))
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe('https://api.openai.com/v1/audio/speech')
const body = JSON.parse(String(init.body)) as Record<string, unknown>
expect(body.model).toBe('gpt-4o-mini-tts')
expect(body.voice).toBe('nova')
expect(body.input).toBe('Hello there.')
expect(body.speed).toBeUndefined()
// Server-resolved tts.openai extras (consent_attestation for cloned voices) reach the wire.
expect(body.consent_attestation).toBe('I own this voice')
})
it('speaks the elevenlabs tts shape with the voice in the path', async () => {
const fetchMock = vi.fn(async () => new Response(new ArrayBuffer(4), { status: 200 }))
vi.stubGlobal('fetch', fetchMock)
await synthesizeSpeechClientDirect(
{
...openaiTts,
wire: 'elevenlabs-tts',
provider: 'elevenlabs',
base_url: 'https://api.elevenlabs.io/v1',
model: 'eleven_turbo_v2',
voice: 'voice123'
},
'Hi.'
)
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe('https://api.elevenlabs.io/v1/text-to-speech/voice123')
expect((init.headers as Record<string, string>)['xi-api-key']).toBe('sk_tts')
})
it('throws on provider rejection', async () => {
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('quota exceeded', { status: 429 }))
)
await expect(synthesizeSpeechClientDirect(openaiTts, 'Hi.')).rejects.toThrow(/openai TTS error.*429/)
})
})
describe('transcriptFromOpenAiMultipartBody', () => {
it('keeps Groq/OpenAI plain-text bodies', () => {
expect(transcriptFromOpenAiMultipartBody(' hello world ')).toBe('hello world')
})
it('extracts .text from a Voxtral JSON envelope', () => {
expect(
transcriptFromOpenAiMultipartBody(
JSON.stringify({
model: 'voxtral-mini-latest',
text: 'Hallo, bist du da?',
language: null,
segments: [],
usage: { prompt_audio_seconds: 4 }
})
)
).toBe('Hallo, bist du da?')
})
it('treats a JSON envelope with empty text as silence', () => {
expect(transcriptFromOpenAiMultipartBody(JSON.stringify({ text: ' ', model: 'voxtral-mini-latest' }))).toBe('')
})
it('leaves a non-envelope JSON object intact', () => {
expect(transcriptFromOpenAiMultipartBody('{"error":"nope"}')).toBe('{"error":"nope"}')
})
})