The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
import { hermesApiAs, type OwnerScope, ownerScoped, type ResolvedOwner } from '@/api/client'
import { getApiRequestConnection, getApiRequestProfile, hermesApi } from '@/hermes'
/**
* Client-direct voice: call the active profile's STT/TTS providers straight
* from the desktop, cutting the audio relay hop through the gateway.
*
* The gateway stays the single source of truth for WHICH provider and WHICH
* credentials to use — `GET /api/audio/voice-config` returns the same
* resolution the gateway's own relay endpoints would apply (see
* `tools/voice_client_config.py`). This module only executes the provider
* call locally. Direction of travel:
*
* mic → provider (audio up, once) → text → gateway (STT)
* gateway → text (already streaming) → provider → speaker (TTS)
*
* Keys live in renderer MEMORY only — never persisted, never logged. A
* provider that can only run on the gateway host resolves to
* `{mode:'relay'}` and callers fall back to the existing relay endpoints.
*/
export interface DirectSttConfig {
mode: 'direct'
wire: 'elevenlabs-stt' | 'openai-multipart' | 'xai-stt'
provider: string
base_url: string
api_key: string
model: null | string
language: null | string
/** Seconds the gateway allows one transcription request (`stt.openai.timeout`); absent on older backends. */
timeout_s?: null | number
/** Silence-hallucination contract the relay path applies (`is_whisper_hallucination`);
* absent on older backends — a matching transcript is still returned as-is then. */
hallucination_filter?: null | { phrases: string[]; repeat_regex: string }
}
export interface DirectTtsConfig {
mode: 'direct'
wire: 'elevenlabs-tts' | 'openai-speech'
provider: string
base_url: string
api_key: string
model: null | string
voice: null | string
speed: null | number
/** tts.streaming.min_len — shortest first sentence (chars) cut on its own; absent on older backends. */
min_len?: null | number
/** Optional tts.openai fields the server forwards verbatim (lang_code, consent_attestation). */
extra_body?: Record<string, unknown>
}
interface RelayConfig {
mode: 'relay'
reason?: string
}
export interface VoiceClientConfig {
/** `streaming`: the host serves live dictation over /api/audio/transcribe-stream (stt.streaming). */
stt: (DirectSttConfig | RelayConfig) & { streaming?: boolean }
tts: DirectTtsConfig | RelayConfig
}
// ---------------------------------------------------------------------------
// Config fetch + cache. Keyed by (connection, profile) so a profile/backend
// switch never reuses another scope's credentials; TTL'd so a config change
// on the gateway propagates within a minute without a per-utterance fetch.
// ---------------------------------------------------------------------------
const CONFIG_TTL_MS = 60_000
// Per-request cap on a direct STT upload; the gateway's stt timeout is not part of the
// client config, so this mirrors its 60s default rather than hanging dictation forever.
const STT_REQUEST_TIMEOUT_MS = 60_000
let cached: { key: string; at: number; config: VoiceClientConfig } | null = null
let inflight: { key: string; promise: Promise<null | VoiceClientConfig> } | null = null
// `owner` is the speaking session's (connection, profile) — a Bot chat runs
// on its own profile, on its own gateway; missing halves → the active scope.
function scopeKey(owner?: OwnerScope): string {
return `${owner?.connectionId || getApiRequestConnection() || 'local'}::${owner?.profile || getApiRequestProfile() || 'default'}`
}
/** Drop cached credentials (used by tests; scope changes rotate the key). */
export function clearVoiceClientConfigCache(): void {
cached = null
inflight = null
}
export async function fetchVoiceClientConfig(owner?: OwnerScope): Promise<null | VoiceClientConfig> {
// hermesApi carries connectionScoped(); profileScoped() adds the profile —
// the same routing every relay audio call uses, so the config comes from
// the backend the user is actually talking to.
return loadVoiceClientConfig(scopeKey(owner), () =>
hermesApi<VoiceConfigResponse>({ ...ownerScoped(owner), path: '/api/audio/voice-config' })
)
}
/** The config for an owner resolved once for a whole voice operation: the
* lookup cannot drift to whatever scope is ambient by the time it runs. */
export async function fetchVoiceClientConfigFor(owner: ResolvedOwner): Promise<null | VoiceClientConfig> {
return loadVoiceClientConfig(`${owner.connectionId || 'local'}::${owner.profile || 'default'}`, () =>
hermesApiAs<VoiceConfigResponse>(owner, { path: '/api/audio/voice-config' })
)
}
type VoiceConfigResponse = { ok: boolean } & VoiceClientConfig
async function loadVoiceClientConfig(
key: string,
fetchConfig: () => Promise<VoiceConfigResponse>
): Promise<null | VoiceClientConfig> {
if (cached && cached.key === key && Date.now() - cached.at < CONFIG_TTL_MS) {
return cached.config
}
if (inflight && inflight.key === key) {
return inflight.promise
}
const promise = (async () => {
try {
const response = await fetchConfig()
if (!response?.ok || !response.stt || !response.tts) {
return null
}
const config: VoiceClientConfig = { stt: response.stt, tts: response.tts }
cached = { key, at: Date.now(), config }
return config
} catch {
// Older backend without the endpoint / transient failure → relay.
return null
} finally {
inflight = null
}
})()
inflight = { key, promise }
return promise
}
// ---------------------------------------------------------------------------
// STT — audio blob → transcript, provider-direct.
// ---------------------------------------------------------------------------
function sttFileName(audio: Blob): string {
const subtype = (audio.type.split(';')[0].split('/')[1] || 'webm').toLowerCase()
return `recording.${subtype === 'mpeg' ? 'mp3' : subtype}`
}
async function providerErrorText(response: Response): Promise<string> {
const body = await response.text().catch(() => '')
try {
const parsed = JSON.parse(body)
const detail = parsed?.error?.message ?? parsed?.detail ?? parsed?.error
if (typeof detail === 'string' && detail) {
return detail
}
} catch {
// fall through to raw body
}
return body.slice(0, 300)
}
/**
* Normalize an OpenAI-compatible transcription HTTP body to spoken text.
*
* Groq and OpenAI honor `response_format=text` and return a bare string.
* Mistral Voxtral ignores that flag and returns JSON
* `{ text, model, usage, ... }` — dumping that object into the Desktop
* composer is the dictation regression (plain speech becomes raw JSON).
*/
export function transcriptFromOpenAiMultipartBody(body: string): string {
const trimmed = String(body || '').trim()
if (!trimmed) {
return ''
}
if (trimmed.startsWith('{')) {
try {
const parsed = JSON.parse(trimmed) as { text?: unknown }
if (typeof parsed?.text === 'string') {
return parsed.text.trim()
}
} catch {
// Not JSON — treat the body as the transcript.
}
}
return trimmed
}
const DEFAULT_STT_TIMEOUT_S = 60
/** Same budget the gateway's own transcription client uses (`stt.openai.timeout`, default 60 s). */
export function sttTimeoutSeconds(stt: Pick<DirectSttConfig, 'timeout_s'>): number {
const value = Number(stt.timeout_s)
return Number.isFinite(value) && value > 0 ? value : DEFAULT_STT_TIMEOUT_S
}
/**
* The relay path's Whisper-silence filter (`is_whisper_hallucination`,
* tools/voice_mode_transcript.py): empty, an exact known hallucination
* (lowercased, trailing `.!` stripped), or repetitive filler like
* "Thank you. Thank you." A client-direct transcript must agree with a
* relayed one instead of submitting "thank you" on silence as a real turn.
* No filter on the config (older backend) → the transcript passes through.
*/
export function isSttSilenceHallucination(
transcript: string,
filter: DirectSttConfig['hallucination_filter']
): boolean {
if (!filter) {
return false
}
const cleaned = transcript.trim().toLowerCase()
if (!cleaned) {
return true
}
// Trailing `.!` only — the relay strips `cleaned.rstrip('.!')`, so an
// internal period (`thank. you`) stays internal and the transcript stays
// a real turn on both paths, never just one.
if (filter.phrases.includes(cleaned.replace(/[.!]+$/, ''))) {
return true
}
try {
return new RegExp(filter.repeat_regex, 'i').test(cleaned)
} catch {
return false
}
}
/**
* `fetch` with the STT deadline. A slow or wedged endpoint otherwise keeps the
* dictation UI in "transcribing" forever — the browser applies no timeout of
* its own to a POST that never answers.
*/
async function sttFetch(stt: DirectSttConfig, url: string, init: RequestInit): Promise<Response> {
const seconds = sttTimeoutSeconds(stt)
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), seconds * 1000)
try {
return await fetch(url, { ...init, signal: controller.signal })
} catch (error) {
if (controller.signal.aborted) {
throw new Error(`Transcription timed out after ${seconds}s (${stt.provider} did not answer)`)
}
throw error
} finally {
clearTimeout(timer)
}
}
/** Multipart body for xAI `POST /v1/stt`. */
function xaiSttForm(audio: Blob, stt: DirectSttConfig): FormData {
const form = new FormData()
form.set('file', audio, sttFileName(audio))
if (stt.model) {
form.set('model', stt.model)
}
// xAI rejects format=true without a language (HTTP 400), so auto-detect drops the flag.
if (stt.language) {
form.set('language', stt.language)
form.set('format', 'true')
}
return form
}
/**
* Transcribe provider-direct. Returns the transcript ('' = silence), or null
* when the profile's provider isn't client-callable — the caller relays.
* Provider REJECTIONS throw: the configured provider said no, and silently
* re-running the same request through the gateway would just fail again
* slower and hide the real error. `owner` pins the provider config to the
* recording's owner (resolved when the mic opened); omitted → the active scope.
*/
export async function transcribeAudioClientDirect(audio: Blob, owner?: ResolvedOwner): Promise<null | string> {
const config = await (owner ? fetchVoiceClientConfigFor(owner) : fetchVoiceClientConfig())
const stt = config?.stt
if (!stt || stt.mode !== 'direct') {
return null
}
if (stt.wire === 'openai-multipart') {
const form = new FormData()
form.set('file', audio, sttFileName(audio))
if (stt.model) {
form.set('model', stt.model)
}
form.set('response_format', 'text')
if (stt.language) {
form.set('language', stt.language)
}
const response = await sttFetch(stt, `${stt.base_url.replace(/\/+$/, '')}/audio/transcriptions`, {
method: 'POST',
headers: { Authorization: `Bearer ${stt.api_key}` },
body: form,
signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS)
})
if (!response.ok) {
throw new Error(`${stt.provider} STT error (HTTP ${response.status}): ${await providerErrorText(response)}`)
}
const transcript = transcriptFromOpenAiMultipartBody(await response.text())
// Silence hallucination ("thank you" on quiet audio): treat as silence,
// exactly like the relay endpoint, instead of submitting a phantom turn.
return isSttSilenceHallucination(transcript, stt.hallucination_filter) ? '' : transcript
}
if (stt.wire === 'xai-stt') {
const response = await sttFetch(stt, `${stt.base_url.replace(/\/+$/, '')}/stt`, {
method: 'POST',
headers: { Authorization: `Bearer ${stt.api_key}` },
body: xaiSttForm(audio, stt),
signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS)
})
if (!response.ok) {
throw new Error(`xAI STT error (HTTP ${response.status}): ${await providerErrorText(response)}`)
}
const result = (await response.json()) as { text?: string }
const transcript = (result.text || '').trim()
// Silence hallucination: same contract as the relay endpoint.
return isSttSilenceHallucination(transcript, stt.hallucination_filter) ? '' : transcript
}
if (stt.wire === 'elevenlabs-stt') {
const form = new FormData()
form.set('file', audio, sttFileName(audio))
if (stt.model) {
form.set('model_id', stt.model)
}
if (stt.language) {
form.set('language_code', stt.language)
}
const response = await sttFetch(stt, `${stt.base_url.replace(/\/+$/, '')}/speech-to-text`, {
method: 'POST',
headers: { 'xi-api-key': stt.api_key },
body: form,
signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS)
})
if (!response.ok) {
throw new Error(`ElevenLabs STT error (HTTP ${response.status}): ${await providerErrorText(response)}`)
}
const result = (await response.json()) as { text?: string }
const transcript = (result.text || '').trim()
// Silence hallucination: same contract as the relay endpoint.
return isSttSilenceHallucination(transcript, stt.hallucination_filter) ? '' : transcript
}
return null
}
// ---------------------------------------------------------------------------
// TTS — text → audio bytes, provider-direct. One call per sentence/segment;
// the playback queue in voice-playback.ts owns ordering and barge-in.
// ---------------------------------------------------------------------------
/** Resolve the profile's TTS config when it is client-callable, else null. */
export async function directTtsConfig(owner?: OwnerScope): Promise<DirectTtsConfig | null> {
const config = await fetchVoiceClientConfig(owner)
return config?.tts && config.tts.mode === 'direct' ? config.tts : null
}
/** Synthesize one text segment to audio bytes (mp3). Throws on provider rejection. */
export async function synthesizeSpeechClientDirect(tts: DirectTtsConfig, text: string): Promise<ArrayBuffer> {
if (tts.wire === 'openai-speech') {
const body: Record<string, unknown> = {
...(tts.extra_body ?? {}),
model: tts.model,
voice: tts.voice,
input: text,
response_format: 'mp3'
}
if (tts.speed && tts.speed !== 1) {
body.speed = tts.speed
}
const response = await fetch(`${tts.base_url.replace(/\/+$/, '')}/audio/speech`, {
method: 'POST',
headers: {
Authorization: `Bearer ${tts.api_key}`,
'Content-Type': 'application/json'
},
body: JSON.stringify(body)
})
if (!response.ok) {
throw new Error(`${tts.provider} TTS error (HTTP ${response.status}): ${await providerErrorText(response)}`)
}
return response.arrayBuffer()
}
if (tts.wire === 'elevenlabs-tts') {
const response = await fetch(
`${tts.base_url.replace(/\/+$/, '')}/text-to-speech/${encodeURIComponent(tts.voice || '')}`,
{
method: 'POST',
headers: {
'xi-api-key': tts.api_key,
'Content-Type': 'application/json',
Accept: 'audio/mpeg'
},
body: JSON.stringify({ text, model_id: tts.model })
}
)
if (!response.ok) {
throw new Error(`ElevenLabs TTS error (HTTP ${response.status}): ${await providerErrorText(response)}`)
}
return response.arrayBuffer()
}
throw new Error(`Unknown TTS wire: ${(tts as { wire?: string }).wire}`)
}