The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
import { resolveGatewayWsUrl } from '@hermes/shared'
import { openGatewaySocket } from '@/api/canonical-protocol'
import type { OwnerScope } from '@/api/client'
import { getApiRequestConnection, getApiRequestProfile, speakText } from '@/hermes'
import { directTtsConfig, type DirectTtsConfig, synthesizeSpeechClientDirect } from '@/lib/voice-client-direct'
import { RECONNECT_ATTEMPT_TIMEOUT_MS, withTimeout } from '@/lib/with-timeout'
import {
$voicePlayback,
setVoicePlaybackState,
type VoicePlaybackSource,
type VoicePlaybackState
} from '@/store/voice-playback'
import { cutSentences, sanitizeTextForSpeech } from './speech-text'
// Free Edge TTS occasionally hands back audio that never fires `playing`/`ended`
// nor `error` — leaving voice mode stuck "speaking" forever. Reject if playback
// fails to start or stalls mid-stream for this long (rearmed on each progress
// tick, so legitimately long speech is never cut off).
const PLAYBACK_STALL_MS = 15_000
let currentAudio: HTMLAudioElement | null = null
// Every live playback registers its barge-in stop here: streaming sessions
// (kill the WebSocket + AudioContext) and data-URL audio elements (cut
// playback). A single slot cannot represent two overlapping playbacks — the
// overwritten session keeps its socket open, keeps scheduling buffers, and
// its audio resurfaces on a later turn (#91991). Draining the whole set keeps
// stopVoicePlayback() deterministic no matter how playbacks overlap.
const liveStops = new Set<() => void>()
let sequence = 0
let claimTurnKey: string | null = null
let inFlight: { done: Promise<boolean>; turnKey: string } | null = null
// A shared, lazily-created AudioContext used only to nudge the browser's
// autoplay state out of "suspended". A wake-word-started voice turn has no
// preceding user gesture, so the first HTMLAudioElement.play() can be rejected
// with NotAllowedError. resume()-ing a context is the documented way to recover
// once the app is allowed to make sound; on Electron chat windows the
// no-user-gesture-required policy means this is already unlocked, so this is a
// cheap no-op fallback for other surfaces.
let unlockCtx: AudioContext | null = null
async function unlockAutoplay(): Promise<void> {
if (typeof window === 'undefined') {
return
}
const Ctor =
window.AudioContext || (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
if (!Ctor) {
return
}
if (!unlockCtx) {
unlockCtx = new Ctor()
}
if (unlockCtx.state === 'suspended') {
await unlockCtx.resume()
}
}
function currentState(
status: VoicePlaybackState['status'],
options?: VoicePlaybackOptions,
audioElement: HTMLAudioElement | null = null
): VoicePlaybackState {
return {
audioElement,
messageId: options?.messageId ?? null,
sequence,
source: options?.source ?? null,
status
}
}
/** The speaking session's owner: a Bot chat synthesizes with its own
* profile's TTS voice, minted against the Bot's own connection. Omitted
* halves → the active (connection, profile). */
export interface VoicePlaybackOptions extends OwnerScope {
messageId?: string | null
/** Skip the client-direct/stream rungs and POST straight to /api/audio/speak.
* For callers whose stream path (client-direct, else WS relay) already
* answered `fallback` this reply; the relay may not have been probed. */
syncOnly?: boolean
source: VoicePlaybackSource
/** Stable across a live-id rewrite. A second start of this turn must not stop the first. */
turnKey?: string
}
export function stopVoicePlayback() {
inFlight = null
sequence += 1
// Drain EVERY live playback, not just the newest. Each stop is idempotent
// and unregisters itself; the snapshot keeps the iteration safe.
for (const stop of [...liveStops]) {
stop()
}
liveStops.clear()
if (currentAudio) {
currentAudio.pause()
currentAudio.src = ''
currentAudio.load()
currentAudio = null
}
setVoicePlaybackState({
audioElement: null,
messageId: null,
sequence,
source: null,
status: 'idle'
})
}
// ---------------------------------------------------------------------------
// Streaming path — /api/audio/speak-stream WebSocket, raw int16 PCM frames
// scheduled through Web Audio. Speech starts on the provider's first chunk
// instead of after full synthesis + base64 transfer.
// ---------------------------------------------------------------------------
/** Exported for tests: the (connection, profile) routing contract below is
* exactly what broke in the desktop-remote voice report — keep it pinned. */
export async function resolveSpeakStreamUrl(owner?: OwnerScope): Promise<null | string> {
const desktop = window.hermesDesktop
if (!desktop?.getConnection) {
return null
}
try {
// Mint a fresh credential (single-use ticket in OAuth mode) for the
// ACTIVE (connection, profile) backend, then swap the gateway endpoint
// for the PCM one — auth is shared across WS routes. A registry-scoped
// remote MUST resolve through the *For bridges (same seam as
// store/gateway's openSecondary): the bare getConnection/getGatewayWsUrl
// pair answers for the v1 primary backend, which — when a registry
// remote rides over a local install — is the LOCAL machine, so spoken
// replies would synthesize with the local (often unconfigured) TTS
// instead of the profile the user is actually talking to (#90051-adjacent
// desktop-remote voice report, Aug 2026).
const profile = owner?.profile || getApiRequestProfile()
const connectionId = owner?.connectionId || getApiRequestConnection()
// A host whose gateway ticket opens only `/api/ws` mints the stream its own ticket.
if (desktop.getAudioWsUrl) {
return await withTimeout(
desktop.getAudioWsUrl({ connectionId, path: '/api/audio/speak-stream', profile }),
RECONNECT_ATTEMPT_TIMEOUT_MS,
`Timed out minting the speech stream URL for profile "${profile}"`
)
}
// Both awaits below are IPC round-trips into the main process with no
// timeout of their own (#93454) — a wedged main-process round-trip
// otherwise hangs voice mode's "speaking" state forever instead of
// falling back to playSpeechText. Bound the same way
// store/gateway's openSecondary bounds the same *For/plain pair.
const conn =
connectionId && desktop.getConnectionFor
? await withTimeout(
desktop.getConnectionFor({ connectionId, profile }),
RECONNECT_ATTEMPT_TIMEOUT_MS,
`Timed out connecting to profile "${profile}"`
)
: await withTimeout(
desktop.getConnection(profile),
RECONNECT_ATTEMPT_TIMEOUT_MS,
`Timed out connecting to profile "${profile}"`
)
const wsDeps =
connectionId && desktop.getGatewayWsUrlFor
? { getGatewayWsUrl: () => desktop.getGatewayWsUrlFor!({ connectionId, profile }) }
: connectionId
? {}
: desktop
const wsUrl = await withTimeout(
resolveGatewayWsUrl(wsDeps, conn),
RECONNECT_ATTEMPT_TIMEOUT_MS,
`Timed out re-minting the gateway WebSocket URL for profile "${profile}"`
)
const url = new URL(wsUrl)
if (!url.pathname.endsWith('/api/ws')) {
return null
}
url.pathname = url.pathname.replace(/\/api\/ws$/, '/api/audio/speak-stream')
// The backend resolves the TTS provider chain from this profile's
// config/.env (same seam as /api/pty?profile=). A registry-minted URL may
// already carry the BACKEND-namespace profile (sharedRemote scoping, SSH
// remoteProfile aliasing) — never overwrite it with the desktop-side
// routing alias.
if (profile && !url.searchParams.has('profile')) {
url.searchParams.set('profile', profile)
}
return url.toString()
} catch {
return null
}
}
export interface SpeechStreamSession {
/** Feed more reply text as it streams in. Safe after `finish` (no-op). */
append: (text: string) => void
/** Release a sealed bubble's tail without ending the turn (client-direct). */
flush?: () => void
/** No more text coming — resolves `done` once the audio drains. */
finish: () => void
/**
* 'done' — audio fully played (or barged via stopVoicePlayback)
* 'fallback'— no audio ever produced; caller should speak the accumulated
* text through `playSpeechText` instead.
*/
done: Promise<'done' | 'fallback'>
}
// ---------------------------------------------------------------------------
// Client-direct path — synthesize on the DESKTOP with the profile's own TTS
// provider (config + key fetched from the connected gateway). Reply text is
// already streaming to the renderer over the chat socket, so the gateway
// link carries no audio at all: text → provider → speaker, one hop.
// Sentence-cut like the server pipeline; sequential playback; barge-in via
// the same stopVoicePlayback() sequence bump.
// ---------------------------------------------------------------------------
function openClientDirectSpeechSession(tts: DirectTtsConfig, options: VoicePlaybackOptions): SpeechStreamSession {
let buffer = ''
let finished = false
let settled = false
let started = false
const queue: string[] = []
let synthesizing = false
let playing: HTMLAudioElement | null = null
let settle: (value: 'done' | 'fallback') => void = () => undefined
// stopVoicePlayback() → immediate barge-in: settle this session so its
// queued audio never resumes. Registered until settle, so a barge always
// reaches this session even while another playback is live alongside it
// (#91991).
const stop = () => settle(started ? 'done' : 'fallback')
const done = new Promise<'done' | 'fallback'>(resolve => {
settle = value => {
if (settled) {
return
}
settled = true
liveStops.delete(stop)
if (playing) {
playing.pause()
playing.src = ''
playing = null
}
resolve(value)
}
})
liveStops.add(stop)
const pump = async () => {
if (synthesizing || settled) {
return
}
synthesizing = true
try {
while (queue.length > 0 && !settled) {
const sentence = queue.shift()!
let bytes: ArrayBuffer
try {
bytes = await synthesizeSpeechClientDirect(tts, sentence)
} catch {
// Provider rejected mid-reply. Nothing played yet → let the caller
// fall back to the relay with the full text. Mid-playback → treat
// what played as the playback (replaying would stutter).
settle(started ? 'done' : 'fallback')
return
}
if (settled) {
return
}
if (!started) {
started = true
setVoicePlaybackState(currentState('speaking', options))
}
const url = URL.createObjectURL(new Blob([bytes], { type: 'audio/mpeg' }))
try {
await new Promise<void>((resolve, reject) => {
const audio = new Audio(url)
playing = audio
audio.addEventListener('ended', () => resolve(), { once: true })
audio.addEventListener('error', () => reject(new Error('Playback failed')), { once: true })
void audio.play().catch(reject)
})
} catch {
settle(started ? 'done' : 'fallback')
return
} finally {
playing = null
URL.revokeObjectURL(url)
}
}
if (finished && queue.length === 0 && !settled) {
settle(started ? 'done' : 'fallback')
}
} finally {
synthesizing = false
// Deltas that arrived while the last sentence was playing.
if (!settled && queue.length > 0) {
void pump()
} else if (!settled && finished && queue.length === 0) {
settle(started ? 'done' : 'fallback')
}
}
}
const ingest = (flush: boolean) => {
const cut = cutSentences(buffer, flush, tts.min_len)
buffer = cut.rest
if (cut.sentences.length > 0) {
// Sanitize per sentence — same granularity as the server pipeline
// (markdown constructs can span delta boundaries, sentences can't).
for (const sentence of cut.sentences) {
const speakable = sanitizeTextForSpeech(sentence)
if (speakable) {
queue.push(speakable)
}
}
void pump()
} else if (flush && finished && queue.length === 0 && !synthesizing) {
settle(started ? 'done' : 'fallback')
}
}
return {
append: text => {
if (text && !finished && !settled) {
buffer += text
ingest(false)
}
},
flush: () => {
if (!finished && !settled) {
ingest(true)
}
},
finish: () => {
if (!finished && !settled) {
finished = true
ingest(true)
}
},
done
}
}
/**
* Open a live speech session: one WebSocket + one AudioContext for a whole
* reply. Text is appended as LLM deltas arrive; the server cuts sentences and
* streams PCM back while generation continues, so speech overlaps the text
* stream (ChatGPT-style) with no per-sentence connection or synthesis gaps.
*/
function openSpeechStream(wsUrl: string, options: VoicePlaybackOptions): SpeechStreamSession {
const ws = openGatewaySocket(wsUrl)
ws.binaryType = 'arraybuffer'
let context: AudioContext | null = null
let streamRate = 24_000
let nextStartAt = 0
let carry: null | Uint8Array = null
let started = false
let settled = false
let finished = false
const pendingSends: string[] = []
let settle: (value: 'done' | 'fallback') => void = () => undefined
// stopVoicePlayback() → immediate barge-in: kill the socket (the server
// aborts synthesis on disconnect) and the audio context (cuts sound now).
// Registered until settle, so a barge always reaches this session even
// while another playback is live alongside it (#91991).
const stop = () => settle('done')
const done = new Promise<'done' | 'fallback'>(resolve => {
settle = value => {
if (settled) {
return
}
settled = true
liveStops.delete(stop)
try {
ws.close()
} catch {
// already closed
}
void context?.close().catch(() => undefined)
context = null
resolve(value)
}
})
const send = (frame: object) => {
const data = JSON.stringify(frame)
if (ws.readyState === WebSocket.OPEN) {
ws.send(data)
} else if (ws.readyState === WebSocket.CONNECTING) {
pendingSends.push(data)
}
}
liveStops.add(stop)
const finishWhenDrained = () => {
const remainingMs = context ? Math.max(0, nextStartAt - context.currentTime) * 1_000 : 0
window.setTimeout(() => settle('done'), remainingMs + 100)
}
const schedule = (data: ArrayBuffer) => {
if (!context) {
return
}
// Provider chunks are not sample-aligned — carry any odd byte over.
let bytes = new Uint8Array(data)
if (carry) {
const joined = new Uint8Array(carry.length + bytes.length)
joined.set(carry)
joined.set(bytes, carry.length)
bytes = joined
carry = null
}
const usable = bytes.length - (bytes.length % 2)
if (bytes.length !== usable) {
carry = bytes.slice(usable)
}
if (!usable) {
return
}
const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, usable / 2)
const buffer = context.createBuffer(1, pcm.length, streamRate)
const channel = buffer.getChannelData(0)
for (let index = 0; index < pcm.length; index += 1) {
channel[index] = pcm[index] / 32_768
}
const source = context.createBufferSource()
source.buffer = buffer
source.connect(context.destination)
const startAt = Math.max(context.currentTime + 0.05, nextStartAt)
source.start(startAt)
nextStartAt = startAt + buffer.duration
if (!started) {
started = true
setVoicePlaybackState(currentState('speaking', options))
}
}
ws.onopen = () => {
pendingSends.splice(0).forEach(data => ws.send(data))
}
ws.onmessage = event => {
if (typeof event.data !== 'string') {
schedule(event.data as ArrayBuffer)
return
}
let frame: { channels?: number; sample_rate?: number; type?: string }
try {
frame = JSON.parse(event.data) as typeof frame
} catch {
return
}
if (frame.type === 'start') {
streamRate = frame.sample_rate || 24_000
context = new AudioContext()
// Autoplay policy can hand back a suspended context when playback wasn't
// started by a user gesture (e.g. a wake-word-started voice turn). Resume
// it so the first reply is audible instead of silently buffering. Electron
// chat windows also set autoplayPolicy: no-user-gesture-required, but the
// dashboard-embedded surface relies on this resume.
if (context.state === 'suspended') {
void context.resume().catch(() => undefined)
}
nextStartAt = 0
} else if (frame.type === 'end') {
finishWhenDrained()
} else if (frame.type === 'fallback') {
settle(started ? 'done' : 'fallback')
}
}
// A drop before any audio means the endpoint is unavailable (old backend,
// auth, network) → fall back. After audio started, replaying the whole
// message via POST would stutter — treat what played as the playback.
ws.onerror = () => settle(started ? 'done' : 'fallback')
ws.onclose = () => (started ? finishWhenDrained() : settle('fallback'))
return {
// Raw deltas — the server strips markdown/emoji per *sentence*, which is
// the only safe granularity when constructs span delta boundaries.
append: text => {
if (text && !finished && !settled) {
send({ text })
}
},
finish: () => {
if (!finished && !settled) {
finished = true
send({ done: true })
}
},
done
}
}
/**
* Live-speak an in-progress reply: open a session, then `append` deltas and
* `finish` when generation completes. Ladder: client-direct synthesis with
* the profile's own TTS (lowest hops — reply text is already streaming here,
* audio goes provider → speaker without touching the gateway link) → the
* gateway speak-stream WS relay → null (caller falls back to whole-text
* `playSpeechText`).
*/
export async function startSpeechStream(options: VoicePlaybackOptions): Promise<null | SpeechStreamSession> {
const startSequence = sequence
const direct = await directTtsConfig(options).catch(() => null)
if (direct) {
// A stop (barge-in, Stop button, another playback) landed while the
// direct TTS config was resolving — do not resurrect stopped playback.
if (sequence !== startSequence) {
return null
}
stopVoicePlayback()
setVoicePlaybackState(currentState('preparing', options))
const session = openClientDirectSpeechSession(direct, options)
const sessionSequence = sequence
void session.done.then(outcome => {
// A settled session must not reset the state of a NEWER playback —
// only write idle when no other playback started since this one.
if (outcome === 'done' && sessionSequence === sequence) {
setVoicePlaybackState(currentState('idle'))
}
})
return session
}
const wsUrl = await resolveSpeakStreamUrl(options)
if (!wsUrl) {
return null
}
// A stop (barge-in, Stop button, another playback) landed while the
// stream URL was resolving — do not resurrect playback that was already
// stopped. This is the reported overlap (#91991): barge + next-turn open
// crossing the async discovery gap previously orphaned the older session.
if (sequence !== startSequence) {
return null
}
stopVoicePlayback()
setVoicePlaybackState(currentState('preparing', options))
const session = openSpeechStream(wsUrl, options)
const sessionSequence = sequence
void session.done.then(outcome => {
// A settled session must not reset the state of a NEWER playback — only
// write idle when no other playback started since this one was created.
if (outcome === 'done' && sessionSequence === sequence) {
setVoicePlaybackState(currentState('idle'))
}
})
return session
}
/** One-shot playback of complete text over the streaming WS. */
function playSpeechStream(wsUrl: string, text: string, options: VoicePlaybackOptions): Promise<'fallback' | 'played'> {
const session = openSpeechStream(wsUrl, options)
session.append(text)
session.finish()
return session.done.then(outcome => (outcome === 'done' ? 'played' : 'fallback'))
}
async function playSpeechDataUrl(
speakableText: string,
options: VoicePlaybackOptions,
isCurrent: () => boolean
): Promise<boolean> {
const response = await speakText(speakableText, options)
if (!isCurrent()) {
return false
}
const audio = new Audio(response.data_url)
currentAudio = audio
setVoicePlaybackState(currentState('speaking', options, audio))
await new Promise<void>((resolve, reject) => {
let stall: number | null = null
// Registered until cleanup so stopVoicePlayback() reaches this element
// even while a streaming session is live alongside it (#91991).
const stop = () => {
cleanup()
resolve()
}
const cleanup = () => {
if (stall !== null) {
window.clearTimeout(stall)
stall = null
}
audio.removeEventListener('ended', onEnded)
audio.removeEventListener('error', onError)
audio.removeEventListener('timeupdate', armStall)
liveStops.delete(stop)
}
const armStall = () => {
if (stall !== null) {
window.clearTimeout(stall)
}
stall = window.setTimeout(() => {
cleanup()
reject(new Error('Playback stalled'))
}, PLAYBACK_STALL_MS)
}
const onEnded = () => {
cleanup()
resolve()
}
const onError = () => {
cleanup()
reject(new Error('Playback failed'))
}
liveStops.add(stop)
audio.addEventListener('ended', onEnded, { once: true })
audio.addEventListener('error', onError, { once: true })
audio.addEventListener('timeupdate', armStall)
armStall()
// A wake-word-started turn has no user gesture, so the autoplay policy can
// reject the first play() with NotAllowedError. Electron chat windows set
// autoplayPolicy: no-user-gesture-required to prevent this, but retry once
// after resuming a shared AudioContext as a fallback for other surfaces
// (dashboard-embedded) so the first reply isn't silently dropped.
void audio.play().catch(async () => {
try {
await unlockAutoplay()
await audio.play()
} catch {
onError()
}
})
})
if (!isCurrent()) {
return false
}
currentAudio = null
return true
}
export async function playSpeechText(text: string, options: VoicePlaybackOptions): Promise<boolean> {
if (options.turnKey && (claimTurnKey === options.turnKey || inFlight?.turnKey === options.turnKey)) {
return inFlight?.turnKey === options.turnKey ? inFlight.done : Promise.resolve(true)
}
const previousClaim = claimTurnKey
claimTurnKey = options.turnKey ?? null
try {
stopVoicePlayback()
const done = startSpeechText(text, options)
if (options.turnKey) {
inFlight = { done, turnKey: options.turnKey }
void done.finally(() => {
if (inFlight?.done === done) {
inFlight = null
}
})
}
return done
} finally {
if (claimTurnKey === (options.turnKey ?? null)) {
claimTurnKey = previousClaim
}
}
}
async function startSpeechText(text: string, options: VoicePlaybackOptions): Promise<boolean> {
const speakableText = sanitizeTextForSpeech(text)
if (!speakableText) {
return false
}
const ownSequence = sequence
const isCurrent = () => ownSequence === sequence
setVoicePlaybackState(currentState('preparing', options))
try {
// Ladder: client-direct synthesis (profile's own TTS, no gateway audio
// hop) → streaming WS relay → POST data-URL fallback.
const direct = options.syncOnly ? null : await directTtsConfig(options).catch(() => null)
if (direct && isCurrent()) {
const session = openClientDirectSpeechSession(direct, options)
session.append(speakableText)
session.finish()
const outcome = await session.done
if (outcome === 'done') {
if (!isCurrent()) {
return false
}
setVoicePlaybackState(currentState('idle'))
return true
}
}
if (!isCurrent()) {
return false
}
const streamUrl = options.syncOnly ? null : await resolveSpeakStreamUrl(options)
if (streamUrl && isCurrent()) {
const outcome = await playSpeechStream(streamUrl, speakableText, options)
if (outcome === 'played') {
if (!isCurrent()) {
return false
}
setVoicePlaybackState(currentState('idle'))
return true
}
}
if (!isCurrent()) {
return false
}
const played = await playSpeechDataUrl(speakableText, options, isCurrent)
if (played) {
setVoicePlaybackState(currentState('idle'))
}
return played
} catch (error) {
if (isCurrent()) {
currentAudio = null
setVoicePlaybackState(currentState('idle'))
}
throw error
}
}
export function isVoicePlaybackActive() {
return $voicePlayback.get().status !== 'idle'
}
// ---------------------------------------------------------------------------
// Interruption latch — the next prompt.submit carries `interrupted: true` so
// the model knows its spoken reply was cut off (it can react: "rude!").
// Marked by the barge-in paths (VAD, typing over playback); TTL'd so a stale
// barge never annotates an unrelated message minutes later.
// ---------------------------------------------------------------------------
const INTERRUPT_TTL_MS = 120_000
let interruptedAt: null | number = null
export function markVoicePlaybackInterrupted() {
interruptedAt = Date.now()
}
export function takeVoicePlaybackInterrupted(): boolean {
const at = interruptedAt
interruptedAt = null
return at !== null && Date.now() - at < INTERRUPT_TTL_MS
}