Skip to content

The agent that grows with you

15.2 KB
Raw Blame
import { hermesApiAs, type OwnerScope, ownerScoped, type ResolvedOwner } from '@/api/client'
import { getApiRequestConnection, getApiRequestProfile, hermesApi } from '@/hermes'

/**
 * Client-direct voice: call the active profile's STT/TTS providers straight
 * from the desktop, cutting the audio relay hop through the gateway.
 *
 * The gateway stays the single source of truth for WHICH provider and WHICH
 * credentials to use — `GET /api/audio/voice-config` returns the same
 * resolution the gateway's own relay endpoints would apply (see
 * `tools/voice_client_config.py`). This module only executes the provider
 * call locally. Direction of travel:
 *
 *   mic → provider (audio up, once) → text → gateway   (STT)
 *   gateway → text (already streaming) → provider → speaker   (TTS)
 *
 * Keys live in renderer MEMORY only — never persisted, never logged. A
 * provider that can only run on the gateway host resolves to
 * `{mode:'relay'}` and callers fall back to the existing relay endpoints.
 */

export interface DirectSttConfig {
  mode: 'direct'
  wire: 'elevenlabs-stt' | 'openai-multipart' | 'xai-stt'
  provider: string
  base_url: string
  api_key: string
  model: null | string
  language: null | string
  /** Seconds the gateway allows one transcription request (`stt.openai.timeout`); absent on older backends. */
  timeout_s?: null | number
  /** Silence-hallucination contract the relay path applies (`is_whisper_hallucination`);
   *  absent on older backends — a matching transcript is still returned as-is then. */
  hallucination_filter?: null | { phrases: string[]; repeat_regex: string }
}

export interface DirectTtsConfig {
  mode: 'direct'
  wire: 'elevenlabs-tts' | 'openai-speech'
  provider: string
  base_url: string
  api_key: string
  model: null | string
  voice: null | string
  speed: null | number
  /** tts.streaming.min_len — shortest first sentence (chars) cut on its own; absent on older backends. */
  min_len?: null | number
  /** Optional tts.openai fields the server forwards verbatim (lang_code, consent_attestation). */
  extra_body?: Record<string, unknown>
}

interface RelayConfig {
  mode: 'relay'
  reason?: string
}

export interface VoiceClientConfig {
  /** `streaming`: the host serves live dictation over /api/audio/transcribe-stream (stt.streaming). */
  stt: (DirectSttConfig | RelayConfig) & { streaming?: boolean }
  tts: DirectTtsConfig | RelayConfig
}

// ---------------------------------------------------------------------------
// Config fetch + cache. Keyed by (connection, profile) so a profile/backend
// switch never reuses another scope's credentials; TTL'd so a config change
// on the gateway propagates within a minute without a per-utterance fetch.
// ---------------------------------------------------------------------------

const CONFIG_TTL_MS = 60_000
// Per-request cap on a direct STT upload; the gateway's stt timeout is not part of the
// client config, so this mirrors its 60s default rather than hanging dictation forever.
const STT_REQUEST_TIMEOUT_MS = 60_000

let cached: { key: string; at: number; config: VoiceClientConfig } | null = null
let inflight: { key: string; promise: Promise<null | VoiceClientConfig> } | null = null

// `owner` is the speaking session's (connection, profile) — a Bot chat runs
// on its own profile, on its own gateway; missing halves → the active scope.
function scopeKey(owner?: OwnerScope): string {
  return `${owner?.connectionId || getApiRequestConnection() || 'local'}::${owner?.profile || getApiRequestProfile() || 'default'}`
}

/** Drop cached credentials (used by tests; scope changes rotate the key). */
export function clearVoiceClientConfigCache(): void {
  cached = null
  inflight = null
}

export async function fetchVoiceClientConfig(owner?: OwnerScope): Promise<null | VoiceClientConfig> {
  // hermesApi carries connectionScoped(); profileScoped() adds the profile —
  // the same routing every relay audio call uses, so the config comes from
  // the backend the user is actually talking to.
  return loadVoiceClientConfig(scopeKey(owner), () =>
    hermesApi<VoiceConfigResponse>({ ...ownerScoped(owner), path: '/api/audio/voice-config' })
  )
}

/** The config for an owner resolved once for a whole voice operation: the
 *  lookup cannot drift to whatever scope is ambient by the time it runs. */
export async function fetchVoiceClientConfigFor(owner: ResolvedOwner): Promise<null | VoiceClientConfig> {
  return loadVoiceClientConfig(`${owner.connectionId || 'local'}::${owner.profile || 'default'}`, () =>
    hermesApiAs<VoiceConfigResponse>(owner, { path: '/api/audio/voice-config' })
  )
}

type VoiceConfigResponse = { ok: boolean } & VoiceClientConfig

async function loadVoiceClientConfig(
  key: string,
  fetchConfig: () => Promise<VoiceConfigResponse>
): Promise<null | VoiceClientConfig> {
  if (cached && cached.key === key && Date.now() - cached.at < CONFIG_TTL_MS) {
    return cached.config
  }

  if (inflight && inflight.key === key) {
    return inflight.promise
  }

  const promise = (async () => {
    try {
      const response = await fetchConfig()

      if (!response?.ok || !response.stt || !response.tts) {
        return null
      }

      const config: VoiceClientConfig = { stt: response.stt, tts: response.tts }
      cached = { key, at: Date.now(), config }

      return config
    } catch {
      // Older backend without the endpoint / transient failure → relay.
      return null
    } finally {
      inflight = null
    }
  })()

  inflight = { key, promise }

  return promise
}

// ---------------------------------------------------------------------------
// STT — audio blob → transcript, provider-direct.
// ---------------------------------------------------------------------------

function sttFileName(audio: Blob): string {
  const subtype = (audio.type.split(';')[0].split('/')[1] || 'webm').toLowerCase()

  return `recording.${subtype === 'mpeg' ? 'mp3' : subtype}`
}

async function providerErrorText(response: Response): Promise<string> {
  const body = await response.text().catch(() => '')

  try {
    const parsed = JSON.parse(body)
    const detail = parsed?.error?.message ?? parsed?.detail ?? parsed?.error

    if (typeof detail === 'string' && detail) {
      return detail
    }
  } catch {
    // fall through to raw body
  }

  return body.slice(0, 300)
}

/**
 * Normalize an OpenAI-compatible transcription HTTP body to spoken text.
 *
 * Groq and OpenAI honor `response_format=text` and return a bare string.
 * Mistral Voxtral ignores that flag and returns JSON
 * `{ text, model, usage, ... }` — dumping that object into the Desktop
 * composer is the dictation regression (plain speech becomes raw JSON).
 */
export function transcriptFromOpenAiMultipartBody(body: string): string {
  const trimmed = String(body || '').trim()

  if (!trimmed) {
    return ''
  }

  if (trimmed.startsWith('{')) {
    try {
      const parsed = JSON.parse(trimmed) as { text?: unknown }

      if (typeof parsed?.text === 'string') {
        return parsed.text.trim()
      }
    } catch {
      // Not JSON — treat the body as the transcript.
    }
  }

  return trimmed
}

const DEFAULT_STT_TIMEOUT_S = 60

/** Same budget the gateway's own transcription client uses (`stt.openai.timeout`, default 60 s). */
export function sttTimeoutSeconds(stt: Pick<DirectSttConfig, 'timeout_s'>): number {
  const value = Number(stt.timeout_s)

  return Number.isFinite(value) && value > 0 ? value : DEFAULT_STT_TIMEOUT_S
}

/**
 * The relay path's Whisper-silence filter (`is_whisper_hallucination`,
 * tools/voice_mode_transcript.py): empty, an exact known hallucination
 * (lowercased, trailing `.!` stripped), or repetitive filler like
 * "Thank you. Thank you." A client-direct transcript must agree with a
 * relayed one instead of submitting "thank you" on silence as a real turn.
 * No filter on the config (older backend) → the transcript passes through.
 */
export function isSttSilenceHallucination(
  transcript: string,
  filter: DirectSttConfig['hallucination_filter']
): boolean {
  if (!filter) {
    return false
  }

  const cleaned = transcript.trim().toLowerCase()

  if (!cleaned) {
    return true
  }

  // Trailing `.!` only — the relay strips `cleaned.rstrip('.!')`, so an
  // internal period (`thank. you`) stays internal and the transcript stays
  // a real turn on both paths, never just one.
  if (filter.phrases.includes(cleaned.replace(/[.!]+$/, ''))) {
    return true
  }

  try {
    return new RegExp(filter.repeat_regex, 'i').test(cleaned)
  } catch {
    return false
  }
}

/**
 * `fetch` with the STT deadline. A slow or wedged endpoint otherwise keeps the
 * dictation UI in "transcribing" forever — the browser applies no timeout of
 * its own to a POST that never answers.
 */
async function sttFetch(stt: DirectSttConfig, url: string, init: RequestInit): Promise<Response> {
  const seconds = sttTimeoutSeconds(stt)
  const controller = new AbortController()
  const timer = setTimeout(() => controller.abort(), seconds * 1000)

  try {
    return await fetch(url, { ...init, signal: controller.signal })
  } catch (error) {
    if (controller.signal.aborted) {
      throw new Error(`Transcription timed out after ${seconds}s (${stt.provider} did not answer)`)
    }

    throw error
  } finally {
    clearTimeout(timer)
  }
}

/** Multipart body for xAI `POST /v1/stt`. */
function xaiSttForm(audio: Blob, stt: DirectSttConfig): FormData {
  const form = new FormData()
  form.set('file', audio, sttFileName(audio))

  if (stt.model) {
    form.set('model', stt.model)
  }

  // xAI rejects format=true without a language (HTTP 400), so auto-detect drops the flag.
  if (stt.language) {
    form.set('language', stt.language)
    form.set('format', 'true')
  }

  return form
}

/**
 * Transcribe provider-direct. Returns the transcript ('' = silence), or null
 * when the profile's provider isn't client-callable — the caller relays.
 * Provider REJECTIONS throw: the configured provider said no, and silently
 * re-running the same request through the gateway would just fail again
 * slower and hide the real error. `owner` pins the provider config to the
 * recording's owner (resolved when the mic opened); omitted → the active scope.
 */
export async function transcribeAudioClientDirect(audio: Blob, owner?: ResolvedOwner): Promise<null | string> {
  const config = await (owner ? fetchVoiceClientConfigFor(owner) : fetchVoiceClientConfig())
  const stt = config?.stt

  if (!stt || stt.mode !== 'direct') {
    return null
  }

  if (stt.wire === 'openai-multipart') {
    const form = new FormData()
    form.set('file', audio, sttFileName(audio))

    if (stt.model) {
      form.set('model', stt.model)
    }

    form.set('response_format', 'text')

    if (stt.language) {
      form.set('language', stt.language)
    }

    const response = await sttFetch(stt, `${stt.base_url.replace(/\/+$/, '')}/audio/transcriptions`, {
      method: 'POST',
      headers: { Authorization: `Bearer ${stt.api_key}` },
      body: form,
      signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS)
    })

    if (!response.ok) {
      throw new Error(`${stt.provider} STT error (HTTP ${response.status}): ${await providerErrorText(response)}`)
    }

    const transcript = transcriptFromOpenAiMultipartBody(await response.text())

    // Silence hallucination ("thank you" on quiet audio): treat as silence,
    // exactly like the relay endpoint, instead of submitting a phantom turn.
    return isSttSilenceHallucination(transcript, stt.hallucination_filter) ? '' : transcript
  }

  if (stt.wire === 'xai-stt') {
    const response = await sttFetch(stt, `${stt.base_url.replace(/\/+$/, '')}/stt`, {
      method: 'POST',
      headers: { Authorization: `Bearer ${stt.api_key}` },
      body: xaiSttForm(audio, stt),
      signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS)
    })

    if (!response.ok) {
      throw new Error(`xAI STT error (HTTP ${response.status}): ${await providerErrorText(response)}`)
    }

    const result = (await response.json()) as { text?: string }
    const transcript = (result.text || '').trim()

    // Silence hallucination: same contract as the relay endpoint.
    return isSttSilenceHallucination(transcript, stt.hallucination_filter) ? '' : transcript
  }

  if (stt.wire === 'elevenlabs-stt') {
    const form = new FormData()
    form.set('file', audio, sttFileName(audio))

    if (stt.model) {
      form.set('model_id', stt.model)
    }

    if (stt.language) {
      form.set('language_code', stt.language)
    }

    const response = await sttFetch(stt, `${stt.base_url.replace(/\/+$/, '')}/speech-to-text`, {
      method: 'POST',
      headers: { 'xi-api-key': stt.api_key },
      body: form,
      signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS)
    })

    if (!response.ok) {
      throw new Error(`ElevenLabs STT error (HTTP ${response.status}): ${await providerErrorText(response)}`)
    }

    const result = (await response.json()) as { text?: string }
    const transcript = (result.text || '').trim()

    // Silence hallucination: same contract as the relay endpoint.
    return isSttSilenceHallucination(transcript, stt.hallucination_filter) ? '' : transcript
  }

  return null
}

// ---------------------------------------------------------------------------
// TTS — text → audio bytes, provider-direct. One call per sentence/segment;
// the playback queue in voice-playback.ts owns ordering and barge-in.
// ---------------------------------------------------------------------------

/** Resolve the profile's TTS config when it is client-callable, else null. */
export async function directTtsConfig(owner?: OwnerScope): Promise<DirectTtsConfig | null> {
  const config = await fetchVoiceClientConfig(owner)

  return config?.tts && config.tts.mode === 'direct' ? config.tts : null
}

/** Synthesize one text segment to audio bytes (mp3). Throws on provider rejection. */
export async function synthesizeSpeechClientDirect(tts: DirectTtsConfig, text: string): Promise<ArrayBuffer> {
  if (tts.wire === 'openai-speech') {
    const body: Record<string, unknown> = {
      ...(tts.extra_body ?? {}),
      model: tts.model,
      voice: tts.voice,
      input: text,
      response_format: 'mp3'
    }

    if (tts.speed && tts.speed !== 1) {
      body.speed = tts.speed
    }

    const response = await fetch(`${tts.base_url.replace(/\/+$/, '')}/audio/speech`, {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${tts.api_key}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify(body)
    })

    if (!response.ok) {
      throw new Error(`${tts.provider} TTS error (HTTP ${response.status}): ${await providerErrorText(response)}`)
    }

    return response.arrayBuffer()
  }

  if (tts.wire === 'elevenlabs-tts') {
    const response = await fetch(
      `${tts.base_url.replace(/\/+$/, '')}/text-to-speech/${encodeURIComponent(tts.voice || '')}`,
      {
        method: 'POST',
        headers: {
          'xi-api-key': tts.api_key,
          'Content-Type': 'application/json',
          Accept: 'audio/mpeg'
        },
        body: JSON.stringify({ text, model_id: tts.model })
      }
    )

    if (!response.ok) {
      throw new Error(`ElevenLabs TTS error (HTTP ${response.status}): ${await providerErrorText(response)}`)
    }

    return response.arrayBuffer()
  }

  throw new Error(`Unknown TTS wire: ${(tts as { wire?: string }).wire}`)
}