The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
/**
* Client-side mic capture for remote wake word.
*
* When the backend arms with `capture: "client"`, PortAudio runs on a headless
* VM with no mic. The desktop opens getUserMedia here, resamples to 16 kHz
* mono int16 frames, and pushes them via `wake.feed` so openWakeWord still
* runs server-side without requiring a server sound device.
*/
import { bytesToBase64 } from './base64'
const TARGET_RATE = 16_000
const DEFAULT_FRAME = 1280 // 80 ms @ 16 kHz — matches tools/wake_word.py
// Health-monitor tuning for the continuous capture chain. A platform capture
// failure (macOS PLAS IPC delegate error → StopSourceOnError, #119089) leaves
// getUserMedia resolved but the PCM dead: ended track, halted callbacks, or
// endless digital zeros. Without a watchdog the ear shows "listening" forever
// and the gateway detector can never fire.
/** -60 dBFS: live mic noise floors sit well above this; exact zeros do not. */
const SILENCE_PEAK = 0.001
/** ~8 s of digital zeros at 80 ms/frame before the ear is declared deaf. */
const DEFAULT_SILENCE_FRAMES = 100
const DEFAULT_STALL_TIMEOUT_MS = 3000
const DEFAULT_MAX_FEED_FAILURES = 5
export type WakeFeedRequester = (method: string, params?: Record<string, unknown>) => Promise<unknown>
export interface ClientWakeCaptureOptions {
/** Samples per frame at 16 kHz (from wake.start response). */
frameLength?: number
request: WakeFeedRequester
/**
* Fatal capture-chain failures only (dead track, stalled graph, sustained
* silence, refused feeds). Isolated feed RPC blips are retried silently —
* reporting every one would flap the ear on ordinary network jitter.
*/
onError?: (error: Error) => void
/** Consecutive near-silent 16 kHz frames before the ear is declared deaf. */
silenceFramesThreshold?: number
/** ms without an onaudioprocess callback before the graph is declared stalled. */
stallTimeoutMs?: number
/** Consecutive rejected/failed wake.feed calls before escalating. */
maxConsecutiveFeedFailures?: number
}
export interface ClientWakeCaptureHandle {
stop: () => void
readonly active: boolean
}
function downsampleTo16k(input: Float32Array, inputRate: number): Float32Array {
if (inputRate === TARGET_RATE) {
return input
}
if (inputRate <= 0) {
return new Float32Array(0)
}
const ratio = inputRate / TARGET_RATE
const outLen = Math.max(1, Math.floor(input.length / ratio))
const out = new Float32Array(outLen)
for (let i = 0; i < outLen; i++) {
const start = Math.floor(i * ratio)
const end = Math.min(input.length, Math.floor((i + 1) * ratio))
let sum = 0
let count = 0
for (let j = start; j < end; j++) {
sum += input[j] ?? 0
count++
}
out[i] = count > 0 ? sum / count : 0
}
return out
}
function floatToInt16LE(input: Float32Array): ArrayBuffer {
const buf = new ArrayBuffer(input.length * 2)
const view = new DataView(buf)
for (let i = 0; i < input.length; i++) {
const s = Math.max(-1, Math.min(1, input[i] ?? 0))
view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true)
}
return buf
}
/**
* Start streaming the default microphone to `wake.feed`.
* Returns a handle whose `stop()` ends tracks + audio graph.
*/
export async function startClientWakeCapture(options: ClientWakeCaptureOptions): Promise<ClientWakeCaptureHandle> {
const frameLength = Math.max(160, Math.trunc(options.frameLength || DEFAULT_FRAME))
const audioWindow = window as Window & { webkitAudioContext?: typeof AudioContext }
const AudioContextCtor = window.AudioContext || audioWindow.webkitAudioContext
if (!AudioContextCtor) {
throw new Error('AudioContext unavailable for client wake capture')
}
if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('getUserMedia unavailable for client wake capture')
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true
},
video: false
})
// getUserMedia can resolve with a stillborn stream (no audio track, or an
// already-ended one after a platform capture error). Starting the graph on
// that feeds the detector silence forever — fail loudly instead (#119089).
const audioTracks = stream.getAudioTracks()
if (audioTracks.length === 0 || audioTracks.every(track => track.readyState === 'ended')) {
stream.getTracks().forEach(track => track.stop())
throw new Error('microphone track unavailable for client wake capture')
}
const silenceFramesThreshold = Math.max(1, Math.trunc(options.silenceFramesThreshold ?? DEFAULT_SILENCE_FRAMES))
const stallTimeoutMs = Math.max(250, options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS)
const maxFeedFailures = Math.max(1, Math.trunc(options.maxConsecutiveFeedFailures ?? DEFAULT_MAX_FEED_FAILURES))
const context = new AudioContextCtor()
const source = context.createMediaStreamSource(stream)
// ScriptProcessor is deprecated but widely available and simple for PCM export.
// Buffer size 4096 keeps callback rate reasonable on desktop.
const processor = context.createScriptProcessor(4096, 1, 1)
const mute = context.createGain()
mute.gain.value = 0
let pending = new Float32Array(0)
let stopped = false
let failed = false
let silentFrames = 0
let feedFailures = 0
let stallTimer: ReturnType<typeof setTimeout> | undefined
const handle: ClientWakeCaptureHandle = {
get active() {
return !stopped
},
stop() {
if (stopped) {
return
}
stopped = true
queue.length = 0
if (stallTimer !== undefined) {
clearTimeout(stallTimer)
stallTimer = undefined
}
try {
processor.disconnect()
source.disconnect()
mute.disconnect()
} catch {
// ignore
}
void context.close().catch(() => undefined)
stream.getTracks().forEach(t => t.stop())
}
}
// Fatal capture-chain failure: tear the graph down exactly once and report.
// stop() is idempotent, so late track-ended events after a manual stop stay silent.
const fail = (error: Error) => {
if (failed || stopped) {
return
}
failed = true
handle.stop()
options.onError?.(error)
}
const armStallTimer = () => {
if (stopped || failed) {
return
}
if (stallTimer !== undefined) {
clearTimeout(stallTimer)
}
stallTimer = setTimeout(() => {
fail(new Error('client wake capture stalled: no microphone audio callbacks — the OS capture chain may have died'))
}, stallTimeoutMs)
}
// A platform capture error (macOS StopSourceOnError) ends the track after a
// successful getUserMedia. Without this the graph feeds zeros forever.
for (const track of audioTracks) {
track.onended = () => {
fail(new Error('microphone track ended during client wake capture — the OS capture chain may have died'))
}
}
// Bounded ordered queue of 16 kHz frames. We never drop the frame that is
// currently being sent; under remote latency we drop the oldest queued
// frames so the detector still sees contiguous recent PCM rather than gaps
// from fire-and-forget discard-while-inflight.
const MAX_QUEUED_FRAMES = 24 // ~1.9s at 80 ms/frame
// Coalesce queued frames into one wake.feed call (backend splits them back
// into engine frames). 4 × 80 ms ≈ 3 RPCs/s steady-state instead of 12.5.
const MAX_FRAMES_PER_FEED = 4
const queue: Float32Array[] = []
let draining = false
const noteFeedResult = (accepted: boolean, reason: string | null) => {
if (stopped || failed) {
return
}
if (accepted) {
feedFailures = 0
return
}
// The server dropped our PCM (lease lost, wrong owner, detector gone).
// One blip is ordinary network jitter; a run of them means the ear is
// armed but permanently deaf — escalate instead of staying "listening".
feedFailures += 1
if (feedFailures >= maxFeedFailures) {
fail(
new Error(
`client wake capture deaf: wake.feed refused ${feedFailures} consecutive frames` +
(reason ? ` (${reason})` : '') +
' — re-toggle the ear to re-arm'
)
)
}
}
const drainQueue = async () => {
if (draining) {
return
}
draining = true
try {
while (!stopped && !failed && queue.length > 0) {
const batch = queue.splice(0, MAX_FRAMES_PER_FEED)
if (batch.length === 0) {
break
}
try {
const merged = new Float32Array(batch.length * frameLength)
batch.forEach((frame, i) => merged.set(frame, i * frameLength))
const pcm = floatToInt16LE(merged)
const result = (await options.request('wake.feed', {
pcm: bytesToBase64(pcm),
sample_rate: TARGET_RATE
})) as { fed?: boolean; reason?: string | null } | null | undefined
// Absent on older backends that predate the {fed} envelope — only an
// explicit fed:false counts as a refusal. Thrown RPC errors land below.
noteFeedResult(!result || result.fed !== false, result?.reason ?? null)
} catch {
// Keep draining later frames; one failed RPC should not freeze the ear.
noteFeedResult(false, null)
}
}
} finally {
draining = false
if (!stopped && !failed && queue.length > 0) {
void drainQueue()
}
}
}
const enqueueFrame = (frame: Float32Array) => {
if (stopped) {
return
}
queue.push(frame)
while (queue.length > MAX_QUEUED_FRAMES) {
queue.shift()
}
void drainQueue()
}
processor.onaudioprocess = event => {
if (stopped || failed) {
return
}
armStallTimer()
const input = event.inputBuffer.getChannelData(0)
const at16k = downsampleTo16k(input, context.sampleRate)
// Append to pending and emit full frames
const merged = new Float32Array(pending.length + at16k.length)
merged.set(pending, 0)
merged.set(at16k, pending.length)
let offset = 0
while (offset + frameLength <= merged.length) {
const frame = merged.subarray(offset, offset + frameLength)
offset += frameLength
// A dead capture chain delivers endless digital zeros. Live mics never
// do — their noise floor sits far above SILENCE_PEAK — so a long run of
// zeros means the detector is being fed silence, not speech (#119089).
let peak = 0
for (let i = 0; i < frame.length; i++) {
const abs = Math.abs(frame[i] ?? 0)
if (abs > peak) {
peak = abs
}
}
if (peak < SILENCE_PEAK) {
silentFrames += 1
if (silentFrames >= silenceFramesThreshold) {
fail(
new Error(
'client wake capture hears only silence — the microphone delivers no audio; check OS mic access and re-toggle the ear'
)
)
return
}
continue
}
silentFrames = 0
enqueueFrame(new Float32Array(frame))
}
pending = merged.subarray(offset)
}
source.connect(processor)
processor.connect(mute)
mute.connect(context.destination)
if (context.state === 'suspended') {
await context.resume().catch(() => undefined)
}
armStallTimer()
return handle
}