Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(home): let the owner talk to the house and hear it answer

nirholasfd6bc5dd0d78
diff --git a/pages/voice-home.html b/pages/voice-home.html
new file mode 100644
index 000000000..d20b61ceb
--- /dev/null
+++ b/pages/voice-home.html
@@ -0,0 +1,157 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <script>/* three.ws theme boot: no-flash */(function(){try{var m=localStorage.getItem('twx_theme');var l=m==='auto'?(window.matchMedia&&window.matchMedia('(prefers-color-scheme: light)').matches):m==='light';document.documentElement.setAttribute('data-theme',l?'light':'dark');}catch(e){document.documentElement.setAttribute('data-theme','dark');}})();</script>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
+ <title>Hands-free voice for your home · three.ws</title>
+ <meta name="description" content="Talk to your house without finding a phone. A wake word matched on your own device, real barge-in, and a spoken confirmation that a background yes can never satisfy." />
+ <link rel="canonical" href="https://three.ws/voice/home" />
+ <link rel="icon" href="/favicon.ico" sizes="any" />
+ <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
+ <link rel="apple-touch-icon" href="/apple-touch-icon.png" />
+ <link rel="manifest" href="/manifest.webmanifest" />
+ <link rel="stylesheet" href="/style.css" />
+ <link rel="stylesheet" href="/nav.css" />
+ <link rel="stylesheet" href="/mobile.css" />
+ <link rel="stylesheet" href="/home-voice.css" />
+ <meta property="og:type" content="website" />
+ <meta property="og:title" content="Hands-free voice for your home · three.ws" />
+ <meta property="og:description" content="A wake word matched on your own device, real barge-in, and a spoken confirmation a background yes can never satisfy." />
+ <meta property="og:url" content="https://three.ws/voice/home" />
+ <meta name="twitter:card" content="summary_large_image" />
+ <meta name="twitter:title" content="Hands-free voice for your home · three.ws" />
+ <meta name="twitter:description" content="Talk to your house without finding a phone. The wake word never leaves your device." />
+ <meta name="twitter:site" content="@trythreews" />
+ <meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1" />
+ <style>
+ * { box-sizing: border-box; }
+ html, body { margin: 0; background: var(--bg-0); color: var(--ink); }
+ body { font-family: var(--font-body, 'Inter', system-ui, sans-serif); min-height: 100vh; }
+ .site-header { position: sticky; top: 0; z-index: 50; background: color-mix(in srgb, var(--bg-0) 82%, transparent); backdrop-filter: blur(10px); }
+ .vh-wrap { max-width: 1080px; margin: 0 auto; padding: 0 18px 96px; }
+ .vh-hero { padding: 40px 0 20px; }
+ .vh-kicker {
+ display: inline-flex; align-items: center; gap: 7px;
+ font-size: var(--text-2xs); letter-spacing: 0.09em; text-transform: uppercase;
+ color: var(--ink-dim); border: 1px solid var(--stroke); border-radius: var(--radius-pill);
+ padding: 5px 11px; margin-bottom: 14px; background: var(--surface-1);
+ }
+ .vh-kicker::before { content: ''; width: 6px; height: 6px; border-radius: 50%; background: var(--success); }
+ .vh-hero h1 { font-size: clamp(1.9rem, 5vw, 2.9rem); line-height: 1.1; margin: 0 0 12px; letter-spacing: -0.025em; color: var(--ink-bright); }
+ .vh-hero p { margin: 0; max-width: 66ch; color: var(--ink-dim); line-height: 1.65; font-size: 1.02rem; }
+ .vh-grid { display: grid; grid-template-columns: minmax(0, 1.35fr) minmax(0, 1fr); gap: var(--space-lg); margin-top: var(--space-lg); align-items: start; }
+ @media (max-width: 860px) { .vh-grid { grid-template-columns: 1fr; } }
+ .vh-aside { display: flex; flex-direction: column; gap: var(--space-md); }
+ .vh-card {
+ border: 1px solid var(--stroke); border-radius: var(--radius-lg);
+ background: var(--surface-1); padding: var(--space-lg);
+ }
+ .vh-card h2 { margin: 0 0 var(--space-sm); font: 600 var(--text-lg)/1.25 var(--font-display); color: var(--ink-bright); }
+ .vh-card p, .vh-card li { color: var(--ink-dim); line-height: 1.6; font-size: var(--text-ui); }
+ .vh-card p { margin: 0 0 var(--space-sm); max-width: 62ch; }
+ .vh-card ul { margin: 0; padding-left: 1.15em; display: flex; flex-direction: column; gap: var(--space-xs); }
+ .vh-pipe { font: var(--text-sm)/1.7 var(--font-mono); color: var(--ink-faint); white-space: pre; overflow-x: auto; margin: 0; }
+ .vh-budget { width: 100%; border-collapse: collapse; font-size: var(--text-sm); }
+ .vh-budget th, .vh-budget td { text-align: left; padding: var(--space-xs) var(--space-sm); border-bottom: 1px solid var(--stroke); }
+ .vh-budget th { color: var(--ink-dim); font-weight: 500; }
+ .vh-budget td:last-child { font-family: var(--font-mono); color: var(--ink); text-align: right; }
+ .vh-scroll { overflow-x: auto; }
+ .vh-states { margin-top: var(--space-2xl); }
+ .vh-states > h2 { font: 600 var(--text-xl)/1.2 var(--font-display); color: var(--ink-bright); margin: 0 0 var(--space-xs); }
+ .vh-states > p { color: var(--ink-dim); max-width: 66ch; margin: 0 0 var(--space-lg); line-height: 1.6; }
+ .vh-gallery { display: grid; grid-template-columns: repeat(auto-fill, minmax(21rem, 1fr)); gap: var(--space-lg); }
+ .vh-state { display: flex; flex-direction: column; gap: var(--space-sm); }
+ .vh-state > h3 { margin: 0; font: 500 var(--text-sm)/1.3 var(--font-mono); color: var(--ink-faint); letter-spacing: 0.04em; }
+ .vh-links { margin-top: var(--space-2xl); display: flex; gap: var(--space-md); flex-wrap: wrap; }
+ .vh-links a { color: var(--ink-dim); text-decoration: none; border-bottom: 1px solid var(--stroke-strong); padding-bottom: 2px; }
+ .vh-links a:hover, .vh-links a:focus-visible { color: var(--ink-bright); border-color: var(--accent); }
+ </style>
+</head>
+<body>
+ <header class="site-header"><div id="nav-container"></div></header>
+ <script src="/nav.js"></script>
+
+ <main id="main-content" class="vh-wrap">
+ <section class="vh-hero">
+ <span class="vh-kicker">Hands-free</span>
+ <h1>Talk to your house without finding a phone</h1>
+ <p>
+ A wake word matched by a model running in this tab, an interruption that actually stops the agent
+ mid-sentence, and a spoken confirmation narrow enough that somebody agreeing with somebody else in the
+ same room can never unlock your front door.
+ </p>
+ </section>
+
+ <div class="vh-grid">
+ <div id="voice-panel"></div>
+
+ <aside class="vh-aside">
+ <section class="vh-card">
+ <h2>What leaves this device</h2>
+ <p>
+ The wake word runs on openWakeWord and the endpointing on silero, both downloaded from three.ws
+ and executed in this tab. That audio is never uploaded. Only the sentence after the wake word is
+ sent, and only to be transcribed.
+ </p>
+ <pre class="vh-pipe">mic
+ -> silero VAD on device
+ -> openWakeWord on device
+ -> /api/asr the utterance only
+ -> agent turn
+ -> /api/tts/speak
+ -> the agent speaks</pre>
+ </section>
+
+ <section class="vh-card">
+ <h2>The latency budget</h2>
+ <div class="vh-scroll">
+ <table class="vh-budget">
+ <thead><tr><th scope="col">Leg</th><th scope="col">Budget</th></tr></thead>
+ <tbody>
+ <tr><td>Wake word detection</td><td>200 ms</td></tr>
+ <tr><td>End of speech</td><td>400 ms</td></tr>
+ <tr><td>Transcription round trip</td><td>900 ms</td></tr>
+ <tr><td>Agent turn</td><td>1200 ms</td></tr>
+ <tr><td>Action to device change</td><td>700 ms</td></tr>
+ <tr><td>First audible reply</td><td>1800 ms</td></tr>
+ </tbody>
+ </table>
+ </div>
+ <p>Measured live in the panel while you use it, not asserted here.</p>
+ </section>
+
+ <section class="vh-card">
+ <h2>Why a door is different</h2>
+ <p>
+ Turning a light off happens the moment you ask. Unlocking a door does not: the agent says the whole
+ sentence, the entity appears on screen, and only the word <strong>confirm</strong> goes through. A
+ general yes does not, and on a device with no screen a guarded action is refused outright.
+ </p>
+ </section>
+ </aside>
+ </div>
+
+ <section class="vh-states">
+ <h2>Every state</h2>
+ <p>
+ The same panel component, driven into each of the twelve states it can be in. Off is the default, and
+ nothing about listening is downloaded until you turn it on.
+ </p>
+ <div class="vh-gallery" id="state-gallery"></div>
+ </section>
+
+ <nav class="vh-links" aria-label="Related">
+ <a href="/docs/home-voice">Voice loop documentation</a>
+ <a href="/lipsync/mic">Audio-driven lipsync</a>
+ <a href="/voice">Voice and cloning</a>
+ </nav>
+ </main>
+
+ <div id="footer-container"></div>
+ <script src="/footer.js"></script>
+
+ <script type="module" src="/src/voice-home.js"></script>
+ <script type="module" src="/i18n.js"></script>
+</body>
+</html>
diff --git a/public/home-voice.css b/public/home-voice.css
new file mode 100644
index 000000000..76fdf60be
--- /dev/null
+++ b/public/home-voice.css
@@ -0,0 +1,371 @@
+/* The hands-free panel (src/voice/home-voice-ui.js).
+ *
+ * Every colour, space and radius here is a token from public/tokens.css. The one
+ * value that is NOT a token is the listening indicator's red: it deliberately
+ * refuses the monochrome palette, because a microphone that is live has to be
+ * legible as such at a glance across a kitchen, and a tasteful grey dot is not.
+ */
+
+.hv {
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-md);
+ padding: var(--space-lg);
+ background: var(--surface-1);
+ border: 1px solid var(--stroke);
+ border-radius: var(--radius-lg);
+ font: var(--text-md) / 1.55 var(--font-body);
+ color: var(--ink);
+ container-type: inline-size;
+}
+
+/* ── the indicator ─────────────────────────────────────────────────────────
+ * Always present, never conditional on a setting, and driven by the live
+ * MediaStreamTrack rather than by the state machine. */
+.hv-indicator {
+ display: flex;
+ align-items: center;
+ gap: var(--space-sm);
+ padding: var(--space-sm) var(--space-md);
+ border-radius: var(--radius-pill);
+ background: var(--surface-2);
+ border: 1px solid var(--stroke);
+ font-size: var(--text-sm);
+ color: var(--ink-dim);
+ align-self: flex-start;
+ transition: background var(--duration-fast) var(--ease-standard), color var(--duration-fast) var(--ease-standard);
+}
+.hv-indicator[data-live='true'] {
+ background: rgba(239, 68, 68, 0.14);
+ border-color: rgba(239, 68, 68, 0.5);
+ color: #fca5a5;
+}
+.hv-dot {
+ width: 9px;
+ height: 9px;
+ border-radius: var(--radius-pill);
+ background: var(--ink-dim);
+ flex: none;
+}
+.hv-indicator[data-live='true'] .hv-dot {
+ background: #ef4444;
+ animation: hv-pulse 1.6s var(--ease-standard) infinite;
+}
+@keyframes hv-pulse {
+ 0%,
+ 100% {
+ opacity: 1;
+ transform: scale(1);
+ }
+ 50% {
+ opacity: 0.45;
+ transform: scale(0.82);
+ }
+}
+@media (prefers-reduced-motion: reduce) {
+ .hv-indicator[data-live='true'] .hv-dot {
+ animation: none;
+ }
+}
+
+/* ── head ─────────────────────────────────────────────────────────────────── */
+.hv-head {
+ display: flex;
+ gap: var(--space-md);
+ justify-content: space-between;
+ align-items: flex-start;
+ flex-wrap: wrap;
+}
+.hv-badge {
+ display: inline-block;
+ font-size: var(--text-2xs);
+ letter-spacing: 0.04em;
+ text-transform: uppercase;
+ color: var(--ink-dim);
+ background: var(--surface-2);
+ border: 1px solid var(--stroke);
+ border-radius: var(--radius-pill);
+ padding: var(--space-3xs) var(--space-sm);
+}
+.hv-badge[data-state='idle'],
+.hv-badge[data-state='capturing'] {
+ color: var(--success);
+ border-color: rgba(74, 222, 128, 0.35);
+}
+.hv-badge[data-state='confirm-pending'] {
+ color: var(--warn);
+ border-color: rgba(251, 191, 36, 0.4);
+}
+.hv-badge[data-state='error'],
+.hv-badge[data-state='permission-denied'] {
+ color: var(--danger);
+ border-color: rgba(248, 113, 113, 0.4);
+}
+.hv-title {
+ font: 600 var(--text-lg) / 1.25 var(--font-display);
+ margin: var(--space-xs) 0 var(--space-2xs);
+ color: var(--ink-bright);
+}
+.hv-body {
+ margin: 0;
+ color: var(--ink-dim);
+ max-width: 60ch;
+}
+
+/* ── controls ─────────────────────────────────────────────────────────────── */
+.hv-actions,
+.hv-consent-actions,
+.hv-confirm-actions {
+ display: flex;
+ gap: var(--space-sm);
+ flex-wrap: wrap;
+}
+.hv-btn {
+ font: 500 var(--text-ui) / 1 var(--font-body);
+ padding: var(--space-sm) var(--space-md);
+ border-radius: var(--radius-md);
+ border: 1px solid var(--stroke-strong);
+ background: var(--surface-2);
+ color: var(--ink);
+ cursor: pointer;
+ transition:
+ background var(--duration-fast) var(--ease-standard),
+ border-color var(--duration-fast) var(--ease-standard),
+ transform var(--duration-instant) var(--ease-standard);
+}
+.hv-btn:hover {
+ background: var(--surface-3);
+ border-color: var(--accent);
+}
+.hv-btn:active {
+ transform: translateY(1px);
+}
+.hv-btn:focus-visible {
+ outline: 2px solid var(--accent);
+ outline-offset: 2px;
+}
+.hv-btn-primary {
+ background: var(--ink-bright);
+ color: var(--bg-0);
+ border-color: var(--ink-bright);
+}
+.hv-btn-primary:hover {
+ background: var(--ink);
+}
+
+/* ── consent ──────────────────────────────────────────────────────────────── */
+.hv-consent {
+ background: var(--surface-2);
+ border: 1px solid var(--stroke);
+ border-radius: var(--radius-md);
+ padding: var(--space-md);
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-sm);
+ animation: hv-rise var(--duration-base) var(--ease-emphasized);
+}
+.hv-consent h3 {
+ margin: 0;
+ font: 600 var(--text-ui) / 1.3 var(--font-display);
+ color: var(--ink-bright);
+}
+.hv-consent-list {
+ margin: 0;
+ padding-left: 1.15em;
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-xs);
+ color: var(--ink-dim);
+ max-width: 68ch;
+}
+@keyframes hv-rise {
+ from {
+ opacity: 0;
+ transform: translateY(6px);
+ }
+ to {
+ opacity: 1;
+ transform: none;
+ }
+}
+@media (prefers-reduced-motion: reduce) {
+ .hv-consent {
+ animation: none;
+ }
+}
+
+.hv-recovery {
+ background: var(--surface-2);
+ border: 1px solid rgba(248, 113, 113, 0.35);
+ border-radius: var(--radius-md);
+ padding: var(--space-md);
+ color: var(--ink);
+ max-width: 68ch;
+}
+
+/* ── the confirmation card ────────────────────────────────────────────────── */
+.hv-confirm {
+ background: var(--surface-2);
+ border: 1px solid rgba(251, 191, 36, 0.45);
+ border-radius: var(--radius-md);
+ padding: var(--space-md);
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-sm);
+}
+.hv-confirm-sentence {
+ margin: 0;
+ font: 600 var(--text-base) / 1.4 var(--font-display);
+ color: var(--ink-bright);
+}
+.hv-confirm-entities {
+ list-style: none;
+ margin: 0;
+ padding: 0;
+ display: flex;
+ gap: var(--space-xs);
+ flex-wrap: wrap;
+}
+.hv-confirm-entities li {
+ font: var(--text-2xs) / 1.4 var(--font-mono);
+ background: var(--surface-3);
+ border: 1px solid var(--stroke);
+ border-radius: var(--radius-sm);
+ padding: var(--space-3xs) var(--space-xs);
+ color: var(--ink-dim);
+ word-break: break-all;
+}
+.hv-confirm-hint {
+ margin: 0;
+ color: var(--ink-dim);
+ font-size: var(--text-sm);
+}
+.hv-confirm-hint strong {
+ color: var(--warn);
+}
+.hv-confirm-timer {
+ height: 3px;
+ border-radius: var(--radius-pill);
+ background: var(--surface-3);
+ overflow: hidden;
+}
+.hv-confirm-timer span {
+ display: block;
+ height: 100%;
+ background: var(--warn);
+ transform-origin: left center;
+ transform: scaleX(1);
+}
+
+/* ── wake-word meter ──────────────────────────────────────────────────────── */
+.hv-meter {
+ height: 4px;
+ border-radius: var(--radius-pill);
+ background: var(--surface-3);
+ overflow: hidden;
+}
+.hv-meter-fill {
+ display: block;
+ height: 100%;
+ background: var(--success);
+ transform-origin: left center;
+ transform: scaleX(0);
+ transition: transform var(--duration-instant) linear;
+}
+
+/* ── transcript ───────────────────────────────────────────────────────────── */
+.hv-transcript {
+ list-style: none;
+ margin: 0;
+ padding: 0;
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-xs);
+ max-height: 16rem;
+ overflow-y: auto;
+ overscroll-behavior: contain;
+}
+.hv-transcript:empty {
+ display: none;
+}
+.hv-line {
+ padding: var(--space-xs) var(--space-sm);
+ border-radius: var(--radius-sm);
+ background: var(--surface-2);
+ max-width: 68ch;
+}
+.hv-line-you {
+ align-self: flex-end;
+ background: var(--surface-3);
+}
+.hv-line-system {
+ background: transparent;
+ color: var(--ink-dim);
+ font-size: var(--text-sm);
+ padding-left: 0;
+}
+
+/* ── settings and latency ─────────────────────────────────────────────────── */
+.hv-settings {
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-2xs);
+ border-top: 1px solid var(--stroke);
+ padding-top: var(--space-md);
+}
+.hv-field {
+ display: flex;
+ align-items: center;
+ gap: var(--space-sm);
+ font-size: var(--text-sm);
+ color: var(--ink-dim);
+}
+.hv-wake {
+ font: var(--text-md) / 1 var(--font-body);
+ background: var(--surface-2);
+ color: var(--ink);
+ border: 1px solid var(--stroke-strong);
+ border-radius: var(--radius-sm);
+ padding: var(--space-xs) var(--space-sm);
+}
+.hv-wake:focus-visible {
+ outline: 2px solid var(--accent);
+ outline-offset: 2px;
+}
+.hv-wake-hint {
+ margin: 0;
+ color: var(--ink-faint);
+ font-size: var(--text-sm);
+ max-width: 60ch;
+}
+
+.hv-latency {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
+ gap: var(--space-xs) var(--space-md);
+ margin: 0;
+ font-size: var(--text-sm);
+}
+.hv-latency:empty {
+ display: none;
+}
+.hv-latency dt {
+ color: var(--ink-dim);
+}
+.hv-latency dd {
+ margin: 0 0 var(--space-sm);
+ font-family: var(--font-mono);
+ color: var(--success);
+}
+.hv-latency dd[data-over='true'] {
+ color: var(--warn);
+}
+
+@container (max-width: 30rem) {
+ .hv {
+ padding: var(--space-md);
+ }
+ .hv-head {
+ flex-direction: column;
+ }
+}
diff --git a/scripts/check-home-voice.mjs b/scripts/check-home-voice.mjs
new file mode 100644
index 000000000..864d16bf9
--- /dev/null
+++ b/scripts/check-home-voice.mjs
@@ -0,0 +1,605 @@
+#!/usr/bin/env node
+/**
+ * The browser proof for the hands-free voice loop (src/voice/home-voice.js).
+ *
+ * Everything a claim about this feature rests on is measured here, in a real
+ * Chromium, against the real speech lanes:
+ *
+ * 1 cold load nothing about listening is fetched before opt-in
+ * 2 happy path wake word, capture, transcription, agent turn, reply
+ * 3 barge-in playback stops after the user starts talking
+ * 4 self-trigger the agent's own wake word does not wake it
+ * 5 guarded (refused) an ambient "yeah" does not confirm
+ * 6 guarded (accepted) the word "confirm" does
+ * 7 mute capture stops at the track, not behind a flag
+ * 8 unavailable a deployment with no speech lane degrades honestly
+ * 9 permission denied a refused microphone has a way back
+ * 10 state gallery a frame of each of the twelve states
+ *
+ * The microphone is a real MediaStream fed from a WAV by Chromium's fake capture
+ * device, and the speech in those WAVs is synthesized by the platform's own TTS
+ * lane rather than hand-picked from a recording. /api/asr and /api/tts/speak are
+ * the real endpoints throughout.
+ *
+ * ONE thing is substituted, and only in scenarios 5 and 6: the /api/chat
+ * response that carries a pending_confirmation. The server-side home tools that
+ * mint a real confirmation are order 04 of the home campaign and have not landed
+ * yet, so the payload is supplied here to the documented shape. Every other leg
+ * of those two scenarios (speech, VAD, ASR, the grammar, the redemption request)
+ * is the real path. When order 04 lands, delete the route handler and the
+ * scenarios keep working against a real confirmation.
+ *
+ * Usage:
+ * node scripts/check-home-voice.mjs [--port 3457] [--out .cache/home-voice]
+ * node scripts/check-home-voice.mjs --headed # watch it run
+ *
+ * Exit code is 0 when every assertion held, 1 otherwise. The measured legs and
+ * the assertions are written to <out>/report.json.
+ */
+
+import { chromium } from 'playwright';
+import { mkdirSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
+import { resolve, join } from 'node:path';
+
+const args = process.argv.slice(2);
+const flag = (name, fallback) => {
+ const i = args.indexOf(`--${name}`);
+ return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : fallback;
+};
+const PORT = Number(flag('port', process.env.HOME_VOICE_PORT || 3457));
+const BASE = `http://localhost:${PORT}`;
+const OUT = resolve(flag('out', '.cache/home-voice'));
+const HEADED = args.includes('--headed');
+const AUDIO = join(OUT, 'audio');
+
+mkdirSync(AUDIO, { recursive: true });
+
+const results = [];
+function check(name, pass, detail) {
+ results.push({ name, pass: !!pass, detail });
+ console.log(`${pass ? 'PASS' : 'FAIL'} ${name}${detail === undefined ? '' : ` ${JSON.stringify(detail)}`}`);
+}
+
+// ── fixtures: real synthesized speech, cached on disk ───────────────────────
+
+/**
+ * Utterances the scenarios need. leadMs is silence before the speech, which is
+ * what buys the page time to load its models before the microphone matters, and
+ * tailMs is the trailing silence that lets the VAD call the utterance finished.
+ */
+const CLIPS = {
+ command: { text: 'Hey Jarvis, turn the kitchen light off.', leadMs: 1200, tailMs: 2500 },
+ selfTrigger: { text: 'Hey Jarvis. That is my own wake word, spoken by me.', leadMs: 1200, tailMs: 2000 },
+ bargeUser: { text: 'Actually stop, I changed my mind about that.', leadMs: 5000, tailMs: 2500 },
+ ambientYeah: { text: 'Yeah.', leadMs: 8000, tailMs: 2500 },
+ confirmToken: { text: 'Confirm.', leadMs: 8000, tailMs: 2500 },
+ silence: { text: '', leadMs: 12000, tailMs: 0 },
+};
+
+async function ensureClips() {
+ for (const [name, spec] of Object.entries(CLIPS)) {
+ const path = join(AUDIO, `${name}.wav`);
+ spec.path = path;
+ if (existsSync(path)) continue;
+ const speech = spec.text ? await synthesize(spec.text) : { samples: new Float32Array(0), rate: 16000 };
+ const lead = Math.round((spec.leadMs / 1000) * 16000);
+ const tail = Math.round((spec.tailMs / 1000) * 16000);
+ const resampled = resample(speech.samples, speech.rate, 16000);
+ const out = new Float32Array(lead + resampled.length + tail);
+ out.set(resampled, lead);
+ writeFileSync(path, wav16(out, 16000));
+ spec.speechStartMs = spec.leadMs;
+ console.log(`[fixture] ${name}.wav ${(out.length / 16000).toFixed(2)}s`);
+ }
+}
+
+/** The platform's own TTS lane, so the test speech is speech the product makes. */
+async function synthesize(text) {
+ const res = await fetch(`${BASE}/api/tts/speak`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ text, format: 'wav' }),
+ });
+ if (!res.ok) throw new Error(`tts ${res.status}: ${await res.text()}`);
+ return decodeWav(Buffer.from(await res.arrayBuffer()));
+}
+
+function decodeWav(buf) {
+ const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+ let off = 12;
+ let rate = 16000;
+ let bits = 16;
+ let channels = 1;
+ let dataOff = 0;
+ let dataLen = 0;
+ while (off + 8 <= buf.byteLength) {
+ const id = String.fromCharCode(buf[off], buf[off + 1], buf[off + 2], buf[off + 3]);
+ const size = dv.getUint32(off + 4, true);
+ if (id === 'fmt ') {
+ channels = dv.getUint16(off + 10, true);
+ rate = dv.getUint32(off + 12, true);
+ bits = dv.getUint16(off + 22, true);
+ }
+ if (id === 'data') {
+ dataOff = off + 8;
+ dataLen = size;
+ break;
+ }
+ off += 8 + size + (size % 2);
+ }
+ const frames = Math.floor(dataLen / (bits / 8) / channels);
+ const out = new Float32Array(frames);
+ for (let i = 0; i < frames; i++) out[i] = dv.getInt16(dataOff + i * 2 * channels, true) / 32768;
+ return { samples: out, rate };
+}
+
+function resample(x, from, to) {
+ if (from === to || !x.length) return x;
+ const ratio = from / to;
+ const n = Math.floor(x.length / ratio);
+ const out = new Float32Array(n);
+ for (let i = 0; i < n; i++) {
+ const p = i * ratio;
+ const k = Math.floor(p);
+ const f = p - k;
+ const a = x[k] || 0;
+ const b = x[k + 1] !== undefined ? x[k + 1] : a;
+ out[i] = a + (b - a) * f;
+ }
+ return out;
+}
+
+function wav16(samples, rate) {
+ const n = samples.length;
+ const buf = Buffer.alloc(44 + n * 2);
+ buf.write('RIFF', 0);
+ buf.writeUInt32LE(36 + n * 2, 4);
+ buf.write('WAVE', 8);
+ buf.write('fmt ', 12);
+ buf.writeUInt32LE(16, 16);
+ buf.writeUInt16LE(1, 20);
+ buf.writeUInt16LE(1, 22);
+ buf.writeUInt32LE(rate, 24);
+ buf.writeUInt32LE(rate * 2, 28);
+ buf.writeUInt16LE(2, 32);
+ buf.writeUInt16LE(16, 34);
+ buf.write('data', 36);
+ buf.writeUInt32LE(n * 2, 40);
+ for (let i = 0; i < n; i++) {
+ buf.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i] * 32767))), 44 + i * 2);
+ }
+ return buf;
+}
+
+// ── browser plumbing ────────────────────────────────────────────────────────
+
+async function launch(audioPath) {
+ const chromiumArgs = [
+ '--use-fake-device-for-media-stream',
+ '--use-fake-ui-for-media-stream',
+ '--autoplay-policy=no-user-gesture-required',
+ ];
+ if (audioPath) chromiumArgs.push(`--use-file-for-fake-audio-capture=${audioPath}%noloop`);
+ const browser = await chromium.launch({ headless: !HEADED, args: chromiumArgs });
+ const context = await browser.newContext({ permissions: ['microphone'] });
+ const page = await context.newPage();
+ const requests = [];
+ page.on('request', (r) => requests.push({ url: r.url(), at: Date.now() }));
+ const consoleErrors = [];
+ // The dev server's own HMR socket cannot reach a Codespace-forwarded origin,
+ // which is noise from the harness rather than from the page under test.
+ const isDevNoise = (text) => /\[vite\]|WebSocket|hmr/i.test(text);
+ page.on('console', (m) => {
+ if (m.type() === 'error' && !isDevNoise(m.text())) consoleErrors.push(m.text());
+ });
+ page.on('pageerror', (e) => {
+ if (!isDevNoise(String(e))) consoleErrors.push(String(e));
+ });
+ return { browser, context, page, requests, consoleErrors };
+}
+
+/** Mirror the loop's own event stream into the page so the script can await it. */
+const TAP = `
+ window.__hv = { events: [], states: [] };
+ const install = () => {
+ if (!window.homeVoice) return false;
+ const loop = window.homeVoice.loop;
+ const priorEvent = loop.onEvent;
+ loop.onEvent = (e) => { priorEvent?.(e); window.__hv.events.push({ ...e, t: performance.now() }); };
+ const priorState = loop.onState;
+ loop.onState = (s, d) => { priorState?.(s, d); window.__hv.states.push({ state: s, t: performance.now() }); };
+ return true;
+ };
+ if (!install()) {
+ const timer = setInterval(() => { if (install()) clearInterval(timer); }, 25);
+ }
+`;
+
+async function bootPage(page, { query = '' } = {}) {
+ // Concurrent agents edit vite.config.js in this worktree, which restarts the
+ // dev server mid-run. Retry rather than fail a real assertion on that.
+ let lastError = null;
+ for (let attempt = 0; attempt < 6; attempt++) {
+ try {
+ await page.goto(`${BASE}/voice/home${query}`, { waitUntil: 'domcontentloaded' });
+ await page.waitForFunction(() => !!window.homeVoice, null, { timeout: 30000 });
+ await page.evaluate(TAP);
+ return;
+ } catch (err) {
+ lastError = err;
+ await page.waitForTimeout(4000);
+ }
+ }
+ throw lastError;
+}
+
+async function optIn(page) {
+ await page.evaluate(async () => {
+ const { loop } = window.homeVoice;
+ loop.grantConsent();
+ await loop.enable();
+ });
+}
+
+const waitForEvent = (page, type, timeout = 45000) =>
+ page.waitForFunction((t) => window.__hv.events.some((e) => e.type === t), type, { timeout, polling: 50 });
+
+const events = (page) => page.evaluate(() => window.__hv.events);
+const legs = (page) => page.evaluate(() => window.homeVoice.loop.latencySummary());
+
+// ── scenarios ───────────────────────────────────────────────────────────────
+
+async function scenarioColdLoad() {
+ const { browser, page, requests, consoleErrors } = await launch(null);
+ try {
+ await bootPage(page);
+ await page.waitForTimeout(3000);
+ const voiceAssets = requests.filter((r) =>
+ /models\/voice|onnxruntime|vad-web|wake-word\.js|silero/i.test(r.url),
+ );
+ check('cold load fetches nothing about listening before opt-in', voiceAssets.length === 0, {
+ fetched: voiceAssets.map((r) => r.url.replace(BASE, '')),
+ });
+ const state = await page.evaluate(() => window.homeVoice.loop.state);
+ check('the loop rests in a non-listening state on a cold load', state === 'off' || state === 'unavailable', {
+ state,
+ });
+ const micLive = await page.evaluate(() => window.homeVoice.loop.micLive);
+ check('no microphone is open on a cold load', micLive === false);
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, 'cold-load.png') });
+
+ // The same cold browser, after opting in: now the models arrive.
+ await optIn(page);
+ await page.waitForTimeout(1500);
+ const afterOptIn = requests.filter((r) => /models\/voice/i.test(r.url));
+ check('the models are fetched only once the user opts in', afterOptIn.length > 0, {
+ count: afterOptIn.length,
+ sample: afterOptIn.slice(0, 3).map((r) => r.url.replace(BASE, '')),
+ });
+ check('no console errors on the voice surface', consoleErrors.length === 0, { errors: consoleErrors.slice(0, 4) });
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioHappyPath() {
+ const { browser, page } = await launch(CLIPS.command.path);
+ try {
+ await bootPage(page);
+ await optIn(page);
+ await waitForEvent(page, 'wake', 45000);
+ const wake = (await events(page)).find((e) => e.type === 'wake');
+ check('the wake word fires on real speech', wake.score >= 0.5, { score: Number(wake.score.toFixed(4)) });
+
+ await waitForEvent(page, 'transcript', 60000);
+ const transcript = (await events(page)).find((e) => e.type === 'transcript');
+ check('the utterance after the wake word is transcribed by the real lane', /kitchen light/i.test(transcript.text), {
+ text: transcript.text,
+ });
+
+ // The turn runs against the real /api/chat. Whether it can reach a device is
+ // order 04's business; what is measured here is that the loop completes.
+ await page
+ .waitForFunction(
+ () => ['idle', 'speaking', 'error', 'confirm-pending'].includes(window.homeVoice.loop.state),
+ null,
+ { timeout: 90000 },
+ )
+ .catch(() => {});
+
+ const measured = await legs(page);
+ check('the wake-word leg is inside its 200 ms budget', (measured.wake?.median ?? 1e9) <= 200, measured.wake);
+ check('the end-of-speech leg is inside its 400 ms budget', (measured.endpoint?.median ?? 1e9) <= 400, measured.endpoint);
+ check('transcription round trip is measured', !!measured.asr, measured.asr);
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, 'happy-path.png') });
+ return measured;
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioBargeIn() {
+ const { browser, page } = await launch(CLIPS.bargeUser.path);
+ try {
+ await bootPage(page);
+ await optIn(page);
+ // Speak a long enough answer that the user's interruption lands inside it.
+ await page.evaluate(() => {
+ window.homeVoice.loop._speak(
+ 'The kitchen light is off, the hallway light is off, the thermostat is holding at twenty degrees, ' +
+ 'the front door is locked, and the garage is closed. Nothing else has changed since this morning.',
+ );
+ });
+ await page.waitForFunction(() => window.homeVoice.loop.state === 'speaking', null, { timeout: 45000 });
+ await waitForEvent(page, 'barge-in', 45000);
+ const all = await events(page);
+ const stop = all.find((e) => e.type === 'playback-stopped' && e.reason === 'barge-in');
+ check('the user talking over the agent stops playback', !!stop);
+ const measured = await legs(page);
+ check('playback stops within 200 ms of the user starting to talk', (measured.bargeIn?.median ?? 1e9) <= 200, measured.bargeIn);
+ const state = await page.evaluate(() => window.homeVoice.loop.state);
+ check('the loop is capturing again immediately after a barge-in', state === 'capturing' || state === 'thinking', {
+ state,
+ });
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, 'barge-in.png') });
+ return measured;
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioSelfTrigger() {
+ const { browser, page } = await launch(CLIPS.selfTrigger.path);
+ try {
+ await bootPage(page);
+ await optIn(page);
+ // Hold the loop in the state it is in while its own voice is playing.
+ await page.evaluate(() => {
+ const { loop } = window.homeVoice;
+ loop._wake.suppressed = true;
+ loop._setState('speaking', { text: 'Hey Jarvis is my wake word.' });
+ // The wake word only reads frames while idle, so for this proof the
+ // frames are handed to it directly: the guard, not the state gate, is
+ // what is under test.
+ loop._onFrame = function (frame) {
+ this._wake.push(frame);
+ }.bind(loop);
+ loop._vad.onFrame = (frame) => loop._onFrame(frame, 0);
+ });
+ await page.waitForFunction(() => window.homeVoice.loop._wake.suppressedPeak > 0.5, null, { timeout: 45000 });
+ const peak = await page.evaluate(() => window.homeVoice.loop._wake.suppressedPeak);
+ const wakes = (await events(page)).filter((e) => e.type === 'wake');
+ check('the agent hearing its own wake word scores high enough to wake', peak >= 0.5, { peak: Number(peak.toFixed(4)) });
+ check('and does not wake, because the guard is absolute', wakes.length === 0, { wakes: wakes.length });
+ return { peak, wakes: wakes.length };
+ } finally {
+ await browser.close();
+ }
+}
+
+/**
+ * The pending_confirmation payload order 04 will mint server-side. Supplied here
+ * so the client half can be proven now; see the note at the top of this file.
+ */
+const PENDING_CONFIRMATION = {
+ pending_confirmation: {
+ confirmation_id: 'check-home-voice-confirmation',
+ home_id: 'check-home',
+ sentence: 'This will unlock the Front Door.',
+ entity_ids: ['lock.front_door'],
+ risk: 'opens the house',
+ expires_in_ms: 90000,
+ },
+};
+
+async function scenarioGuarded({ clip, expectRedeemed, label }) {
+ const { browser, page } = await launch(clip.path);
+ const redemptions = [];
+ try {
+ await page.route('**/api/chat', (route) =>
+ route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(PENDING_CONFIRMATION) }),
+ );
+ await page.route('**/api/csrf-token', (route) =>
+ route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ data: { token: 'check' } }) }),
+ );
+ await page.route('**/api/home/*/confirm', (route) => {
+ redemptions.push(JSON.parse(route.request().postData() || '{}'));
+ route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify({ message: 'Unlocked the Front Door.' }),
+ });
+ });
+
+ await bootPage(page, { query: '?home=check-home' });
+ await optIn(page);
+ await page.evaluate(() => window.homeVoice.loop.say('unlock the front door'));
+ await waitForEvent(page, 'confirmation-open', 45000);
+ const spoken = await page.evaluate(() => window.homeVoice.loop.stateDetail.confirmation?.sentence);
+ check(`${label}: the whole action is spoken and shown before anything happens`, spoken === 'This will unlock the Front Door.', {
+ spoken,
+ });
+ const shownEntities = await page.locator('.hv-confirm-entities li').allTextContents();
+ check(`${label}: the entity is on screen while the confirmation is open`, shownEntities.includes('lock.front_door'), {
+ shownEntities,
+ });
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, `guarded-${expectRedeemed ? 'confirm' : 'yeah'}.png`) });
+
+ await waitForEvent(page, 'transcript', 90000);
+ const transcript = (await events(page)).find((e) => e.type === 'transcript');
+ await page.waitForTimeout(2500);
+
+ if (expectRedeemed) {
+ check(`${label}: the spoken token is transcribed as the token`, /confirm/i.test(transcript.text), {
+ text: transcript.text,
+ });
+ check(`${label}: the door is unlocked, by id alone`, redemptions.length === 1, { redemptions });
+ check(
+ `${label}: the redemption carries the confirmation id and nothing else`,
+ redemptions[0] && Object.keys(redemptions[0]).join(',') === 'confirmation_id',
+ redemptions[0],
+ );
+ } else {
+ check(`${label}: the ambient word is transcribed`, transcript.text.trim().length > 0, { text: transcript.text });
+ check(`${label}: nothing is unlocked`, redemptions.length === 0, { redemptions });
+ const notToken = (await events(page)).some((e) => e.type === 'confirmation-not-token');
+ check(`${label}: the loop says out loud that it was not the token`, notToken);
+ }
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioMute() {
+ const { browser, page } = await launch(CLIPS.silence.path);
+ try {
+ await bootPage(page);
+ await optIn(page);
+ const before = await page.evaluate(() => ({
+ tracks: window.homeVoice.loop.trackStates(),
+ live: window.homeVoice.loop.micLive,
+ indicator: document.querySelector('.hv-indicator').dataset.live,
+ }));
+ check('the indicator reads live while the microphone is live', before.live && before.indicator === 'true', before);
+
+ await page.click('[data-act="mute"]');
+ await page.waitForFunction(() => window.homeVoice.loop.state === 'muted', null, { timeout: 15000 });
+ await page.waitForTimeout(400);
+ const after = await page.evaluate(() => ({
+ tracks: window.homeVoice.loop.trackStates(),
+ live: window.homeVoice.loop.micLive,
+ indicator: document.querySelector('.hv-indicator').dataset.live,
+ text: document.querySelector('.hv-indicator-text').textContent,
+ }));
+ check('mute ends every track at the device', after.tracks.every((s) => s === 'ended'), after.tracks);
+ check('and the indicator follows the track, not a flag', after.live === false && after.indicator === 'false', after);
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, 'muted.png') });
+ return after;
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioUnavailable() {
+ const { browser, page } = await launch(null);
+ try {
+ await page.route('**/api/asr', (route) =>
+ route.request().method() === 'GET'
+ ? route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify({ configured: false, encodings: [], sampleRate: 16000 }),
+ })
+ : route.continue(),
+ );
+ await bootPage(page);
+ await page.waitForFunction(() => window.homeVoice.loop.state === 'unavailable', null, { timeout: 20000 });
+ const detail = await page.evaluate(() => ({
+ state: window.homeVoice.loop.state,
+ reason: window.homeVoice.loop.stateDetail.reason,
+ micLive: window.homeVoice.loop.micLive,
+ body: document.querySelector('#voice-panel .hv-body').textContent,
+ }));
+ check('an unconfigured speech lane lands in the unavailable state', detail.state === 'unavailable');
+ check('with an honest reason, and no microphone', /not available/i.test(detail.body) && !detail.micLive, detail);
+
+ // Even a user who insists gets no microphone.
+ await page.evaluate(async () => {
+ const { loop } = window.homeVoice;
+ loop.grantConsent();
+ await loop.enable();
+ });
+ const stillOff = await page.evaluate(() => window.homeVoice.loop.micLive);
+ check('and opting in anyway still opens no microphone', stillOff === false);
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, 'unavailable.png') });
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioPermissionDenied() {
+ const { browser, page } = await launch(null);
+ try {
+ // The browser refusing the microphone, which is the one failure the loop
+ // cannot route around and must therefore explain.
+ await page.addInitScript(() => {
+ Object.defineProperty(navigator.mediaDevices, 'getUserMedia', {
+ configurable: true,
+ value: async () => {
+ const err = new Error('Permission denied');
+ err.name = 'NotAllowedError';
+ throw err;
+ },
+ });
+ });
+ await bootPage(page);
+ await optIn(page);
+ await page.waitForFunction(() => window.homeVoice.loop.state === 'permission-denied', null, { timeout: 20000 });
+ const recovery = await page.locator('#voice-panel .hv-recovery').textContent();
+ check('a denied microphone is a designed state, not a dead end', /Allow|address bar|settings/i.test(recovery), {
+ recovery,
+ });
+ const retry = await page.locator('#voice-panel [data-act="retry"]').count();
+ check('and it offers a way back', retry === 1);
+ await page.locator('#voice-panel').screenshot({ path: join(OUT, 'permission-denied.png') });
+ } finally {
+ await browser.close();
+ }
+}
+
+async function scenarioStateGallery() {
+ const { browser, page } = await launch(null);
+ try {
+ await page.goto(`${BASE}/voice/home`, { waitUntil: 'networkidle' });
+ const sections = page.locator('.vh-state');
+ const count = await sections.count();
+ check('the state gallery renders every one of the twelve states', count === 12, { count });
+ for (let i = 0; i < count; i++) {
+ const section = sections.nth(i);
+ const name = (await section.locator('h3').textContent()).trim();
+ await section.screenshot({ path: join(OUT, `state-${String(i + 1).padStart(2, '0')}-${name}.png`) });
+ }
+ await page.screenshot({ path: join(OUT, 'page-full.png'), fullPage: true });
+ } finally {
+ await browser.close();
+ }
+}
+
+// ── run ─────────────────────────────────────────────────────────────────────
+
+async function main() {
+ const probe = await fetch(`${BASE}/voice/home`).catch(() => null);
+ if (!probe?.ok) {
+ console.error(`No dev server at ${BASE}. Start one with: npx vite --port ${PORT}`);
+ process.exit(2);
+ }
+ await ensureClips();
+
+ const measured = {};
+ await scenarioColdLoad();
+ measured.happy = await scenarioHappyPath();
+ measured.barge = await scenarioBargeIn();
+ measured.selfTrigger = await scenarioSelfTrigger();
+ await scenarioGuarded({ clip: CLIPS.ambientYeah, expectRedeemed: false, label: 'ambient yes' });
+ await scenarioGuarded({ clip: CLIPS.confirmToken, expectRedeemed: true, label: 'the token' });
+ measured.mute = await scenarioMute();
+ await scenarioUnavailable();
+ await scenarioPermissionDenied();
+ await scenarioStateGallery();
+
+ const failed = results.filter((r) => !r.pass);
+ writeFileSync(
+ join(OUT, 'report.json'),
+ JSON.stringify({ base: BASE, at: new Date().toISOString(), results, measured }, null, 2),
+ );
+ console.log(`\n${results.length - failed.length}/${results.length} checks passed. Report: ${join(OUT, 'report.json')}`);
+ if (failed.length) {
+ for (const f of failed) console.log(` FAILED: ${f.name}`);
+ process.exit(1);
+ }
+}
+
+main().catch((err) => {
+ console.error(err);
+ process.exit(1);
+});
diff --git a/src/home/pcm-player.js b/src/home/pcm-player.js
new file mode 100644
index 000000000..45811e447
--- /dev/null
+++ b/src/home/pcm-player.js
@@ -0,0 +1,158 @@
+/**
+ * Streaming PCM playback with an analyser the lipsync driver can read.
+ *
+ * Home Assistant's pipeline sends its answer as raw 16-bit PCM in Wyoming audio
+ * chunks, not as a file. There is no `<audio>` element to hand to
+ * `tapAudioElement()`, so this builds the same shape of graph a level down:
+ *
+ * AudioBufferSourceNode (one per chunk, scheduled back to back)
+ * └─▶ GainNode ─▶ AnalyserNode ─▶ destination
+ * └─▶ LipsyncDriver reads this every frame
+ *
+ * Scheduling is the whole job. Chunks arrive over a network at whatever rate
+ * the pipeline produces them, and playing each one "now" on arrival produces
+ * gaps and overlaps that sound like a bad phone line. Each buffer is instead
+ * scheduled at the end of the previous one, on the AudioContext's own clock,
+ * with a small lead so a late chunk does not get scheduled in the past.
+ */
+
+/** Start playback this far ahead of `currentTime` so the first chunk is not late. */
+const START_LEAD_SECONDS = 0.08;
+
+/** If the schedule falls behind the clock, restart from here rather than the past. */
+const CATCHUP_LEAD_SECONDS = 0.04;
+
+export class PcmPlayer {
+ /**
+ * @param {object} [options]
+ * @param {AudioContext} [options.context] Reused across utterances.
+ * @param {number} [options.fftSize=256] Matches the lipsync driver's default.
+ */
+ constructor({ context = null, fftSize = 256 } = {}) {
+ this.context = context;
+ this._fftSize = fftSize;
+ this.analyser = null;
+ this._gain = null;
+ this._format = null;
+ this._nextTime = 0;
+ this._pending = new Set();
+ this._endsAt = 0;
+ this._playing = false;
+ }
+
+ get playing() {
+ return this._playing;
+ }
+
+ /** Seconds of audio scheduled but not yet played. */
+ get bufferedSeconds() {
+ if (!this.context) return 0;
+ return Math.max(0, this._nextTime - this.context.currentTime);
+ }
+
+ /**
+ * Open a stream. Safe to call again for a new utterance.
+ * @param {{rate:number, width:number, channels:number}} format
+ */
+ async start(format) {
+ const AC = window.AudioContext || window.webkitAudioContext;
+ if (!AC) throw new Error('This browser has no Web Audio support, so the agent cannot speak here.');
+ if (!this.context) this.context = new AC();
+ // Browsers suspend a context created outside a gesture. The satellite view
+ // only ever starts one after the user has pressed something, but a tab
+ // restored from the back/forward cache comes back suspended too.
+ if (this.context.state === 'suspended') await this.context.resume().catch(() => {});
+
+ if (!this.analyser) {
+ this.analyser = this.context.createAnalyser();
+ this.analyser.fftSize = this._fftSize;
+ this.analyser.smoothingTimeConstant = 0.4;
+ this._gain = this.context.createGain();
+ this._gain.connect(this.analyser);
+ this.analyser.connect(this.context.destination);
+ }
+
+ this._format = format;
+ this._nextTime = this.context.currentTime + START_LEAD_SECONDS;
+ this._endsAt = this._nextTime;
+ this._playing = true;
+ }
+
+ /**
+ * Queue one chunk of interleaved little-endian PCM.
+ * @param {ArrayBuffer|Uint8Array} bytes
+ */
+ push(bytes) {
+ if (!this._playing || !this.context || !this._format) return false;
+ const { rate, width, channels } = this._format;
+ const view = bytes instanceof ArrayBuffer ? new Uint8Array(bytes) : bytes;
+ const frames = Math.floor(view.byteLength / (width * channels));
+ if (frames <= 0) return false;
+
+ const buffer = this.context.createBuffer(channels, frames, rate);
+ const data = new DataView(view.buffer, view.byteOffset, view.byteLength);
+ for (let ch = 0; ch < channels; ch += 1) {
+ const out = buffer.getChannelData(ch);
+ for (let i = 0; i < frames; i += 1) {
+ const offset = (i * channels + ch) * width;
+ out[i] = width === 2
+ ? data.getInt16(offset, true) / 32768
+ : width === 1
+ ? (data.getUint8(offset) - 128) / 128
+ : data.getInt32(offset, true) / 2147483648;
+ }
+ }
+
+ const source = this.context.createBufferSource();
+ source.buffer = buffer;
+ source.connect(this._gain);
+ const startAt = Math.max(this._nextTime, this.context.currentTime + CATCHUP_LEAD_SECONDS);
+ source.start(startAt);
+ this._nextTime = startAt + buffer.duration;
+ this._endsAt = this._nextTime;
+ this._pending.add(source);
+ source.onended = () => this._pending.delete(source);
+ return true;
+ }
+
+ /**
+ * Stop accepting audio and resolve once everything queued has played.
+ * Resolving on the schedule rather than on the last `onended` matters: a
+ * suspended tab never fires `onended`, and the caller uses this to tell Home
+ * Assistant its answer finished. It must always resolve.
+ * @returns {Promise<void>}
+ */
+ async finish() {
+ if (!this._playing || !this.context) return;
+ this._playing = false;
+ const remaining = Math.max(0, this._endsAt - this.context.currentTime);
+ if (remaining > 0) await new Promise((resolve) => setTimeout(resolve, remaining * 1000 + 40));
+ }
+
+ /** Cut playback off immediately. Used when a run is cancelled or errors. */
+ stop() {
+ this._playing = false;
+ for (const source of this._pending) {
+ try {
+ source.stop();
+ } catch {
+ /* already ended */
+ }
+ }
+ this._pending.clear();
+ if (this.context) this._nextTime = this.context.currentTime;
+ }
+
+ /** Release the graph. The AudioContext is kept: reopening one costs a gesture. */
+ dispose() {
+ this.stop();
+ try {
+ this._gain?.disconnect();
+ this.analyser?.disconnect();
+ } catch {
+ /* already disconnected */
+ }
+ this._gain = null;
+ this.analyser = null;
+ }
+}
diff --git a/src/voice-home.js b/src/voice-home.js
new file mode 100644
index 000000000..e801f606e
--- /dev/null
+++ b/src/voice-home.js
@@ -0,0 +1,89 @@
+/**
+ * /voice/home: the hands-free voice loop, mounted on its own surface.
+ *
+ * The panel here is the same HomeVoicePanel that /home/:id mounts over the live
+ * 3D house. Keeping one component and two hosts is what stops the voice path and
+ * the scene path from drifting into two different confirmation flows, which is
+ * the exact failure that ends with a door opening on a "yeah".
+ *
+ * The state gallery below is not a mockup. It builds twelve real panels over
+ * twelve real loops and drives each into one state through the same code path
+ * the live loop uses, so what is rendered is what a user gets. No microphone is
+ * opened for any of them, and no model is downloaded.
+ */
+
+import { HomeVoiceLoop, STATES, STATE_ORDER, normalizePendingConfirmation } from './voice/home-voice.js';
+import { HomeVoicePanel } from './voice/home-voice-ui.js';
+import { log } from './shared/log.js';
+
+const params = new URLSearchParams(location.search);
+
+const loop = new HomeVoiceLoop({
+ // /home/:id passes the real connection. On this surface the home is optional:
+ // the loop is fully exercisable without one, and every leg except the device
+ // action is measured either way.
+ homeId: params.get('home'),
+ surface: params.get('surface') === 'screenless' ? 'screenless' : 'display',
+ onEvent: (event) => {
+ if (event.type === 'latency') log.info('[voice-home]', event.leg, event.ms + 'ms');
+ },
+});
+
+const mount = document.getElementById('voice-panel');
+const panel = new HomeVoicePanel({ mount, loop });
+
+// Probe before anything claims to listen. A deployment without a speech lane
+// lands in the unavailable state and says so, rather than lighting an indicator.
+void loop.probeAsr();
+
+// Exposed so a browser check and the /home scene can read the measured legs
+// without scraping the DOM.
+window.homeVoice = { loop, panel };
+
+// ── the state gallery ───────────────────────────────────────────────────────
+
+const SAMPLE_CONFIRMATION = normalizePendingConfirmation({
+ confirmation_id: 'preview',
+ sentence: 'This will unlock the Front Door.',
+ entity_ids: ['lock.front_door'],
+ risk: 'opens the house',
+ expires_in_ms: 90000,
+});
+
+const DETAILS = {
+ [STATES.PERMISSION_DENIED]: {
+ recovery: 'Chrome: click the icon at the left of the address bar, set Microphone to Allow, then reload this page.',
+ },
+ [STATES.CONFIRM_PENDING]: { confirmation: SAMPLE_CONFIRMATION },
+ [STATES.UNAVAILABLE]: {
+ reason:
+ 'Speech recognition is not available in this deployment, so the microphone stays off. ' +
+ 'Type to the agent instead: everything voice can do, text can do.',
+ },
+ [STATES.ERROR]: {
+ message: 'Speech recognition failed: 503 upstream unavailable. Say it again, or type it.',
+ retryable: true,
+ },
+ [STATES.IDLE]: { wakeWord: 'hey_jarvis' },
+ [STATES.CAPTURING]: { wakeWord: 'hey_jarvis', score: 0.99 },
+ [STATES.SPEAKING]: { text: 'The kitchen light is off.' },
+ [STATES.MUTED]: { tracks: ['ended'] },
+};
+
+const gallery = document.getElementById('state-gallery');
+for (const state of STATE_ORDER) {
+ const wrap = document.createElement('section');
+ wrap.className = 'vh-state';
+ const heading = document.createElement('h3');
+ heading.textContent = state;
+ const host = document.createElement('div');
+ wrap.append(heading, host);
+ gallery.appendChild(wrap);
+
+ const previewLoop = new HomeVoiceLoop({ surface: 'display' });
+ const previewPanel = new HomeVoicePanel({ mount: host, loop: previewLoop });
+ if (state === STATES.CONFIRM_PENDING) previewLoop.pendingConfirmation = SAMPLE_CONFIRMATION;
+ previewLoop._setState(state, DETAILS[state] || {});
+ // Keep a reference so the panels are not collected while their timers run.
+ wrap._panel = previewPanel;
+}
diff --git a/src/voice/home-voice-ui.js b/src/voice/home-voice-ui.js
new file mode 100644
index 000000000..e6dd907c5
--- /dev/null
+++ b/src/voice/home-voice-ui.js
@@ -0,0 +1,420 @@
+/**
+ * The panel that renders HomeVoiceLoop's twelve states.
+ *
+ * Split from the loop deliberately: the loop is the machine and this is the
+ * face, so /home/:id, a wall display and the demo surface can all mount the same
+ * loop and dress it differently. Everything user-facing about an always-on
+ * microphone lives here, which is why it is the file to read when asking "can
+ * this thing be listening without me knowing".
+ *
+ * Four things it guarantees, and they are the four that make always-on listening
+ * defensible at all:
+ *
+ * - Off by default. The first thing a new user sees is the explanation and a
+ * button, not a live microphone.
+ * - A permanent indicator whenever a track is live. It is not a setting, there
+ * is no way to hide it, and it reads the MediaStreamTrack rather than a flag.
+ * - One tap to mute, which really stops capture. The button reports the track
+ * state back to the user in plain words.
+ * - The pending action is spoken AND shown before it can be confirmed, with the
+ * entity ids visible, because agreeing to something you cannot see is not
+ * agreement.
+ */
+
+import { STATES } from './home-voice.js';
+import { WAKE_WORDS } from './wake-words.js';
+
+/** Everything the panel says, in one place, so the copy can be read as copy. */
+const COPY = {
+ [STATES.OFF]: {
+ badge: 'Off',
+ title: 'Hands-free is off',
+ body: 'Turn it on and you can talk to your house without finding a phone. It stays off until you say so.',
+ },
+ [STATES.PERMISSION_PENDING]: {
+ badge: 'Waiting',
+ title: 'Your browser is asking',
+ body: 'Choose Allow in the microphone prompt. Nothing is captured until you do.',
+ },
+ [STATES.PERMISSION_DENIED]: {
+ badge: 'Blocked',
+ title: 'The microphone is blocked',
+ body: 'Hands-free needs the mic. You can keep using the agent by typing in the meantime.',
+ },
+ [STATES.IDLE]: {
+ badge: 'Listening for the wake word',
+ title: 'Say the wake word',
+ body: 'Everything is being matched on this device. Nothing is sent anywhere until you say it.',
+ },
+ [STATES.CAPTURING]: {
+ badge: 'Listening',
+ title: 'Go ahead',
+ body: 'This part of what you say is sent for transcription, and nothing else is.',
+ },
+ [STATES.THINKING]: { badge: 'Thinking', title: 'Working on it', body: 'Reading the house and deciding what to do.' },
+ [STATES.SPEAKING]: { badge: 'Speaking', title: 'Answering', body: 'Talk over it any time and it will stop.' },
+ [STATES.BARGED_IN]: { badge: 'Interrupted', title: 'Stopped', body: 'Listening to you instead.' },
+ [STATES.CONFIRM_PENDING]: {
+ badge: 'Waiting for you',
+ title: 'This one needs a yes',
+ body: 'Say the word confirm, or press the button. Nothing else counts.',
+ },
+ [STATES.UNAVAILABLE]: {
+ badge: 'Unavailable',
+ title: 'Voice is not available here',
+ body: 'Speech recognition is not configured in this deployment, so the microphone stays off.',
+ },
+ [STATES.MUTED]: {
+ badge: 'Muted',
+ title: 'The microphone is stopped',
+ body: 'Capture is stopped at the device, not hidden behind a setting.',
+ },
+ [STATES.ERROR]: { badge: 'Problem', title: 'Something went wrong', body: '' },
+};
+
+const CONSENT_POINTS = [
+ 'The wake word is matched here, in this tab, by a model downloaded to your device. That audio never leaves.',
+ 'Only what you say after the wake word is uploaded, and only to transcribe it.',
+ 'Nothing is kept. The audio is not stored on our servers once the request is answered.',
+ 'While the microphone is live there is an indicator on this page that cannot be turned off.',
+ 'One tap on Mute stops the microphone at the device. You can turn hands-free off completely at any time.',
+];
+
+export class HomeVoicePanel {
+ /**
+ * @param {object} opts
+ * @param {HTMLElement} opts.mount
+ * @param {import('./home-voice.js').HomeVoiceLoop} opts.loop
+ */
+ constructor({ mount, loop }) {
+ if (!mount) throw new Error('HomeVoicePanel requires a mount element');
+ if (!loop) throw new Error('HomeVoicePanel requires a HomeVoiceLoop');
+ this.mount = mount;
+ this.loop = loop;
+ this.transcript = [];
+ this._indicatorTimer = 0;
+ this._wakeScore = 0;
+
+ this.mount.classList.add('hv');
+ this._build();
+
+ loop.onState = (state, detail) => this._render(state, detail);
+ const priorEvent = loop.onEvent;
+ loop.onEvent = (event) => {
+ priorEvent?.(event);
+ this._onEvent(event);
+ };
+ this._render(loop.state, loop.stateDetail);
+ // The indicator reads the live track, not the state machine, so a stuck
+ // state can never leave a live microphone unlabelled.
+ this._indicatorTimer = setInterval(() => this._syncIndicator(), 250);
+ }
+
+ destroy() {
+ clearInterval(this._indicatorTimer);
+ this.mount.replaceChildren();
+ this.mount.classList.remove('hv');
+ }
+
+ _build() {
+ this.mount.replaceChildren();
+ this.mount.innerHTML = `
+ <div class="hv-indicator" data-live="false" role="status" aria-live="polite">
+ <span class="hv-dot" aria-hidden="true"></span>
+ <span class="hv-indicator-text">Microphone off</span>
+ </div>
+ <div class="hv-head">
+ <div>
+ <span class="hv-badge" data-state="off">Off</span>
+ <h2 class="hv-title">Hands-free is off</h2>
+ <p class="hv-body"></p>
+ </div>
+ <div class="hv-actions"></div>
+ </div>
+ <div class="hv-consent" hidden>
+ <h3>Before you turn this on</h3>
+ <ul class="hv-consent-list"></ul>
+ <div class="hv-consent-actions">
+ <button type="button" class="hv-btn hv-btn-primary" data-act="consent">Turn on hands-free</button>
+ <button type="button" class="hv-btn" data-act="cancel-consent">Not now</button>
+ </div>
+ </div>
+ <div class="hv-recovery" hidden></div>
+ <div class="hv-confirm" hidden>
+ <p class="hv-confirm-sentence"></p>
+ <ul class="hv-confirm-entities"></ul>
+ <p class="hv-confirm-hint">Say <strong>confirm</strong> to continue. A general yes does not count.</p>
+ <div class="hv-confirm-actions">
+ <button type="button" class="hv-btn hv-btn-primary" data-act="confirm">Confirm</button>
+ <button type="button" class="hv-btn" data-act="cancel-confirm">Cancel</button>
+ </div>
+ <div class="hv-confirm-timer"><span></span></div>
+ </div>
+ <div class="hv-meter" hidden aria-hidden="true"><span class="hv-meter-fill"></span></div>
+ <ol class="hv-transcript" aria-label="Conversation"></ol>
+ <div class="hv-settings">
+ <label class="hv-field">
+ <span>Wake word</span>
+ <select class="hv-wake" aria-label="Wake word"></select>
+ </label>
+ <p class="hv-wake-hint"></p>
+ </div>
+ <dl class="hv-latency" aria-label="Measured latency"></dl>
+ `;
+
+ this.el = {
+ indicator: this.mount.querySelector('.hv-indicator'),
+ indicatorText: this.mount.querySelector('.hv-indicator-text'),
+ badge: this.mount.querySelector('.hv-badge'),
+ title: this.mount.querySelector('.hv-title'),
+ body: this.mount.querySelector('.hv-body'),
+ actions: this.mount.querySelector('.hv-actions'),
+ consent: this.mount.querySelector('.hv-consent'),
+ consentList: this.mount.querySelector('.hv-consent-list'),
+ recovery: this.mount.querySelector('.hv-recovery'),
+ confirm: this.mount.querySelector('.hv-confirm'),
+ confirmSentence: this.mount.querySelector('.hv-confirm-sentence'),
+ confirmEntities: this.mount.querySelector('.hv-confirm-entities'),
+ confirmTimer: this.mount.querySelector('.hv-confirm-timer span'),
+ meter: this.mount.querySelector('.hv-meter'),
+ meterFill: this.mount.querySelector('.hv-meter-fill'),
+ transcript: this.mount.querySelector('.hv-transcript'),
+ wake: this.mount.querySelector('.hv-wake'),
+ wakeHint: this.mount.querySelector('.hv-wake-hint'),
+ latency: this.mount.querySelector('.hv-latency'),
+ };
+
+ for (const point of CONSENT_POINTS) {
+ const li = document.createElement('li');
+ li.textContent = point;
+ this.el.consentList.appendChild(li);
+ }
+
+ for (const w of WAKE_WORDS) {
+ const option = document.createElement('option');
+ option.value = w.id;
+ option.textContent = w.phrase;
+ this.el.wake.appendChild(option);
+ }
+ this.el.wake.value = this.loop.settings.wakeWord;
+ this._renderWakeHint();
+ this.el.wake.addEventListener('change', () => {
+ this.loop.saveSettings({ wakeWord: this.el.wake.value });
+ this._renderWakeHint();
+ });
+
+ this.mount.addEventListener('click', (event) => this._onClick(event));
+ }
+
+ _renderWakeHint() {
+ const def = WAKE_WORDS.find((w) => w.id === this.el.wake.value);
+ this.el.wakeHint.textContent = def ? def.hint : '';
+ }
+
+ async _onClick(event) {
+ const act = event.target.closest('[data-act]')?.dataset.act;
+ if (!act) return;
+ switch (act) {
+ case 'ask-consent':
+ this.el.consent.hidden = false;
+ this.el.consent.querySelector('[data-act="consent"]')?.focus();
+ break;
+ case 'cancel-consent':
+ this.el.consent.hidden = true;
+ break;
+ case 'consent':
+ this.el.consent.hidden = true;
+ this.loop.grantConsent();
+ await this.loop.enable();
+ break;
+ case 'mute':
+ await this.loop.mute();
+ break;
+ case 'unmute':
+ await this.loop.unmute();
+ break;
+ case 'off':
+ await this.loop.revokeConsent();
+ break;
+ case 'retry':
+ await this.loop.enable();
+ break;
+ case 'confirm':
+ await this.loop.confirmPending();
+ break;
+ case 'cancel-confirm':
+ this.loop.cancelConfirmation();
+ break;
+ default:
+ break;
+ }
+ }
+
+ _onEvent(event) {
+ switch (event.type) {
+ case 'wake-score':
+ this._wakeScore = event.score;
+ this.el.meterFill.style.transform = `scaleX(${Math.min(1, Math.max(0, event.score))})`;
+ break;
+ case 'transcript':
+ this._appendLine('you', event.text);
+ break;
+ case 'wake':
+ this._appendLine('system', `Woke on "${wakePhrase(event.wakeWord)}" (${event.score.toFixed(2)}).`);
+ break;
+ case 'barge-in':
+ this._appendLine('system', 'You interrupted, so it stopped talking.');
+ break;
+ case 'guarded-refused':
+ this._appendLine('system', 'Refused: a guarded action cannot be confirmed by voice on a screenless device.');
+ break;
+ case 'confirmation-not-token':
+ this._appendLine('system', 'That was not the word confirm, so nothing was unlocked.');
+ break;
+ case 'confirmation-executed':
+ this._appendLine('system', 'Confirmed and carried out.');
+ break;
+ case 'confirmation-closed':
+ if (event.reason === 'expired') this._appendLine('system', 'The confirmation expired. Nothing changed.');
+ if (event.reason === 'cancelled') this._appendLine('system', 'Cancelled. Nothing changed.');
+ break;
+ case 'tts-failed':
+ this._appendLine('agent', event.text);
+ break;
+ case 'latency':
+ this._renderLatency();
+ break;
+ default:
+ break;
+ }
+ }
+
+ _appendLine(who, text) {
+ const li = document.createElement('li');
+ li.className = `hv-line hv-line-${who}`;
+ li.textContent = text;
+ this.el.transcript.appendChild(li);
+ while (this.el.transcript.children.length > 40) this.el.transcript.firstElementChild.remove();
+ this.el.transcript.scrollTop = this.el.transcript.scrollHeight;
+ }
+
+ _syncIndicator() {
+ const live = this.loop.micLive;
+ this.el.indicator.dataset.live = String(live);
+ this.el.indicatorText.textContent = live
+ ? this.loop.state === STATES.CAPTURING
+ ? 'Microphone live, and this is being sent for transcription'
+ : 'Microphone live, matching the wake word on this device'
+ : this.loop.muted
+ ? 'Microphone stopped'
+ : 'Microphone off';
+ }
+
+ _render(state, detail) {
+ const copy = COPY[state] || COPY[STATES.ERROR];
+ this.el.badge.textContent = copy.badge;
+ this.el.badge.dataset.state = state;
+ this.el.title.textContent = copy.title;
+ this.el.body.textContent = detail?.message || detail?.reason || detail?.note || copy.body;
+ this.mount.dataset.state = state;
+ this._syncIndicator();
+
+ this.el.meter.hidden = state !== STATES.IDLE;
+ this.el.recovery.hidden = true;
+ this.el.confirm.hidden = state !== STATES.CONFIRM_PENDING;
+
+ if (state === STATES.PERMISSION_DENIED && detail?.recovery) {
+ this.el.recovery.hidden = false;
+ this.el.recovery.textContent = detail.recovery;
+ }
+
+ if (state === STATES.CONFIRM_PENDING && detail?.confirmation) {
+ this._renderConfirmation(detail.confirmation);
+ }
+
+ if (state === STATES.SPEAKING && detail?.text) this._appendLine('agent', detail.text);
+
+ this._renderActions(state);
+ }
+
+ _renderConfirmation(confirmation) {
+ this.el.confirmSentence.textContent = confirmation.sentence;
+ this.el.confirmEntities.replaceChildren();
+ for (const id of confirmation.entityIds) {
+ const li = document.createElement('li');
+ // textContent, always: an entity name is a string a device or another
+ // household member controls, and it is rendered as data, never markup.
+ li.textContent = id;
+ this.el.confirmEntities.appendChild(li);
+ }
+ const started = Date.now();
+ const total = confirmation.expiresInMs;
+ const tick = () => {
+ if (this.loop.pendingConfirmation !== confirmation) return;
+ const left = Math.max(0, total - (Date.now() - started));
+ this.el.confirmTimer.style.transform = `scaleX(${left / total})`;
+ if (left > 0) requestAnimationFrame(tick);
+ };
+ requestAnimationFrame(tick);
+ }
+
+ _renderActions(state) {
+ const buttons = [];
+ if (state === STATES.OFF) {
+ buttons.push(['ask-consent', 'Turn on hands-free', true]);
+ } else if (state === STATES.PERMISSION_DENIED || state === STATES.ERROR) {
+ buttons.push(['retry', 'Try again', true]);
+ buttons.push(['off', 'Turn hands-free off', false]);
+ } else if (state === STATES.MUTED) {
+ buttons.push(['unmute', 'Unmute', true]);
+ buttons.push(['off', 'Turn hands-free off', false]);
+ } else if (state !== STATES.UNAVAILABLE && state !== STATES.PERMISSION_PENDING) {
+ buttons.push(['mute', 'Mute', false]);
+ buttons.push(['off', 'Turn hands-free off', false]);
+ }
+ this.el.actions.replaceChildren();
+ for (const [act, label, primary] of buttons) {
+ const btn = document.createElement('button');
+ btn.type = 'button';
+ btn.className = `hv-btn${primary ? ' hv-btn-primary' : ''}`;
+ btn.dataset.act = act;
+ btn.textContent = label;
+ this.el.actions.appendChild(btn);
+ }
+ }
+
+ _renderLatency() {
+ const summary = this.loop.latencySummary();
+ const budgets = {
+ wake: 200,
+ endpoint: 400,
+ asr: 900,
+ turn: 1200,
+ action: 700,
+ firstAudio: 1800,
+ };
+ const labels = {
+ wake: 'Wake word',
+ endpoint: 'End of speech',
+ asr: 'Transcription',
+ turn: 'Agent turn',
+ action: 'Action to device',
+ firstAudio: 'First audible reply',
+ tts: 'Speech synthesis',
+ };
+ this.el.latency.replaceChildren();
+ for (const [leg, stats] of Object.entries(summary)) {
+ const dt = document.createElement('dt');
+ dt.textContent = labels[leg] || leg;
+ const dd = document.createElement('dd');
+ dd.textContent = `${stats.median} ms`;
+ const budget = budgets[leg];
+ if (budget) dd.dataset.over = String(stats.median > budget);
+ this.el.latency.append(dt, dd);
+ }
+ }
+}
+
+function wakePhrase(id) {
+ return WAKE_WORDS.find((w) => w.id === id)?.phrase || id;
+}
diff --git a/src/voice/home-voice.js b/src/voice/home-voice.js
new file mode 100644
index 000000000..ef95b925e
--- /dev/null
+++ b/src/voice/home-voice.js
@@ -0,0 +1,1069 @@
+/**
+ * The browser voice loop for the house.
+ *
+ * mic -> silero VAD -> openWakeWord -> capture -> /api/asr
+ * -> agent turn (/api/chat with the home tools)
+ * -> /api/tts/speak -> playback -> LipsyncDriver -> the 3D agent speaks
+ *
+ * One microphone, one audio context, one indicator. The VAD, the wake word and
+ * the utterance capture all read the same stream, because three getUserMedia
+ * calls would mean three things to remember to stop and "mute" has to stop
+ * everything that exists.
+ *
+ * Three properties this file exists to guarantee, in order of how much damage
+ * their absence does:
+ *
+ * 1. Nothing about listening runs until the user opts in. The VAD module, the
+ * wake-word module, onnxruntime and every model byte are behind dynamic
+ * imports reached only from enable(). A cold load of a page that mounts this
+ * loop fetches none of it.
+ * 2. The microphone's state is never in doubt. mute() stops the MediaStream
+ * tracks, so the browser's own recording indicator goes out and
+ * track.readyState reads "ended". It is not a flag that hides a dot.
+ * 3. Speech cannot satisfy a guarded action on its own. The confirmation
+ * grammar is a narrow token, the pending action comes from a server-minted
+ * confirmationId, and on a surface with no display a guarded action is
+ * refused outright. ASR is not a gate on a door: recognizers mishear, and
+ * the failure mode of mishearing "confirm" is an open front door.
+ *
+ * The twelve states in STATES are the whole external surface of the loop. A host
+ * (the /home scene, the demo page, a wall display) renders them; this module
+ * never touches the DOM.
+ */
+
+import { log } from '../shared/log.js';
+import { VoiceActivityDetector, float32ToWav, now, VAD_SAMPLE_RATE, DEFAULT_REDEMPTION_MS } from './vad.js';
+
+/** The twelve states. A host must handle every one of them. */
+export const STATES = {
+ /** Default. Listening has never been turned on. */
+ OFF: 'off',
+ /** getUserMedia is open and the user has not answered. */
+ PERMISSION_PENDING: 'permission-pending',
+ /** The user or the browser refused the microphone. */
+ PERMISSION_DENIED: 'permission-denied',
+ /** Live, listening for the wake word, uploading nothing. */
+ IDLE: 'idle',
+ /** Woken. Capturing one utterance. */
+ CAPTURING: 'capturing',
+ /** Transcribed, and the agent is deciding. */
+ THINKING: 'thinking',
+ /** The agent is speaking and barge-in is armed. */
+ SPEAKING: 'speaking',
+ /** The user interrupted; playback was cut. */
+ BARGED_IN: 'barged-in',
+ /** A guarded action is waiting for the explicit spoken token. */
+ CONFIRM_PENDING: 'confirm-pending',
+ /** Speech-to-text is not available. The loop degrades to text and says so. */
+ UNAVAILABLE: 'unavailable',
+ /** Capture is stopped at the track level. */
+ MUTED: 'muted',
+ /** Something failed in a way the user has to know about. */
+ ERROR: 'error',
+};
+
+/** Every state, in the order a host should present them. Used by the demo surface. */
+export const STATE_ORDER = [
+ STATES.OFF,
+ STATES.PERMISSION_PENDING,
+ STATES.PERMISSION_DENIED,
+ STATES.IDLE,
+ STATES.CAPTURING,
+ STATES.THINKING,
+ STATES.SPEAKING,
+ STATES.BARGED_IN,
+ STATES.CONFIRM_PENDING,
+ STATES.UNAVAILABLE,
+ STATES.MUTED,
+ STATES.ERROR,
+];
+
+/** Bumped when the consent copy changes materially, which re-asks everyone. */
+export const CONSENT_VERSION = 1;
+const CONSENT_KEY = 'tws:home-voice:consent';
+const SETTINGS_KEY = 'tws:home-voice:settings';
+
+/**
+ * Barge-in sensitivity while the agent is speaking. Four consecutive frames
+ * (128 ms) above a high probability, rather than one frame at the idle
+ * threshold. One frame would fire on the agent's own voice leaking past echo
+ * cancellation on a laptop speaker at volume, and an agent that interrupts
+ * itself is worse than one that cannot be interrupted.
+ */
+const BARGE_IN_PROBABILITY = 0.65;
+const BARGE_IN_FRAMES = 4;
+
+/** An utterance longer than this is a stuck mic or a television, not a request. */
+const MAX_UTTERANCE_SEC = 20;
+
+/**
+ * The spoken confirmation token, and the words that are NOT it.
+ *
+ * The rule, and the reason it is a narrow token: a background "yeah" in somebody
+ * else's conversation must never unlock a door. A general affirmative is exactly
+ * what a recognizer produces from ambient speech, so the grammar accepts one
+ * deliberate word and its smallest natural pairings, and nothing else.
+ */
+const CONFIRM_PHRASES = new Set([
+ 'confirm',
+ 'confirm it',
+ 'confirm that',
+ 'confirm this',
+ 'i confirm',
+ 'yes confirm',
+ 'okay confirm',
+ 'ok confirm',
+ 'confirm yes',
+ 'confirmed',
+]);
+
+/**
+ * Listed explicitly so the intent is readable rather than implied by a regex:
+ * every one of these is a word a person says while agreeing with somebody else
+ * in the same room, and not one of them confirms anything here.
+ */
+const REJECTED_AFFIRMATIVES = new Set([
+ 'yes',
+ 'yeah',
+ 'yep',
+ 'yup',
+ 'ya',
+ 'sure',
+ 'ok',
+ 'okay',
+ 'alright',
+ 'all right',
+ 'fine',
+ 'do it',
+ 'go ahead',
+ 'go for it',
+ 'please',
+ 'please do',
+ 'absolutely',
+ 'definitely',
+ 'of course',
+ 'uh huh',
+ 'mhm',
+ 'right',
+ 'correct',
+ 'affirmative',
+ 'yes please',
+ 'yeah sure',
+ 'sounds good',
+ 'why not',
+]);
+
+const CANCEL_PHRASES = new Set([
+ 'cancel',
+ 'stop',
+ 'no',
+ 'nope',
+ 'never mind',
+ 'nevermind',
+ 'forget it',
+ 'dont',
+ 'do not',
+ 'cancel that',
+ 'cancel it',
+ 'abort',
+]);
+
+/**
+ * Normalize a transcript for grammar matching: lowercase, strip punctuation and
+ * apostrophes, collapse whitespace. Deliberately strict about word count: the
+ * grammar's safety comes from matching a whole short utterance rather than from
+ * finding a word buried inside a long one.
+ */
+export function normalizeTranscript(text) {
+ return String(text || '')
+ .toLowerCase()
+ .replace(/['’]/g, '')
+ .replace(/[^a-z0-9\s]/g, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+/**
+ * Classify an utterance heard while a confirmation is outstanding.
+ *
+ * @param {string} text
+ * @returns {'confirm'|'cancel'|'other'}
+ */
+export function classifyConfirmation(text) {
+ const norm = normalizeTranscript(text);
+ if (!norm) return 'other';
+ // A general affirmative is tested first and by name. It is not a near miss to
+ // be forgiven, it is the exact thing this grammar exists to refuse.
+ if (REJECTED_AFFIRMATIVES.has(norm)) return 'other';
+ if (CONFIRM_PHRASES.has(norm)) return 'confirm';
+ if (CANCEL_PHRASES.has(norm)) return 'cancel';
+ return 'other';
+}
+
+/** Whether an utterance would satisfy a guarded action. Exported for tests. */
+export function isConfirmationToken(text) {
+ return classifyConfirmation(text) === 'confirm';
+}
+
+function safeStorage() {
+ try {
+ return typeof localStorage !== 'undefined' ? localStorage : null;
+ } catch {
+ return null;
+ }
+}
+
+/** Read the stored consent record, tolerating a wiped or hostile localStorage. */
+export function readConsent(storage = safeStorage()) {
+ try {
+ const raw = storage?.getItem(CONSENT_KEY);
+ if (!raw) return null;
+ const parsed = JSON.parse(raw);
+ if (parsed?.version !== CONSENT_VERSION) return null;
+ return parsed;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Per-browser recovery for a denied microphone. A dead end here is the whole
+ * feature dead, and "check your browser settings" helps nobody.
+ */
+export function permissionRecovery(userAgent = typeof navigator !== 'undefined' ? navigator.userAgent : '') {
+ const ua = String(userAgent);
+ if (/Firefox\//.test(ua)) {
+ return 'Firefox: click the crossed-out microphone in the address bar, choose Blocked Temporarily or Remove, then reload this page.';
+ }
+ if (/Edg\//.test(ua)) {
+ return 'Edge: click the lock icon in the address bar, set Microphone to Allow, then reload this page.';
+ }
+ if (/Chrome\//.test(ua)) {
+ return 'Chrome: click the icon at the left of the address bar, set Microphone to Allow, then reload this page.';
+ }
+ if (/Safari\//.test(ua)) {
+ return 'Safari: open Settings for This Website from the Safari menu, set Microphone to Allow, then reload.';
+ }
+ return 'Allow microphone access for this site in your browser settings, then reload this page.';
+}
+
+export class HomeVoiceLoop {
+ /**
+ * @param {object} opts
+ * @param {string} [opts.homeId] The home connection this loop acts on.
+ * @param {'display'|'screenless'} [opts.surface]
+ * Whether the user can SEE what they are confirming. A screenless
+ * surface refuses guarded actions; see _openConfirmation().
+ * @param {(state: string, detail: object) => void} [opts.onState]
+ * @param {(event: {type: string}) => void} [opts.onEvent]
+ * @param {(leg: string, ms: number) => void} [opts.onLatency]
+ * @param {{ setMouthShape(s: object): void }} [opts.mouthTarget]
+ * @param {typeof fetch} [opts.fetchImpl]
+ */
+ constructor({ homeId = null, surface = 'display', onState, onEvent, onLatency, mouthTarget = null, fetchImpl } = {}) {
+ this.homeId = homeId;
+ this.surface = surface;
+ this.onState = onState || (() => {});
+ this.onEvent = onEvent || (() => {});
+ this.onLatency = onLatency || (() => {});
+ this.mouthTarget = mouthTarget;
+ this.fetch = fetchImpl || ((...args) => fetch(...args));
+
+ this.state = STATES.OFF;
+ this.stateDetail = {};
+ /** Every leg measured this session, newest last. */
+ this.latency = [];
+ /** ASR capability, read from /api/asr before anything claims to listen. */
+ this.asr = { probed: false, configured: false, languages: [] };
+ this.settings = this._readSettings();
+
+ this._stream = null;
+ this._audioContext = null;
+ this._vad = null;
+ this._wake = null;
+ this._lipsync = null;
+ this._playback = null;
+ this._playbackAbort = null;
+ this._turnAbort = null;
+ this._bargeFrames = 0;
+ this._captureUntil = 0;
+ this._enabled = false;
+ this._muted = false;
+ this._destroyed = false;
+ /** The outstanding server-minted confirmation, or null. */
+ this.pendingConfirmation = null;
+ this._confirmTimer = 0;
+ /** Marks used to derive the leg timings. */
+ this._marks = {};
+ }
+
+ get enabled() {
+ return this._enabled;
+ }
+
+ get muted() {
+ return this._muted;
+ }
+
+ /** True when a live microphone track exists. The indicator reads this, only this. */
+ get micLive() {
+ return !!this._stream?.getAudioTracks().some((t) => t.readyState === 'live');
+ }
+
+ /** The readyState of every track this loop holds. The mute proof reads this. */
+ trackStates() {
+ return (this._stream?.getAudioTracks() || []).map((t) => t.readyState);
+ }
+
+ /** Whether the user has already opted in on this device. */
+ hasConsent() {
+ return !!readConsent();
+ }
+
+ /**
+ * Ask /api/asr whether speech-to-text exists in this deployment. Cheap,
+ * unauthenticated, and safe to call before consent: it uploads nothing and
+ * touches no microphone. A loop that cannot transcribe must say so rather than
+ * light an indicator and listen to a house for nothing.
+ */
+ async probeAsr() {
+ try {
+ const res = await this.fetch('/api/asr', { headers: { accept: 'application/json' } });
+ if (!res.ok) throw new Error(`asr probe ${res.status}`);
+ const body = await res.json();
+ this.asr = {
+ probed: true,
+ configured: !!body.configured,
+ languages: Array.isArray(body.languages) ? body.languages : [],
+ sampleRate: body.sampleRate || VAD_SAMPLE_RATE,
+ };
+ } catch (err) {
+ this.asr = { probed: true, configured: false, languages: [], error: err?.message };
+ }
+ if (!this.asr.configured && this.state === STATES.OFF) {
+ this._setState(STATES.UNAVAILABLE, { reason: UNAVAILABLE_REASON });
+ }
+ return this.asr;
+ }
+
+ /**
+ * Record the opt-in. Separate from enable() on purpose: consent is a decision
+ * with a timestamp, and turning the mic on is an action. A host that stores the
+ * first without doing the second is behaving correctly.
+ */
+ grantConsent() {
+ const record = { version: CONSENT_VERSION, grantedAt: new Date().toISOString() };
+ try {
+ safeStorage()?.setItem(CONSENT_KEY, JSON.stringify(record));
+ } catch {}
+ this.onEvent({ type: 'consent-granted', ...record });
+ return record;
+ }
+
+ /** Withdraw the opt-in and stop everything. One tap, no confirmation dialog. */
+ async revokeConsent() {
+ try {
+ safeStorage()?.removeItem(CONSENT_KEY);
+ } catch {}
+ await this.disable();
+ this.onEvent({ type: 'consent-revoked' });
+ }
+
+ /** Persisted, non-sensitive preferences: the chosen wake word and language. */
+ _readSettings() {
+ const defaults = { wakeWord: 'hey_jarvis', language: 'en-US' };
+ try {
+ const raw = safeStorage()?.getItem(SETTINGS_KEY);
+ return raw ? { ...defaults, ...JSON.parse(raw) } : defaults;
+ } catch {
+ return defaults;
+ }
+ }
+
+ saveSettings(patch) {
+ this.settings = { ...this.settings, ...patch };
+ try {
+ safeStorage()?.setItem(SETTINGS_KEY, JSON.stringify(this.settings));
+ } catch {}
+ if (patch.wakeWord && this._wake) void this._wake.setWakeWord(patch.wakeWord);
+ this.onEvent({ type: 'settings', settings: this.settings });
+ }
+
+ /**
+ * Turn listening on: acquire the microphone, then load the VAD and the wake
+ * word. Requires consent; call grantConsent() first.
+ *
+ * Every module and model byte below is fetched here and nowhere earlier.
+ */
+ async enable() {
+ if (this._destroyed) throw new Error('HomeVoiceLoop was destroyed');
+ if (this._enabled) return;
+ if (!this.hasConsent()) throw new Error('HomeVoiceLoop.enable() requires consent');
+ if (!this.asr.probed) await this.probeAsr();
+ if (!this.asr.configured) {
+ this._setState(STATES.UNAVAILABLE, { reason: UNAVAILABLE_REASON });
+ return;
+ }
+
+ this._setState(STATES.PERMISSION_PENDING, {});
+ try {
+ this._stream = await navigator.mediaDevices.getUserMedia({ audio: MIC_CONSTRAINTS });
+ } catch (err) {
+ const name = err?.name || '';
+ if (name === 'NotAllowedError' || name === 'SecurityError') {
+ this._setState(STATES.PERMISSION_DENIED, { recovery: permissionRecovery() });
+ return;
+ }
+ this._setState(STATES.ERROR, {
+ message:
+ name === 'NotFoundError'
+ ? 'No microphone was found. Plug one in, or keep talking to the agent by text.'
+ : `The microphone could not start: ${err?.message || name || 'unknown error'}`,
+ retryable: true,
+ });
+ return;
+ }
+
+ const AC = window.AudioContext || window.webkitAudioContext;
+ this._audioContext = new AC();
+ if (this._audioContext.state === 'suspended') await this._audioContext.resume().catch(() => {});
+
+ try {
+ const { WakeWordDetector } = await import('./wake-word.js');
+ this._wake = new WakeWordDetector({
+ wakeWord: this.settings.wakeWord,
+ onWake: (d) => this._onWake(d),
+ onScore: (score) => this.onEvent({ type: 'wake-score', score }),
+ });
+ await this._wake.load();
+ await this._startVad();
+ } catch (err) {
+ await this._teardownAudio();
+ this._setState(STATES.ERROR, {
+ message: `The listening models could not load: ${err?.message || 'unknown error'}`,
+ retryable: true,
+ });
+ return;
+ }
+
+ this._enabled = true;
+ this._muted = false;
+ this._setState(STATES.IDLE, { wakeWord: this.settings.wakeWord });
+ }
+
+ async _startVad() {
+ this._vad = new VoiceActivityDetector({
+ stream: this._stream,
+ audioContext: this._audioContext,
+ redemptionMs: DEFAULT_REDEMPTION_MS,
+ onFrame: (frame, prob) => this._onFrame(frame, prob),
+ onSpeechStart: () => this._onSpeechStart(),
+ onSpeechEnd: (audio) => this._onSpeechEnd(audio),
+ onMisfire: () => this.onEvent({ type: 'vad-misfire' }),
+ });
+ await this._vad.start();
+ }
+
+ /**
+ * Stop capture at the track level. The browser's own recording indicator goes
+ * out, and track.readyState reads "ended" for every track. Nothing here hides
+ * an indicator over a live microphone.
+ */
+ async mute() {
+ if (!this._enabled || this._muted) return;
+ this._muted = true;
+ this._cancelPlayback('muted');
+ await this._vad?.pause();
+ for (const track of this._stream?.getAudioTracks() || []) {
+ try {
+ track.stop();
+ } catch {}
+ }
+ this._wake?.reset();
+ this._setState(STATES.MUTED, { tracks: this.trackStates() });
+ }
+
+ /** Re-acquire the microphone. A stopped track cannot be restarted, so this is a new one. */
+ async unmute() {
+ if (!this._enabled || !this._muted) return;
+ try {
+ this._stream = await navigator.mediaDevices.getUserMedia({ audio: MIC_CONSTRAINTS });
+ } catch {
+ this._setState(STATES.PERMISSION_DENIED, { recovery: permissionRecovery() });
+ return;
+ }
+ // The VAD holds the old, ended stream: rebuild it against the new one.
+ await this._vad?.destroy();
+ await this._startVad();
+ this._muted = false;
+ this._wake?.reset();
+ this._setState(STATES.IDLE, { wakeWord: this.settings.wakeWord });
+ }
+
+ /** Turn the whole loop off and release every resource. */
+ async disable() {
+ this._enabled = false;
+ this._muted = false;
+ this._cancelPlayback('disabled');
+ this._clearConfirmation('disabled', { silent: true });
+ await this._teardownAudio();
+ this._setState(STATES.OFF, {});
+ }
+
+ async destroy() {
+ this._destroyed = true;
+ await this.disable();
+ }
+
+ async _teardownAudio() {
+ try {
+ await this._vad?.destroy();
+ } catch {}
+ this._vad = null;
+ try {
+ await this._wake?.destroy();
+ } catch {}
+ this._wake = null;
+ for (const track of this._stream?.getAudioTracks() || []) {
+ try {
+ track.stop();
+ } catch {}
+ }
+ this._stream = null;
+ if (this._audioContext && this._audioContext.state !== 'closed') {
+ await this._audioContext.close().catch(() => {});
+ }
+ this._audioContext = null;
+ }
+
+ // -- the loop ------------------------------------------------------------
+
+ _onFrame(frame, probability) {
+ // The wake word only ever sees audio while the loop is idle. During a
+ // capture it is irrelevant, and during playback it must not run at all:
+ // see the suppression in _speak().
+ if (this.state === STATES.IDLE) this._wake?.push(frame);
+
+ if (this.state === STATES.SPEAKING) {
+ if (probability >= BARGE_IN_PROBABILITY) {
+ // The first frame of the qualifying run is when the user actually
+ // started talking, and it is what the barge-in budget is measured
+ // against. Measuring from the fourth frame would flatter the number
+ // by exactly the confirmation window.
+ if (this._bargeFrames === 0) this._marks.bargeFirstFrame = now();
+ this._bargeFrames++;
+ if (this._bargeFrames >= BARGE_IN_FRAMES) this._bargeIn();
+ } else {
+ this._bargeFrames = 0;
+ }
+ }
+ }
+
+ _onSpeechStart() {
+ this._marks.speechStart = now();
+ this.onEvent({ type: 'speech-start' });
+ }
+
+ _onWake({ score, latencyMs, wakeWord }) {
+ if (this.state !== STATES.IDLE) return;
+ this._marks.wake = now();
+ this._record('wake', latencyMs);
+ this.onEvent({ type: 'wake', score, wakeWord, latencyMs });
+ this._captureUntil = now() + MAX_UTTERANCE_SEC * 1000;
+ this._setState(STATES.CAPTURING, { wakeWord, score });
+ }
+
+ async _onSpeechEnd(audio) {
+ const endedAt = now();
+ // Trailing silence actually spent before the utterance was called finished.
+ if (this._vad?.lastSpeechFrameAt) this._record('endpoint', endedAt - this._vad.lastSpeechFrameAt);
+ this._marks.speechEnd = endedAt;
+
+ if (this.state !== STATES.CAPTURING && this.state !== STATES.CONFIRM_PENDING) return;
+ if (this.state === STATES.CAPTURING && endedAt > this._captureUntil) {
+ this._setState(STATES.IDLE, {
+ wakeWord: this.settings.wakeWord,
+ note: 'That went on too long, so it was dropped.',
+ });
+ return;
+ }
+
+ const wasConfirming = this.state === STATES.CONFIRM_PENDING;
+ if (!wasConfirming) this._setState(STATES.THINKING, {});
+
+ let transcript = '';
+ try {
+ transcript = await this._transcribe(audio);
+ } catch (err) {
+ this._setState(STATES.ERROR, {
+ message: `Speech recognition failed: ${err?.message || 'unknown error'}. Say it again, or type it.`,
+ retryable: true,
+ });
+ this._recover();
+ return;
+ }
+
+ if (!transcript.trim()) {
+ this.onEvent({ type: 'empty-transcript' });
+ if (!wasConfirming) this._setState(STATES.IDLE, { wakeWord: this.settings.wakeWord });
+ return;
+ }
+ this.onEvent({ type: 'transcript', text: transcript, forConfirmation: wasConfirming });
+
+ if (wasConfirming) {
+ await this._handleSpokenConfirmation(transcript);
+ return;
+ }
+ await this.say(transcript);
+ }
+
+ /** Upload one utterance to the real ASR lane and time the round trip. */
+ async _transcribe(audio) {
+ const wav = float32ToWav(audio, VAD_SAMPLE_RATE);
+ const started = now();
+ const res = await this.fetch('/api/asr', {
+ method: 'POST',
+ headers: { 'content-type': 'audio/wav' },
+ body: wav,
+ credentials: 'same-origin',
+ });
+ this._record('asr', now() - started);
+ if (!res.ok) {
+ const detail = await res.text().catch(() => '');
+ throw new Error(`${res.status} ${detail.slice(0, 160)}`);
+ }
+ const body = await res.json();
+ return String(body.text || '');
+ }
+
+ /**
+ * Run one agent turn over a transcript. Public so a host can feed the same
+ * loop from a text box: the degraded path is the voice path minus the audio,
+ * not a second implementation.
+ */
+ async say(text) {
+ this._setState(STATES.THINKING, { text });
+ this._turnAbort?.abort();
+ const controller = new AbortController();
+ this._turnAbort = controller;
+ const started = now();
+ let result;
+ try {
+ result = await this._turn(text, controller.signal);
+ } catch (err) {
+ if (controller.signal.aborted) return;
+ this._setState(STATES.ERROR, {
+ message: `The agent could not answer: ${err?.message || 'unknown error'}`,
+ retryable: true,
+ });
+ this._recover();
+ return;
+ }
+ this._record('turn', now() - started);
+
+ if (result.pendingConfirmation) {
+ await this._openConfirmation(result.pendingConfirmation);
+ return;
+ }
+ if (result.reply) await this._speak(result.reply);
+ else this._recover();
+ }
+
+ /**
+ * One turn against the platform's chat lane, scoped to this home.
+ *
+ * The home tools, the gate and the confirmation record all live server-side.
+ * This client sends the transcript and the home id and reads back either an
+ * answer or a pending_confirmation payload; it never decides whether an action
+ * is guarded, and it has no way to set `confirmed`.
+ */
+ async _turn(text, signal) {
+ const res = await this.fetch('/api/chat', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ credentials: 'same-origin',
+ signal,
+ body: JSON.stringify({
+ messages: [{ role: 'user', content: text }],
+ home_id: this.homeId,
+ surface: 'home-voice',
+ }),
+ });
+ if (!res.ok) {
+ const detail = await res.text().catch(() => '');
+ throw new Error(`${res.status} ${detail.slice(0, 200)}`);
+ }
+ const body = await res.json().catch(() => ({}));
+ const reply = String(body.reply ?? body.content ?? body.text ?? '').trim();
+ const pending = body.pending_confirmation || body.pendingConfirmation || null;
+ return { reply, pendingConfirmation: pending ? normalizePendingConfirmation(pending) : null };
+ }
+
+ // -- guarded actions -----------------------------------------------------
+
+ /**
+ * Open a pending confirmation: say the whole sentence, show the entity, and
+ * wait for the token.
+ *
+ * On a surface with no display the answer is no. A user who cannot SEE what
+ * they are agreeing to is being asked to trust a recognizer with a lock, and a
+ * recognizer is not trustworthy enough for that. This refusal is deliberate
+ * and stays in the code.
+ */
+ async _openConfirmation(pending) {
+ if (this.surface === 'screenless') {
+ this.onEvent({ type: 'guarded-refused', reason: 'screenless', confirmation: pending });
+ await this._speak(
+ `${pending.sentence} I cannot take that one by voice alone on a device with no screen, because it would ` +
+ 'mean trusting speech recognition with a lock. Confirm it on your phone and it will go through.',
+ );
+ return;
+ }
+
+ this.pendingConfirmation = pending;
+ clearTimeout(this._confirmTimer);
+ this._confirmTimer = setTimeout(() => this._clearConfirmation('expired'), Math.max(1000, pending.expiresInMs));
+ this.onEvent({ type: 'confirmation-open', confirmation: pending });
+ this._setState(STATES.CONFIRM_PENDING, { confirmation: pending });
+
+ // The full sentence, then the exact word required. Never "OK?", and never a
+ // yes/no about an action that was not named.
+ await this._speak(`${pending.sentence} Say confirm to continue, or cancel to leave it alone.`, {
+ keepState: STATES.CONFIRM_PENDING,
+ });
+ }
+
+ async _handleSpokenConfirmation(transcript) {
+ const verdict = classifyConfirmation(transcript);
+ const pending = this.pendingConfirmation;
+ if (!pending) {
+ this._recover();
+ return;
+ }
+
+ if (verdict === 'confirm') {
+ this._clearConfirmation('redeemed', { silent: true });
+ await this._redeem(pending);
+ return;
+ }
+ if (verdict === 'cancel') {
+ this._clearConfirmation('cancelled');
+ await this._speak('Cancelled. Nothing changed.');
+ return;
+ }
+ // Anything else, a general "yeah" included, does not confirm. The pending
+ // action is dropped rather than left hanging over a conversation that has
+ // moved on, and the utterance is treated as the new request it looks like.
+ this._clearConfirmation('superseded');
+ this.onEvent({ type: 'confirmation-not-token', text: transcript });
+ await this.say(transcript);
+ }
+
+ /**
+ * Redeem the server-minted confirmation. The id is all that travels: the action
+ * itself was stored server-side when the confirmation was minted, so a
+ * confirmation for one lock can never execute against another.
+ */
+ async _redeem(pending) {
+ this._setState(STATES.THINKING, { redeeming: pending });
+ try {
+ // Minted before the clock starts: the action leg measures the house
+ // responding, not our own token round trip.
+ const csrf = await csrfHeader(this.fetch);
+ const started = now();
+ const res = await this.fetch(`/api/home/${encodeURIComponent(pending.homeId || this.homeId)}/confirm`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', ...csrf },
+ credentials: 'same-origin',
+ body: JSON.stringify({ confirmation_id: pending.confirmationId }),
+ });
+ this._record('action', now() - started);
+ const body = await res.json().catch(() => ({}));
+ if (!res.ok) {
+ const message = body?.message || body?.error || `confirmation refused (${res.status})`;
+ this.onEvent({ type: 'confirmation-refused', message, status: res.status });
+ await this._speak(`That did not go through: ${message}`);
+ return;
+ }
+ this.onEvent({ type: 'confirmation-executed', result: body });
+ await this._speak(body.spoken || body.message || 'Done.');
+ } catch (err) {
+ this.onEvent({ type: 'confirmation-refused', message: err?.message });
+ await this._speak(`That did not go through: ${err?.message || 'the house did not answer'}`);
+ }
+ }
+
+ _clearConfirmation(reason, { silent = false } = {}) {
+ clearTimeout(this._confirmTimer);
+ this._confirmTimer = 0;
+ const had = this.pendingConfirmation;
+ this.pendingConfirmation = null;
+ if (!had) return;
+ if (!silent) this.onEvent({ type: 'confirmation-closed', reason, confirmation: had });
+ if (reason === 'expired') {
+ void this._speak('That confirmation expired, so nothing changed. Ask again if you still want it.');
+ }
+ }
+
+ /** Cancel from the UI rather than by voice. Same effect, no recognizer involved. */
+ cancelConfirmation() {
+ this._clearConfirmation('cancelled');
+ this._recover();
+ }
+
+ /** Confirm from the UI: a real button press, the strongest signal there is. */
+ async confirmPending() {
+ const pending = this.pendingConfirmation;
+ if (!pending) return;
+ this._clearConfirmation('redeemed', { silent: true });
+ await this._redeem(pending);
+ }
+
+ // -- speaking, and being interrupted --------------------------------------
+
+ /** Synthesize and play, with barge-in armed for the whole of it. */
+ async _speak(text, { keepState = null } = {}) {
+ if (!text) {
+ this._recover(keepState);
+ return;
+ }
+ this._cancelPlayback('superseded');
+ this._bargeFrames = 0;
+ // The agent must not hear itself. Echo cancellation removes most of the
+ // far-end signal, but "most" is not a guarantee on a laptop speaker at
+ // volume, and the failure it produces is the agent waking itself.
+ if (this._wake) {
+ this._wake.suppressed = true;
+ this._wake.reset();
+ }
+ this._setState(keepState || STATES.SPEAKING, { text });
+
+ const controller = new AbortController();
+ this._playbackAbort = controller;
+ const requestedAt = now();
+ try {
+ const res = await this.fetch('/api/tts/speak', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ credentials: 'same-origin',
+ signal: controller.signal,
+ body: JSON.stringify({ text, format: 'wav', language: this.settings.language }),
+ });
+ if (!res.ok) throw new Error(`tts ${res.status}`);
+ const buffer = await res.arrayBuffer();
+ if (controller.signal.aborted) return;
+ const audio = await this._audioContext.decodeAudioData(buffer);
+ if (controller.signal.aborted) return;
+ await this._play(audio, keepState, requestedAt);
+ } catch (err) {
+ if (controller.signal.aborted) return;
+ // A voice that fails is not a loop that fails: the answer still exists,
+ // it just has to be read instead of heard.
+ this.onEvent({ type: 'tts-failed', message: err?.message, text });
+ this._releaseSuppression();
+ this._recover(keepState);
+ }
+ }
+
+ _play(audioBuffer, keepState, requestedAt) {
+ return new Promise((resolve) => {
+ const ctx = this._audioContext;
+ if (!ctx) {
+ resolve();
+ return;
+ }
+ const source = ctx.createBufferSource();
+ source.buffer = audioBuffer;
+ const analyser = ctx.createAnalyser();
+ analyser.fftSize = 256;
+ source.connect(analyser);
+ analyser.connect(ctx.destination);
+ this._playback = source;
+
+ if (this.mouthTarget) void this._startLipsync(analyser);
+
+ source.onended = () => {
+ if (this._playback !== source) return;
+ this._playback = null;
+ this._stopLipsync();
+ this._releaseSuppression();
+ this._recover(keepState);
+ resolve();
+ };
+ source.start();
+ // The number a user actually feels: the end of their sentence to the
+ // first sound coming back.
+ if (this._marks.speechEnd) this._record('firstAudio', now() - this._marks.speechEnd);
+ this._record('tts', now() - requestedAt);
+ });
+ }
+
+ async _startLipsync(analyser) {
+ try {
+ const { LipsyncDriver } = await import('./lipsync-driver.js');
+ this._lipsync = new LipsyncDriver({ analyser, target: this.mouthTarget });
+ this._lipsync.start();
+ } catch (err) {
+ log.warn('[home-voice] lipsync unavailable', err?.message);
+ }
+ }
+
+ _stopLipsync() {
+ try {
+ this._lipsync?.stop();
+ } catch {}
+ this._lipsync = null;
+ }
+
+ /**
+ * The user started talking over the agent. Cut the sound now, drop whatever
+ * synthesis is still in flight, and start listening again in the same breath.
+ */
+ _bargeIn() {
+ if (this.state !== STATES.SPEAKING) return;
+ const at = now();
+ this._cancelPlayback('barge-in');
+ if (this._marks.bargeFirstFrame) this._record('bargeIn', at - this._marks.bargeFirstFrame);
+ this._releaseSuppression();
+ this.onEvent({ type: 'barge-in', at });
+ this._setState(STATES.BARGED_IN, {});
+ // The utterance that interrupted is the next request: the VAD is already
+ // capturing it, and its speech-end runs a turn without a second wake word.
+ this._captureUntil = at + MAX_UTTERANCE_SEC * 1000;
+ this._setState(STATES.CAPTURING, { viaBargeIn: true });
+ }
+
+ _cancelPlayback(reason) {
+ this._playbackAbort?.abort();
+ this._playbackAbort = null;
+ const source = this._playback;
+ this._playback = null;
+ this._stopLipsync();
+ if (!source) return;
+ try {
+ source.onended = null;
+ source.stop();
+ } catch {}
+ this.onEvent({ type: 'playback-stopped', reason });
+ }
+
+ _releaseSuppression() {
+ if (!this._wake) return;
+ this._wake.suppressed = false;
+ // Whatever leaked past echo cancellation stays in the ring buffers for
+ // almost two seconds. Clearing them is what stops the tail of the agent's
+ // own sentence from scoring after the guard lifts.
+ this._wake.reset();
+ this._bargeFrames = 0;
+ }
+
+ /** Back to whichever resting state is correct right now. */
+ _recover(keepState = null) {
+ if (!this._enabled) {
+ this._setState(STATES.OFF, {});
+ return;
+ }
+ if (this._muted) {
+ this._setState(STATES.MUTED, { tracks: this.trackStates() });
+ return;
+ }
+ if (keepState === STATES.CONFIRM_PENDING && this.pendingConfirmation) {
+ this._setState(STATES.CONFIRM_PENDING, { confirmation: this.pendingConfirmation });
+ return;
+ }
+ this._setState(STATES.IDLE, { wakeWord: this.settings.wakeWord });
+ }
+
+ _setState(state, detail) {
+ this.state = state;
+ this.stateDetail = detail || {};
+ this.onState(state, this.stateDetail);
+ }
+
+ _record(leg, ms) {
+ const entry = { leg, ms: Math.round(ms), at: Date.now() };
+ this.latency.push(entry);
+ if (this.latency.length > 200) this.latency.shift();
+ this.onLatency(leg, entry.ms);
+ this.onEvent({ type: 'latency', ...entry });
+ }
+
+ /** Every leg measured this session, as medians. What the report table is built from. */
+ latencySummary() {
+ const byLeg = new Map();
+ for (const { leg, ms } of this.latency) {
+ if (!byLeg.has(leg)) byLeg.set(leg, []);
+ byLeg.get(leg).push(ms);
+ }
+ const out = {};
+ for (const [leg, values] of byLeg) {
+ const sorted = [...values].sort((a, b) => a - b);
+ out[leg] = { count: sorted.length, median: sorted[Math.floor(sorted.length / 2)], worst: sorted[sorted.length - 1] };
+ }
+ return out;
+ }
+}
+
+const MIC_CONSTRAINTS = {
+ channelCount: 1,
+ // Full duplex depends on all three: without echo cancellation the agent's own
+ // voice arrives back in the capture and barge-in fires on it, which reads to a
+ // user as the agent interrupting itself.
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+};
+
+const UNAVAILABLE_REASON =
+ 'Speech recognition is not available in this deployment, so the microphone stays off. ' +
+ 'Type to the agent instead: everything voice can do, text can do.';
+
+/**
+ * Reduce a server pending_confirmation payload to what the client is allowed to
+ * act on. Nothing here is trusted as instruction: `sentence` is spoken and shown,
+ * never fed back into a model, and the entity ids are rendered as data.
+ */
+export function normalizePendingConfirmation(raw) {
+ return {
+ confirmationId: String(raw.confirmation_id ?? raw.confirmationId ?? ''),
+ homeId: raw.home_id ?? raw.homeId ?? null,
+ sentence: capText(raw.sentence ?? raw.message ?? 'This will change something in your home.', 240),
+ entityIds: (raw.entity_ids ?? raw.entityIds ?? []).slice(0, 24).map((id) => capText(id, 128)),
+ risk: capText(raw.risk ?? 'unknown', 32),
+ expiresInMs: Number(raw.expires_in_ms ?? raw.expiresInMs ?? 90000),
+ };
+}
+
+/**
+ * Entity and area names are attacker-controlled: they come from devices, from
+ * integrations and from other people in the household. Cap the length and strip
+ * control characters before anything renders or speaks them.
+ */
+export function capText(value, max) {
+ return String(value ?? '')
+ .replace(CONTROL_CHARS, ' ')
+ .slice(0, max);
+}
+
+// Built from codepoints rather than written as a literal so the source file
+// never itself contains the control characters it is stripping.
+const CONTROL_CHARS = new RegExp('[\\u0000-\\u001f\\u007f-\\u009f]', 'g');
+
+/**
+ * A fresh CSRF token for the confirm endpoint, minted the way every other write
+ * on the platform mints one (GET /api/csrf-token, echo it in the header). Each
+ * token is single use, so it is fetched per redemption rather than cached: a
+ * confirmation is at most one request per guarded action, and reusing a spent
+ * token would fail exactly when a lock is waiting.
+ */
+async function csrfHeader(fetchImpl) {
+ try {
+ const res = await fetchImpl('/api/csrf-token', { credentials: 'include' });
+ if (!res.ok) return {};
+ const body = await res.json();
+ const token = body?.data?.token ?? body?.token ?? null;
+ return token ? { 'x-csrf-token': token } : {};
+ } catch {
+ return {};
+ }
+}
diff --git a/src/voice/mic-capture.js b/src/voice/mic-capture.js
index 379ba9cfd..38829b7c9 100644
--- a/src/voice/mic-capture.js
+++ b/src/voice/mic-capture.js
@@ -24,6 +24,16 @@
* const interim = mic.snapshotWav(); // Blob | null — audio so far, for a partial pass
* const final = await mic.stop(); // Blob | null — the full utterance as a 16 kHz WAV
* mic.dispose(); // idempotent teardown
+ *
+ * Streaming, for callers that push audio somewhere live instead of recording it
+ * (the Home Assistant voice satellite, src/home/satellite.js):
+ *
+ * const down = new PcmDownsampler(48000);
+ * const mic = new MicCapture({ retain: false, onFrame: (f) => send(down.push(f)) });
+ *
+ * `retain: false` is what keeps an always-on microphone from growing without
+ * bound; `PcmDownsampler` is the streaming counterpart of the WAV builder's
+ * resample and carries its phase across frames so the output has no seams.
*/
const TARGET_RATE = 16000;
@@ -42,7 +52,22 @@ registerProcessor('mic-capture', MicCaptureProcessor);
`;
export class MicCapture {
- constructor() {
+ /**
+ * @param {object} [options]
+ * @param {(frame: Float32Array) => void} [options.onFrame]
+ * Live capture. Every frame the audio thread produces is handed over
+ * at the device's native rate, as it arrives. Used by anything that
+ * streams rather than records: the Home Assistant voice satellite
+ * (src/home/satellite.js) pushes these straight into a pipeline.
+ * @param {boolean} [options.retain=true]
+ * Keep every frame so `stop()` and `snapshotWav()` can build a WAV.
+ * A streaming caller passes false: an always-on satellite that
+ * retained its microphone would grow by 5 MB an hour forever, and it
+ * never asks for the WAV.
+ */
+ constructor({ onFrame = null, retain = true } = {}) {
+ this._onFrame = typeof onFrame === 'function' ? onFrame : null;
+ this._retain = retain !== false;
this._stream = null;
this._ctx = null;
this._source = null;
@@ -60,6 +85,15 @@ export class MicCapture {
return this._started;
}
+ /**
+ * The device's native capture rate, known once `start()` has built the audio
+ * context. A streaming caller needs it to size its resampler, and guessing
+ * it wrong produces a transcript at the wrong pitch rather than an error.
+ */
+ get sampleRate() {
+ return this._sourceRate;
+ }
+
/** Whether the environment can capture at all (no mic UI on insecure origins). */
static isSupported() {
return (
@@ -112,8 +146,20 @@ export class MicCapture {
if (!this._started) return;
// Copy — the worklet transfers a view backed by a reused buffer.
const copy = frame instanceof Float32Array ? frame : new Float32Array(frame);
- this._chunks.push(copy.slice ? copy.slice(0) : new Float32Array(copy));
- this._length += copy.length;
+ const owned = copy.slice ? copy.slice(0) : new Float32Array(copy);
+ if (this._retain) {
+ this._chunks.push(owned);
+ this._length += owned.length;
+ }
+ // A throw from a subscriber must not kill the capture graph: the
+ // microphone would go silent with no error anywhere near the cause.
+ if (this._onFrame) {
+ try {
+ this._onFrame(owned);
+ } catch {
+ /* a subscriber's failure is its own problem */
+ }
+ }
};
let usedWorklet = false;
@@ -197,6 +243,7 @@ export class MicCapture {
if (this._disposed) return;
this._disposed = true;
this._started = false;
+ this._onFrame = null;
try {
if (this._node) {
this._node.onaudioprocess = null;
@@ -247,16 +294,77 @@ export class MicCapture {
}
}
+/**
+ * Streaming version of the conversion above.
+ *
+ * Resampling frame by frame with a stateless call is subtly wrong: 128 native
+ * samples at 48 kHz is 42.67 output samples, and a function that floors that on
+ * every frame drops two thirds of a sample 375 times a second. Over a sentence
+ * that is an audible click track and a transcript that quietly gets worse. This
+ * class carries the fractional read position and one sample of overlap across
+ * calls, so the output is a continuous resample of a continuous input.
+ */
+export class PcmDownsampler {
+ /** @param {number} sourceRate The AudioContext's native sample rate. */
+ constructor(sourceRate) {
+ this.ratio = Math.max(0.01, sourceRate / TARGET_RATE);
+ this._phase = 0;
+ this._tail = new Float32Array(0);
+ }
+
+ /**
+ * Convert one frame. Returns little-endian s16 at 16 kHz, mono, ready to be
+ * put on a wire. An empty result is normal for very short frames.
+ * @param {Float32Array} frame
+ * @returns {Int16Array}
+ */
+ push(frame) {
+ if (!frame?.length) return new Int16Array(0);
+ const src = new Float32Array(this._tail.length + frame.length);
+ src.set(this._tail, 0);
+ src.set(frame, this._tail.length);
+
+ const out = [];
+ let pos = this._phase;
+ // Stop one sample short of the end so the interpolation always has a
+ // right-hand neighbour; the remainder is carried into the next call.
+ while (pos < src.length - 1) {
+ const idx = Math.floor(pos);
+ const frac = pos - idx;
+ const s = src[idx] + (src[idx + 1] - src[idx]) * frac;
+ out.push(Math.max(-32768, Math.min(32767, Math.round(s * 32767))));
+ pos += this.ratio;
+ }
+
+ const consumed = Math.floor(pos);
+ this._phase = pos - consumed;
+ this._tail = src.slice(Math.min(consumed, src.length));
+ return Int16Array.from(out);
+ }
+
+ /** Forget the carried state, e.g. between two separate utterances. */
+ reset() {
+ this._phase = 0;
+ this._tail = new Float32Array(0);
+ }
+}
+
function codedError(message, code) {
const err = new Error(message);
err.code = code;
return err;
}
-// Linear-interpolating resample of mono Float32 from `inRate` to 16 kHz, then
-// quantize to little-endian s16. Linear interpolation is more than adequate for
-// speech recognition and avoids pulling in a filter library.
-function floatTo16kPcm(samples, inRate) {
+/**
+ * Linear-interpolating resample of mono Float32 from `inRate` to 16 kHz, then
+ * quantize to little-endian s16. Linear interpolation is more than adequate for
+ * speech recognition and avoids pulling in a filter library.
+ *
+ * Exported because streaming callers need the same conversion the WAV builder
+ * uses; see PcmDownsampler below, which is the one to reach for when the audio
+ * arrives in frames rather than all at once.
+ */
+export function floatTo16kPcm(samples, inRate) {
if (!samples.length) return new Int16Array(0);
const ratio = inRate / TARGET_RATE;
const outLen = ratio <= 1 ? samples.length : Math.floor(samples.length / ratio);
diff --git a/src/voice/wake-word.js b/src/voice/wake-word.js
index 0ed867fcc..9332a088a 100644
--- a/src/voice/wake-word.js
+++ b/src/voice/wake-word.js
@@ -31,6 +31,12 @@
import { log } from '../shared/log.js';
import { now } from './vad.js';
+import { WAKE_WORDS, DEFAULT_WAKE_WORD, wakeWordById } from './wake-words.js';
+
+// Re-exported so a caller that already needs the detector does not need a second
+// import for the catalog. The catalog itself lives in wake-words.js so the panel
+// can render its picker without pulling this module in.
+export { WAKE_WORDS, DEFAULT_WAKE_WORD, wakeWordById };
/** Where the committed openWakeWord models are served from. */
const MODEL_PATH = '/models/voice/wake-word/';
@@ -63,46 +69,6 @@ export const DEFAULT_THRESHOLD = 0.5;
*/
const REFRACTORY_MS = 1500;
-/**
- * The wake words shipped with the product. Every one is a pre-trained upstream
- * model: we offer a choice among what exists rather than training a phrase of our
- * own, because a custom phrase trained on synthetic speech would be measurably
- * worse than these and the user would pay for it in false wakes.
- */
-export const WAKE_WORDS = [
- {
- id: 'hey_jarvis',
- file: 'hey_jarvis_v0.1.onnx',
- phrase: 'Hey Jarvis',
- hint: 'Two syllables and an uncommon name, so it survives a noisy kitchen best.',
- },
- {
- id: 'hey_mycroft',
- file: 'hey_mycroft_v0.1.onnx',
- phrase: 'Hey Mycroft',
- hint: 'The Mycroft project phrase. Rare enough in conversation to rarely misfire.',
- },
- {
- id: 'alexa',
- file: 'alexa_v0.1.onnx',
- phrase: 'Alexa',
- hint: 'Pick this only if no Echo is in earshot: it will wake both.',
- },
- {
- id: 'hey_rhasspy',
- file: 'hey_rhasspy_v0.1.onnx',
- phrase: 'Hey Rhasspy',
- hint: 'The Rhasspy phrase, and the smallest model of the four.',
- },
-];
-
-export const DEFAULT_WAKE_WORD = 'hey_jarvis';
-
-/** @param {string} id */
-export function wakeWordById(id) {
- return WAKE_WORDS.find((w) => w.id === id) || WAKE_WORDS.find((w) => w.id === DEFAULT_WAKE_WORD);
-}
-
let ortPromise = null;
/** Load onnxruntime-web once, on demand, pointed at our own wasm. */
@@ -156,6 +122,28 @@ class RowRing {
}
}
+/**
+ * Whether one score fires a wake. Pure, and separated from the inference so the
+ * security-relevant decision can be tested without a model:
+ *
+ * - Suppressed never wakes. This is the self-trigger guard, and it is absolute:
+ * while the agent is speaking, a perfect score changes nothing.
+ * - Only a rising edge wakes. A score that stays above the threshold across
+ * several chunks is one wake, not five.
+ * - The refractory window keeps the trailing audio of the wake word itself,
+ * which is still in the ring buffer, from scoring a second time.
+ *
+ * @param {{score:number, threshold:number, above:boolean, suppressed:boolean, sinceLastWakeMs:number, refractoryMs?:number}} input
+ * @returns {{wake: boolean, above: boolean}}
+ */
+export function decideWake({ score, threshold, above, suppressed, sinceLastWakeMs, refractoryMs = REFRACTORY_MS }) {
+ const nowAbove = score >= threshold;
+ if (suppressed) return { wake: false, above: nowAbove };
+ if (!nowAbove || above) return { wake: false, above: nowAbove };
+ if (sinceLastWakeMs < refractoryMs) return { wake: false, above: nowAbove };
+ return { wake: true, above: nowAbove };
+}
+
export class WakeWordDetector {
/**
* @param {object} opts
@@ -365,21 +353,20 @@ export class WakeWordDetector {
this.lastScore = score;
this.onScore(score);
- if (this.suppressed) {
- this.suppressedPeak = Math.max(this.suppressedPeak, score);
- this._above = score >= this.threshold;
- return;
- }
-
- const crossed = score >= this.threshold && !this._above;
- this._above = score >= this.threshold;
- if (!crossed) return;
const t = now();
- if (t - this._lastWakeAt < REFRACTORY_MS) return;
- this._lastWakeAt = t;
- // Measured from the frame that completed the chunk, so the number
- // covers exactly what this module is responsible for: the inference,
- // not the audio that had not arrived yet.
+ const decision = decideWake({
+ score,
+ threshold: this.threshold,
+ above: this._above,
+ suppressed: this.suppressed,
+ sinceLastWakeMs: t - this._lastWakeAt,
+ });
+ this._above = decision.above;
+ if (this.suppressed) this.suppressedPeak = Math.max(this.suppressedPeak, score);
+ if (!decision.wake) return;
+ // Measured from the frame that completed the chunk, so the number covers
+ // exactly what this module is responsible for: the inference, not the
+ // audio that had not arrived yet.
const latencyMs = t - chunkAt;
this.reset();
this._lastWakeAt = t;
diff --git a/tests/home-voice.test.js b/tests/home-voice.test.js
new file mode 100644
index 000000000..721bf9182
--- /dev/null
+++ b/tests/home-voice.test.js
@@ -0,0 +1,404 @@
+/**
+ * The browser voice loop: the parts that must not be wrong.
+ *
+ * Three of these tests exist because of a specific way this feature can hurt
+ * somebody, and they are the reason the file is here at all:
+ *
+ * - the confirmation grammar, because a background "yeah" must never unlock a
+ * door;
+ * - the self-trigger guard, because an agent that hears its own wake word wakes
+ * itself in a loop;
+ * - the degraded path, because a loop that cannot transcribe must say so rather
+ * than light a microphone indicator and listen to a house for nothing.
+ *
+ * The wake-word models are exercised end to end against real speech in the
+ * browser check (scripts/check-home-voice.mjs); what is unit-tested here is the
+ * decision logic that sits above them, which is where the safety lives.
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import {
+ HomeVoiceLoop,
+ STATES,
+ STATE_ORDER,
+ classifyConfirmation,
+ isConfirmationToken,
+ normalizeTranscript,
+ normalizePendingConfirmation,
+ permissionRecovery,
+ capText,
+ CONSENT_VERSION,
+} from '../src/voice/home-voice.js';
+import { decideWake, DEFAULT_THRESHOLD } from '../src/voice/wake-word.js';
+import { WAKE_WORDS, wakeWordById } from '../src/voice/wake-words.js';
+
+// A localStorage good enough for the consent record, since vitest runs in node.
+function installStorage() {
+ const map = new Map();
+ globalThis.localStorage = {
+ getItem: (k) => (map.has(k) ? map.get(k) : null),
+ setItem: (k, v) => map.set(k, String(v)),
+ removeItem: (k) => map.delete(k),
+ clear: () => map.clear(),
+ };
+ return map;
+}
+
+function jsonResponse(body, { ok = true, status = 200 } = {}) {
+ return {
+ ok,
+ status,
+ json: async () => body,
+ text: async () => JSON.stringify(body),
+ arrayBuffer: async () => new ArrayBuffer(8),
+ };
+}
+
+describe('the spoken confirmation grammar', () => {
+ it('accepts the explicit token and its smallest natural pairings', () => {
+ for (const phrase of ['confirm', 'Confirm.', 'confirm it', 'Confirm that', 'yes confirm', 'okay confirm', 'confirmed']) {
+ expect(isConfirmationToken(phrase), phrase).toBe(true);
+ }
+ });
+
+ it('refuses every general affirmative, which is the whole point of the grammar', () => {
+ const ambient = [
+ 'yeah',
+ 'Yeah.',
+ 'yes',
+ 'yep',
+ 'sure',
+ 'ok',
+ 'okay',
+ 'alright',
+ 'do it',
+ 'go ahead',
+ 'absolutely',
+ 'uh huh',
+ 'sounds good',
+ 'yes please',
+ 'why not',
+ ];
+ for (const phrase of ambient) {
+ expect(isConfirmationToken(phrase), phrase).toBe(false);
+ expect(classifyConfirmation(phrase), phrase).toBe('other');
+ }
+ });
+
+ it('refuses the token buried in a sentence, so overheard speech cannot confirm', () => {
+ for (const phrase of [
+ 'can you confirm the kitchen light is off',
+ 'i want to confirm something else later',
+ 'she said confirm the booking',
+ 'confirm the front door is not what i meant',
+ ]) {
+ expect(isConfirmationToken(phrase), phrase).toBe(false);
+ }
+ });
+
+ it('recognises a cancellation, and a cancellation is never a confirmation', () => {
+ for (const phrase of ['cancel', 'stop', 'no', 'never mind', 'forget it', 'abort']) {
+ expect(classifyConfirmation(phrase), phrase).toBe('cancel');
+ expect(isConfirmationToken(phrase), phrase).toBe(false);
+ }
+ });
+
+ it('normalizes punctuation, case and curly apostrophes before matching', () => {
+ expect(normalizeTranscript(' CONFIRM!! ')).toBe('confirm');
+ expect(normalizeTranscript('Don’t')).toBe('dont');
+ expect(normalizeTranscript('')).toBe('');
+ expect(classifyConfirmation(null)).toBe('other');
+ expect(classifyConfirmation(undefined)).toBe('other');
+ });
+});
+
+describe('the self-trigger guard', () => {
+ it('never wakes while the agent is speaking, however high the score', () => {
+ for (const score of [0.5, 0.9, 1]) {
+ const d = decideWake({ score, threshold: DEFAULT_THRESHOLD, above: false, suppressed: true, sinceLastWakeMs: 1e6 });
+ expect(d.wake).toBe(false);
+ }
+ });
+
+ it('wakes on a rising edge once the agent has stopped', () => {
+ const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: false, suppressed: false, sinceLastWakeMs: 1e6 });
+ expect(d.wake).toBe(true);
+ expect(d.above).toBe(true);
+ });
+
+ it('does not wake again while the score stays high', () => {
+ const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: true, suppressed: false, sinceLastWakeMs: 1e6 });
+ expect(d.wake).toBe(false);
+ });
+
+ it('holds the refractory window so the wake word cannot score twice', () => {
+ const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: false, suppressed: false, sinceLastWakeMs: 300 });
+ expect(d.wake).toBe(false);
+ });
+
+ it('tracks the threshold crossing even while suppressed, so the edge is not lost', () => {
+ const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: false, suppressed: true, sinceLastWakeMs: 1e6 });
+ expect(d.above).toBe(true);
+ });
+});
+
+describe('the wake-word catalog', () => {
+ it('offers only pre-trained upstream models, each with a real file', () => {
+ expect(WAKE_WORDS.length).toBeGreaterThanOrEqual(4);
+ for (const w of WAKE_WORDS) {
+ expect(w.file).toMatch(/^[a-z_]+_v0\.1\.onnx$/);
+ expect(w.phrase.length).toBeGreaterThan(2);
+ expect(w.hint.length).toBeGreaterThan(10);
+ }
+ });
+
+ it('falls back to the default rather than returning nothing for an unknown id', () => {
+ expect(wakeWordById('not-a-wake-word').id).toBe('hey_jarvis');
+ });
+});
+
+describe('the degraded path', () => {
+ beforeEach(() => installStorage());
+
+ it('lands in the unavailable state when the speech lane is not configured', async () => {
+ const states = [];
+ const loop = new HomeVoiceLoop({
+ onState: (s) => states.push(s),
+ fetchImpl: async () => jsonResponse({ configured: false, languages: [] }),
+ });
+ await loop.probeAsr();
+ expect(loop.asr.configured).toBe(false);
+ expect(loop.state).toBe(STATES.UNAVAILABLE);
+ expect(loop.stateDetail.reason).toMatch(/not available/i);
+ expect(states).toContain(STATES.UNAVAILABLE);
+ });
+
+ it('degrades honestly when the probe itself fails, rather than claiming to listen', async () => {
+ const loop = new HomeVoiceLoop({
+ fetchImpl: async () => {
+ throw new Error('offline');
+ },
+ });
+ await loop.probeAsr();
+ expect(loop.asr.configured).toBe(false);
+ expect(loop.state).toBe(STATES.UNAVAILABLE);
+ });
+
+ it('refuses to open a microphone when speech recognition is unavailable', async () => {
+ const loop = new HomeVoiceLoop({ fetchImpl: async () => jsonResponse({ configured: false }) });
+ loop.grantConsent();
+ const getUserMedia = vi.fn();
+ // navigator is a getter-only global in node, so patch mediaDevices onto the
+ // real object rather than replacing it.
+ Object.defineProperty(globalThis.navigator, 'mediaDevices', {
+ value: { getUserMedia },
+ configurable: true,
+ });
+ await loop.enable();
+ expect(getUserMedia).not.toHaveBeenCalled();
+ expect(loop.state).toBe(STATES.UNAVAILABLE);
+ expect(loop.micLive).toBe(false);
+ });
+
+ it('will not enable without a recorded consent', async () => {
+ const loop = new HomeVoiceLoop({ fetchImpl: async () => jsonResponse({ configured: true }) });
+ await expect(loop.enable()).rejects.toThrow(/consent/i);
+ });
+});
+
+describe('consent', () => {
+ beforeEach(() => installStorage());
+
+ it('is off by default and records a version and a timestamp when granted', () => {
+ const loop = new HomeVoiceLoop({});
+ expect(loop.hasConsent()).toBe(false);
+ const record = loop.grantConsent();
+ expect(record.version).toBe(CONSENT_VERSION);
+ expect(Date.parse(record.grantedAt)).toBeGreaterThan(0);
+ expect(loop.hasConsent()).toBe(true);
+ });
+
+ it('is invalidated by a version bump rather than silently carried forward', () => {
+ const loop = new HomeVoiceLoop({});
+ localStorage.setItem('tws:home-voice:consent', JSON.stringify({ version: CONSENT_VERSION - 1 }));
+ expect(loop.hasConsent()).toBe(false);
+ });
+
+ it('survives a localStorage that throws', () => {
+ globalThis.localStorage = {
+ getItem() {
+ throw new Error('blocked');
+ },
+ setItem() {
+ throw new Error('blocked');
+ },
+ removeItem() {
+ throw new Error('blocked');
+ },
+ };
+ const loop = new HomeVoiceLoop({});
+ expect(loop.hasConsent()).toBe(false);
+ expect(() => loop.grantConsent()).not.toThrow();
+ });
+});
+
+describe('mute stops capture at the track level', () => {
+ beforeEach(() => installStorage());
+
+ function fakeStream() {
+ const track = {
+ readyState: 'live',
+ stop() {
+ this.readyState = 'ended';
+ },
+ };
+ return { getAudioTracks: () => [track], _track: track };
+ }
+
+ it('ends every track and reports the readyState, not a flag', async () => {
+ const loop = new HomeVoiceLoop({});
+ const stream = fakeStream();
+ loop._stream = stream;
+ loop._enabled = true;
+ expect(loop.micLive).toBe(true);
+
+ await loop.mute();
+
+ expect(stream._track.readyState).toBe('ended');
+ expect(loop.micLive).toBe(false);
+ expect(loop.trackStates()).toEqual(['ended']);
+ expect(loop.state).toBe(STATES.MUTED);
+ expect(loop.stateDetail.tracks).toEqual(['ended']);
+ });
+
+ it('leaves the microphone dead until it is explicitly re-acquired', async () => {
+ const loop = new HomeVoiceLoop({});
+ loop._stream = fakeStream();
+ loop._enabled = true;
+ await loop.mute();
+ await loop.mute();
+ // The indicator reads micLive and nothing else, so this is the property the
+ // UI depends on to never label a live microphone as off.
+ expect(loop.micLive).toBe(false);
+ });
+});
+
+describe('guarded actions', () => {
+ beforeEach(() => installStorage());
+
+ it('refuses a guarded action outright on a surface with no display', async () => {
+ const events = [];
+ const loop = new HomeVoiceLoop({
+ surface: 'screenless',
+ onEvent: (e) => events.push(e),
+ fetchImpl: async () => jsonResponse({ configured: true }),
+ });
+ await loop._openConfirmation(
+ normalizePendingConfirmation({ confirmation_id: 'abc', sentence: 'This will unlock the Front Door.' }),
+ );
+ expect(loop.pendingConfirmation).toBeNull();
+ const refusal = events.find((e) => e.type === 'guarded-refused');
+ expect(refusal?.reason).toBe('screenless');
+ });
+
+ it('drops the pending action when the utterance is not the token', async () => {
+ const events = [];
+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e), fetchImpl: async () => jsonResponse({}) });
+ loop.pendingConfirmation = normalizePendingConfirmation({
+ confirmation_id: 'abc',
+ sentence: 'This will unlock the Front Door.',
+ });
+ loop.say = vi.fn(async () => {});
+ await loop._handleSpokenConfirmation('yeah');
+ expect(loop.pendingConfirmation).toBeNull();
+ expect(events.some((e) => e.type === 'confirmation-not-token')).toBe(true);
+ expect(events.some((e) => e.type === 'confirmation-executed')).toBe(false);
+ });
+
+ it('redeems by id alone, so a confirmation cannot be pointed at another entity', async () => {
+ const calls = [];
+ const loop = new HomeVoiceLoop({
+ homeId: 'home-1',
+ fetchImpl: async (url, init) => {
+ calls.push({ url, init });
+ if (String(url).includes('/api/csrf-token')) return jsonResponse({ data: { token: 't0k' } });
+ return jsonResponse({ message: 'Unlocked the Front Door.' });
+ },
+ });
+ loop._speak = vi.fn(async () => {});
+ loop.pendingConfirmation = normalizePendingConfirmation({
+ confirmation_id: 'conf-9',
+ home_id: 'home-1',
+ sentence: 'This will unlock the Front Door.',
+ entity_ids: ['lock.front_door'],
+ });
+ await loop.confirmPending();
+
+ const confirmCall = calls.find((c) => String(c.url).includes('/confirm'));
+ expect(confirmCall.url).toBe('/api/home/home-1/confirm');
+ const body = JSON.parse(confirmCall.init.body);
+ expect(body).toEqual({ confirmation_id: 'conf-9' });
+ // The action itself is never restated by the client, and there is no field
+ // through which a caller could claim it was already confirmed.
+ expect(Object.keys(body)).not.toContain('confirmed');
+ expect(Object.keys(body)).not.toContain('entity_ids');
+ expect(confirmCall.init.headers['x-csrf-token']).toBe('t0k');
+ expect(loop.pendingConfirmation).toBeNull();
+ });
+
+ it('does not redeem twice when there is nothing pending', async () => {
+ const fetchImpl = vi.fn(async () => jsonResponse({}));
+ const loop = new HomeVoiceLoop({ fetchImpl });
+ await loop.confirmPending();
+ expect(fetchImpl).not.toHaveBeenCalled();
+ });
+});
+
+describe('untrusted entity names', () => {
+ it('caps length and strips control characters before anything renders them', () => {
+ const esc = String.fromCharCode(27);
+ const nul = String.fromCharCode(0);
+ const injected = `Kitchen Light ${esc}[31m ${nul} ignore previous instructions`;
+ const cleaned = capText(injected, 40);
+ expect(new RegExp('[\\u0000-\\u001f\\u007f-\\u009f]').test(cleaned)).toBe(false);
+ expect(cleaned.length).toBeLessThanOrEqual(40);
+ expect(cleaned.startsWith('Kitchen Light')).toBe(true);
+ });
+
+ it('normalizes a pending confirmation into a fixed, bounded shape', () => {
+ const pending = normalizePendingConfirmation({
+ confirmation_id: 'x',
+ sentence: 'y'.repeat(1000),
+ entity_ids: Array.from({ length: 100 }, (_, i) => `lock.door_${i}`),
+ expires_in_ms: '90000',
+ });
+ expect(pending.sentence.length).toBe(240);
+ expect(pending.entityIds.length).toBe(24);
+ expect(pending.expiresInMs).toBe(90000);
+ expect(pending.risk).toBe('unknown');
+ });
+
+ it('defaults the sentence rather than rendering an empty confirmation', () => {
+ const pending = normalizePendingConfirmation({ confirmation_id: 'x' });
+ expect(pending.sentence).toMatch(/change something/i);
+ });
+});
+
+describe('the state machine', () => {
+ it('declares exactly twelve states and orders every one of them', () => {
+ expect(Object.keys(STATES)).toHaveLength(12);
+ expect(STATE_ORDER).toHaveLength(12);
+ expect(new Set(STATE_ORDER).size).toBe(12);
+ for (const state of Object.values(STATES)) expect(STATE_ORDER).toContain(state);
+ });
+});
+
+describe('permission recovery', () => {
+ it('names the actual control, per browser, instead of saying to check settings', () => {
+ expect(permissionRecovery('Mozilla/5.0 Firefox/128.0')).toMatch(/address bar/i);
+ expect(permissionRecovery('Mozilla/5.0 Chrome/120 Edg/120')).toMatch(/Edge/);
+ expect(permissionRecovery('Mozilla/5.0 Chrome/120 Safari/537')).toMatch(/Chrome/);
+ expect(permissionRecovery('Mozilla/5.0 Version/17 Safari/605')).toMatch(/Safari/);
+ expect(permissionRecovery('some-unknown-agent')).toMatch(/browser settings/i);
+ });
+});