Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(home): report whether interrupted speech was actually audible
nirholasdd8cb15aa138
diff --git a/scripts/check-home-voice.mjs b/scripts/check-home-voice.mjsindex af08c4135..3975cac57 100644--- a/scripts/check-home-voice.mjs+++ b/scripts/check-home-voice.mjs@@ -478,6 +478,14 @@ async function scenarioHappyPath() {async function scenarioBargeIn() {const { browser, page } = await launch(CLIPS.bargeUser.path);try {+ // The agent's own speech comes from the disk cache here, which is the same+ // bytes the real lane produced, returned in milliseconds. The live lane+ // takes seconds to synthesize a sentence this long, and the interrupting+ // clip starts talking 5 s in: against the live lane the interruption+ // reliably lands BEFORE the first sample plays, which measures TTS latency+ // rather than barge-in. Cached, the agent is genuinely audible when the+ // user talks over it, which is the thing this scenario exists to prove.+ await routeCachedSpeech(page);await bootPage(page);await optIn(page);// Speak a long enough answer that the user's interruption lands inside it.@@ -499,14 +507,28 @@ async function scenarioBargeIn() {if (ttsFailure) throw new Error(`the agent could not speak, so barge-in cannot be measured: ${ttsFailure.message}`);await waitForEvent(page, 'barge-in', 45000);const all = await events(page);+ const barge = all.find((e) => e.type === 'barge-in');const stop = all.find((e) => e.type === 'playback-stopped' && e.reason === 'barge-in');- check('the user talking over the agent stops playback', !!stop);+ check('the user talking over the agent stops playback', !!stop, {+ audible: stop?.audible,+ });+ check('and the sound it stopped was already coming out of the speaker', stop?.audible === true, {+ audible: stop?.audible,+ });const measured = await legs(page);check('playback stops within 200 ms of the user starting to talk', (measured.bargeIn?.median ?? 1e9) <= 200, measured.bargeIn);- const state = await page.evaluate(() => window.homeVoice.loop.state);- check('the loop is capturing again immediately after a barge-in', state === 'capturing' || state === 'thinking', {- state,- });+ // Read the transitions the loop made AT the barge-in, from the recorded+ // log. A live read races the real turn that the interrupting utterance+ // legitimately kicks off: by the time it runs the loop may already be+ // thinking, or speaking the answer, and neither of those is a failure.+ // The pair is asserted rather than the destination alone because state 8+ // existing but being skipped would leave a host with no way to render it.+ const after = await page.evaluate((t) => window.__hv.states.filter((s) => s.t >= t).map((s) => s.state), barge.t);+ check(+ 'the loop passes through barged-in and is capturing again immediately',+ after[0] === 'barged-in' && after[1] === 'capturing',+ { after: after.slice(0, 4).join(' -> ') },+ );await page.locator('#voice-panel').screenshot({ path: join(OUT, 'barge-in.png') });return measured;} finally {diff --git a/src/voice/home-voice.js b/src/voice/home-voice.jsindex d00d6a3ff..96b5b1129 100644--- a/src/voice/home-voice.js+++ b/src/voice/home-voice.js@@ -1003,6 +1003,7 @@ export class HomeVoiceLoop {if (controller.signal.aborted) return;// A voice that fails is not a loop that fails: the answer still exists,// it just has to be read instead of heard.+ this._playbackAbort = null;this.onEvent({ type: 'tts-failed', message: err?.message, text });this._releaseSuppression();this._recover(keepState);@@ -1029,6 +1030,9 @@ export class HomeVoiceLoop {source.onended = () => {if (this._playback !== source) return;this._playback = null;+ // The utterance is over, so there is nothing left for a later+ // _cancelPlayback to cancel.+ this._playbackAbort = null;this._stopLipsync();this._releaseSuppression();this._recover(keepState);@@ -1077,18 +1081,32 @@ export class HomeVoiceLoop {this._setState(STATES.CAPTURING, { viaBargeIn: true });}+ /**+ * Stop the current utterance, whether it is already audible or still being+ * synthesized, and say so.+ *+ * The event fires in both cases on purpose. An interruption that lands inside+ * the synthesis window cancels a sentence the user never heard, and a host+ * that clears its speaking affordance on `playback-stopped` would otherwise+ * sit on that affordance forever waiting for audio that was aborted in+ * flight. `audible` tells the two apart for a host that renders them+ * differently; both are a stopped utterance.+ */_cancelPlayback(reason) {+ const pendingSynthesis = !!this._playbackAbort;this._playbackAbort?.abort();this._playbackAbort = null;const source = this._playback;this._playback = null;this._stopLipsync();- if (!source) return;- try {- source.onended = null;- source.stop();- } catch {}- this.onEvent({ type: 'playback-stopped', reason });+ if (!source && !pendingSynthesis) return;+ if (source) {+ try {+ source.onended = null;+ source.stop();+ } catch {}+ }+ this.onEvent({ type: 'playback-stopped', reason, audible: !!source });}_releaseSuppression() {diff --git a/tests/home-voice.test.js b/tests/home-voice.test.jsindex 6230e23c8..1153b3365 100644--- a/tests/home-voice.test.js+++ b/tests/home-voice.test.js@@ -536,3 +536,54 @@ describe('the agent turn over Server-Sent Events', () => {await expect(loop._turn('hello', new AbortController().signal)).rejects.toThrow(/stream interrupted/);});});++describe('interrupting an utterance that is not audible yet', () => {+ beforeEach(() => installStorage());++ /**+ * A barge-in that lands inside the synthesis window cancels a sentence the+ * user never heard. The host still has to be told, because a surface that+ * clears its speaking affordance on `playback-stopped` would otherwise hold+ * that affordance forever waiting for audio that was aborted in flight.+ */+ it('reports the stop even when no sound had started, and marks it inaudible', () => {+ const events = [];+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });+ loop._playbackAbort = new AbortController();++ loop._cancelPlayback('barge-in');++ const stop = events.find((e) => e.type === 'playback-stopped');+ expect(stop).toBeTruthy();+ expect(stop.reason).toBe('barge-in');+ expect(stop.audible).toBe(false);+ expect(loop._playbackAbort).toBeNull();+ });++ it('marks the stop audible when a source was actually playing', () => {+ const events = [];+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });+ let stopped = false;+ loop._playbackAbort = new AbortController();+ loop._playback = {+ onended: () => {},+ stop() {+ stopped = true;+ },+ };++ loop._cancelPlayback('barge-in');++ expect(stopped).toBe(true);+ expect(events.find((e) => e.type === 'playback-stopped').audible).toBe(true);+ });++ it('stays silent when there was nothing to cancel', () => {+ const events = [];+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });++ loop._cancelPlayback('superseded');++ expect(events.filter((e) => e.type === 'playback-stopped')).toEqual([]);+ });+});