Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(home): report whether interrupted speech was actually audible

nirholasdd8cb15aa138
diff --git a/scripts/check-home-voice.mjs b/scripts/check-home-voice.mjs
index af08c4135..3975cac57 100644
--- a/scripts/check-home-voice.mjs
+++ b/scripts/check-home-voice.mjs
@@ -478,6 +478,14 @@ async function scenarioHappyPath() {
async function scenarioBargeIn() {
const { browser, page } = await launch(CLIPS.bargeUser.path);
try {
+ // The agent's own speech comes from the disk cache here, which is the same
+ // bytes the real lane produced, returned in milliseconds. The live lane
+ // takes seconds to synthesize a sentence this long, and the interrupting
+ // clip starts talking 5 s in: against the live lane the interruption
+ // reliably lands BEFORE the first sample plays, which measures TTS latency
+ // rather than barge-in. Cached, the agent is genuinely audible when the
+ // user talks over it, which is the thing this scenario exists to prove.
+ await routeCachedSpeech(page);
await bootPage(page);
await optIn(page);
// Speak a long enough answer that the user's interruption lands inside it.
@@ -499,14 +507,28 @@ async function scenarioBargeIn() {
if (ttsFailure) throw new Error(`the agent could not speak, so barge-in cannot be measured: ${ttsFailure.message}`);
await waitForEvent(page, 'barge-in', 45000);
const all = await events(page);
+ const barge = all.find((e) => e.type === 'barge-in');
const stop = all.find((e) => e.type === 'playback-stopped' && e.reason === 'barge-in');
- check('the user talking over the agent stops playback', !!stop);
+ check('the user talking over the agent stops playback', !!stop, {
+ audible: stop?.audible,
+ });
+ check('and the sound it stopped was already coming out of the speaker', stop?.audible === true, {
+ audible: stop?.audible,
+ });
const measured = await legs(page);
check('playback stops within 200 ms of the user starting to talk', (measured.bargeIn?.median ?? 1e9) <= 200, measured.bargeIn);
- const state = await page.evaluate(() => window.homeVoice.loop.state);
- check('the loop is capturing again immediately after a barge-in', state === 'capturing' || state === 'thinking', {
- state,
- });
+ // Read the transitions the loop made AT the barge-in, from the recorded
+ // log. A live read races the real turn that the interrupting utterance
+ // legitimately kicks off: by the time it runs the loop may already be
+ // thinking, or speaking the answer, and neither of those is a failure.
+ // The pair is asserted rather than the destination alone because state 8
+ // existing but being skipped would leave a host with no way to render it.
+ const after = await page.evaluate((t) => window.__hv.states.filter((s) => s.t >= t).map((s) => s.state), barge.t);
+ check(
+ 'the loop passes through barged-in and is capturing again immediately',
+ after[0] === 'barged-in' && after[1] === 'capturing',
+ { after: after.slice(0, 4).join(' -> ') },
+ );
await page.locator('#voice-panel').screenshot({ path: join(OUT, 'barge-in.png') });
return measured;
} finally {
diff --git a/src/voice/home-voice.js b/src/voice/home-voice.js
index d00d6a3ff..96b5b1129 100644
--- a/src/voice/home-voice.js
+++ b/src/voice/home-voice.js
@@ -1003,6 +1003,7 @@ export class HomeVoiceLoop {
if (controller.signal.aborted) return;
// A voice that fails is not a loop that fails: the answer still exists,
// it just has to be read instead of heard.
+ this._playbackAbort = null;
this.onEvent({ type: 'tts-failed', message: err?.message, text });
this._releaseSuppression();
this._recover(keepState);
@@ -1029,6 +1030,9 @@ export class HomeVoiceLoop {
source.onended = () => {
if (this._playback !== source) return;
this._playback = null;
+ // The utterance is over, so there is nothing left for a later
+ // _cancelPlayback to cancel.
+ this._playbackAbort = null;
this._stopLipsync();
this._releaseSuppression();
this._recover(keepState);
@@ -1077,18 +1081,32 @@ export class HomeVoiceLoop {
this._setState(STATES.CAPTURING, { viaBargeIn: true });
}
+ /**
+ * Stop the current utterance, whether it is already audible or still being
+ * synthesized, and say so.
+ *
+ * The event fires in both cases on purpose. An interruption that lands inside
+ * the synthesis window cancels a sentence the user never heard, and a host
+ * that clears its speaking affordance on `playback-stopped` would otherwise
+ * sit on that affordance forever waiting for audio that was aborted in
+ * flight. `audible` tells the two apart for a host that renders them
+ * differently; both are a stopped utterance.
+ */
_cancelPlayback(reason) {
+ const pendingSynthesis = !!this._playbackAbort;
this._playbackAbort?.abort();
this._playbackAbort = null;
const source = this._playback;
this._playback = null;
this._stopLipsync();
- if (!source) return;
- try {
- source.onended = null;
- source.stop();
- } catch {}
- this.onEvent({ type: 'playback-stopped', reason });
+ if (!source && !pendingSynthesis) return;
+ if (source) {
+ try {
+ source.onended = null;
+ source.stop();
+ } catch {}
+ }
+ this.onEvent({ type: 'playback-stopped', reason, audible: !!source });
}
_releaseSuppression() {
diff --git a/tests/home-voice.test.js b/tests/home-voice.test.js
index 6230e23c8..1153b3365 100644
--- a/tests/home-voice.test.js
+++ b/tests/home-voice.test.js
@@ -536,3 +536,54 @@ describe('the agent turn over Server-Sent Events', () => {
await expect(loop._turn('hello', new AbortController().signal)).rejects.toThrow(/stream interrupted/);
});
});
+
+describe('interrupting an utterance that is not audible yet', () => {
+ beforeEach(() => installStorage());
+
+ /**
+ * A barge-in that lands inside the synthesis window cancels a sentence the
+ * user never heard. The host still has to be told, because a surface that
+ * clears its speaking affordance on `playback-stopped` would otherwise hold
+ * that affordance forever waiting for audio that was aborted in flight.
+ */
+ it('reports the stop even when no sound had started, and marks it inaudible', () => {
+ const events = [];
+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });
+ loop._playbackAbort = new AbortController();
+
+ loop._cancelPlayback('barge-in');
+
+ const stop = events.find((e) => e.type === 'playback-stopped');
+ expect(stop).toBeTruthy();
+ expect(stop.reason).toBe('barge-in');
+ expect(stop.audible).toBe(false);
+ expect(loop._playbackAbort).toBeNull();
+ });
+
+ it('marks the stop audible when a source was actually playing', () => {
+ const events = [];
+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });
+ let stopped = false;
+ loop._playbackAbort = new AbortController();
+ loop._playback = {
+ onended: () => {},
+ stop() {
+ stopped = true;
+ },
+ };
+
+ loop._cancelPlayback('barge-in');
+
+ expect(stopped).toBe(true);
+ expect(events.find((e) => e.type === 'playback-stopped').audible).toBe(true);
+ });
+
+ it('stays silent when there was nothing to cancel', () => {
+ const events = [];
+ const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });
+
+ loop._cancelPlayback('superseded');
+
+ expect(events.filter((e) => e.type === 'playback-stopped')).toEqual([]);
+ });
+});