Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

20.5 KB
Raw Blame
/**
 * The browser voice loop: the parts that must not be wrong.
 *
 * Three of these tests exist because of a specific way this feature can hurt
 * somebody, and they are the reason the file is here at all:
 *
 *  - the confirmation grammar, because a background "yeah" must never unlock a
 *    door;
 *  - the self-trigger guard, because an agent that hears its own wake word wakes
 *    itself in a loop;
 *  - the degraded path, because a loop that cannot transcribe must say so rather
 *    than light a microphone indicator and listen to a house for nothing.
 *
 * The wake-word models are exercised end to end against real speech in the
 * browser check (scripts/check-home-voice.mjs); what is unit-tested here is the
 * decision logic that sits above them, which is where the safety lives.
 */

import { describe, it, expect, beforeEach, vi } from 'vitest';
import {
	HomeVoiceLoop,
	STATES,
	STATE_ORDER,
	classifyConfirmation,
	isConfirmationToken,
	normalizeTranscript,
	normalizePendingConfirmation,
	permissionRecovery,
	capText,
	CONSENT_VERSION,
} from '../src/voice/home-voice.js';
import { decideWake, DEFAULT_THRESHOLD } from '../src/voice/wake-word.js';
import { WAKE_WORDS, wakeWordById } from '../src/voice/wake-words.js';

// A localStorage good enough for the consent record, since vitest runs in node.
function installStorage() {
	const map = new Map();
	globalThis.localStorage = {
		getItem: (k) => (map.has(k) ? map.get(k) : null),
		setItem: (k, v) => map.set(k, String(v)),
		removeItem: (k) => map.delete(k),
		clear: () => map.clear(),
	};
	return map;
}

function jsonResponse(body, { ok = true, status = 200 } = {}) {
	return {
		ok,
		status,
		json: async () => body,
		text: async () => JSON.stringify(body),
		arrayBuffer: async () => new ArrayBuffer(8),
	};
}

describe('the spoken confirmation grammar', () => {
	it('accepts the explicit token and its smallest natural pairings', () => {
		for (const phrase of ['confirm', 'Confirm.', 'confirm it', 'Confirm that', 'yes confirm', 'okay confirm', 'confirmed']) {
			expect(isConfirmationToken(phrase), phrase).toBe(true);
		}
	});

	it('refuses every general affirmative, which is the whole point of the grammar', () => {
		const ambient = [
			'yeah',
			'Yeah.',
			'yes',
			'yep',
			'sure',
			'ok',
			'okay',
			'alright',
			'do it',
			'go ahead',
			'absolutely',
			'uh huh',
			'sounds good',
			'yes please',
			'why not',
		];
		for (const phrase of ambient) {
			expect(isConfirmationToken(phrase), phrase).toBe(false);
			expect(classifyConfirmation(phrase), phrase).toBe('other');
		}
	});

	it('refuses the token buried in a sentence, so overheard speech cannot confirm', () => {
		for (const phrase of [
			'can you confirm the kitchen light is off',
			'i want to confirm something else later',
			'she said confirm the booking',
			'confirm the front door is not what i meant',
		]) {
			expect(isConfirmationToken(phrase), phrase).toBe(false);
		}
	});

	it('recognises a cancellation, and a cancellation is never a confirmation', () => {
		for (const phrase of ['cancel', 'stop', 'no', 'never mind', 'forget it', 'abort']) {
			expect(classifyConfirmation(phrase), phrase).toBe('cancel');
			expect(isConfirmationToken(phrase), phrase).toBe(false);
		}
	});

	it('normalizes punctuation, case and curly apostrophes before matching', () => {
		expect(normalizeTranscript('  CONFIRM!!  ')).toBe('confirm');
		expect(normalizeTranscript('Don’t')).toBe('dont');
		expect(normalizeTranscript('')).toBe('');
		expect(classifyConfirmation(null)).toBe('other');
		expect(classifyConfirmation(undefined)).toBe('other');
	});
});

describe('the self-trigger guard', () => {
	it('never wakes while the agent is speaking, however high the score', () => {
		for (const score of [0.5, 0.9, 1]) {
			const d = decideWake({ score, threshold: DEFAULT_THRESHOLD, above: false, suppressed: true, sinceLastWakeMs: 1e6 });
			expect(d.wake).toBe(false);
		}
	});

	it('wakes on a rising edge once the agent has stopped', () => {
		const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: false, suppressed: false, sinceLastWakeMs: 1e6 });
		expect(d.wake).toBe(true);
		expect(d.above).toBe(true);
	});

	it('does not wake again while the score stays high', () => {
		const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: true, suppressed: false, sinceLastWakeMs: 1e6 });
		expect(d.wake).toBe(false);
	});

	it('holds the refractory window so the wake word cannot score twice', () => {
		const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: false, suppressed: false, sinceLastWakeMs: 300 });
		expect(d.wake).toBe(false);
	});

	it('tracks the threshold crossing even while suppressed, so the edge is not lost', () => {
		const d = decideWake({ score: 0.99, threshold: DEFAULT_THRESHOLD, above: false, suppressed: true, sinceLastWakeMs: 1e6 });
		expect(d.above).toBe(true);
	});
});

describe('the wake-word catalog', () => {
	it('offers only pre-trained upstream models, each with a real file', () => {
		expect(WAKE_WORDS.length).toBeGreaterThanOrEqual(4);
		for (const w of WAKE_WORDS) {
			expect(w.file).toMatch(/^[a-z_]+_v0\.1\.onnx$/);
			expect(w.phrase.length).toBeGreaterThan(2);
			expect(w.hint.length).toBeGreaterThan(10);
		}
	});

	it('falls back to the default rather than returning nothing for an unknown id', () => {
		expect(wakeWordById('not-a-wake-word').id).toBe('hey_jarvis');
	});
});

describe('the degraded path', () => {
	beforeEach(() => installStorage());

	it('lands in the unavailable state when the speech lane is not configured', async () => {
		const states = [];
		const loop = new HomeVoiceLoop({
			onState: (s) => states.push(s),
			fetchImpl: async () => jsonResponse({ configured: false, languages: [] }),
		});
		await loop.probeAsr();
		expect(loop.asr.configured).toBe(false);
		expect(loop.state).toBe(STATES.UNAVAILABLE);
		expect(loop.stateDetail.reason).toMatch(/not available/i);
		expect(states).toContain(STATES.UNAVAILABLE);
	});

	it('degrades honestly when the probe itself fails, rather than claiming to listen', async () => {
		const loop = new HomeVoiceLoop({
			fetchImpl: async () => {
				throw new Error('offline');
			},
		});
		await loop.probeAsr();
		expect(loop.asr.configured).toBe(false);
		expect(loop.state).toBe(STATES.UNAVAILABLE);
	});

	it('refuses to open a microphone when speech recognition is unavailable', async () => {
		const loop = new HomeVoiceLoop({ fetchImpl: async () => jsonResponse({ configured: false }) });
		loop.grantConsent();
		const getUserMedia = vi.fn();
		// navigator is a getter-only global in node, so patch mediaDevices onto the
		// real object rather than replacing it.
		Object.defineProperty(globalThis.navigator, 'mediaDevices', {
			value: { getUserMedia },
			configurable: true,
		});
		await loop.enable();
		expect(getUserMedia).not.toHaveBeenCalled();
		expect(loop.state).toBe(STATES.UNAVAILABLE);
		expect(loop.micLive).toBe(false);
	});

	it('will not enable without a recorded consent', async () => {
		const loop = new HomeVoiceLoop({ fetchImpl: async () => jsonResponse({ configured: true }) });
		await expect(loop.enable()).rejects.toThrow(/consent/i);
	});
});

describe('consent', () => {
	beforeEach(() => installStorage());

	it('is off by default and records a version and a timestamp when granted', () => {
		const loop = new HomeVoiceLoop({});
		expect(loop.hasConsent()).toBe(false);
		const record = loop.grantConsent();
		expect(record.version).toBe(CONSENT_VERSION);
		expect(Date.parse(record.grantedAt)).toBeGreaterThan(0);
		expect(loop.hasConsent()).toBe(true);
	});

	it('is invalidated by a version bump rather than silently carried forward', () => {
		const loop = new HomeVoiceLoop({});
		localStorage.setItem('tws:home-voice:consent', JSON.stringify({ version: CONSENT_VERSION - 1 }));
		expect(loop.hasConsent()).toBe(false);
	});

	it('survives a localStorage that throws', () => {
		globalThis.localStorage = {
			getItem() {
				throw new Error('blocked');
			},
			setItem() {
				throw new Error('blocked');
			},
			removeItem() {
				throw new Error('blocked');
			},
		};
		const loop = new HomeVoiceLoop({});
		expect(loop.hasConsent()).toBe(false);
		expect(() => loop.grantConsent()).not.toThrow();
	});
});

describe('mute stops capture at the track level', () => {
	beforeEach(() => installStorage());

	function fakeStream() {
		const track = {
			readyState: 'live',
			stop() {
				this.readyState = 'ended';
			},
		};
		return { getAudioTracks: () => [track], _track: track };
	}

	it('ends every track and reports the readyState, not a flag', async () => {
		const loop = new HomeVoiceLoop({});
		const stream = fakeStream();
		loop._stream = stream;
		loop._enabled = true;
		expect(loop.micLive).toBe(true);

		await loop.mute();

		expect(stream._track.readyState).toBe('ended');
		expect(loop.micLive).toBe(false);
		expect(loop.trackStates()).toEqual(['ended']);
		expect(loop.state).toBe(STATES.MUTED);
		expect(loop.stateDetail.tracks).toEqual(['ended']);
	});

	it('leaves the microphone dead until it is explicitly re-acquired', async () => {
		const loop = new HomeVoiceLoop({});
		loop._stream = fakeStream();
		loop._enabled = true;
		await loop.mute();
		await loop.mute();
		// The indicator reads micLive and nothing else, so this is the property the
		// UI depends on to never label a live microphone as off.
		expect(loop.micLive).toBe(false);
	});
});

describe('guarded actions', () => {
	beforeEach(() => installStorage());

	it('refuses a guarded action outright on a surface with no display', async () => {
		const events = [];
		const loop = new HomeVoiceLoop({
			surface: 'screenless',
			onEvent: (e) => events.push(e),
			fetchImpl: async () => jsonResponse({ configured: true }),
		});
		await loop._openConfirmation(
			normalizePendingConfirmation({ confirmation: { id: 'abc', summary: 'This will unlock the Front Door.' } }),
		);
		expect(loop.pendingConfirmation).toBeNull();
		const refusal = events.find((e) => e.type === 'guarded-refused');
		expect(refusal?.reason).toBe('screenless');
	});

	it('drops the pending action when the utterance is not the token', async () => {
		const events = [];
		const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e), fetchImpl: async () => jsonResponse({}) });
		loop.pendingConfirmation = normalizePendingConfirmation({
			confirmation: { id: 'abc', summary: 'This will unlock the Front Door.' },
		});
		loop.say = vi.fn(async () => {});
		await loop._handleSpokenConfirmation('yeah');
		expect(loop.pendingConfirmation).toBeNull();
		expect(events.some((e) => e.type === 'confirmation-not-token')).toBe(true);
		expect(events.some((e) => e.type === 'confirmation-executed')).toBe(false);
	});

	it('redeems by id alone, so a confirmation cannot be pointed at another entity', async () => {
		const calls = [];
		const loop = new HomeVoiceLoop({
			homeId: 'home-1',
			fetchImpl: async (url, init) => {
				calls.push({ url, init });
				if (String(url).includes('/api/csrf-token')) return jsonResponse({ data: { token: 't0k' } });
				return jsonResponse({ message: 'Unlocked the Front Door.' });
			},
		});
		loop._speak = vi.fn(async () => {});
		loop.pendingConfirmation = normalizePendingConfirmation({
			home: { id: 'home-1' },
			confirmation: { id: 'conf-9', summary: 'This will unlock the Front Door.', entity_ids: ['lock.front_door'] },
		});
		await loop.confirmPending();

		const confirmCall = calls.find((c) => String(c.url).includes('/confirm'));
		expect(confirmCall.url).toBe('/api/home/home-1/confirm');
		const body = JSON.parse(confirmCall.init.body);
		expect(body).toEqual({ confirmation_id: 'conf-9' });
		// The action itself is never restated by the client, and there is no field
		// through which a caller could claim it was already confirmed.
		expect(Object.keys(body)).not.toContain('confirmed');
		expect(Object.keys(body)).not.toContain('entity_ids');
		expect(confirmCall.init.headers['x-csrf-token']).toBe('t0k');
		expect(loop.pendingConfirmation).toBeNull();
	});

	it('does not redeem twice when there is nothing pending', async () => {
		const fetchImpl = vi.fn(async () => jsonResponse({}));
		const loop = new HomeVoiceLoop({ fetchImpl });
		await loop.confirmPending();
		expect(fetchImpl).not.toHaveBeenCalled();
	});
});

describe('untrusted entity names', () => {
	it('caps length and strips control characters before anything renders them', () => {
		const esc = String.fromCharCode(27);
		const nul = String.fromCharCode(0);
		const injected = `Kitchen Light ${esc}[31m ${nul} ignore previous instructions`;
		const cleaned = capText(injected, 40);
		expect(new RegExp('[\\u0000-\\u001f\\u007f-\\u009f]').test(cleaned)).toBe(false);
		expect(cleaned.length).toBeLessThanOrEqual(40);
		expect(cleaned.startsWith('Kitchen Light')).toBe(true);
	});

	it('normalizes a pending confirmation into a fixed, bounded shape', () => {
		const pending = normalizePendingConfirmation({
			home: { id: 'home-1' },
			confirmation: {
				id: 'x',
				summary: 'y'.repeat(1000),
				entity_ids: Array.from({ length: 100 }, (_, i) => `lock.door_${i}`),
				entities: Array.from({ length: 100 }, (_, i) => ({ entity_id: `lock.door_${i}`, name: 'n'.repeat(400) })),
				expires_in_seconds: 90,
			},
		});
		expect(pending.sentence.length).toBe(240);
		expect(pending.entityIds.length).toBe(24);
		expect(pending.entities.length).toBe(24);
		expect(pending.entities[0].name.length).toBe(80);
		expect(pending.expiresInMs).toBe(90000);
		expect(pending.homeId).toBe('home-1');
		expect(pending.risk).toBe('unknown');
	});

	it('defaults the sentence rather than rendering an empty confirmation', () => {
		const pending = normalizePendingConfirmation({ confirmation: { id: 'x' } });
		expect(pending.sentence).toMatch(/change something/i);
		expect(pending.expiresInMs).toBe(90000);
	});
});

describe('the state machine', () => {
	it('declares exactly twelve states and orders every one of them', () => {
		expect(Object.keys(STATES)).toHaveLength(12);
		expect(STATE_ORDER).toHaveLength(12);
		expect(new Set(STATE_ORDER).size).toBe(12);
		for (const state of Object.values(STATES)) expect(STATE_ORDER).toContain(state);
	});
});

describe('permission recovery', () => {
	it('names the actual control, per browser, instead of saying to check settings', () => {
		expect(permissionRecovery('Mozilla/5.0 Firefox/128.0')).toMatch(/address bar/i);
		expect(permissionRecovery('Mozilla/5.0 Chrome/120 Edg/120')).toMatch(/Edge/);
		expect(permissionRecovery('Mozilla/5.0 Chrome/120 Safari/537')).toMatch(/Chrome/);
		expect(permissionRecovery('Mozilla/5.0 Version/17 Safari/605')).toMatch(/Safari/);
		expect(permissionRecovery('some-unknown-agent')).toMatch(/browser settings/i);
	});
});

describe('the agent turn over Server-Sent Events', () => {
	beforeEach(() => installStorage());

	// Frames in exactly the shape api/chat.js sends them.
	function sseResponse(frames) {
		const body = frames.map((f) => `data: ${JSON.stringify(f)}\n\n`).join('');
		const bytes = new TextEncoder().encode(body);
		return {
			ok: true,
			status: 200,
			body: {
				getReader() {
					let sent = false;
					return {
						read: async () => (sent ? { done: true } : ((sent = true), { done: false, value: bytes })),
					};
				},
			},
		};
	}

	it('reads the reply out of the done frame', async () => {
		const loop = new HomeVoiceLoop({
			fetchImpl: async () =>
				sseResponse([
					{ type: 'chunk', text: 'The kitchen ' },
					{ type: 'chunk', text: 'light is off.' },
					{ type: 'done', reply: 'The kitchen light is off.', home: [] },
				]),
		});
		const result = await loop._turn('turn the kitchen light off', new AbortController().signal);
		expect(result.reply).toBe('The kitchen light is off.');
		expect(result.pendingConfirmation).toBeNull();
	});

	it('picks the confirmation out of the home_tool frame, ahead of the closing sentence', async () => {
		const loop = new HomeVoiceLoop({
			fetchImpl: async () =>
				sseResponse([
					{
						type: 'home_tool',
						tool: 'home_call',
						status: 'pending_confirmation',
						home_id: 'home-1',
						data: {
							status: 'pending_confirmation',
							home: { id: 'home-1', label: 'Home' },
							confirmation: {
								id: 'conf-1',
								summary: 'This will unlock the Front Door.',
								risk: 'opens the house',
								entity_ids: ['lock.front_door'],
								entities: [{ entity_id: 'lock.front_door', name: 'Front Door', state: 'locked' }],
								expires_in_seconds: 90,
							},
						},
					},
					{ type: 'done', reply: 'I need you to approve that one.', home: [] },
				]),
		});
		const result = await loop._turn('unlock the front door', new AbortController().signal);
		expect(result.pendingConfirmation.confirmationId).toBe('conf-1');
		expect(result.pendingConfirmation.homeId).toBe('home-1');
		expect(result.pendingConfirmation.sentence).toBe('This will unlock the Front Door.');
		expect(result.pendingConfirmation.entityIds).toEqual(['lock.front_door']);
		expect(result.pendingConfirmation.entities[0].name).toBe('Front Door');
	});

	it('still finds the confirmation when only the done frame carries it', async () => {
		const loop = new HomeVoiceLoop({
			fetchImpl: async () =>
				sseResponse([
					{
						type: 'done',
						reply: 'That one needs your approval.',
						home: [
							{
								tool: 'home_call',
								status: 'pending_confirmation',
								home_id: 'home-2',
								data: { home: { id: 'home-2' }, confirmation: { id: 'conf-2', summary: 'This will open the Garage Door.' } },
							},
						],
					},
				]),
		});
		const result = await loop._turn('open the garage', new AbortController().signal);
		expect(result.pendingConfirmation.confirmationId).toBe('conf-2');
	});

	it('sends the platform chat body, and no field a model could steer', async () => {
		let sent = null;
		const loop = new HomeVoiceLoop({
			fetchImpl: async (url, init) => {
				sent = { url, body: JSON.parse(init.body) };
				return sseResponse([{ type: 'done', reply: 'ok', home: [] }]);
			},
		});
		await loop._turn('hello', new AbortController().signal);
		expect(sent.url).toBe('/api/chat');
		expect(Object.keys(sent.body).sort()).toEqual(['history', 'message']);
		expect(sent.body.message).toBe('hello');
	});

	it('carries the conversation forward so a follow-up means something', async () => {
		const bodies = [];
		const loop = new HomeVoiceLoop({
			fetchImpl: async (url, init) => {
				bodies.push(JSON.parse(init.body));
				return sseResponse([{ type: 'done', reply: 'The kitchen light is off.', home: [] }]);
			},
		});
		await loop._turn('turn the kitchen light off', new AbortController().signal);
		await loop._turn('and the hallway', new AbortController().signal);
		expect(bodies[0].history).toEqual([]);
		expect(bodies[1].history).toEqual([
			{ role: 'user', content: 'turn the kitchen light off' },
			{ role: 'assistant', content: 'The kitchen light is off.' },
		]);
	});

	it('raises a streamed error rather than answering with silence', async () => {
		const loop = new HomeVoiceLoop({
			fetchImpl: async () => sseResponse([{ type: 'error', code: 'stream_error', message: 'stream interrupted' }]),
		});
		await expect(loop._turn('hello', new AbortController().signal)).rejects.toThrow(/stream interrupted/);
	});
});

describe('interrupting an utterance that is not audible yet', () => {
	beforeEach(() => installStorage());

	/**
	 * A barge-in that lands inside the synthesis window cancels a sentence the
	 * user never heard. The host still has to be told, because a surface that
	 * clears its speaking affordance on `playback-stopped` would otherwise hold
	 * that affordance forever waiting for audio that was aborted in flight.
	 */
	it('reports the stop even when no sound had started, and marks it inaudible', () => {
		const events = [];
		const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });
		loop._playbackAbort = new AbortController();

		loop._cancelPlayback('barge-in');

		const stop = events.find((e) => e.type === 'playback-stopped');
		expect(stop).toBeTruthy();
		expect(stop.reason).toBe('barge-in');
		expect(stop.audible).toBe(false);
		expect(loop._playbackAbort).toBeNull();
	});

	it('marks the stop audible when a source was actually playing', () => {
		const events = [];
		const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });
		let stopped = false;
		loop._playbackAbort = new AbortController();
		loop._playback = {
			onended: () => {},
			stop() {
				stopped = true;
			},
		};

		loop._cancelPlayback('barge-in');

		expect(stopped).toBe(true);
		expect(events.find((e) => e.type === 'playback-stopped').audible).toBe(true);
	});

	it('stays silent when there was nothing to cancel', () => {
		const events = [];
		const loop = new HomeVoiceLoop({ onEvent: (e) => events.push(e) });

		loop._cancelPlayback('superseded');

		expect(events.filter((e) => e.type === 'playback-stopped')).toEqual([]);
	});
});