Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// LiveKit realtime voice — bidirectional audio via a LiveKit room.
// The agent server joins the same room, handles VAD/STT/TTS, and sends
// data-channel messages with transcript payloads.
import { Room, RoomEvent, Track } from 'livekit-client';
import { ACTION_TYPES } from '../agent-protocol.js';
export class LiveKitVoice {
/**
* @param {object} opts
* @param {string} opts.serverUrl LiveKit server WebSocket URL (wss://…)
* @param {string} opts.token Room access JWT
* @param {import('../agent-protocol.js').AgentProtocol} opts.protocol
*/
constructor({ serverUrl, token, protocol }) {
this._serverUrl = serverUrl;
this._token = token;
this._protocol = protocol;
this._room = null;
this._audioEls = []; // { track, el }[]
this._userSpeaking = false;
}
async connect() {
const room = new Room();
this._room = room;
// Agent TTS audio — attach <audio> element when track arrives.
// In the slim widget shell we still need the element to exist so the
// browser plays the track, but it must be hidden (no DOM leak) and
// detached from any user-visible layout. The audio is what the
// embedder asked for; the <audio> tag itself is invisible chrome.
room.on(RoomEvent.TrackSubscribed, (track, _pub, _participant) => {
if (track.kind !== Track.Kind.Audio) return;
const el = track.attach();
if (window.__WIDGET_SHELL) {
el.style.display = 'none';
el.setAttribute('aria-hidden', 'true');
}
document.body.appendChild(el);
this._audioEls.push({ track, el });
});
room.on(RoomEvent.TrackUnsubscribed, (track) => {
const idx = this._audioEls.findIndex((e) => e.track === track);
if (idx === -1) return;
const { el } = this._audioEls.splice(idx, 1)[0];
try { track.detach(el); } catch {}
el.remove();
});
// Data messages from the agent server
room.on(RoomEvent.DataReceived, (data, _participant) => {
let msg;
try {
msg = JSON.parse(new TextDecoder().decode(data));
} catch {
return;
}
// Agent speaking — drive avatar lipsync + chat bubble
if (msg.type === 'transcript' && msg.text) {
this._protocol.emit({
type: 'speak',
payload: { text: msg.text, sentiment: 0 },
});
}
// Named body gesture (e.g. from the LiveKit avatar plugin)
if (msg.type === 'gesture' && msg.name) {
this._protocol.emit({
type: ACTION_TYPES.GESTURE,
payload: { name: msg.name },
});
}
// User speech boundaries (VAD from agent server)
if (msg.type === 'speech_started') {
this._userSpeaking = true;
this._protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });
}
if (msg.type === 'speech_ended') {
this._userSpeaking = false;
this._protocol.emit({
type: ACTION_TYPES.LISTEN_END,
payload: { text: msg.transcript || '' },
});
}
});
// Fallback: use LiveKit active-speaker VAD when agent doesn't send speech events
room.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
const localId = room.localParticipant?.identity;
const active = speakers.some((p) => p.identity === localId);
if (active && !this._userSpeaking) {
this._userSpeaking = true;
this._protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });
} else if (!active && this._userSpeaking) {
this._userSpeaking = false;
this._protocol.emit({ type: ACTION_TYPES.LISTEN_END, payload: { text: '' } });
}
});
await room.connect(this._serverUrl, this._token);
await room.localParticipant.setMicrophoneEnabled(true);
}
async disconnect() {
for (const { track, el } of this._audioEls) {
try { track.detach(el); } catch {}
try { el.remove(); } catch {}
}
this._audioEls = [];
if (this._room) {
try { await this._room.disconnect(); } catch {}
this._room = null;
}
this._userSpeaking = false;
}
get connected() {
return this._room?.state === 'connected';
}
}