Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
import type {
ExternalExecutionResultEvent,
UserConfirmResultEvent,
} from '@agentscope-ai/agentscope/event';
import { realtimeApi } from '@/api';
export type RealtimeConnectionState = 'idle' | 'connecting' | 'active' | 'error';
interface BrowserWebRTCTransportOptions {
agentId: string;
sessionId: string;
onStateChange: (state: RealtimeConnectionState) => void;
onError: (error: Error) => void;
}
interface AudioStartFrame {
type: 'audio_start';
item_id: string;
track_time_ms: number;
}
interface AudioDurationFrame {
type: 'audio_duration';
item_id: string;
duration_ms: number;
}
interface ClearAudioFrame {
type: 'clear_audio';
request_id: string;
resume_track_time_ms: number;
}
interface ErrorFrame {
type: 'error';
detail: string;
}
type ServerFrame = AudioStartFrame | AudioDurationFrame | ClearAudioFrame | ErrorFrame;
const ICE_GATHERING_TIMEOUT_MS = 10_000;
const CONNECTION_TIMEOUT_MS = 20_000;
const PLAYOUT_FENCE_POLL_MS = 20;
const PLAYOUT_FENCE_POLL_TIMEOUT_MS = 5_000;
/** WebRTC microphone, model audio, and DataChannel control transport. */
export class BrowserWebRTCTransport {
private readonly options: BrowserWebRTCTransportOptions;
private peerConnection: RTCPeerConnection | null = null;
private controlChannel: RTCDataChannel | null = null;
private localStream: MediaStream | null = null;
private remoteStream: MediaStream | null = null;
private audioElement: HTMLAudioElement | null = null;
private progressTimer: number | null = null;
private playoutFenceTimer: number | null = null;
private playoutFenceAudio: HTMLAudioElement | null = null;
private playoutFenceListener: (() => void) | null = null;
private readonly itemStarts: AudioStartFrame[] = [];
private readonly durationsMs = new Map<string, number>();
private intentionalClose = false;
private failed = false;
private active = false;
private failureError: Error | null = null;
constructor(options: BrowserWebRTCTransportOptions) {
this.options = options;
}
async start(): Promise<void> {
this.intentionalClose = false;
this.failed = false;
this.active = false;
this.failureError = null;
this.options.onStateChange('connecting');
if (!navigator.mediaDevices?.getUserMedia || !window.RTCPeerConnection) {
throw new Error('WebRTC microphone capture is not available in this browser.');
}
const { ice_servers: iceServers } = await realtimeApi.config();
this.throwIfStopped();
const localStream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
this.localStream = localStream;
this.throwIfStopped();
const peerConnection = new RTCPeerConnection({ iceServers });
this.peerConnection = peerConnection;
const controlChannel = peerConnection.createDataChannel('control', {
ordered: true,
});
this.controlChannel = controlChannel;
controlChannel.onmessage = (event) => this.handleMessage(event.data);
controlChannel.onclose = () => {
if (!this.intentionalClose) {
this.fail(new Error('The realtime control channel was closed.'));
}
};
peerConnection.ontrack = (event) => this.attachRemoteTrack(event.track);
peerConnection.onconnectionstatechange = () => {
if (
!this.intentionalClose &&
(peerConnection.connectionState === 'failed' ||
peerConnection.connectionState === 'closed')
) {
this.fail(new Error('The realtime media connection was lost.'));
}
};
for (const track of localStream.getTracks()) {
peerConnection.addTrack(track, localStream);
}
const channelReady = this.waitForControlChannel(controlChannel);
void channelReady.catch(() => undefined);
const offer = await peerConnection.createOffer();
this.throwIfStopped();
await peerConnection.setLocalDescription(offer);
await this.waitForIceGathering(peerConnection);
this.throwIfStopped();
const localDescription = peerConnection.localDescription;
if (!localDescription || localDescription.type !== 'offer') {
throw new Error('The browser did not produce a WebRTC offer.');
}
const answer = await realtimeApi.offer(this.options.sessionId, {
agent_id: this.options.agentId,
sdp: localDescription.sdp,
type: 'offer',
});
this.throwIfStopped();
await peerConnection.setRemoteDescription({
sdp: answer.sdp,
type: answer.type,
});
await channelReady;
this.throwIfStopped();
this.progressTimer = window.setInterval(() => this.reportPlayout(), 100);
this.active = true;
this.options.onStateChange('active');
}
async stop(): Promise<void> {
this.intentionalClose = true;
if (this.progressTimer !== null) {
window.clearInterval(this.progressTimer);
this.progressTimer = null;
}
this.clearPlayoutFence();
this.sendJson({ type: 'close' });
this.controlChannel?.close();
this.peerConnection?.close();
this.localStream?.getTracks().forEach((track) => track.stop());
this.remoteStream?.getTracks().forEach((track) => track.stop());
if (this.audioElement) {
this.audioElement.pause();
this.audioElement.srcObject = null;
}
this.controlChannel = null;
this.peerConnection = null;
this.localStream = null;
this.remoteStream = null;
this.audioElement = null;
this.itemStarts.length = 0;
this.durationsMs.clear();
this.active = false;
this.options.onStateChange('idle');
}
userConfirm(event: UserConfirmResultEvent): void {
if (this.controlChannel?.readyState !== 'open') {
throw new Error('The realtime control channel is not open.');
}
this.controlChannel.send(
JSON.stringify({ type: 'control', control: 'user_confirm', data: event }),
);
}
externalExecutionResult(event: ExternalExecutionResultEvent): void {
if (this.controlChannel?.readyState !== 'open') {
throw new Error('The realtime control channel is not open.');
}
this.controlChannel.send(
JSON.stringify({
type: 'control',
control: 'external_execution_result',
data: event,
}),
);
}
private attachRemoteTrack(track: MediaStreamTrack): void {
if (track.kind !== 'audio') return;
this.clearPlayoutFence();
this.remoteStream = new MediaStream([track]);
const audio = new Audio();
audio.autoplay = true;
audio.srcObject = this.remoteStream;
this.audioElement = audio;
void audio.play().catch(() => {
this.fail(new Error('The browser blocked realtime audio playback.'));
});
}
private handleMessage(raw: unknown): void {
if (typeof raw !== 'string') return;
let frame: ServerFrame;
try {
frame = JSON.parse(raw) as ServerFrame;
} catch {
this.fail(new Error('The server sent an invalid realtime control frame.'));
return;
}
if (frame.type === 'audio_start') {
this.itemStarts.push(frame);
this.itemStarts.sort((left, right) => left.track_time_ms - right.track_time_ms);
} else if (frame.type === 'audio_duration') {
this.durationsMs.set(frame.item_id, frame.duration_ms);
} else if (frame.type === 'clear_audio') {
this.acknowledgeClear(frame);
} else if (frame.type === 'error') {
this.fail(new Error(frame.detail));
}
}
private acknowledgeClear(frame: ClearAudioFrame): void {
const position = this.currentPosition();
this.muteUntilTrackTime(frame.resume_track_time_ms);
this.sendJson({
type: 'playout_cleared',
request_id: frame.request_id,
...position,
});
this.itemStarts.length = 0;
this.durationsMs.clear();
}
private muteUntilTrackTime(trackTimeMs: number): void {
const audio = this.audioElement;
if (!audio || !Number.isFinite(trackTimeMs)) return;
this.clearPlayoutFence();
audio.muted = true;
const deadline = performance.now() + PLAYOUT_FENCE_POLL_TIMEOUT_MS;
const releaseIfCaughtUp = () => {
if (this.audioElement !== audio) {
this.clearPlayoutFence();
return true;
}
if (audio.currentTime * 1000 < trackTimeMs) return false;
audio.muted = false;
this.clearPlayoutFence();
return true;
};
const release = () => {
if (releaseIfCaughtUp()) return;
if (performance.now() >= deadline) {
this.playoutFenceTimer = null;
this.playoutFenceAudio = audio;
this.playoutFenceListener = () => {
releaseIfCaughtUp();
};
audio.addEventListener('playing', this.playoutFenceListener);
audio.addEventListener('timeupdate', this.playoutFenceListener);
return;
}
this.playoutFenceTimer = window.setTimeout(release, PLAYOUT_FENCE_POLL_MS);
};
release();
}
private clearPlayoutFence(): void {
if (this.playoutFenceTimer !== null) {
window.clearTimeout(this.playoutFenceTimer);
this.playoutFenceTimer = null;
}
if (this.playoutFenceAudio && this.playoutFenceListener) {
this.playoutFenceAudio.removeEventListener('playing', this.playoutFenceListener);
this.playoutFenceAudio.removeEventListener('timeupdate', this.playoutFenceListener);
}
this.playoutFenceAudio = null;
this.playoutFenceListener = null;
}
private currentPosition(): { item_id: string; played_ms: number } {
const currentTimeMs = (this.audioElement?.currentTime ?? 0) * 1000;
if (!Number.isFinite(currentTimeMs)) {
return { item_id: '', played_ms: 0 };
}
let currentIndex = -1;
for (let index = 0; index < this.itemStarts.length; index += 1) {
if (this.itemStarts[index].track_time_ms > currentTimeMs) break;
currentIndex = index;
}
if (currentIndex < 0) return { item_id: '', played_ms: 0 };
const current = this.itemStarts[currentIndex];
for (const completed of this.itemStarts.splice(0, currentIndex)) {
this.durationsMs.delete(completed.item_id);
}
const elapsed = Math.max(0, currentTimeMs - current.track_time_ms);
const duration = this.durationsMs.get(current.item_id) ?? 0;
return {
item_id: current.item_id,
played_ms: Math.round(Math.min(elapsed, duration)),
};
}
private reportPlayout(): void {
const position = this.currentPosition();
if (position.item_id) this.sendJson({ type: 'playout', ...position });
}
private sendJson(payload: Record<string, unknown>): void {
if (this.controlChannel?.readyState === 'open') {
this.controlChannel.send(JSON.stringify(payload));
}
}
private throwIfStopped(): void {
if (this.intentionalClose) {
throw this.failureError ?? new Error('The realtime connection was stopped.');
}
}
private waitForIceGathering(peerConnection: RTCPeerConnection): Promise<void> {
if (peerConnection.iceGatheringState === 'complete') return Promise.resolve();
return new Promise((resolve, reject) => {
const timeout = window.setTimeout(() => {
peerConnection.removeEventListener('icegatheringstatechange', onChange);
reject(new Error('Timed out while gathering WebRTC network candidates.'));
}, ICE_GATHERING_TIMEOUT_MS);
const onChange = () => {
if (peerConnection.iceGatheringState !== 'complete') return;
window.clearTimeout(timeout);
peerConnection.removeEventListener('icegatheringstatechange', onChange);
resolve();
};
peerConnection.addEventListener('icegatheringstatechange', onChange);
});
}
private waitForControlChannel(channel: RTCDataChannel): Promise<void> {
if (channel.readyState === 'open') return Promise.resolve();
return new Promise((resolve, reject) => {
const cleanup = () => {
window.clearTimeout(timeout);
channel.removeEventListener('open', onOpen);
channel.removeEventListener('close', onClose);
};
const timeout = window.setTimeout(() => {
cleanup();
reject(new Error('Timed out while opening the realtime connection.'));
}, CONNECTION_TIMEOUT_MS);
const onOpen = () => {
cleanup();
resolve();
};
const onClose = () => {
cleanup();
reject(this.failureError ?? new Error('The realtime connection was closed.'));
};
channel.addEventListener('open', onOpen);
channel.addEventListener('close', onClose);
});
}
private fail(error: Error): void {
if (this.failed || this.intentionalClose) return;
this.failed = true;
this.failureError = error;
if (this.active) this.options.onError(error);
void this.stop().finally(() => this.options.onStateChange('error'));
}
}