Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
import { StopSoundActivity, type AsrEvent, type AsrProvider, type AudioTrace } from '@cindy/voice-input-core';
import type { StoredMobileVoiceCredential } from '@/session/mobileVoiceCredentialStore';
import { redactMobileVoiceCredentialText } from '@/session/mobileVoiceCredentialRedaction';
import { isMobileVoiceRateLimited } from '@/session/mobileVoiceInput';
import { classifyMobileVoiceFailure, logMobileVoice } from '@/session/mobileVoiceDiagnostics';
import { gzip, ungzip } from 'pako';
import { i18n } from '@/i18n';
type MobileRealtimeProtocolProfile =
| 'openai-transcription-manual'
| 'qwen-asr-server-vad';
type MobileRealtimeAsrProviderOptions = {
credential: StoredMobileVoiceCredential;
sourceLanguage?: string;
websocketFactory?: WebSocketFactory;
connectTimeoutMs?: number;
flushTimeoutMs?: number;
connectionProvider?: (provider: string) => Promise<{
websocketUrl: string;
authorizationToken: string;
}>;
};
type WebSocketFactory = new (
url: string,
protocols?: string | string[] | null,
options?: { headers?: Record<string, string> },
) => WebSocketLike;
type WebSocketLike = {
readyState: number;
binaryType?: string;
send(data: string | ArrayBuffer): void;
close(): void;
onopen: ((event: unknown) => void) | null;
onmessage: ((event: { data: unknown }) => void) | null;
onerror: ((event: { message?: string; error?: unknown }) => void) | null;
onclose: ((event: { code?: number; reason?: string }) => void) | null;
};
const OPEN = 1;
const DEFAULT_CONNECT_TIMEOUT_MS = 5_000;
const DEFAULT_FLUSH_TIMEOUT_MS = 4_000;
const RECOVER_TIMEOUT_MS = 3_000;
const MIN_COMMIT_AUDIO_MS = 100;
const CONFIRMED_AUDIO_RETENTION_MS = 1_500;
const QWEN_SERVER_VAD_THRESHOLD = 0.0;
const QWEN_SERVER_VAD_SILENCE_DURATION_MS = 400;
const LOCAL_TURN_VOICED_RMS_THRESHOLD = 300;
const VOLCENGINE_MODEL_NAME = 'bigmodel';
const VOLCENGINE_NONSTREAM_END_WINDOW_MS = 300;
// Quiet time after the last local sound before stop may skip the final response.
const VOLCENGINE_FAST_FINISH_SILENCE_MS = 2_000;
const CLOSED = 3;
const VOLC_PROTOCOL_VERSION = 0x1;
const VOLC_HEADER_SIZE_WORDS = 0x1;
const VOLC_SERIALIZATION_NONE = 0x0;
const VOLC_SERIALIZATION_JSON = 0x1;
const VOLC_COMPRESSION_NONE = 0x0;
const VOLC_COMPRESSION_GZIP = 0x1;
const VOLC_MESSAGE_TYPE_FULL_CLIENT_REQUEST = 0x1;
const VOLC_MESSAGE_TYPE_AUDIO_ONLY_REQUEST = 0x2;
const VOLC_MESSAGE_TYPE_FULL_SERVER_RESPONSE = 0x9;
const VOLC_MESSAGE_TYPE_SERVER_ACK = 0xb;
const VOLC_MESSAGE_TYPE_SERVER_ERROR = 0xf;
const VOLC_FLAG_NO_SEQUENCE = 0x0;
const VOLC_FLAG_POSITIVE_SEQUENCE = 0x1;
const VOLC_FLAG_NEGATIVE_SEQUENCE = 0x3;
type ParsedVolcengineMessage = {
messageType: number;
flags: number;
sequence?: number;
payload?: unknown;
payloadText?: string;
};
type RealtimeReplayAudioChunk = {
audioBase64: string;
durationMs: number;
addedAt: number;
sent: boolean;
localTurnId?: number;
commitId?: number;
};
type VolcengineReplayAudioChunk = {
pcm: Uint8Array;
durationMs: number;
addedAt: number;
sent: boolean;
localTurnId?: number;
};
type PendingConnect = {
reject(error: Error): void;
};
export function createMobileAsrProvider(
credential: StoredMobileVoiceCredential,
options: Omit<MobileRealtimeAsrProviderOptions, 'credential'> = {},
): AsrProvider {
const chain = mobileAsrCredentialChain(credential);
if (chain.length > 1) {
return new MobileFallbackAsrProvider(
chain.map((asr) => ({
provider: asr.provider,
create: () => createSingleMobileAsrProvider({ ...credential, asr }, options),
})),
);
}
return createSingleMobileAsrProvider({ ...credential, asr: chain[0] ?? credential.asr }, options);
}
function createSingleMobileAsrProvider(
credential: StoredMobileVoiceCredential,
options: Omit<MobileRealtimeAsrProviderOptions, 'credential'> = {},
): AsrProvider {
if (credential.asr.mode === 'realtime-websocket') {
return new MobileRealtimeAsrProvider({ ...options, credential });
}
if (credential.asr.mode === 'provider-native-websocket' && credential.asr.protocolProfile === 'volcengine-sauc-duration') {
return new MobileVolcengineSaucAsrProvider({ ...options, credential });
}
throw new Error(i18n.t('composer.voice.realtimeUnsupported', { detail: `${credential.asr.mode} / ${credential.asr.protocolProfile ?? 'unknown'}` }));
}
type MobileAsrCandidate = {
provider: string;
create: () => AsrProvider;
};
class MobileFallbackAsrProvider implements AsrProvider {
private readonly candidates: MobileAsrCandidate[];
private readonly callbacks: Array<(event: AsrEvent) => void> = [];
private active: AsrProvider | null = null;
private pending: AsrProvider | null = null;
private activeProvider = '';
private stopped = false;
recover?: () => Promise<void>;
constructor(candidates: MobileAsrCandidate[]) {
if (candidates.length === 0) throw new Error('Mobile ASR fallback requires at least one candidate.');
this.candidates = candidates;
}
async start(): Promise<void> {
this.stopped = false;
let lastError: unknown = null;
for (const candidate of this.candidates) {
if (this.stopped) throw new Error('Mobile ASR fallback provider stopped.');
const provider = candidate.create();
this.pending = provider;
provider.onEvent((event) => {
if (this.active === provider) this.callbacks.forEach((callback) => callback(event));
});
try {
await provider.start();
} catch (err) {
if (this.pending === provider) this.pending = null;
lastError = err;
await provider.dispose?.().catch(() => undefined);
if (this.stopped) throw err;
const rateLimited = isMobileVoiceRateLimited(err);
logMobileVoice('warn', 'asr candidate failed', {
provider: candidate.provider,
attempt: this.candidates.indexOf(candidate) + 1,
totalCandidates: this.candidates.length,
rateLimited,
reason: classifyMobileVoiceFailure(err),
});
// Managed candidates share one account quota: the next candidate would
// hit the same limit, so surface it instead of switching.
if (rateLimited) throw err;
continue;
}
if (this.pending === provider) this.pending = null;
if (this.stopped) {
await provider.stop().catch(() => undefined);
throw new Error('Mobile ASR fallback provider stopped.');
}
this.active = provider;
this.activeProvider = candidate.provider;
logMobileVoice('info', 'asr provider selected', {
provider: candidate.provider,
attempt: this.candidates.indexOf(candidate) + 1,
});
this.recover = provider.recover ? () => provider.recover!() : undefined;
return;
}
throw lastError instanceof Error ? lastError : new Error('All mobile ASR providers failed to start.');
}
appendAudio(chunk: ArrayBuffer, trace?: AudioTrace): void {
this.active?.appendAudio(chunk, trace);
}
async flushAudio(): Promise<void> {
await this.active?.flushAudio();
}
async stop(): Promise<void> {
this.stopped = true;
const providers = new Set([this.pending, this.active]);
await Promise.all([...providers].map((provider) => provider?.stop()));
}
async dispose(): Promise<void> {
await this.stop();
const providers = new Set([this.pending, this.active]);
await Promise.all([...providers].map((provider) => provider?.dispose?.()));
}
onEvent(callback: (event: AsrEvent) => void): void {
this.callbacks.push(callback);
}
get activeProviderKind(): string {
return this.activeProvider;
}
}
function mobileAsrCredentialChain(
credential: StoredMobileVoiceCredential,
): Array<StoredMobileVoiceCredential['asr']> {
const chain = credential.asrProviderChain?.length ? credential.asrProviderChain : [credential.asr];
const seen = new Set<string>();
const result: Array<StoredMobileVoiceCredential['asr']> = [];
for (const item of chain) {
if (seen.has(item.provider)) continue;
seen.add(item.provider);
result.push(item);
}
return result;
}
export class MobileRealtimeAsrProvider implements AsrProvider {
private readonly credential: StoredMobileVoiceCredential;
private readonly sourceLanguage: string;
private readonly websocketFactory: WebSocketFactory;
private readonly connectTimeoutMs: number;
private readonly flushTimeoutMs: number;
private readonly protocolProfile: MobileRealtimeProtocolProfile;
private readonly pcmSampleRate: number;
private readonly connectionProvider?: MobileRealtimeAsrProviderOptions['connectionProvider'];
private callback: (event: AsrEvent) => void = () => {};
private socket: WebSocketLike | null = null;
private sessionReady = false;
private bufferedMs = 0;
private pendingCommitCount = 0;
private nextAudioCommitId = 1;
private pendingAudioCommitIds: number[] = [];
private audioCommitIdsByItem = new Map<string, number>();
private unboundAudioItemTurnIds = new Map<string, number>();
private currentLocalAudioTurnId = 1;
private localTurnHasVoice = false;
private localTurnSilenceMs = 0;
private unconfirmedAudio: RealtimeReplayAudioChunk[] = [];
private flushResolvers: Array<() => void> = [];
private itemOrder: string[] = [];
private partialsByItem = new Map<string, string>();
private finalsByItem = new Map<string, string>();
private serverVadFinishRequested = false;
private stopped = false;
private inRecovery = false;
private recoveryPromise?: Promise<void>;
private pendingConnect?: PendingConnect;
constructor(options: MobileRealtimeAsrProviderOptions) {
this.credential = options.credential;
this.sourceLanguage = options.sourceLanguage ?? this.credential.settings?.language ?? 'auto';
this.websocketFactory = options.websocketFactory ?? (globalThis.WebSocket as unknown as WebSocketFactory);
this.connectTimeoutMs = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
this.flushTimeoutMs = options.flushTimeoutMs ?? DEFAULT_FLUSH_TIMEOUT_MS;
this.protocolProfile = readRealtimeProtocolProfile(this.credential);
this.pcmSampleRate = this.credential.asr.pcmSampleRate ?? 16_000;
this.connectionProvider = options.connectionProvider;
}
onEvent(callback: (event: AsrEvent) => void): void {
this.callback = callback;
}
async start(): Promise<void> {
if (this.credential.asr.mode !== 'realtime-websocket') {
throw new Error(i18n.t('composer.voice.realtimeUnsupported', { detail: this.credential.asr.mode }));
}
if (this.credential.asr.auth !== 'api-key') {
throw new Error(i18n.t('composer.voice.realtimeAuthUnsupported', { auth: this.credential.asr.auth }));
}
this.resetTranscriptState();
this.stopped = false;
await this.connect();
}
private async connect(): Promise<void> {
// 托管 ticket 是唯一建连方式:BYOK/穿透 key 直拨上游的路径已删除。
if (!this.connectionProvider) {
throw new Error(i18n.t('composer.voice.missingConnectionProvider'));
}
const dynamicConnection = await this.connectionProvider(this.credential.asr.provider);
if (this.stopped) throw new Error('Realtime ASR connection stopped.');
const { websocketUrl, authorizationToken } = dynamicConnection;
await new Promise<void>((resolve, reject) => {
const socket = new this.websocketFactory(
websocketUrl,
null,
{ headers: { Authorization: `Bearer ${authorizationToken}` } },
);
this.socket = socket;
let settled = false;
let pendingConnect: PendingConnect | undefined;
let pendingConnectError: Error | null = null;
let pendingConnectErrorTimer: ReturnType<typeof setTimeout> | null = null;
const timer = setTimeout(() => {
fail(new Error(`Realtime ASR connection timed out after ${this.connectTimeoutMs}ms`));
socket.close();
}, this.connectTimeoutMs);
const cleanup = (): void => {
clearTimeout(timer);
if (pendingConnect && this.pendingConnect === pendingConnect) this.pendingConnect = undefined;
if (pendingConnectErrorTimer) clearTimeout(pendingConnectErrorTimer);
};
const fail = (error: Error): void => {
if (settled) return;
settled = true;
cleanup();
this.socket = null;
this.sessionReady = false;
reject(error);
};
const ready = (): void => {
if (settled) return;
settled = true;
cleanup();
resolve();
};
pendingConnect = { reject: fail };
this.pendingConnect = pendingConnect;
socket.onopen = () => {
if (this.socket !== socket) return;
this.sendSessionUpdate();
};
socket.onmessage = (event) => {
if (this.socket !== socket) return;
this.handleMessage(String(event.data), ready);
};
socket.onerror = (event) => {
if (this.socket !== socket) return;
const { message, specific } = webSocketErrorMessage(event, 'Realtime ASR connection failed.');
const redacted = this.redactError(message);
if (!settled) {
pendingConnectError = new Error(redacted);
if (specific) {
fail(pendingConnectError);
} else {
pendingConnectErrorTimer = setTimeout(() => {
if (pendingConnectError) fail(pendingConnectError);
}, 250);
}
} else {
this.sessionReady = false;
this.resolveFlushWaiters();
this.callback({ type: 'error', message: redacted, at: Date.now() });
}
};
socket.onclose = (event) => {
if (this.socket !== socket) return;
this.sessionReady = false;
this.resolveFlushWaiters();
if (!settled) {
fail(new Error(this.redactError(webSocketCloseBeforeReadyMessage(
event,
'Realtime ASR connection closed before it was ready.',
pendingConnectError?.message,
))));
} else if (!this.stopped) this.callback({ type: 'disconnected', at: Date.now() });
};
});
}
appendAudio(chunk: ArrayBuffer, trace?: AudioTrace): void {
const durationMs = trace?.durationMs ?? estimatePcmDurationMs(chunk.byteLength, trace?.sampleRate ?? this.pcmSampleRate);
const entry = this.bufferUnconfirmedAudio(
arrayBufferToBase64(chunk),
durationMs,
this.assignRealtimeAudioTurnId(chunk, durationMs),
);
this.sendRealtimeAudioEntry(entry);
}
async flushAudio(): Promise<void> {
if (this.recoveryPromise) {
await Promise.race([
this.recoveryPromise.catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, RECOVER_TIMEOUT_MS)),
]);
}
if (this.protocolProfile === 'qwen-asr-server-vad') {
const sent = this.sendWs(buildFinishSessionMessage(this.protocolProfile));
if (sent) {
this.bufferedMs = 0;
this.serverVadFinishRequested = true;
await this.waitForFlush();
}
return;
}
if (this.bufferedMs < MIN_COMMIT_AUDIO_MS) {
this.bufferedMs = 0;
return;
}
if (this.sendWs({ type: 'input_audio_buffer.commit' })) {
this.queueRealtimeAudioCommit();
this.pendingCommitCount += 1;
this.bufferedMs = 0;
await this.waitForFlush();
}
}
async stop(): Promise<void> {
this.stopped = true;
this.sessionReady = false;
this.resolveFlushWaiters();
this.clearUnconfirmedAudio();
const socket = this.socket;
this.pendingConnect?.reject(new Error('Realtime ASR connection stopped.'));
this.socket = null;
if (socket && socket.readyState !== CLOSED) socket.close();
if (this.recoveryPromise) {
await Promise.race([
this.recoveryPromise.catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, RECOVER_TIMEOUT_MS)),
]);
}
this.callback({ type: 'disconnected', at: Date.now() });
}
async recover(): Promise<void> {
if (this.stopped) return;
if (this.recoveryPromise) return this.recoveryPromise;
this.recoveryPromise = this.performRecover().finally(() => {
this.recoveryPromise = undefined;
});
return this.recoveryPromise;
}
private sendSessionUpdate(): void {
this.sendWs(buildSessionUpdateMessage(
this.credential.asr.model,
this.sourceLanguage,
this.pcmSampleRate,
this.protocolProfile,
));
}
private sendWs(message: Record<string, unknown>): boolean {
const socket = this.socket;
if (!socket || socket.readyState !== OPEN || (!this.sessionReady && message.type !== 'session.update')) return false;
socket.send(JSON.stringify(message));
return true;
}
private sendRealtimeAudioEntry(entry: RealtimeReplayAudioChunk): boolean {
if (!this.sessionReady) return false;
const sent = this.sendWs(buildAppendAudioMessage(entry.audioBase64, this.protocolProfile));
if (!sent) return false;
entry.sent = true;
this.bufferedMs += entry.durationMs;
return true;
}
private flushPendingRealtimeAudio(): void {
for (const entry of this.unconfirmedAudio) {
if (!entry.sent) this.sendRealtimeAudioEntry(entry);
}
}
private handleMessage(raw: string, markReady?: () => void): void {
const event = parseJsonObject(raw);
if (!isRecord(event) || typeof event.type !== 'string') return;
switch (event.type) {
case 'session.updated':
this.sessionReady = true;
this.callback({ type: 'connected', at: Date.now() });
markReady?.();
if (!this.inRecovery) this.flushPendingRealtimeAudio();
break;
case 'input_audio_buffer.committed':
if (typeof event.item_id === 'string') {
this.registerItem(event.item_id);
this.bindCommittedAudioToItem(event.item_id);
}
if (this.protocolProfile === 'qwen-asr-server-vad') this.bufferedMs = 0;
break;
case 'conversation.item.input_audio_transcription.delta':
this.handleDelta(event);
break;
case 'conversation.item.input_audio_transcription.text':
this.handleText(event);
break;
case 'conversation.item.input_audio_transcription.completed':
this.handleCompleted(event);
break;
case 'conversation.item.input_audio_transcription.failed':
this.callback({ type: 'error', message: this.redactError(realtimeErrorMessage(event, i18n.t('composer.voice.realtimeAsrFailed'))), at: Date.now() });
this.resolveFlushWaiters();
break;
case 'error': {
const message = this.redactError(realtimeErrorMessage(event, i18n.t('composer.voice.realtimeAsrFailed')));
if (this.isNonFatalServerVadFinishError(message)) {
this.resolveFlushWaiters();
break;
}
this.callback({ type: 'error', message, at: Date.now() });
this.resolveFlushWaiters();
break;
}
case 'session.finished':
this.resolveFlushWaiters();
break;
}
}
private handleDelta(event: Record<string, unknown>): void {
if (typeof event.item_id !== 'string' || typeof event.delta !== 'string') return;
this.registerItem(event.item_id);
const next = `${this.partialsByItem.get(event.item_id) ?? ''}${event.delta}`;
this.partialsByItem.set(event.item_id, next);
this.callback({ type: 'partial', text: this.aggregateTranscript(), at: Date.now() });
}
private handleText(event: Record<string, unknown>): void {
if (typeof event.item_id !== 'string' || typeof event.text !== 'string') return;
this.registerItem(event.item_id);
const preview = `${event.text}${typeof event.stash === 'string' ? event.stash : ''}`;
this.partialsByItem.set(event.item_id, preview);
this.callback({ type: 'partial', text: this.aggregateTranscript(), at: Date.now() });
}
private handleCompleted(event: Record<string, unknown>): void {
const itemTranscript = typeof event.transcript === 'string'
? event.transcript
: typeof event.text === 'string'
? event.text
: undefined;
if (typeof event.item_id !== 'string' || typeof itemTranscript !== 'string') return;
this.registerItem(event.item_id);
this.finalsByItem.set(event.item_id, itemTranscript);
this.partialsByItem.delete(event.item_id);
if (this.pendingCommitCount > 0) this.pendingCommitCount -= 1;
const aggregate = this.aggregateTranscript();
if (aggregate) this.callback({ type: 'stable', text: aggregate, at: Date.now() });
this.finishRealtimeAudio(event.item_id, itemTranscript.trim().length > 0);
if (this.pendingCommitCount === 0) this.resolveFlushWaiters();
}
private async performRecover(): Promise<void> {
if (this.inRecovery || this.stopped) return;
this.inRecovery = true;
const partialOnlyItemIds = new Set(this.partialsByItem.keys());
this.itemOrder = this.itemOrder.filter((itemId) => !partialOnlyItemIds.has(itemId) || this.finalsByItem.has(itemId));
this.partialsByItem.clear();
this.sessionReady = false;
this.bufferedMs = 0;
this.pendingCommitCount = 0;
this.pendingAudioCommitIds = [];
this.audioCommitIdsByItem.clear();
this.unboundAudioItemTurnIds.clear();
this.serverVadFinishRequested = false;
this.resolveFlushWaiters();
this.resetUnconfirmedAudioForReplay();
const socket = this.socket;
this.socket = null;
if (socket && socket.readyState === OPEN) socket.close();
try {
await this.connect();
if (this.stopped) return;
for (const entry of this.unconfirmedAudio) {
if (!this.sendRealtimeAudioEntry(entry)) {
throw new Error('Realtime ASR recovery replay failed.');
}
}
} finally {
this.inRecovery = false;
}
}
private isNonFatalServerVadFinishError(message: string): boolean {
if (this.protocolProfile !== 'qwen-asr-server-vad') return false;
if (!this.serverVadFinishRequested) return false;
if (!this.aggregateTranscript()) return false;
return /input_audio_buffer|commit/i.test(message);
}
private registerItem(itemId: string): void {
if (!this.itemOrder.includes(itemId)) this.itemOrder.push(itemId);
}
private aggregateTranscript(): string {
return this.itemOrder
.map((itemId) => this.finalsByItem.get(itemId) ?? this.partialsByItem.get(itemId) ?? '')
.join('')
.replace(/\s+/g, ' ')
.trim();
}
private waitForFlush(): Promise<void> {
return new Promise((resolve) => {
const timer = setTimeout(resolve, this.flushTimeoutMs);
this.flushResolvers.push(() => {
clearTimeout(timer);
resolve();
});
});
}
private resolveFlushWaiters(): void {
this.flushResolvers.splice(0).forEach((resolve) => resolve());
}
private bufferUnconfirmedAudio(
audioBase64: string,
durationMs: number,
localTurnId?: number,
): RealtimeReplayAudioChunk {
const entry: RealtimeReplayAudioChunk = {
audioBase64,
durationMs,
addedAt: Date.now(),
sent: false,
localTurnId,
};
this.unconfirmedAudio.push(entry);
return entry;
}
private queueRealtimeAudioCommit(maxAddedAt?: number, localTurnId?: number): number | undefined {
const commitId = this.nextAudioCommitId;
let hasAudio = false;
for (const entry of this.unconfirmedAudio) {
if (!entry.sent || entry.commitId !== undefined) continue;
if (maxAddedAt !== undefined && entry.addedAt > maxAddedAt) continue;
if (localTurnId !== undefined && entry.localTurnId !== localTurnId) continue;
entry.commitId = commitId;
hasAudio = true;
}
if (!hasAudio) return undefined;
this.nextAudioCommitId += 1;
this.pendingAudioCommitIds.push(commitId);
return commitId;
}
private bindCommittedAudioToItem(itemId: string): void {
const committedAt = Date.now();
let committedTurnId: number | undefined;
let commitId = this.pendingAudioCommitIds.shift();
if (commitId === undefined) {
if (this.protocolProfile === 'qwen-asr-server-vad') {
committedTurnId = this.oldestUncommittedSentAudioTurnId();
commitId = this.queueRealtimeAudioCommit(
committedAt - CONFIRMED_AUDIO_RETENTION_MS,
committedTurnId,
);
} else {
commitId = this.queueRealtimeAudioCommit();
}
}
if (commitId !== undefined) {
this.audioCommitIdsByItem.set(itemId, commitId);
this.unboundAudioItemTurnIds.delete(itemId);
if (committedTurnId !== undefined) this.advanceLocalAudioTurnAfterServerCommit(committedTurnId);
} else {
const turnId = this.oldestUncommittedSentAudioTurnId();
if (turnId !== undefined) {
this.unboundAudioItemTurnIds.set(itemId, turnId);
this.advanceLocalAudioTurnAfterServerCommit(turnId);
}
}
}
private finishRealtimeAudio(itemId: string, hasTranscript: boolean): void {
// Completion ends the commit bookkeeping, but an empty result does not
// confirm its audio. Keep those bytes for the one stop-time replay.
let commitId = this.audioCommitIdsByItem.get(itemId);
if (commitId === undefined) {
const turnId = this.unboundAudioItemTurnIds.get(itemId);
if (turnId !== undefined) {
this.unboundAudioItemTurnIds.delete(itemId);
if (hasTranscript) this.clearUnboundRealtimeAudioTurn(turnId);
return;
}
commitId = this.pendingAudioCommitIds.shift();
}
if (commitId === undefined) {
if (hasTranscript) this.trimUnconfirmedAudioOlderThan(Date.now() - CONFIRMED_AUDIO_RETENTION_MS);
return;
}
this.audioCommitIdsByItem.delete(itemId);
this.unboundAudioItemTurnIds.delete(itemId);
this.pendingAudioCommitIds = this.pendingAudioCommitIds.filter((id) => id !== commitId);
if (hasTranscript) this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => entry.commitId !== commitId);
}
private clearUnboundRealtimeAudioTurn(turnId: number): void {
this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => {
if (!entry.sent || entry.commitId !== undefined) return true;
return entry.localTurnId !== turnId;
});
}
private oldestUncommittedSentAudioTurnId(): number | undefined {
return this.unconfirmedAudio.find((entry) => (
entry.sent
&& entry.commitId === undefined
&& entry.localTurnId !== undefined
))?.localTurnId;
}
private assignRealtimeAudioTurnId(chunk: ArrayBuffer, durationMs: number): number | undefined {
if (this.protocolProfile !== 'qwen-asr-server-vad') return undefined;
const voiced = isPcm16ChunkVoiced(chunk);
if (voiced && this.localTurnHasVoice && this.localTurnSilenceMs >= QWEN_SERVER_VAD_SILENCE_DURATION_MS) {
this.currentLocalAudioTurnId += 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
}
const turnId = this.currentLocalAudioTurnId;
if (voiced) {
this.localTurnHasVoice = true;
this.localTurnSilenceMs = 0;
} else if (this.localTurnHasVoice) {
this.localTurnSilenceMs += durationMs;
}
return turnId;
}
private advanceLocalAudioTurnAfterServerCommit(turnId: number): void {
if (this.currentLocalAudioTurnId > turnId) return;
this.currentLocalAudioTurnId = turnId + 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
}
private trimUnconfirmedAudioOlderThan(cutoffMs: number): void {
while (this.unconfirmedAudio.length > 0 && this.unconfirmedAudio[0].sent && this.unconfirmedAudio[0].addedAt < cutoffMs) {
const dropped = this.unconfirmedAudio.shift();
if (!dropped) break;
}
}
private clearUnconfirmedAudio(): void {
this.unconfirmedAudio = [];
this.pendingAudioCommitIds = [];
this.audioCommitIdsByItem.clear();
this.unboundAudioItemTurnIds.clear();
this.currentLocalAudioTurnId = 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
}
private resetUnconfirmedAudioForReplay(): void {
for (const entry of this.unconfirmedAudio) {
entry.sent = false;
entry.commitId = undefined;
}
}
private resetTranscriptState(): void {
this.sessionReady = false;
this.bufferedMs = 0;
this.pendingCommitCount = 0;
this.nextAudioCommitId = 1;
this.pendingAudioCommitIds = [];
this.audioCommitIdsByItem = new Map();
this.unboundAudioItemTurnIds = new Map();
this.currentLocalAudioTurnId = 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
this.flushResolvers = [];
this.itemOrder = [];
this.partialsByItem = new Map();
this.finalsByItem = new Map();
this.serverVadFinishRequested = false;
this.clearUnconfirmedAudio();
}
private redactError(message: string): string {
return redactMobileVoiceCredentialText(message, this.credential);
}
}
type VolcengineFlushReason = 'final_response' | 'transport_closed' | 'transport_error' | 'server_error' | 'timeout' | 'stopped' | 'reset';
export class MobileVolcengineSaucAsrProvider implements AsrProvider {
private readonly credential: StoredMobileVoiceCredential;
private readonly sourceLanguage: string;
private readonly websocketFactory: WebSocketFactory;
private readonly connectTimeoutMs: number;
private readonly flushTimeoutMs: number;
private readonly pcmSampleRate: number;
private readonly connectionProvider?: MobileRealtimeAsrProviderOptions['connectionProvider'];
private callback: (event: AsrEvent) => void = () => {};
private socket: WebSocketLike | null = null;
private connected = false;
private sequence = 1;
private sentAudioMs = 0;
private lastTranscript = '';
private finalizedTranscript = '';
private sessionTranscriptPrefix = '';
private unconfirmedAudio: VolcengineReplayAudioChunk[] = [];
private currentLocalAudioTurnId = 1;
private localTurnHasVoice = false;
private localTurnSilenceMs = 0;
private finalRequested = false;
private stableEmitted = false;
// Stop-time fast finish compares positions on the transmitted PCM clock:
// the end of the fully confirmed transcript vs. the last local sound.
private confirmedEndMs = 0;
private readonly soundActivity: StopSoundActivity;
private recovered = false;
private flushResolvers: Array<(reason: VolcengineFlushReason) => void> = [];
private stopped = false;
private recoveryPromise?: Promise<void>;
private pendingConnect?: PendingConnect;
constructor(options: MobileRealtimeAsrProviderOptions) {
this.credential = options.credential;
this.sourceLanguage = options.sourceLanguage ?? this.credential.settings?.language ?? 'auto';
this.websocketFactory = options.websocketFactory ?? (globalThis.WebSocket as unknown as WebSocketFactory);
this.connectTimeoutMs = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
this.flushTimeoutMs = options.flushTimeoutMs ?? DEFAULT_FLUSH_TIMEOUT_MS;
this.pcmSampleRate = this.credential.asr.pcmSampleRate ?? 16_000;
this.soundActivity = new StopSoundActivity(this.pcmSampleRate);
this.connectionProvider = options.connectionProvider;
}
onEvent(callback: (event: AsrEvent) => void): void {
this.callback = callback;
}
async start(): Promise<void> {
if (this.credential.asr.auth !== 'api-key') {
throw new Error(i18n.t('composer.voice.volcengineAuthUnsupported', { auth: this.credential.asr.auth }));
}
if (!this.credential.asr.resourceId) throw new Error(i18n.t('composer.voice.volcengineMissingResourceId'));
const resourceId = this.credential.asr.resourceId;
this.resetState();
await this.openSocket(resourceId);
}
private async openSocket(resourceId: string): Promise<void> {
// 托管 ticket 是唯一建连方式:BYOK/穿透 key 直拨上游的路径已删除。
if (!this.connectionProvider) {
throw new Error(i18n.t('composer.voice.missingConnectionProvider'));
}
const connectStartedAt = Date.now();
const dynamicConnection = await this.connectionProvider(this.credential.asr.provider);
const sessionMs = Date.now() - connectStartedAt;
if (this.stopped) throw new Error('Volcengine SAUC ASR connection stopped.');
const { websocketUrl, authorizationToken } = dynamicConnection;
await new Promise<void>((resolve, reject) => {
const socket = new this.websocketFactory(
websocketUrl,
null,
{
headers: {
Authorization: `Bearer ${authorizationToken}`,
'X-Api-Resource-Id': resourceId,
'X-Api-Connect-Id': buildConnectId(),
},
},
);
socket.binaryType = 'arraybuffer';
this.socket = socket;
let settled = false;
let pendingConnect: PendingConnect | undefined;
let pendingConnectError: Error | null = null;
let pendingConnectErrorTimer: ReturnType<typeof setTimeout> | null = null;
const timer = setTimeout(() => {
fail(new Error(`Volcengine SAUC ASR connection timed out after ${this.connectTimeoutMs}ms`));
socket.close();
}, this.connectTimeoutMs);
const cleanup = (): void => {
clearTimeout(timer);
if (pendingConnect && this.pendingConnect === pendingConnect) this.pendingConnect = undefined;
if (pendingConnectErrorTimer) clearTimeout(pendingConnectErrorTimer);
};
const fail = (error: Error): void => {
if (settled) return;
settled = true;
cleanup();
this.socket = null;
reject(error);
};
pendingConnect = { reject: fail };
this.pendingConnect = pendingConnect;
socket.onopen = () => {
if (this.socket !== socket) return;
if (settled) return;
settled = true;
cleanup();
this.connected = true;
this.sendInitialRequest();
logMobileVoice('debug', 'volcengine sauc connected', {
sessionMs,
socketOpenMs: Date.now() - connectStartedAt - sessionMs,
recovery: this.recovered,
});
this.callback({ type: 'connected', at: Date.now() });
resolve();
};
socket.onmessage = (event) => {
if (this.socket !== socket) return;
this.handleMessage(event.data);
};
socket.onerror = (event) => {
if (this.socket !== socket) return;
const { message, specific } = webSocketErrorMessage(event, 'Volcengine SAUC ASR connection failed.');
const redacted = this.redactError(message);
if (!settled) {
pendingConnectError = new Error(redacted);
if (specific) {
fail(pendingConnectError);
} else {
pendingConnectErrorTimer = setTimeout(() => {
if (pendingConnectError) fail(pendingConnectError);
}, 250);
}
} else {
this.connected = false;
this.resolveFlushWaiters('transport_error');
this.callback({ type: 'error', message: redacted, at: Date.now() });
}
};
socket.onclose = (event) => {
if (this.socket !== socket) return;
this.connected = false;
this.resolveFlushWaiters('transport_closed');
logMobileVoice('debug', 'volcengine sauc transport closed', {
code: typeof event.code === 'number' ? event.code : undefined,
finalRequested: this.finalRequested,
});
if (!settled) {
fail(new Error(this.redactError(webSocketCloseBeforeReadyMessage(
event,
'Volcengine SAUC ASR connection closed before it was ready.',
pendingConnectError?.message,
))));
} else if (!this.stopped) this.callback({ type: 'disconnected', at: Date.now() });
};
});
}
appendAudio(chunk: ArrayBuffer, trace?: AudioTrace): void {
const durationMs = trace?.durationMs ?? estimatePcmDurationMs(chunk.byteLength, trace?.sampleRate ?? this.pcmSampleRate);
const pcm = new Uint8Array(chunk);
const entry = this.addUnconfirmedAudio(
pcm,
durationMs,
this.assignVolcengineAudioTurnId(chunk, durationMs),
);
// Capture already runs at pcmSampleRate. Fast-finish positions use the
// transmitted PCM clock so they match utterance end_time (milliseconds).
this.sentAudioMs += estimatePcmDurationMs(chunk.byteLength, this.pcmSampleRate);
this.soundActivity.append(pcm);
const socket = this.socket;
// flushAudio sends a dedicated final silence packet, so every audio packet
// goes out immediately instead of waiting for the next capture chunk.
if (this.connected && socket?.readyState === OPEN) this.sendVolcengineAudioEntry(entry, socket);
}
async flushAudio(): Promise<void> {
if (this.recoveryPromise) {
await Promise.race([
this.recoveryPromise.catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, RECOVER_TIMEOUT_MS)),
]);
}
if (!this.connected || this.finalRequested || this.sentAudioMs <= 0) return;
const socket = this.socket;
if (!socket || socket.readyState !== OPEN) return;
this.finalRequested = true;
sendBinary(socket, encodeVolcengineAudioOnlyRequest(silencePcm16(this.pcmSampleRate, VOLCENGINE_NONSTREAM_END_WINDOW_MS), -this.nextSequence()));
// The final packet still goes out; only waiting for its response is skipped
// when the whole transcript is already confirmed past the last sound.
const decision = this.stopFinalizationDecision();
logMobileVoice('debug', 'volcengine sauc stop finalization decision', decision);
if (decision.fastFinish) return;
const flushStartedAt = Date.now();
const reason = await this.waitForFlush();
logMobileVoice('debug', 'volcengine sauc flush settled', {
waitMs: Date.now() - flushStartedAt,
hasStable: this.stableEmitted,
transcriptChars: this.lastTranscript.length,
reason,
});
if (this.lastTranscript && !this.stableEmitted) {
this.finalizedTranscript = this.lastTranscript;
this.callback({ type: 'stable', text: this.lastTranscript, at: Date.now() });
this.stableEmitted = true;
}
}
async stop(): Promise<void> {
this.stopped = true;
this.connected = false;
this.resolveFlushWaiters('stopped');
this.clearUnconfirmedAudio();
this.sessionTranscriptPrefix = '';
const socket = this.socket;
this.pendingConnect?.reject(new Error('Volcengine SAUC ASR connection stopped.'));
this.socket = null;
if (socket && socket.readyState !== CLOSED) socket.close();
if (this.recoveryPromise) {
await Promise.race([
this.recoveryPromise.catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, RECOVER_TIMEOUT_MS)),
]);
}
this.callback({ type: 'disconnected', at: Date.now() });
}
async recover(): Promise<void> {
if (this.stopped) return;
if (this.recoveryPromise) return this.recoveryPromise;
this.recoveryPromise = this.performRecover().finally(() => {
this.recoveryPromise = undefined;
});
return this.recoveryPromise;
}
private sendInitialRequest(): void {
const socket = this.socket;
if (!socket) return;
sendBinary(socket, encodeVolcengineFullClientRequest({
user: { uid: 'xdt-maker' },
audio: {
format: 'pcm',
codec: 'raw',
rate: this.pcmSampleRate,
bits: 16,
channel: 1,
},
request: {
model_name: VOLCENGINE_MODEL_NAME,
result_type: 'full',
show_utterances: true,
enable_nonstream: true,
end_window_size: VOLCENGINE_NONSTREAM_END_WINDOW_MS,
enable_punc: true,
enable_itn: true,
...(this.sourceLanguage && this.sourceLanguage.toLowerCase() !== 'auto'
? { language: this.sourceLanguage }
: {}),
},
}));
}
private handleMessage(data: unknown): void {
const message = decodeVolcengineMessage(rawDataToBytes(data));
if (message.messageType === VOLC_MESSAGE_TYPE_SERVER_ERROR) {
const errorMessage = this.redactError(
extractErrorMessage(message.payload) ?? message.payloadText ?? 'Volcengine SAUC ASR failed.',
);
this.callback({ type: 'error', message: errorMessage, at: Date.now() });
this.resolveFlushWaiters('server_error');
const code = (message.payload as { code?: unknown } | undefined)?.code;
logMobileVoice('warn', 'volcengine sauc server error', {
code: typeof code === 'number' ? code : undefined,
});
return;
}
if (message.messageType !== VOLC_MESSAGE_TYPE_FULL_SERVER_RESPONSE && message.messageType !== VOLC_MESSAGE_TYPE_SERVER_ACK) {
return;
}
const rawTranscript = extractTranscript(message.payload);
const transcript = mergeRecoveredTranscript(this.sessionTranscriptPrefix, rawTranscript);
// The stable lane requires every utterance of the aggregate to be definite;
// a confirmed first sentence must not mark an unconfirmed tail as stable.
const confirmation = getVolcengineTranscriptConfirmation(message.payload, rawTranscript);
const isDefinite = confirmation.confirmed;
const isLastResponse = isVolcengineProtocolLastResponse(message);
// Recovery keeps finalizedTranscript as a fixed prefix and replays only the
// audio after it, so it may contain definite text only: an indefinite tail
// must stay correctable by the replay.
const definiteText = isLastResponse || isDefinite
? rawTranscript
: definiteVolcengineTranscriptPrefix(message.payload, rawTranscript);
if (definiteText || (isLastResponse && transcript)) {
this.finalizedTranscript = mergeRecoveredTranscript(this.sessionTranscriptPrefix, definiteText);
this.clearConfirmedVolcengineAudio();
}
if (rawTranscript) this.confirmedEndMs = confirmation.endMs;
// A confirmation change with identical text must still reach the controller.
if (transcript && (transcript !== this.lastTranscript || isDefinite !== this.stableEmitted)) {
this.lastTranscript = transcript;
this.stableEmitted = isDefinite;
this.callback({ type: isDefinite ? 'stable' : 'partial', text: transcript, at: Date.now() });
}
if (isLastResponse) {
logMobileVoice('debug', 'volcengine sauc final response', {
transcriptChars: rawTranscript.length,
finalRequested: this.finalRequested,
messageType: message.messageType,
});
if (this.finalRequested) this.resolveFlushWaiters('final_response');
}
}
/**
* Fast finish needs every condition; missing or odd evidence waits for the
* final response as before. Positions are on the transmitted PCM clock, and
* a recovery replay restarts that clock, so recovered runs always wait.
* Reason names match desktop so logs from both clients compare directly.
*/
private stopFinalizationDecision(): {
fastFinish: boolean;
missedReasons: string[];
lastSoundEndMs: number;
confirmedEndMs: number;
sentAudioMs: number;
silenceMs: number;
} {
const lastSoundEndMs = this.soundActivity.lastSoundEndMs;
const silenceMs = this.sentAudioMs - lastSoundEndMs;
const missedReasons: string[] = [];
if (this.recovered) missedReasons.push('recovered_audio_clock');
if (!this.stableEmitted) missedReasons.push('transcript_not_fully_confirmed');
if (lastSoundEndMs <= 0) missedReasons.push('no_sound_position');
if (this.confirmedEndMs <= 0) missedReasons.push('no_confirmed_audio_position');
if (this.confirmedEndMs < lastSoundEndMs) missedReasons.push('sound_after_confirmed_audio');
if (this.confirmedEndMs > this.sentAudioMs) missedReasons.push('confirmed_position_beyond_sent_audio');
if (silenceMs < VOLCENGINE_FAST_FINISH_SILENCE_MS) missedReasons.push('silence_under_2000ms');
return {
fastFinish: missedReasons.length === 0,
missedReasons,
lastSoundEndMs: Math.round(lastSoundEndMs),
confirmedEndMs: Math.round(this.confirmedEndMs),
sentAudioMs: Math.round(this.sentAudioMs),
silenceMs: Math.round(silenceMs),
};
}
private nextSequence(): number {
this.sequence += 1;
return this.sequence;
}
private sendVolcengineAudioEntry(entry: VolcengineReplayAudioChunk, socket: WebSocketLike): boolean {
if (socket.readyState !== OPEN) return false;
sendBinary(socket, encodeVolcengineAudioOnlyRequest(entry.pcm, this.nextSequence()));
entry.sent = true;
return true;
}
private addUnconfirmedAudio(pcm: Uint8Array, durationMs: number, localTurnId?: number): VolcengineReplayAudioChunk {
const entry: VolcengineReplayAudioChunk = {
pcm,
durationMs,
addedAt: Date.now(),
sent: false,
localTurnId,
};
this.unconfirmedAudio.push(entry);
return entry;
}
private clearUnconfirmedAudio(): void {
this.unconfirmedAudio = [];
this.currentLocalAudioTurnId = 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
}
private assignVolcengineAudioTurnId(chunk: ArrayBuffer, durationMs: number): number {
const voiced = isPcm16ChunkVoiced(chunk);
if (voiced && this.localTurnHasVoice && this.localTurnSilenceMs >= VOLCENGINE_NONSTREAM_END_WINDOW_MS) {
this.currentLocalAudioTurnId += 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
}
const turnId = this.currentLocalAudioTurnId;
if (voiced) {
this.localTurnHasVoice = true;
this.localTurnSilenceMs = 0;
} else if (this.localTurnHasVoice) {
this.localTurnSilenceMs += durationMs;
}
return turnId;
}
private clearConfirmedVolcengineAudio(): void {
const confirmedTurnId = this.unconfirmedAudio.find((entry) => entry.sent)?.localTurnId;
if (confirmedTurnId !== undefined) {
this.unconfirmedAudio = this.unconfirmedAudio.filter((entry) => entry.localTurnId !== confirmedTurnId);
if (this.currentLocalAudioTurnId <= confirmedTurnId) {
this.currentLocalAudioTurnId = confirmedTurnId + 1;
this.localTurnHasVoice = false;
this.localTurnSilenceMs = 0;
}
} else {
this.trimConfirmedVolcengineAudioOlderThan(Date.now() - CONFIRMED_AUDIO_RETENTION_MS);
}
}
private trimConfirmedVolcengineAudioOlderThan(cutoffMs: number): void {
while (this.unconfirmedAudio.length > 0 && this.unconfirmedAudio[0].sent && this.unconfirmedAudio[0].addedAt < cutoffMs) {
this.unconfirmedAudio.shift();
}
}
private async performRecover(): Promise<void> {
const resourceId = this.credential.asr.resourceId;
if (!resourceId) throw new Error(i18n.t('composer.voice.volcengineMissingResourceId'));
// A replay uses a different audio origin. Never compare its timestamps with
// the original capture clock, even when the transcript prefix is empty.
this.recovered = true;
this.confirmedEndMs = 0;
const prefix = this.finalizedTranscript;
this.sessionTranscriptPrefix = prefix;
this.connected = false;
this.sequence = 1;
this.finalRequested = false;
this.stableEmitted = false;
this.resolveFlushWaiters('reset');
const socket = this.socket;
this.socket = null;
if (socket && socket.readyState === OPEN) socket.close();
await this.openSocket(resourceId);
if (this.stopped) return;
const nextSocket = this.currentOpenSocket();
if (!nextSocket) {
throw new Error('Volcengine SAUC ASR recover reconnected without an open socket.');
}
const replay = [...this.unconfirmedAudio];
for (const entry of replay) {
if (!this.sendVolcengineAudioEntry(entry, nextSocket)) {
throw new Error('Volcengine SAUC ASR recovery replay failed.');
}
}
logMobileVoice('info', 'volcengine sauc recovery replayed audio', {
chunks: replay.length,
replayMs: Math.round(replay.reduce((total, entry) => total + entry.durationMs, 0)),
prefixChars: this.sessionTranscriptPrefix.length,
});
}
private currentOpenSocket(): WebSocketLike | null {
const socket = this.socket;
return socket && socket.readyState === OPEN ? socket : null;
}
private waitForFlush(): Promise<VolcengineFlushReason> {
return new Promise((resolve) => {
const finish = (reason: VolcengineFlushReason): void => {
clearTimeout(timer);
this.flushResolvers = this.flushResolvers.filter((item) => item !== finish);
resolve(reason);
};
const timer = setTimeout(() => finish('timeout'), this.flushTimeoutMs);
this.flushResolvers.push(finish);
});
}
private resolveFlushWaiters(reason: VolcengineFlushReason): void {
this.flushResolvers.splice(0).forEach((resolve) => resolve(reason));
}
private resetState(): void {
this.stopped = false;
this.connected = false;
this.sequence = 1;
this.sentAudioMs = 0;
this.confirmedEndMs = 0;
this.soundActivity.reset();
this.recovered = false;
this.lastTranscript = '';
this.finalizedTranscript = '';
this.sessionTranscriptPrefix = '';
this.clearUnconfirmedAudio();
this.finalRequested = false;
this.stableEmitted = false;
this.resolveFlushWaiters('reset');
}
private redactError(message: string): string {
return redactMobileVoiceCredentialText(message, this.credential);
}
}
export function buildSessionUpdateMessage(
model: string,
sourceLanguage: string,
pcmSampleRate: number,
protocolProfile: MobileRealtimeProtocolProfile,
): Record<string, unknown> {
const language = openAiLanguageCode(sourceLanguage);
if (protocolProfile === 'qwen-asr-server-vad') {
return {
event_id: buildRealtimeEventId('session_update'),
type: 'session.update',
session: {
modalities: ['text'],
input_audio_format: 'pcm',
sample_rate: pcmSampleRate,
input_audio_transcription: {
...(language ? { language } : {}),
},
turn_detection: {
type: 'server_vad',
threshold: QWEN_SERVER_VAD_THRESHOLD,
silence_duration_ms: QWEN_SERVER_VAD_SILENCE_DURATION_MS,
},
},
};
}
return {
type: 'session.update',
session: {
type: 'transcription',
audio: {
input: {
format: { type: 'audio/pcm', rate: pcmSampleRate },
transcription: {
model,
...(language ? { language } : {}),
},
turn_detection: null,
},
},
},
};
}
function buildAppendAudioMessage(
audioBase64: string,
protocolProfile: MobileRealtimeProtocolProfile,
): Record<string, unknown> {
return {
...(protocolProfile === 'qwen-asr-server-vad' ? { event_id: buildRealtimeEventId('append') } : {}),
type: 'input_audio_buffer.append',
audio: audioBase64,
};
}
function buildFinishSessionMessage(protocolProfile: MobileRealtimeProtocolProfile): Record<string, unknown> {
return {
...(protocolProfile === 'qwen-asr-server-vad' ? { event_id: buildRealtimeEventId('finish') } : {}),
type: 'session.finish',
};
}
function readRealtimeProtocolProfile(credential: StoredMobileVoiceCredential): MobileRealtimeProtocolProfile {
if (
credential.asr.protocolProfile === 'openai-transcription-manual'
|| credential.asr.protocolProfile === 'qwen-asr-server-vad'
) {
return credential.asr.protocolProfile;
}
throw new Error(i18n.t('composer.voice.realtimeProtocolUnsupported', { detail: credential.asr.protocolProfile ?? credential.asr.mode }));
}
function openAiLanguageCode(sourceLanguage: string): string | undefined {
const normalized = sourceLanguage.trim().toLowerCase();
if (!normalized || normalized === 'auto') return undefined;
switch (normalized) {
case 'chinese':
case 'mandarin':
case 'simplified chinese':
case 'traditional chinese':
return 'zh';
case 'cantonese':
return 'yue';
case 'english':
return 'en';
case 'japanese':
return 'ja';
case 'korean':
return 'ko';
}
const primary = normalized.split(/[-_]/)[0];
return /^[a-z]{2,3}$/.test(primary) ? primary : undefined;
}
function arrayBufferToBase64(buffer: ArrayBuffer): string {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let index = 0; index < bytes.length; index += 1) {
binary += String.fromCharCode(bytes[index]);
}
if (typeof btoa === 'function') return btoa(binary);
return encodeBase64(bytes);
}
function encodeBase64(bytes: Uint8Array): string {
const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
let output = '';
for (let index = 0; index < bytes.length; index += 3) {
const a = bytes[index];
const b = bytes[index + 1] ?? 0;
const c = bytes[index + 2] ?? 0;
const triple = (a << 16) | (b << 8) | c;
output += alphabet[(triple >> 18) & 63];
output += alphabet[(triple >> 12) & 63];
output += index + 1 < bytes.length ? alphabet[(triple >> 6) & 63] : '=';
output += index + 2 < bytes.length ? alphabet[triple & 63] : '=';
}
return output;
}
function estimatePcmDurationMs(byteLength: number, sampleRate: number): number {
return byteLength / (sampleRate * 2 / 1000);
}
function isPcm16ChunkVoiced(chunk: ArrayBuffer): boolean {
if (chunk.byteLength < 2) return false;
const view = new DataView(chunk);
let sumSquares = 0;
let samples = 0;
for (let offset = 0; offset + 1 < chunk.byteLength; offset += 2) {
const sample = view.getInt16(offset, true);
sumSquares += sample * sample;
samples += 1;
}
if (samples === 0) return false;
return Math.sqrt(sumSquares / samples) >= LOCAL_TURN_VOICED_RMS_THRESHOLD;
}
function realtimeErrorMessage(event: Record<string, unknown>, fallbackMessage: string): string {
const error = event.error;
if (isRecord(error) && typeof error.message === 'string') return error.message;
return fallbackMessage;
}
function parseJsonObject(text: string): unknown {
try {
return JSON.parse(text);
} catch {
return null;
}
}
function buildRealtimeEventId(prefix: string): string {
return `${prefix}_${Date.now()}_${Math.random().toString(16).slice(2)}`;
}
export function encodeVolcengineFullClientRequest(payload: Record<string, unknown>): Uint8Array {
return encodeVolcengineRequest({
messageType: VOLC_MESSAGE_TYPE_FULL_CLIENT_REQUEST,
flags: VOLC_FLAG_NO_SEQUENCE,
serialization: VOLC_SERIALIZATION_JSON,
compression: VOLC_COMPRESSION_GZIP,
payload: textToBytes(JSON.stringify(payload)),
});
}
export function encodeVolcengineAudioOnlyRequest(pcm: Uint8Array, sequence: number): Uint8Array {
const hasPayload = pcm.length > 0;
return encodeVolcengineRequest({
messageType: VOLC_MESSAGE_TYPE_AUDIO_ONLY_REQUEST,
flags: sequence < 0 ? VOLC_FLAG_NEGATIVE_SEQUENCE : VOLC_FLAG_POSITIVE_SEQUENCE,
sequence,
serialization: VOLC_SERIALIZATION_NONE,
compression: hasPayload ? VOLC_COMPRESSION_GZIP : VOLC_COMPRESSION_NONE,
payload: pcm,
});
}
function encodeVolcengineRequest(input: {
messageType: number;
flags: number;
sequence?: number;
serialization: number;
compression: number;
payload: Uint8Array;
}): Uint8Array {
const compressedPayload = input.compression === VOLC_COMPRESSION_GZIP
? gzip(input.payload)
: input.payload;
const sequenceBytes = input.flags === VOLC_FLAG_NO_SEQUENCE ? new Uint8Array(0) : new Uint8Array(4);
if (sequenceBytes.length > 0) {
new DataView(sequenceBytes.buffer).setInt32(0, input.sequence ?? 0, false);
}
const payloadSize = new Uint8Array(4);
new DataView(payloadSize.buffer).setUint32(0, compressedPayload.length, false);
return concatBytes(
new Uint8Array([
(VOLC_PROTOCOL_VERSION << 4) | VOLC_HEADER_SIZE_WORDS,
(input.messageType << 4) | input.flags,
(input.serialization << 4) | input.compression,
0x00,
]),
sequenceBytes,
payloadSize,
compressedPayload,
);
}
export function decodeVolcengineMessage(data: Uint8Array): ParsedVolcengineMessage {
if (data.length < 4) return { messageType: 0, flags: 0 };
const headerSize = (data[0] & 0x0f) * 4;
const messageType = data[1] >> 4;
const flags = data[1] & 0x0f;
const serialization = data[2] >> 4;
const compression = data[2] & 0x0f;
let offset = headerSize;
if (messageType === VOLC_MESSAGE_TYPE_SERVER_ERROR) {
if (data.length < offset + 8) return { messageType, flags };
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
const code = view.getInt32(offset, false);
offset += 4;
const payloadSize = view.getUint32(offset, false);
offset += 4;
if (payloadSize <= 0 || data.length < offset + payloadSize) {
return { messageType, flags, payload: { code } };
}
const payloadBytes = decodeVolcenginePayload(data.slice(offset, offset + payloadSize), compression);
const payloadText = bytesToText(payloadBytes);
return { messageType, flags, payload: { code, message: payloadText }, payloadText };
}
let sequence: number | undefined;
if (flags === VOLC_FLAG_POSITIVE_SEQUENCE || flags === VOLC_FLAG_NEGATIVE_SEQUENCE) {
if (data.length < offset + 4) return { messageType, flags };
sequence = new DataView(data.buffer, data.byteOffset, data.byteLength).getInt32(offset, false);
offset += 4;
}
if (data.length < offset + 4) return { messageType, flags, sequence };
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
const payloadSize = view.getUint32(offset, false);
offset += 4;
if (payloadSize <= 0 || data.length < offset + payloadSize) {
return { messageType, flags, sequence };
}
const payloadBytes = decodeVolcenginePayload(data.slice(offset, offset + payloadSize), compression);
const payloadText = bytesToText(payloadBytes);
if (serialization === VOLC_SERIALIZATION_JSON || looksLikeJson(payloadText)) {
try {
return { messageType, flags, sequence, payload: JSON.parse(payloadText), payloadText };
} catch {
return { messageType, flags, sequence, payloadText };
}
}
return { messageType, flags, sequence, payloadText };
}
function decodeVolcenginePayload(payload: Uint8Array, compression: number): Uint8Array {
return compression === VOLC_COMPRESSION_GZIP ? ungzip(payload) : payload;
}
function rawDataToBytes(data: unknown): Uint8Array {
if (data instanceof Uint8Array) return data;
if (data instanceof ArrayBuffer) return new Uint8Array(data);
if (Array.isArray(data)) return concatBytes(...data.map(rawDataToBytes));
if (typeof data === 'string') return textToBytes(data);
return new Uint8Array(0);
}
function extractTranscript(payload: unknown): string {
const values = collectStringFields(payload, new Set(['text', 'transcript', 'sentence', 'asr_text']));
const best = values
.map((value) => value.trim())
.filter(Boolean)
.sort((a, b) => b.length - a.length)[0];
return best ?? '';
}
function extractErrorMessage(payload: unknown): string | undefined {
const values = collectStringFields(payload, new Set(['message', 'error', 'reason']));
return values.map((value) => value.trim()).find(Boolean);
}
const ASCII_WORD_CHARACTER_PATTERN = /[A-Za-z0-9]/;
function isAsciiWordCharacter(character: string | undefined): boolean {
return character !== undefined && ASCII_WORD_CHARACTER_PATTERN.test(character);
}
function canMergeRecoveredTranscriptOverlap(prefix: string, transcript: string, overlapLength: number): boolean {
const overlap = transcript.slice(0, overlapLength);
if (!ASCII_WORD_CHARACTER_PATTERN.test(overlap)) return true;
const prefixOverlapStart = prefix.length - overlapLength;
const prefixBeforeOverlap = prefixOverlapStart > 0 ? prefix.at(prefixOverlapStart - 1) : undefined;
return !isAsciiWordCharacter(prefixBeforeOverlap) && !isAsciiWordCharacter(transcript.at(overlapLength));
}
function mergeRecoveredTranscript(prefix: string, transcript: string): string {
const normalizedPrefix = prefix.trim();
const normalizedTranscript = transcript.trim();
if (!normalizedPrefix) return normalizedTranscript;
if (!normalizedTranscript) return normalizedPrefix;
if (
normalizedTranscript.startsWith(normalizedPrefix)
&& canMergeRecoveredTranscriptOverlap(normalizedPrefix, normalizedTranscript, normalizedPrefix.length)
) {
return normalizedTranscript;
}
const maxOverlap = Math.min(normalizedPrefix.length, normalizedTranscript.length);
for (let length = maxOverlap; length >= 1; length -= 1) {
if (
normalizedPrefix.endsWith(normalizedTranscript.slice(0, length))
&& canMergeRecoveredTranscriptOverlap(normalizedPrefix, normalizedTranscript, length)
) {
return normalizedPrefix + normalizedTranscript.slice(length);
}
}
const left = normalizedPrefix.at(-1) ?? '';
const right = normalizedTranscript.at(0) ?? '';
if (isAsciiWordCharacter(left) && isAsciiWordCharacter(right)) {
return `${normalizedPrefix} ${normalizedTranscript}`;
}
return normalizedPrefix + normalizedTranscript;
}
function hasDefiniteUtterance(payload: unknown): boolean {
if (Array.isArray(payload)) return payload.some((item) => hasDefiniteUtterance(item));
if (!isRecord(payload)) return false;
if (payload.definite === true) return true;
return Object.values(payload).some((value) => hasDefiniteUtterance(value));
}
/**
* Same judgement as desktop's SAUC provider: the transcript is confirmed only
* when the matching aggregate result's utterances are all definite and
* together cover the full text. endMs is the last utterance end_time on the
* transmitted audio clock, or 0 when missing or invalid.
*/
export function getVolcengineTranscriptConfirmation(
payload: unknown,
transcript: string,
): { confirmed: boolean; endMs: number } {
const unknown = { confirmed: false, endMs: 0 };
if (!transcript || !isRecord(payload)) return unknown;
const results = Array.isArray(payload.result) ? payload.result : [payload.result];
// Only accept a matching aggregate, not an arbitrary nested definite flag.
const result = results.find((item) => isRecord(item) && item.text === transcript);
if (!isRecord(result) || !Array.isArray(result.utterances) || !result.utterances.length) return unknown;
const utterances = result.utterances;
if (!utterances.every((item) => isRecord(item) && item.definite === true && typeof item.text === 'string')) return unknown;
const compact = (value: string): string => value.replace(/\s/gu, '');
if (compact(utterances.map((item) => item.text).join('')) !== compact(transcript)) return unknown;
const last = utterances[utterances.length - 1];
const endMs = typeof last.end_time === 'number' && Number.isFinite(last.end_time) && last.end_time > 0
? last.end_time : 0;
return { confirmed: true, endMs };
}
/**
* The part of the aggregate transcript covered by its leading definite
* utterances, or '' when none is definite. A response without an utterance
* list keeps the legacy whole-aggregate `definite` marker.
*/
export function definiteVolcengineTranscriptPrefix(payload: unknown, transcript: string): string {
if (!transcript || !hasDefiniteUtterance(payload)) return '';
const results = isRecord(payload) ? (Array.isArray(payload.result) ? payload.result : [payload.result]) : [];
const result = results.find((item) => isRecord(item) && item.text === transcript);
if (!isRecord(result) || !Array.isArray(result.utterances) || !result.utterances.length) {
return transcript;
}
let definite = '';
for (const item of result.utterances) {
if (!isRecord(item) || item.definite !== true || typeof item.text !== 'string') break;
definite += item.text;
}
// Map the joined utterances onto the aggregate, ignoring whitespace that
// the aggregate may insert between them; unmatched text finalizes nothing.
const expected = definite.replace(/\s/gu, '');
if (!expected) return '';
let matched = 0;
for (let index = 0; index < transcript.length; index += 1) {
const character = transcript[index];
if (/\s/u.test(character)) continue;
if (character !== expected[matched]) return '';
matched += 1;
if (matched === expected.length) return transcript.slice(0, index + 1);
}
return '';
}
function isVolcengineProtocolLastResponse(message: ParsedVolcengineMessage): boolean {
return message.messageType === VOLC_MESSAGE_TYPE_FULL_SERVER_RESPONSE
&& message.flags === VOLC_FLAG_NEGATIVE_SEQUENCE;
}
function collectStringFields(value: unknown, keys: Set<string>): string[] {
if (typeof value === 'string') return [];
if (!value || typeof value !== 'object') return [];
if (Array.isArray(value)) return value.flatMap((item) => collectStringFields(item, keys));
const result: string[] = [];
for (const [key, child] of Object.entries(value)) {
if (keys.has(key.toLowerCase()) && typeof child === 'string') result.push(child);
result.push(...collectStringFields(child, keys));
}
return result;
}
function silencePcm16(sampleRate: number, durationMs: number): Uint8Array {
const sampleCount = Math.max(0, Math.round(sampleRate * durationMs / 1000));
return new Uint8Array(sampleCount * 2);
}
function buildConnectId(): string {
return `xdt-maker-${Date.now()}-${Math.random().toString(16).slice(2)}`;
}
function looksLikeJson(text: string): boolean {
const trimmed = text.trim();
return trimmed.startsWith('{') || trimmed.startsWith('[');
}
function textToBytes(text: string): Uint8Array {
return new TextEncoder().encode(text);
}
function bytesToText(bytes: Uint8Array): string {
return new TextDecoder().decode(bytes);
}
function concatBytes(...chunks: Uint8Array[]): Uint8Array {
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
function sendBinary(socket: WebSocketLike, bytes: Uint8Array): void {
socket.send(toExactArrayBuffer(bytes));
}
function toExactArrayBuffer(bytes: Uint8Array): ArrayBuffer {
const view = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
return view as ArrayBuffer;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null;
}
function webSocketErrorMessage(
event: { message?: string; error?: unknown },
fallbackMessage: string,
): { message: string; specific: boolean } {
if (typeof event.message === 'string' && event.message.trim()) {
return { message: event.message, specific: true };
}
if (event.error instanceof Error && event.error.message.trim()) {
return { message: event.error.message, specific: true };
}
return { message: fallbackMessage, specific: false };
}
function webSocketCloseBeforeReadyMessage(
event: { code?: number; reason?: string },
fallbackMessage: string,
pendingMessage?: string,
): string {
const closeStatus = webSocketAuthStatus(event);
if (closeStatus) {
return i18n.t('composer.voice.sessionExpiredOrForbidden', { status: closeStatus });
}
if (event.reason?.trim()) return `${fallbackMessage} (${event.reason.trim()})`;
return pendingMessage ?? fallbackMessage;
}
function webSocketAuthStatus(event: { code?: number; reason?: string }): string | null {
const text = `${event.code ?? ''} ${event.reason ?? ''}`;
const match = text.match(/\b(401|403)\b/);
return match?.[1] ?? null;
}