Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
import {
BufferedAsrProvider,
DictationRefiner,
hasPcmSound,
VoiceInputController,
VoiceTimelineLogger,
type AsrProvider,
type AudioGate,
type AudioTrace,
type DictationRefinementContext,
type EditableRange,
type RefinementResult,
type VoiceInputErrorCode,
type VoiceInputDraftSource,
type VoiceInputState,
type VoiceTimelineEvent,
} from '@cindy/voice-input-core';
import { i18n } from '@/i18n';
import type { StoredMobileVoiceCredential } from '@/session/mobileVoiceCredentialStore';
import { redactMobileVoiceCredentialText } from '@/session/mobileVoiceCredentialRedaction';
import { createMobileAsrProvider } from '@/session/mobileRealtimeAsrProvider';
import { CINDY_MANAGED_REFINER_PROVIDER } from '@/session/mobileCindyVoiceSession';
import {
appendVoiceTranscriptDraftWithRange,
insertVoiceTranscriptDraftWithRange,
type MobileVoiceSelection,
buildMobileVoiceRefinementContext,
makeMobileRefinerPromptCacheKey,
MobileLiteLlmTextModelClient,
mobileVoiceErrorCodeMessage,
isMobileVoiceRateLimited,
mobileVoiceRateLimitMessage,
mobileVoiceTranscriptKeptError,
type MobileVoiceDraftInsertion,
} from '@/session/mobileVoiceInput';
import { startMobileRealtimeAudio, type MobileRealtimeAudioChunk } from '@/session/mobileRealtimeAudio';
import { MobileVoiceAudioLevels } from '@/session/mobileVoiceAudioLevels';
import { MobilePendingAudioGate } from '@/session/mobilePendingAudioGate';
import { readCachedMobileVoiceDictionary } from '@/session/mobileVoiceDictionaryCache';
import {
classifyMobileVoiceFailure,
createMobileVoiceTimelineRecorder,
logMobileVoice,
shortRunId,
} from '@/session/mobileVoiceDiagnostics';
type StartRealtimeAudio = (options: {
sampleRate: number;
onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
onError?: (error: Error) => void;
}) => Promise<(stoppedAt?: number) => Promise<void>>;
type MobileDictationRefiner = {
refine(input: {
text: string;
runId: string;
segmentIds: string[];
onPartial?: (text: string) => void;
}): Promise<RefinementResult>;
};
export type MobileVoiceControllerSession = {
start(): Promise<void>;
stop(): Promise<string>;
cancel(): Promise<void>;
currentDraft(): string;
};
type MobileVoiceControllerOptions = {
credential: StoredMobileVoiceCredential;
initialDraft: string;
initialSelection?: MobileVoiceSelection;
refinementContext?: DictationRefinementContext;
/**
* Persisted per-host history for refinement context. A getter lets the host
* read storage in the background: it is only consulted when a refinement
* request is built, so it never has to delay opening the microphone.
*/
localVoiceInputHistory?: MobileVoiceInputHistorySource;
asr?: AsrProvider;
refiner?: MobileDictationRefiner | null;
connectionProvider?: (provider: string) => Promise<{
websocketUrl: string;
authorizationToken: string;
}>;
refinerTargetProvider?: (provider: string, options?: { refreshAccessToken?: boolean }) => Promise<{
url: string;
authorization: string;
}>;
/** 托管润色 prompt cache 预热(voice-server refine-warmup);ASR 就绪后 fire-and-forget。 */
warmRefiner?: (input: {
system: string;
user: unknown;
promptCacheKey: string;
}) => Promise<void>;
startAudio?: StartRealtimeAudio;
readCurrentDraft?: () => string;
onDraftChanged: (draft: string, selection?: MobileVoiceSelection, replacement?: MobileVoiceDraftInsertion) => void;
onStateChanged?: (state: VoiceInputState) => void;
onError?: (message: string) => void;
onReadyForStartCue?: () => void;
onReadyForEndCue?: () => void;
recordHistory?: (text: string) => string | null | void | Promise<string | null | void>;
updateHistoryEntry?: (entryId: string, text: string) => void | Promise<void>;
onRefinementApplied?: (input: {
draft: string;
rawTranscriptText: string;
refinedText: string;
start: number;
end: number;
}) => void;
onTimelineEvent?: (event: VoiceTimelineEvent) => void;
/**
* Optional filter between the microphone and the ASR ("recognize my voice").
* Undefined keeps the unfiltered behavior.
*/
audioGate?: AudioGate | Promise<AudioGate | undefined>;
/** Gesture time, before permission / storage work, for end-to-end diagnostics. */
requestedAt?: number;
};
type MobileVoiceInputHistorySource = readonly string[] | (() => readonly string[] | undefined);
const PARTIAL_DRAFT_PUBLISH_INTERVAL_MS = 80;
/**
* Mobile adapter around shared VoiceInputController.
*
* It mirrors the desktop voice-input model: realtime ASR drives visible draft
* text, stop submits the ASR text, and streaming refinement replaces that same
* composer range. The UI supplies rendering and touch gestures; this adapter
* owns only the controller/draft projection boundary.
*/
export function createMobileVoiceControllerSession(
options: MobileVoiceControllerOptions,
): MobileVoiceControllerSession {
const startAudio = options.startAudio ?? ((input) => startMobileRealtimeAudio(input));
// BufferedAsrProvider makes the concurrent start (ASR connect ‖ mic capture)
// safe for ANY provider: audio appended while the connect handshake is still
// in flight is buffered and replayed in order once it settles, and a run
// abandoned mid-handshake tears the just-opened socket down instead of
// leaking it. Without it, the default fallback provider silently drops every
// chunk appended before its inner provider is chosen.
const asr = new BufferedAsrProvider(options.asr ?? createMobileAsrProvider(options.credential, {
connectionProvider: options.connectionProvider,
}));
const refiner = options.refiner === undefined
? createMobileRefiner(options.credential, {
refinementContext: options.refinementContext,
localVoiceInputHistory: options.localVoiceInputHistory,
refinerTargetProvider: options.refinerTargetProvider,
})
: options.refiner ?? undefined;
// 托管 prompt cache 预热:发一个与真实润色共享 prompt 前缀(dictationText 为空)
// 的请求到 voice-server warmup 端点。fire-and-forget——听写主流程绝不等待或
// 因预热失败而失败;promptCacheKey 派生必须与真实润色请求逐字节一致。
const warmManagedRefinerPromptCache = (): void => {
if (!options.warmRefiner || !(refiner instanceof DictationRefiner)) return;
try {
const request = refiner.buildWarmupRequest();
const promptCacheKey = makeMobileRefinerPromptCacheKey({
model: CINDY_MANAGED_REFINER_PROVIDER,
schemaName: 'dictation_refinement',
promptVersion: request.promptVersion,
system: request.system,
scope: mobileRefinerPromptCacheScope(options.credential),
});
void options.warmRefiner({
system: request.system,
user: { schemaName: 'dictation_refinement', input: request.user },
promptCacheKey,
}).then(() => {
logMobileVoice('debug', 'refiner prompt cache warmed');
}, (error) => {
logMobileVoice('warn', 'refiner warmup failed (non-fatal)', { reason: classifyMobileVoiceFailure(error) });
});
} catch (error) {
logMobileVoice('warn', 'refiner warmup skipped', { reason: classifyMobileVoiceFailure(error) });
}
};
const baseDraft = options.initialDraft;
let latestDraft = baseDraft;
let lastPublishedDraft = baseDraft;
let pendingDraftToPublish: string | null = null;
let draftPublishThrottleTimer: ReturnType<typeof setTimeout> | null = null;
let stopAudio: ((stoppedAt?: number) => Promise<void>) | null = null;
let audioOpening: Promise<void> | null = null;
let audioStopping: Promise<void> | null = null;
let captureStopRequestedAt: number | undefined;
let state: VoiceInputState = 'idle';
let controllerError: Error | null = null;
// Run lifecycle as an explicit phase machine instead of a set of overlapping
// boolean flags. Legal transitions:
// idle → running (start())
// running → stopping (stop())
// running | stopping → cancelled (cancel())
// running | stopping → failed (mic failure, or ASR connect failure)
// stopping → idle (stop() completed successfully)
// any → running (a fresh start() begins a new run)
// Every teardown-ish question is answered from the phase, so the answers can
// never disagree with each other the way independent flags could:
// - captureTornDown(): drop ordinary post-stop events. Only an explicit
// native drain receipt, clipped to the user's stop time, may still enter
// the buffer while stopping; a slow ASR handshake cannot extend capture.
// - phase === 'failed': did the run die (mic or connect failure)?
// - phase === 'cancelled': was the run abandoned on purpose (not an error)?
let runPhase: 'idle' | 'running' | 'stopping' | 'cancelled' | 'failed' = 'idle';
// Read through a function where control flow awaits in between: callbacks
// (handleAudioFailure / cancel) mutate runPhase across awaits, which TS's
// narrowing of the `let` cannot see.
const currentRunPhase = (): typeof runPhase => runPhase;
const captureTornDown = (): boolean => runPhase === 'stopping' || runPhase === 'cancelled' || runPhase === 'failed';
// Whether the mic has delivered its first PCM chunk this run. The shared
// controller sets 'listening' synchronously at start, but the AVAudioSession /
// AVAudioEngine take ~150-200ms to warm up before real audio flows; we hold the
// 'listening' state from the UI until capture is actually live (see start()).
// Orthogonal to runPhase: it describes the mic, not the run lifecycle.
let captureLive = false;
let listeningPendingCaptureLive = false;
// Whether captured PCM ever contained sound this run. A silent recording with
// no transcript ends at stop without waiting for the cloud handshake, final
// recognition or refinement. Deliberately conservative: any plausible sound
// (including background noise) keeps the normal path.
let soundDetected = false;
let audioLevels = new MobileVoiceAudioLevels();
let runStartedAt = Date.now();
const runTag = (): string => shortRunId(controller.id);
const recordTimeline = createMobileVoiceTimelineRecorder({ provider: options.credential.asr.provider });
// The ASR connect runs concurrently with capture (see start()), so its failure
// can surface while a stop()/mic-failure races it. Track the settled outcome at
// session scope so both start() and an early stop() observe the same errors.
// Audio buffered during the handshake lives inside BufferedAsrProvider, not here.
let asrStartPromise: Promise<void> | null = null;
let asrStartError: unknown = null;
let audioFailureError: Error | null = null;
let voiceInsertion: MobileVoiceDraftInsertion | null = null;
let voiceInsertionReplacementRange: { start: number; end: number } | null = null;
let publishedVoiceInsertion: MobileVoiceDraftInsertion | null = null;
let voiceInsertionSegmentIds: string[] = [];
let voiceInsertionTouched = false;
let historyEntryId: string | null = null;
let historyEntryPromise: Promise<string | null | void> | null = null;
let pendingHistoryUpdateText: string | null = null;
let submittedTextForLearning: string | null = null;
const doneWaiters: Array<() => void> = [];
const readCurrentDraft = (): string => {
const visibleDraft = options.readCurrentDraft?.() ?? latestDraft;
if (
options.readCurrentDraft && pendingDraftToPublish !== null
&& visibleDraft !== lastPublishedDraft
&& voiceInsertion && publishedVoiceInsertion
&& isInsertionIntact(visibleDraft, publishedVoiceInsertion)
) {
return replaceInsertionText(visibleDraft, publishedVoiceInsertion, voiceInsertion.text);
}
if (
visibleDraft === lastPublishedDraft
&& (pendingDraftToPublish !== null || draftPublishThrottleTimer !== null)
) {
return latestDraft;
}
return visibleDraft;
};
const publishDraftNow = (draft: string): void => {
pendingDraftToPublish = null;
// The first insertion can change selected atoms even when its text is identical.
if (lastPublishedDraft === draft && (!voiceInsertion || publishedVoiceInsertion)) return;
const replacement = voiceInsertion
? publishedVoiceInsertion
? { ...publishedVoiceInsertion, text: voiceInsertion.text }
: voiceInsertionReplacementRange
? { ...voiceInsertionReplacementRange, text: voiceInsertion.text }
: undefined
: undefined;
lastPublishedDraft = draft;
publishedVoiceInsertion = voiceInsertion ? { ...voiceInsertion } : null;
options.onDraftChanged(draft, voiceInsertion
? { start: voiceInsertion.end, end: voiceInsertion.end }
: undefined, replacement);
};
const publishPendingDraftNow = (): void => {
const pending = pendingDraftToPublish;
if (pending === null) return;
const currentDraft = readCurrentDraft();
if (voiceInsertion && !isInsertionIntact(currentDraft, voiceInsertion)) {
voiceInsertionTouched = true;
pendingDraftToPublish = null;
return;
}
latestDraft = currentDraft;
publishDraftNow(currentDraft);
};
const cancelPendingDraftPublish = (): void => {
if (draftPublishThrottleTimer) {
clearTimeout(draftPublishThrottleTimer);
draftPublishThrottleTimer = null;
}
pendingDraftToPublish = null;
};
const notifyReadyForStartCue = (): void => {
try {
options.onReadyForStartCue?.();
} catch {
// Optional feedback only.
}
};
const notifyReadyForEndCue = (): void => {
try {
options.onReadyForEndCue?.();
} catch {
// Optional feedback only.
}
};
// Called on the first PCM chunk of a run. Surfaces the 'listening' state that
// was deferred at start until the mic is actually capturing, so the UI's
// "listening" cue matches real audio flow rather than the ~150-200ms-earlier
// moment the audio session began warming up.
const markCaptureLive = (trace: AudioTrace, timestampSource?: 'hardware' | 'delivery'): void => {
if (captureLive) return;
captureLive = true;
logMobileVoice('debug', 'capture live', {
runId: runTag(), elapsedMs: Date.now() - runStartedAt,
timestampSource: timestampSource ?? 'delivery',
...(timestampSource === 'hardware' ? {
firstSampleAfterRequestMs: trace.capturedAt - runStartedAt,
deliveryDelayMs: Date.now() - trace.capturedAt,
} : {}),
});
if (listeningPendingCaptureLive && state === 'listening') {
listeningPendingCaptureLive = false;
options.onStateChanged?.('listening');
}
};
const publishDraftChange = (draft: string, mode: 'deferred' | 'immediate'): void => {
if (mode === 'immediate') {
cancelPendingDraftPublish();
publishDraftNow(draft);
return;
}
if (!draftPublishThrottleTimer) {
publishDraftNow(draft);
draftPublishThrottleTimer = setTimeout(() => {
draftPublishThrottleTimer = null;
publishPendingDraftNow();
}, PARTIAL_DRAFT_PUBLISH_INTERVAL_MS);
return;
}
pendingDraftToPublish = draft;
};
const publishText = (
text: string,
segmentIds: string[],
mode: 'deferred' | 'immediate',
): EditableRange | undefined => {
const normalized = text.trim();
if (!normalized) return undefined;
const currentDraft = readCurrentDraft();
latestDraft = currentDraft;
if (voiceInsertion) {
if (!isInsertionIntact(currentDraft, voiceInsertion)) {
voiceInsertionTouched = true;
return undefined;
}
const nextDraft = replaceInsertionText(currentDraft, voiceInsertion, normalized);
voiceInsertion = {
start: voiceInsertion.start,
end: voiceInsertion.start + normalized.length,
text: normalized,
};
voiceInsertionSegmentIds = segmentIds;
latestDraft = nextDraft;
publishDraftChange(nextDraft, mode);
return buildEditableRange(voiceInsertion, segmentIds, voiceInsertionTouched);
}
// A selection belongs to its captured draft. If typing changed that draft
// before the first ASR result, preserve the edit instead of replacing stale offsets.
const result = options.initialSelection && currentDraft === baseDraft
? insertVoiceTranscriptDraftWithRange(currentDraft, normalized, options.initialSelection)
: appendVoiceTranscriptDraftWithRange(currentDraft, normalized);
if (!result.insertion) return undefined;
voiceInsertion = result.insertion;
voiceInsertionReplacementRange = options.initialSelection && currentDraft === baseDraft
? 'atomRange' in options.initialSelection
? null
: { start: options.initialSelection.start, end: options.initialSelection.end }
: { start: result.insertion.start, end: result.insertion.start };
voiceInsertionSegmentIds = segmentIds;
latestDraft = result.draft;
publishDraftChange(latestDraft, mode);
return buildEditableRange(result.insertion, segmentIds, false);
};
const controller = new VoiceInputController({
asr,
refiner,
// Refine during a speech pause and show the result right away. The shared
// controller only acts on it when a refiner exists (refinement enabled).
pauseRefinementEnabled: true,
retryEmptyTranscript: true,
// A reconnect allocates another managed session and can hit the account
// limit; report that instead of the generic "stopped receiving" failure.
recoveryErrorMessage: mobileVoiceRateLimitMessage,
audioGate: options.audioGate instanceof Promise
? new MobilePendingAudioGate(options.audioGate, (error) => {
void handleAudioFailure(error instanceof Error ? error : new Error(String(error)));
})
: options.audioGate,
logger: new VoiceTimelineLogger((event) => {
recordTimeline(event);
options.onTimelineEvent?.(event);
}),
callbacks: {
onStateChanged(nextState) {
if (runPhase === 'failed' && nextState === 'done') {
// The teardown after a mic failure drives the shared controller to
// 'done'; the UI already saw 'error' — don't overwrite it.
doneWaiters.splice(0).forEach((resolve) => resolve());
return;
}
state = nextState;
if (nextState === 'listening' && !captureLive) {
// Hold 'listening' from the UI until the mic delivers its first PCM
// (surfaced by markCaptureLive). Showing it during the ~150-200ms audio
// warm-up makes users speak into a mic that isn't recording yet.
listeningPendingCaptureLive = true;
return;
}
options.onStateChanged?.(nextState);
if (nextState === 'done' || nextState === 'error') {
doneWaiters.splice(0).forEach((resolve) => resolve());
}
},
onDraftChanged(text, segment, source: VoiceInputDraftSource) {
publishText(text, [segment.id], source === 'partial' ? 'deferred' : 'immediate');
},
onSubmitted(text, segment) {
const range = publishText(text, [segment.id], 'immediate');
if (range) {
// A pause-time refinement may already be the submitted text; learning
// still compares against the raw transcript it was based on.
submittedTextForLearning = segment.basedOnText ?? text;
recordSubmittedHistory(text);
}
return range;
},
// No onRefinementPreview: a streamed refinement is only a prefix of the
// final text (removed fillers shift everything after them). Like desktop,
// keep the current text until the validated result arrives via
// applyRefinement instead of showing a half-finished hybrid.
applyRefinement(range, refinedText) {
const applied = publishText(refinedText, range.segmentIds, 'immediate') !== undefined;
if (applied) {
updateSubmittedHistory(refinedText);
if (voiceInsertion && submittedTextForLearning) {
options.onRefinementApplied?.({
draft: latestDraft,
rawTranscriptText: submittedTextForLearning,
refinedText,
start: voiceInsertion.start,
end: voiceInsertion.end,
});
}
}
return applied;
},
isRangeUserTouched(range) {
if (range.userTouched || voiceInsertionTouched) return true;
if (!voiceInsertion) return false;
const sameRange = range.startOffset === voiceInsertion.start
&& range.segmentIds.every((id, index) => voiceInsertionSegmentIds[index] === id);
return sameRange && !isInsertionIntact(readCurrentDraft(), voiceInsertion);
},
onError(message, code: VoiceInputErrorCode | undefined, details) {
// 有 code 的是 controller 自己分类的失败(message 是英文调试串),按 code 取
// 本地化文案;没有 code 的来自 provider,那条 message 才是唯一的失败描述。
const cause = code
? mobileVoiceErrorCodeMessage(code)
: redactMobileVoiceCredentialText(message, options.credential);
const localizedMessage = details?.transcriptKept
? mobileVoiceTranscriptKeptError(cause)
: cause;
controllerError = new Error(localizedMessage);
options.onError?.(localizedMessage);
},
},
});
return {
async start() {
submittedTextForLearning = null;
captureLive = false;
listeningPendingCaptureLive = false;
soundDetected = false;
audioLevels = new MobileVoiceAudioLevels();
audioStopping = null;
captureStopRequestedAt = undefined;
runStartedAt = options.requestedAt ?? Date.now();
// A fresh start() begins a new run from ANY previous phase — in particular
// 'failed': a mic interruption must not poison the next recording on the
// same session, so the failure record is cleared here too.
runPhase = 'running';
asrStartError = null;
audioFailureError = null;
controllerError = null;
// Connect the ASR socket and open the mic CONCURRENTLY instead of serially.
// The shared controller sets 'listening' synchronously, so the mic goes live
// immediately instead of waiting for the ~2.4s WebSocket connect + session
// handshake (the previous serial ordering awaited connect BEFORE opening the
// mic). Audio captured during the handshake is buffered and replayed by
// BufferedAsrProvider once the connect settles, so nothing is lost.
asrStartPromise = controller.start().then(() => {
// With a connection prewarmed at press-in, the controller subscribes
// after the provider already connected; this is the moment it is usable.
logMobileVoice('debug', 'asr ready', { runId: runTag(), elapsedMs: Date.now() - runStartedAt });
}, (error: unknown) => {
asrStartError = error;
});
// controller.start() assigns the run id synchronously.
logMobileVoice('info', 'start', {
runId: runTag(),
provider: options.credential.asr.provider,
sampleRate: options.credential.asr.pcmSampleRate ?? 16_000,
refinement: Boolean(refiner),
});
try {
audioOpening = startAudio({
sampleRate: options.credential.asr.pcmSampleRate ?? 16_000,
onChunk: (chunk) => {
// Drop audio emitted after the run began tearing down capture, so an
// early stop that waits for the in-flight handshake can't buffer and
// later flush post-stop speech/noise.
if (captureTornDown() && !(runPhase === 'stopping' && chunk.drained)) return false;
audioLevels.append(chunk.pcm16, chunk.trace.sampleRate ?? options.credential.asr.pcmSampleRate ?? 16_000);
if (!soundDetected && hasPcmSound(chunk.pcm16)) soundDetected = true;
if (runPhase === 'running') markCaptureLive(chunk.trace, chunk.timestampSource);
controller.appendAudio(chunk.pcm16, chunk.trace);
},
onError: (error) => {
void handleAudioFailure(error);
},
}).then((stop) => { stopAudio = stop; });
await audioOpening;
} catch (error) {
runPhase = 'failed';
// Surface the mic failure immediately — do NOT wait for the in-flight
// ASR handshake (a slow/hung connect would otherwise pin the startup
// and delay the error). controller.cancel() tears the handshake down
// without blocking on it (BufferedAsrProvider.stop() is non-blocking
// for an in-flight connect), and asrStartPromise carries its own
// .catch so the late settle can never become an unhandled rejection.
await controller.cancel().catch(() => undefined);
logMobileVoiceFailure('start failed', 'microphone', error);
throw redactMobileVoiceError(error, options.credential);
}
if (captureTornDown() && stopAudio) {
// cancel() (or a mic failure) landed while native capture was still
// opening: its teardown saw stopAudio === null and could not close the
// capture, so close the just-opened handle here — otherwise the mic and
// its event subscriptions stay live even though the run is abandoned.
await stopNativeCapture();
}
// Wait for the ASR connection to settle before declaring the start
// successful, so a failed handshake still surfaces as a startup error (and
// the optional start cue is only played once both capture and ASR are up).
await asrStartPromise;
if (currentRunPhase() === 'cancelled') {
// The run was cancelled (or ended silently) while the handshake was
// still settling; the ASR wrapper already tears the just-opened provider
// down. A handshake outcome for an abandoned run is not an error, so
// return quietly without announcing a successful start.
return;
}
if (asrStartError) {
runPhase = 'failed';
if (stopAudio) {
const stop = stopAudio;
stopAudio = null;
await stop().catch(() => undefined);
}
await controller.cancel().catch(() => undefined);
logMobileVoiceFailure('start failed', 'asr', asrStartError);
throw redactMobileVoiceError(asrStartError, options.credential);
}
if (currentRunPhase() === 'failed') {
// The mic failed while the ASR handshake was still settling.
// handleAudioFailure already surfaced the error and cancelled the
// controller; don't report a successful start (the caller would otherwise
// treat the run as active and could append audio to a cancelled session).
throw redactMobileVoiceError(
audioFailureError ?? new Error('Realtime voice capture was interrupted during startup.'),
options.credential,
);
}
// ASR 会话已建立(refine session 同步就绪),此刻预热润色 prompt cache,
// 让缓存赶在用户停止说话前热起来。
warmManagedRefinerPromptCache();
notifyReadyForStartCue();
},
async stop() {
// Stop native capture FIRST — before awaiting the in-flight ASR handshake.
// Capture runs concurrently with connect, so waiting for the handshake first
// would keep the mic live for the whole remaining WebSocket connect; audio the
// user produces after releasing stop would keep flowing into the buffer and be
// flushed once ASR settles, transcribing post-stop speech/noise. Entering
// 'stopping' also makes onChunk drop any straggler chunk from the teardown.
// Pre-stop audio buffered inside BufferedAsrProvider is still replayed once
// the connect settles, so short utterances are not lost.
if (runPhase === 'running') runPhase = 'stopping';
const stopRequestedAt = Date.now();
captureStopRequestedAt = stopRequestedAt;
logMobileVoice('debug', 'stop requested', {
runId: runTag(),
elapsedMs: stopRequestedAt - runStartedAt,
captureLive,
soundDetected,
...audioLevels.summary(),
hasTranscript: Boolean(voiceInsertion),
});
// An immediate release can beat the native start receipt. Wait only for
// capture (never ASR), then drain before deciding this was a silent run.
if (!stopAudio && !audioStopping) await audioOpening?.catch(() => undefined);
await stopNativeCapture();
if (currentRunPhase() === 'stopping' && !soundDetected && !voiceInsertion) {
logMobileVoice('info', 'silent recording ended without ASR finalization', {
runId: runTag(),
elapsedMs: Date.now() - runStartedAt,
captureLive,
});
// Nothing audible was captured and nothing was recognized: end now
// instead of waiting for a pending cloud start, final ASR or refinement.
// Audio already sent is not recalled. 'cancelled' also lets a start()
// still awaiting the handshake return quietly.
runPhase = 'cancelled';
cancelPendingDraftPublish();
await controller.cancel().catch(() => undefined);
notifyReadyForEndCue();
return latestDraft;
}
// Then wait for the ASR handshake to settle (it runs concurrently with
// capture) so the failure checks below observe its final outcome. The
// pre-stop audio replay + flush ordering is handled by BufferedAsrProvider.
const handshakeWaitStartedAt = Date.now();
if (asrStartPromise) await asrStartPromise.catch(() => undefined);
const handshakeWaitMs = Date.now() - handshakeWaitStartedAt;
if (handshakeWaitMs > 50) logMobileVoice('debug', 'stop waited for ASR connection', { waitMs: handshakeWaitMs });
if (asrStartError || audioFailureError) {
// The run is dead — either the ASR handshake failed during an early stop,
// or the mic failed while the handshake was still settling. Surface it
// instead of returning a "successful" empty draft the caller shows as
// done, and don't submit against an already-cancelled controller.
runPhase = 'failed';
await controller.cancel().catch(() => undefined);
throw redactMobileVoiceError(
asrStartError ?? audioFailureError ?? new Error('Realtime voice capture was interrupted during startup.'),
options.credential,
);
}
notifyReadyForEndCue();
await controller.stop();
if (state === 'error' || controllerError) {
runPhase = 'failed';
throw controllerError ?? new Error(i18n.t('composer.voice.incomplete'));
}
if (state === 'submitting' || state === 'refining') await waitForDone(doneWaiters);
if (runPhase === 'stopping') runPhase = 'idle';
logMobileVoice('debug', 'stop finished', { runId: runTag(), stopToDoneMs: Date.now() - stopRequestedAt, state });
return latestDraft;
},
async cancel() {
cancelPendingDraftPublish();
// 'cancelled' makes onChunk drop straggler capture, and tells a start()
// that is still awaiting the handshake to return quietly. The ASR-side
// teardown (discard buffered audio, close a socket that only finishes
// opening later) is handled by BufferedAsrProvider when controller.cancel()
// reaches its stop().
runPhase = 'cancelled';
try {
if (stopAudio) {
const stop = stopAudio;
stopAudio = null;
await stop();
}
} finally {
// Native drain can reject (overflow / missing history). Cancellation
// still owns filter disposal and closing the ASR connection.
await controller.cancel();
}
},
currentDraft() {
return readCurrentDraft();
},
};
async function handleAudioFailure(error: Error): Promise<void> {
// Only a live run can fail: ignore repeated failures, and ignore mic errors
// surfacing after the run was already cancelled (teardown noise).
if (runPhase !== 'running' && runPhase !== 'stopping') return;
runPhase = 'failed';
audioFailureError = error;
logMobileVoiceFailure('capture failed', 'microphone', error);
cancelPendingDraftPublish();
state = 'error';
options.onStateChanged?.('error');
options.onError?.(redactMobileVoiceCredentialText(error, options.credential));
const stop = stopAudio;
stopAudio = null;
try {
await stop?.();
} finally {
await controller.cancel().catch(() => undefined);
}
}
async function stopNativeCapture(): Promise<void> {
if (audioStopping) return audioStopping;
const stop = stopAudio;
if (!stop) return;
stopAudio = null;
audioStopping = stop(captureStopRequestedAt).catch(async (error: unknown) => {
await handleAudioFailure(error instanceof Error ? error : new Error(String(error)));
});
return audioStopping;
}
function recordSubmittedHistory(text: string): void {
if (!options.recordHistory) return;
// 下面的 .catch 只兜得住异步拒绝;同步抛会顺着 onSubmitted 冒出 controller,
// 被当成「宿主没接住这段文本」——文字其实已经写进草稿了,而 controller 还会
// 据此认定本次听写失败。历史记录是附带能力,不该有这个权力。
let recording: ReturnType<NonNullable<MobileVoiceControllerOptions['recordHistory']>>;
try {
recording = options.recordHistory(text);
} catch (error) {
console.warn(
'[mobile-voice] record history failed (non-fatal):',
error instanceof Error ? error.message : String(error),
);
return;
}
historyEntryPromise = Promise.resolve(recording)
.then((entryId) => {
if (typeof entryId === 'string' && entryId.trim()) {
historyEntryId = entryId.trim();
if (pendingHistoryUpdateText) {
updateHistoryEntryNow(historyEntryId, pendingHistoryUpdateText);
pendingHistoryUpdateText = null;
}
return historyEntryId;
}
return null;
})
.catch(() => null);
}
function updateSubmittedHistory(text: string): void {
if (!options.updateHistoryEntry) return;
pendingHistoryUpdateText = text;
if (historyEntryId) {
updateHistoryEntryNow(historyEntryId, text);
pendingHistoryUpdateText = null;
}
void historyEntryPromise;
}
function updateHistoryEntryNow(entryId: string, text: string): void {
if (!options.updateHistoryEntry) return;
void Promise.resolve(options.updateHistoryEntry(entryId, text)).catch(() => undefined);
}
}
function buildEditableRange(
insertion: MobileVoiceDraftInsertion,
segmentIds: string[],
userTouched: boolean,
): EditableRange {
return {
id: `mobile-voice-range-${segmentIds.join('-')}`,
segmentIds,
startOffset: insertion.start,
endOffset: insertion.end,
userTouched,
};
}
function isInsertionIntact(draft: string, insertion: MobileVoiceDraftInsertion): boolean {
return insertion.start >= 0
&& insertion.end >= insertion.start
&& insertion.end <= draft.length
&& draft.slice(insertion.start, insertion.end) === insertion.text;
}
function replaceInsertionText(
draft: string,
insertion: MobileVoiceDraftInsertion,
text: string,
): string {
return `${draft.slice(0, insertion.start)}${text}${draft.slice(insertion.end)}`;
}
function logMobileVoiceFailure(message: string, stage: 'microphone' | 'asr', error: unknown): void {
logMobileVoice('warn', message, {
stage,
rateLimited: isMobileVoiceRateLimited(error),
reason: classifyMobileVoiceFailure(error),
});
}
function redactMobileVoiceError(error: unknown, credential: StoredMobileVoiceCredential): Error {
// A recognized account limit gets the localized "wait and retry" text instead
// of whatever the server or the last fallback candidate reported.
const rateLimitMessage = mobileVoiceRateLimitMessage(error);
if (rateLimitMessage) return new Error(rateLimitMessage);
const message = redactMobileVoiceCredentialText(
error instanceof Error ? error.message : String(error),
credential,
);
const redacted = new Error(message);
if (error instanceof Error) redacted.name = error.name;
return redacted;
}
function mobileRefinerPromptCacheScope(credential: StoredMobileVoiceCredential): string {
return `mobile-voice:${credential.hostDeviceId}`;
}
function createMobileRefiner(
credential: StoredMobileVoiceCredential,
options: {
refinementContext?: DictationRefinementContext;
localVoiceInputHistory?: MobileVoiceInputHistorySource;
refinerTargetProvider?: (provider: string, options?: { refreshAccessToken?: boolean }) => Promise<{
url: string;
authorization: string;
}>;
},
): DictationRefiner | undefined {
if (credential.settings?.refinementEnabled === false) return undefined;
const refinerTargetProvider = options.refinerTargetProvider;
if (!refinerTargetProvider) {
throw new Error(i18n.t('composer.voice.missingRefinerTargetProvider'));
}
// 托管单请求:模型选择与 failover 在 voice-server(provider/model 都传 'auto',
// 服务端按自己配置的模型链覆盖),客户端不再做多 provider fallback。
return new DictationRefiner({
client: new MobileLiteLlmTextModelClient({
requestTargetProvider: (requestOptions) =>
refinerTargetProvider(CINDY_MANAGED_REFINER_PROVIDER, requestOptions),
}),
model: CINDY_MANAGED_REFINER_PROVIDER,
promptCacheScope: mobileRefinerPromptCacheScope(credential),
contextProvider: () => buildMobileVoiceRefinementContext(credential, {
refinementContext: options.refinementContext,
localVoiceInputHistory: typeof options.localVoiceInputHistory === 'function'
? options.localVoiceInputHistory()
: options.localVoiceInputHistory,
// 词典来自被控桌面的只读快照缓存(拉取在开麦时异步触发)。桌面离线或还没
// 拉到时是空数组 —— 润色照常进行,只是少了术语提示。
dictionaryEntries: readCachedMobileVoiceDictionary(credential.hostDeviceId),
}),
});
}
function waitForDone(waiters: Array<() => void>): Promise<void> {
return new Promise((resolve) => {
waiters.push(resolve);
});
}