Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

37.7 KB
Raw Blame
import {
  BufferedAsrProvider,
  DictationRefiner,
  hasPcmSound,
  VoiceInputController,
  VoiceTimelineLogger,
  type AsrProvider,
  type AudioGate,
  type AudioTrace,
  type DictationRefinementContext,
  type EditableRange,
  type RefinementResult,
  type VoiceInputErrorCode,
  type VoiceInputDraftSource,
  type VoiceInputState,
  type VoiceTimelineEvent,
} from '@cindy/voice-input-core';
import { i18n } from '@/i18n';
import type { StoredMobileVoiceCredential } from '@/session/mobileVoiceCredentialStore';
import { redactMobileVoiceCredentialText } from '@/session/mobileVoiceCredentialRedaction';
import { createMobileAsrProvider } from '@/session/mobileRealtimeAsrProvider';
import { CINDY_MANAGED_REFINER_PROVIDER } from '@/session/mobileCindyVoiceSession';
import {
  appendVoiceTranscriptDraftWithRange,
  insertVoiceTranscriptDraftWithRange,
  type MobileVoiceSelection,
  buildMobileVoiceRefinementContext,
  makeMobileRefinerPromptCacheKey,
  MobileLiteLlmTextModelClient,
  mobileVoiceErrorCodeMessage,
  isMobileVoiceRateLimited,
  mobileVoiceRateLimitMessage,
  mobileVoiceTranscriptKeptError,
  type MobileVoiceDraftInsertion,
} from '@/session/mobileVoiceInput';
import { startMobileRealtimeAudio, type MobileRealtimeAudioChunk } from '@/session/mobileRealtimeAudio';
import { MobileVoiceAudioLevels } from '@/session/mobileVoiceAudioLevels';
import { MobilePendingAudioGate } from '@/session/mobilePendingAudioGate';
import { readCachedMobileVoiceDictionary } from '@/session/mobileVoiceDictionaryCache';
import {
  classifyMobileVoiceFailure,
  createMobileVoiceTimelineRecorder,
  logMobileVoice,
  shortRunId,
} from '@/session/mobileVoiceDiagnostics';

type StartRealtimeAudio = (options: {
  sampleRate: number;
  onChunk: (chunk: MobileRealtimeAudioChunk) => boolean | void;
  onError?: (error: Error) => void;
}) => Promise<(stoppedAt?: number) => Promise<void>>;

type MobileDictationRefiner = {
  refine(input: {
    text: string;
    runId: string;
    segmentIds: string[];
    onPartial?: (text: string) => void;
  }): Promise<RefinementResult>;
};

export type MobileVoiceControllerSession = {
  start(): Promise<void>;
  stop(): Promise<string>;
  cancel(): Promise<void>;
  currentDraft(): string;
};

type MobileVoiceControllerOptions = {
  credential: StoredMobileVoiceCredential;
  initialDraft: string;
  initialSelection?: MobileVoiceSelection;
  refinementContext?: DictationRefinementContext;
  /**
   * Persisted per-host history for refinement context. A getter lets the host
   * read storage in the background: it is only consulted when a refinement
   * request is built, so it never has to delay opening the microphone.
   */
  localVoiceInputHistory?: MobileVoiceInputHistorySource;
  asr?: AsrProvider;
  refiner?: MobileDictationRefiner | null;
  connectionProvider?: (provider: string) => Promise<{
    websocketUrl: string;
    authorizationToken: string;
  }>;
  refinerTargetProvider?: (provider: string, options?: { refreshAccessToken?: boolean }) => Promise<{
    url: string;
    authorization: string;
  }>;
  /** 托管润色 prompt cache 预热(voice-server refine-warmup);ASR 就绪后 fire-and-forget。 */
  warmRefiner?: (input: {
    system: string;
    user: unknown;
    promptCacheKey: string;
  }) => Promise<void>;
  startAudio?: StartRealtimeAudio;
  readCurrentDraft?: () => string;
  onDraftChanged: (draft: string, selection?: MobileVoiceSelection, replacement?: MobileVoiceDraftInsertion) => void;
  onStateChanged?: (state: VoiceInputState) => void;
  onError?: (message: string) => void;
  onReadyForStartCue?: () => void;
  onReadyForEndCue?: () => void;
  recordHistory?: (text: string) => string | null | void | Promise<string | null | void>;
  updateHistoryEntry?: (entryId: string, text: string) => void | Promise<void>;
  onRefinementApplied?: (input: {
    draft: string;
    rawTranscriptText: string;
    refinedText: string;
    start: number;
    end: number;
  }) => void;
  onTimelineEvent?: (event: VoiceTimelineEvent) => void;
  /**
   * Optional filter between the microphone and the ASR ("recognize my voice").
   * Undefined keeps the unfiltered behavior.
   */
  audioGate?: AudioGate | Promise<AudioGate | undefined>;
  /** Gesture time, before permission / storage work, for end-to-end diagnostics. */
  requestedAt?: number;
};

type MobileVoiceInputHistorySource = readonly string[] | (() => readonly string[] | undefined);

const PARTIAL_DRAFT_PUBLISH_INTERVAL_MS = 80;

/**
 * Mobile adapter around shared VoiceInputController.
 *
 * It mirrors the desktop voice-input model: realtime ASR drives visible draft
 * text, stop submits the ASR text, and streaming refinement replaces that same
 * composer range. The UI supplies rendering and touch gestures; this adapter
 * owns only the controller/draft projection boundary.
 */
export function createMobileVoiceControllerSession(
  options: MobileVoiceControllerOptions,
): MobileVoiceControllerSession {
  const startAudio = options.startAudio ?? ((input) => startMobileRealtimeAudio(input));
  // BufferedAsrProvider makes the concurrent start (ASR connect ‖ mic capture)
  // safe for ANY provider: audio appended while the connect handshake is still
  // in flight is buffered and replayed in order once it settles, and a run
  // abandoned mid-handshake tears the just-opened socket down instead of
  // leaking it. Without it, the default fallback provider silently drops every
  // chunk appended before its inner provider is chosen.
  const asr = new BufferedAsrProvider(options.asr ?? createMobileAsrProvider(options.credential, {
    connectionProvider: options.connectionProvider,
  }));
  const refiner = options.refiner === undefined
    ? createMobileRefiner(options.credential, {
      refinementContext: options.refinementContext,
      localVoiceInputHistory: options.localVoiceInputHistory,
      refinerTargetProvider: options.refinerTargetProvider,
    })
    : options.refiner ?? undefined;
  // 托管 prompt cache 预热:发一个与真实润色共享 prompt 前缀(dictationText 为空)
  // 的请求到 voice-server warmup 端点。fire-and-forget——听写主流程绝不等待或
  // 因预热失败而失败;promptCacheKey 派生必须与真实润色请求逐字节一致。
  const warmManagedRefinerPromptCache = (): void => {
    if (!options.warmRefiner || !(refiner instanceof DictationRefiner)) return;
    try {
      const request = refiner.buildWarmupRequest();
      const promptCacheKey = makeMobileRefinerPromptCacheKey({
        model: CINDY_MANAGED_REFINER_PROVIDER,
        schemaName: 'dictation_refinement',
        promptVersion: request.promptVersion,
        system: request.system,
        scope: mobileRefinerPromptCacheScope(options.credential),
      });
      void options.warmRefiner({
        system: request.system,
        user: { schemaName: 'dictation_refinement', input: request.user },
        promptCacheKey,
      }).then(() => {
        logMobileVoice('debug', 'refiner prompt cache warmed');
      }, (error) => {
        logMobileVoice('warn', 'refiner warmup failed (non-fatal)', { reason: classifyMobileVoiceFailure(error) });
      });
    } catch (error) {
      logMobileVoice('warn', 'refiner warmup skipped', { reason: classifyMobileVoiceFailure(error) });
    }
  };
  const baseDraft = options.initialDraft;
  let latestDraft = baseDraft;
  let lastPublishedDraft = baseDraft;
  let pendingDraftToPublish: string | null = null;
  let draftPublishThrottleTimer: ReturnType<typeof setTimeout> | null = null;
  let stopAudio: ((stoppedAt?: number) => Promise<void>) | null = null;
  let audioOpening: Promise<void> | null = null;
  let audioStopping: Promise<void> | null = null;
  let captureStopRequestedAt: number | undefined;
  let state: VoiceInputState = 'idle';
  let controllerError: Error | null = null;
  // Run lifecycle as an explicit phase machine instead of a set of overlapping
  // boolean flags. Legal transitions:
  //   idle → running                     (start())
  //   running → stopping                 (stop())
  //   running | stopping → cancelled     (cancel())
  //   running | stopping → failed        (mic failure, or ASR connect failure)
  //   stopping → idle                    (stop() completed successfully)
  //   any → running                      (a fresh start() begins a new run)
  // Every teardown-ish question is answered from the phase, so the answers can
  // never disagree with each other the way independent flags could:
  //   - captureTornDown(): drop ordinary post-stop events. Only an explicit
  //     native drain receipt, clipped to the user's stop time, may still enter
  //     the buffer while stopping; a slow ASR handshake cannot extend capture.
  //   - phase === 'failed': did the run die (mic or connect failure)?
  //   - phase === 'cancelled': was the run abandoned on purpose (not an error)?
  let runPhase: 'idle' | 'running' | 'stopping' | 'cancelled' | 'failed' = 'idle';
  // Read through a function where control flow awaits in between: callbacks
  // (handleAudioFailure / cancel) mutate runPhase across awaits, which TS's
  // narrowing of the `let` cannot see.
  const currentRunPhase = (): typeof runPhase => runPhase;
  const captureTornDown = (): boolean => runPhase === 'stopping' || runPhase === 'cancelled' || runPhase === 'failed';
  // Whether the mic has delivered its first PCM chunk this run. The shared
  // controller sets 'listening' synchronously at start, but the AVAudioSession /
  // AVAudioEngine take ~150-200ms to warm up before real audio flows; we hold the
  // 'listening' state from the UI until capture is actually live (see start()).
  // Orthogonal to runPhase: it describes the mic, not the run lifecycle.
  let captureLive = false;
  let listeningPendingCaptureLive = false;
  // Whether captured PCM ever contained sound this run. A silent recording with
  // no transcript ends at stop without waiting for the cloud handshake, final
  // recognition or refinement. Deliberately conservative: any plausible sound
  // (including background noise) keeps the normal path.
  let soundDetected = false;
  let audioLevels = new MobileVoiceAudioLevels();
  let runStartedAt = Date.now();
  const runTag = (): string => shortRunId(controller.id);
  const recordTimeline = createMobileVoiceTimelineRecorder({ provider: options.credential.asr.provider });
  // The ASR connect runs concurrently with capture (see start()), so its failure
  // can surface while a stop()/mic-failure races it. Track the settled outcome at
  // session scope so both start() and an early stop() observe the same errors.
  // Audio buffered during the handshake lives inside BufferedAsrProvider, not here.
  let asrStartPromise: Promise<void> | null = null;
  let asrStartError: unknown = null;
  let audioFailureError: Error | null = null;
  let voiceInsertion: MobileVoiceDraftInsertion | null = null;
  let voiceInsertionReplacementRange: { start: number; end: number } | null = null;
  let publishedVoiceInsertion: MobileVoiceDraftInsertion | null = null;
  let voiceInsertionSegmentIds: string[] = [];
  let voiceInsertionTouched = false;
  let historyEntryId: string | null = null;
  let historyEntryPromise: Promise<string | null | void> | null = null;
  let pendingHistoryUpdateText: string | null = null;
  let submittedTextForLearning: string | null = null;
  const doneWaiters: Array<() => void> = [];

  const readCurrentDraft = (): string => {
    const visibleDraft = options.readCurrentDraft?.() ?? latestDraft;
    if (
      options.readCurrentDraft && pendingDraftToPublish !== null
      && visibleDraft !== lastPublishedDraft
      && voiceInsertion && publishedVoiceInsertion
      && isInsertionIntact(visibleDraft, publishedVoiceInsertion)
    ) {
      return replaceInsertionText(visibleDraft, publishedVoiceInsertion, voiceInsertion.text);
    }
    if (
      visibleDraft === lastPublishedDraft
      && (pendingDraftToPublish !== null || draftPublishThrottleTimer !== null)
    ) {
      return latestDraft;
    }
    return visibleDraft;
  };

  const publishDraftNow = (draft: string): void => {
    pendingDraftToPublish = null;
    // The first insertion can change selected atoms even when its text is identical.
    if (lastPublishedDraft === draft && (!voiceInsertion || publishedVoiceInsertion)) return;
    const replacement = voiceInsertion
      ? publishedVoiceInsertion
        ? { ...publishedVoiceInsertion, text: voiceInsertion.text }
        : voiceInsertionReplacementRange
          ? { ...voiceInsertionReplacementRange, text: voiceInsertion.text }
          : undefined
      : undefined;
    lastPublishedDraft = draft;
    publishedVoiceInsertion = voiceInsertion ? { ...voiceInsertion } : null;
    options.onDraftChanged(draft, voiceInsertion
      ? { start: voiceInsertion.end, end: voiceInsertion.end }
      : undefined, replacement);
  };

  const publishPendingDraftNow = (): void => {
    const pending = pendingDraftToPublish;
    if (pending === null) return;
    const currentDraft = readCurrentDraft();
    if (voiceInsertion && !isInsertionIntact(currentDraft, voiceInsertion)) {
      voiceInsertionTouched = true;
      pendingDraftToPublish = null;
      return;
    }
    latestDraft = currentDraft;
    publishDraftNow(currentDraft);
  };

  const cancelPendingDraftPublish = (): void => {
    if (draftPublishThrottleTimer) {
      clearTimeout(draftPublishThrottleTimer);
      draftPublishThrottleTimer = null;
    }
    pendingDraftToPublish = null;
  };

  const notifyReadyForStartCue = (): void => {
    try {
      options.onReadyForStartCue?.();
    } catch {
      // Optional feedback only.
    }
  };

  const notifyReadyForEndCue = (): void => {
    try {
      options.onReadyForEndCue?.();
    } catch {
      // Optional feedback only.
    }
  };

  // Called on the first PCM chunk of a run. Surfaces the 'listening' state that
  // was deferred at start until the mic is actually capturing, so the UI's
  // "listening" cue matches real audio flow rather than the ~150-200ms-earlier
  // moment the audio session began warming up.
  const markCaptureLive = (trace: AudioTrace, timestampSource?: 'hardware' | 'delivery'): void => {
    if (captureLive) return;
    captureLive = true;
    logMobileVoice('debug', 'capture live', {
      runId: runTag(), elapsedMs: Date.now() - runStartedAt,
      timestampSource: timestampSource ?? 'delivery',
      ...(timestampSource === 'hardware' ? {
        firstSampleAfterRequestMs: trace.capturedAt - runStartedAt,
        deliveryDelayMs: Date.now() - trace.capturedAt,
      } : {}),
    });
    if (listeningPendingCaptureLive && state === 'listening') {
      listeningPendingCaptureLive = false;
      options.onStateChanged?.('listening');
    }
  };

  const publishDraftChange = (draft: string, mode: 'deferred' | 'immediate'): void => {
    if (mode === 'immediate') {
      cancelPendingDraftPublish();
      publishDraftNow(draft);
      return;
    }
    if (!draftPublishThrottleTimer) {
      publishDraftNow(draft);
      draftPublishThrottleTimer = setTimeout(() => {
        draftPublishThrottleTimer = null;
        publishPendingDraftNow();
      }, PARTIAL_DRAFT_PUBLISH_INTERVAL_MS);
      return;
    }
    pendingDraftToPublish = draft;
  };

  const publishText = (
    text: string,
    segmentIds: string[],
    mode: 'deferred' | 'immediate',
  ): EditableRange | undefined => {
    const normalized = text.trim();
    if (!normalized) return undefined;

    const currentDraft = readCurrentDraft();
    latestDraft = currentDraft;

    if (voiceInsertion) {
      if (!isInsertionIntact(currentDraft, voiceInsertion)) {
        voiceInsertionTouched = true;
        return undefined;
      }
      const nextDraft = replaceInsertionText(currentDraft, voiceInsertion, normalized);
      voiceInsertion = {
        start: voiceInsertion.start,
        end: voiceInsertion.start + normalized.length,
        text: normalized,
      };
      voiceInsertionSegmentIds = segmentIds;
      latestDraft = nextDraft;
      publishDraftChange(nextDraft, mode);
      return buildEditableRange(voiceInsertion, segmentIds, voiceInsertionTouched);
    }

    // A selection belongs to its captured draft. If typing changed that draft
    // before the first ASR result, preserve the edit instead of replacing stale offsets.
    const result = options.initialSelection && currentDraft === baseDraft
      ? insertVoiceTranscriptDraftWithRange(currentDraft, normalized, options.initialSelection)
      : appendVoiceTranscriptDraftWithRange(currentDraft, normalized);
    if (!result.insertion) return undefined;
    voiceInsertion = result.insertion;
    voiceInsertionReplacementRange = options.initialSelection && currentDraft === baseDraft
      ? 'atomRange' in options.initialSelection
        ? null
        : { start: options.initialSelection.start, end: options.initialSelection.end }
      : { start: result.insertion.start, end: result.insertion.start };
    voiceInsertionSegmentIds = segmentIds;
    latestDraft = result.draft;
    publishDraftChange(latestDraft, mode);
    return buildEditableRange(result.insertion, segmentIds, false);
  };

  const controller = new VoiceInputController({
    asr,
    refiner,
    // Refine during a speech pause and show the result right away. The shared
    // controller only acts on it when a refiner exists (refinement enabled).
    pauseRefinementEnabled: true,
    retryEmptyTranscript: true,
    // A reconnect allocates another managed session and can hit the account
    // limit; report that instead of the generic "stopped receiving" failure.
    recoveryErrorMessage: mobileVoiceRateLimitMessage,
    audioGate: options.audioGate instanceof Promise
      ? new MobilePendingAudioGate(options.audioGate, (error) => {
        void handleAudioFailure(error instanceof Error ? error : new Error(String(error)));
      })
      : options.audioGate,
    logger: new VoiceTimelineLogger((event) => {
      recordTimeline(event);
      options.onTimelineEvent?.(event);
    }),
    callbacks: {
      onStateChanged(nextState) {
        if (runPhase === 'failed' && nextState === 'done') {
          // The teardown after a mic failure drives the shared controller to
          // 'done'; the UI already saw 'error' — don't overwrite it.
          doneWaiters.splice(0).forEach((resolve) => resolve());
          return;
        }
        state = nextState;
        if (nextState === 'listening' && !captureLive) {
          // Hold 'listening' from the UI until the mic delivers its first PCM
          // (surfaced by markCaptureLive). Showing it during the ~150-200ms audio
          // warm-up makes users speak into a mic that isn't recording yet.
          listeningPendingCaptureLive = true;
          return;
        }
        options.onStateChanged?.(nextState);
        if (nextState === 'done' || nextState === 'error') {
          doneWaiters.splice(0).forEach((resolve) => resolve());
        }
      },
      onDraftChanged(text, segment, source: VoiceInputDraftSource) {
        publishText(text, [segment.id], source === 'partial' ? 'deferred' : 'immediate');
      },
      onSubmitted(text, segment) {
        const range = publishText(text, [segment.id], 'immediate');
        if (range) {
          // A pause-time refinement may already be the submitted text; learning
          // still compares against the raw transcript it was based on.
          submittedTextForLearning = segment.basedOnText ?? text;
          recordSubmittedHistory(text);
        }
        return range;
      },
      // No onRefinementPreview: a streamed refinement is only a prefix of the
      // final text (removed fillers shift everything after them). Like desktop,
      // keep the current text until the validated result arrives via
      // applyRefinement instead of showing a half-finished hybrid.
      applyRefinement(range, refinedText) {
        const applied = publishText(refinedText, range.segmentIds, 'immediate') !== undefined;
        if (applied) {
          updateSubmittedHistory(refinedText);
          if (voiceInsertion && submittedTextForLearning) {
            options.onRefinementApplied?.({
              draft: latestDraft,
              rawTranscriptText: submittedTextForLearning,
              refinedText,
              start: voiceInsertion.start,
              end: voiceInsertion.end,
            });
          }
        }
        return applied;
      },
      isRangeUserTouched(range) {
        if (range.userTouched || voiceInsertionTouched) return true;
        if (!voiceInsertion) return false;
        const sameRange = range.startOffset === voiceInsertion.start
          && range.segmentIds.every((id, index) => voiceInsertionSegmentIds[index] === id);
        return sameRange && !isInsertionIntact(readCurrentDraft(), voiceInsertion);
      },
      onError(message, code: VoiceInputErrorCode | undefined, details) {
        // 有 code 的是 controller 自己分类的失败(message 是英文调试串),按 code 取
        // 本地化文案;没有 code 的来自 provider,那条 message 才是唯一的失败描述。
        const cause = code
          ? mobileVoiceErrorCodeMessage(code)
          : redactMobileVoiceCredentialText(message, options.credential);
        const localizedMessage = details?.transcriptKept
          ? mobileVoiceTranscriptKeptError(cause)
          : cause;
        controllerError = new Error(localizedMessage);
        options.onError?.(localizedMessage);
      },
    },
  });

  return {
    async start() {
      submittedTextForLearning = null;
      captureLive = false;
      listeningPendingCaptureLive = false;
      soundDetected = false;
      audioLevels = new MobileVoiceAudioLevels();
      audioStopping = null;
      captureStopRequestedAt = undefined;
      runStartedAt = options.requestedAt ?? Date.now();
      // A fresh start() begins a new run from ANY previous phase — in particular
      // 'failed': a mic interruption must not poison the next recording on the
      // same session, so the failure record is cleared here too.
      runPhase = 'running';
      asrStartError = null;
      audioFailureError = null;
      controllerError = null;
      // Connect the ASR socket and open the mic CONCURRENTLY instead of serially.
      // The shared controller sets 'listening' synchronously, so the mic goes live
      // immediately instead of waiting for the ~2.4s WebSocket connect + session
      // handshake (the previous serial ordering awaited connect BEFORE opening the
      // mic). Audio captured during the handshake is buffered and replayed by
      // BufferedAsrProvider once the connect settles, so nothing is lost.
      asrStartPromise = controller.start().then(() => {
        // With a connection prewarmed at press-in, the controller subscribes
        // after the provider already connected; this is the moment it is usable.
        logMobileVoice('debug', 'asr ready', { runId: runTag(), elapsedMs: Date.now() - runStartedAt });
      }, (error: unknown) => {
        asrStartError = error;
      });
      // controller.start() assigns the run id synchronously.
      logMobileVoice('info', 'start', {
        runId: runTag(),
        provider: options.credential.asr.provider,
        sampleRate: options.credential.asr.pcmSampleRate ?? 16_000,
        refinement: Boolean(refiner),
      });
      try {
        audioOpening = startAudio({
          sampleRate: options.credential.asr.pcmSampleRate ?? 16_000,
          onChunk: (chunk) => {
            // Drop audio emitted after the run began tearing down capture, so an
            // early stop that waits for the in-flight handshake can't buffer and
            // later flush post-stop speech/noise.
            if (captureTornDown() && !(runPhase === 'stopping' && chunk.drained)) return false;
            audioLevels.append(chunk.pcm16, chunk.trace.sampleRate ?? options.credential.asr.pcmSampleRate ?? 16_000);
            if (!soundDetected && hasPcmSound(chunk.pcm16)) soundDetected = true;
            if (runPhase === 'running') markCaptureLive(chunk.trace, chunk.timestampSource);
            controller.appendAudio(chunk.pcm16, chunk.trace);
          },
          onError: (error) => {
            void handleAudioFailure(error);
          },
        }).then((stop) => { stopAudio = stop; });
        await audioOpening;
      } catch (error) {
        runPhase = 'failed';
        // Surface the mic failure immediately — do NOT wait for the in-flight
        // ASR handshake (a slow/hung connect would otherwise pin the startup
        // and delay the error). controller.cancel() tears the handshake down
        // without blocking on it (BufferedAsrProvider.stop() is non-blocking
        // for an in-flight connect), and asrStartPromise carries its own
        // .catch so the late settle can never become an unhandled rejection.
        await controller.cancel().catch(() => undefined);
        logMobileVoiceFailure('start failed', 'microphone', error);
        throw redactMobileVoiceError(error, options.credential);
      }
      if (captureTornDown() && stopAudio) {
        // cancel() (or a mic failure) landed while native capture was still
        // opening: its teardown saw stopAudio === null and could not close the
        // capture, so close the just-opened handle here — otherwise the mic and
        // its event subscriptions stay live even though the run is abandoned.
        await stopNativeCapture();
      }
      // Wait for the ASR connection to settle before declaring the start
      // successful, so a failed handshake still surfaces as a startup error (and
      // the optional start cue is only played once both capture and ASR are up).
      await asrStartPromise;
      if (currentRunPhase() === 'cancelled') {
        // The run was cancelled (or ended silently) while the handshake was
        // still settling; the ASR wrapper already tears the just-opened provider
        // down. A handshake outcome for an abandoned run is not an error, so
        // return quietly without announcing a successful start.
        return;
      }
      if (asrStartError) {
        runPhase = 'failed';
        if (stopAudio) {
          const stop = stopAudio;
          stopAudio = null;
          await stop().catch(() => undefined);
        }
        await controller.cancel().catch(() => undefined);
        logMobileVoiceFailure('start failed', 'asr', asrStartError);
        throw redactMobileVoiceError(asrStartError, options.credential);
      }
      if (currentRunPhase() === 'failed') {
        // The mic failed while the ASR handshake was still settling.
        // handleAudioFailure already surfaced the error and cancelled the
        // controller; don't report a successful start (the caller would otherwise
        // treat the run as active and could append audio to a cancelled session).
        throw redactMobileVoiceError(
          audioFailureError ?? new Error('Realtime voice capture was interrupted during startup.'),
          options.credential,
        );
      }
      // ASR 会话已建立(refine session 同步就绪),此刻预热润色 prompt cache,
      // 让缓存赶在用户停止说话前热起来。
      warmManagedRefinerPromptCache();
      notifyReadyForStartCue();
    },
    async stop() {
      // Stop native capture FIRST — before awaiting the in-flight ASR handshake.
      // Capture runs concurrently with connect, so waiting for the handshake first
      // would keep the mic live for the whole remaining WebSocket connect; audio the
      // user produces after releasing stop would keep flowing into the buffer and be
      // flushed once ASR settles, transcribing post-stop speech/noise. Entering
      // 'stopping' also makes onChunk drop any straggler chunk from the teardown.
      // Pre-stop audio buffered inside BufferedAsrProvider is still replayed once
      // the connect settles, so short utterances are not lost.
      if (runPhase === 'running') runPhase = 'stopping';
      const stopRequestedAt = Date.now();
      captureStopRequestedAt = stopRequestedAt;
      logMobileVoice('debug', 'stop requested', {
        runId: runTag(),
        elapsedMs: stopRequestedAt - runStartedAt,
        captureLive,
        soundDetected,
        ...audioLevels.summary(),
        hasTranscript: Boolean(voiceInsertion),
      });
      // An immediate release can beat the native start receipt. Wait only for
      // capture (never ASR), then drain before deciding this was a silent run.
      if (!stopAudio && !audioStopping) await audioOpening?.catch(() => undefined);
      await stopNativeCapture();
      if (currentRunPhase() === 'stopping' && !soundDetected && !voiceInsertion) {
        logMobileVoice('info', 'silent recording ended without ASR finalization', {
          runId: runTag(),
          elapsedMs: Date.now() - runStartedAt,
          captureLive,
        });
        // Nothing audible was captured and nothing was recognized: end now
        // instead of waiting for a pending cloud start, final ASR or refinement.
        // Audio already sent is not recalled. 'cancelled' also lets a start()
        // still awaiting the handshake return quietly.
        runPhase = 'cancelled';
        cancelPendingDraftPublish();
        await controller.cancel().catch(() => undefined);
        notifyReadyForEndCue();
        return latestDraft;
      }
      // Then wait for the ASR handshake to settle (it runs concurrently with
      // capture) so the failure checks below observe its final outcome. The
      // pre-stop audio replay + flush ordering is handled by BufferedAsrProvider.
      const handshakeWaitStartedAt = Date.now();
      if (asrStartPromise) await asrStartPromise.catch(() => undefined);
      const handshakeWaitMs = Date.now() - handshakeWaitStartedAt;
      if (handshakeWaitMs > 50) logMobileVoice('debug', 'stop waited for ASR connection', { waitMs: handshakeWaitMs });
      if (asrStartError || audioFailureError) {
        // The run is dead — either the ASR handshake failed during an early stop,
        // or the mic failed while the handshake was still settling. Surface it
        // instead of returning a "successful" empty draft the caller shows as
        // done, and don't submit against an already-cancelled controller.
        runPhase = 'failed';
        await controller.cancel().catch(() => undefined);
        throw redactMobileVoiceError(
          asrStartError ?? audioFailureError ?? new Error('Realtime voice capture was interrupted during startup.'),
          options.credential,
        );
      }
      notifyReadyForEndCue();
      await controller.stop();
      if (state === 'error' || controllerError) {
        runPhase = 'failed';
        throw controllerError ?? new Error(i18n.t('composer.voice.incomplete'));
      }
      if (state === 'submitting' || state === 'refining') await waitForDone(doneWaiters);
      if (runPhase === 'stopping') runPhase = 'idle';
      logMobileVoice('debug', 'stop finished', { runId: runTag(), stopToDoneMs: Date.now() - stopRequestedAt, state });
      return latestDraft;
    },
    async cancel() {
      cancelPendingDraftPublish();
      // 'cancelled' makes onChunk drop straggler capture, and tells a start()
      // that is still awaiting the handshake to return quietly. The ASR-side
      // teardown (discard buffered audio, close a socket that only finishes
      // opening later) is handled by BufferedAsrProvider when controller.cancel()
      // reaches its stop().
      runPhase = 'cancelled';
      try {
        if (stopAudio) {
          const stop = stopAudio;
          stopAudio = null;
          await stop();
        }
      } finally {
        // Native drain can reject (overflow / missing history). Cancellation
        // still owns filter disposal and closing the ASR connection.
        await controller.cancel();
      }
    },
    currentDraft() {
      return readCurrentDraft();
    },
  };

  async function handleAudioFailure(error: Error): Promise<void> {
    // Only a live run can fail: ignore repeated failures, and ignore mic errors
    // surfacing after the run was already cancelled (teardown noise).
    if (runPhase !== 'running' && runPhase !== 'stopping') return;
    runPhase = 'failed';
    audioFailureError = error;
    logMobileVoiceFailure('capture failed', 'microphone', error);
    cancelPendingDraftPublish();
    state = 'error';
    options.onStateChanged?.('error');
    options.onError?.(redactMobileVoiceCredentialText(error, options.credential));
    const stop = stopAudio;
    stopAudio = null;
    try {
      await stop?.();
    } finally {
      await controller.cancel().catch(() => undefined);
    }
  }

  async function stopNativeCapture(): Promise<void> {
    if (audioStopping) return audioStopping;
    const stop = stopAudio;
    if (!stop) return;
    stopAudio = null;
    audioStopping = stop(captureStopRequestedAt).catch(async (error: unknown) => {
      await handleAudioFailure(error instanceof Error ? error : new Error(String(error)));
    });
    return audioStopping;
  }

  function recordSubmittedHistory(text: string): void {
    if (!options.recordHistory) return;
    // 下面的 .catch 只兜得住异步拒绝;同步抛会顺着 onSubmitted 冒出 controller,
    // 被当成「宿主没接住这段文本」——文字其实已经写进草稿了,而 controller 还会
    // 据此认定本次听写失败。历史记录是附带能力,不该有这个权力。
    let recording: ReturnType<NonNullable<MobileVoiceControllerOptions['recordHistory']>>;
    try {
      recording = options.recordHistory(text);
    } catch (error) {
      console.warn(
        '[mobile-voice] record history failed (non-fatal):',
        error instanceof Error ? error.message : String(error),
      );
      return;
    }
    historyEntryPromise = Promise.resolve(recording)
      .then((entryId) => {
        if (typeof entryId === 'string' && entryId.trim()) {
          historyEntryId = entryId.trim();
          if (pendingHistoryUpdateText) {
            updateHistoryEntryNow(historyEntryId, pendingHistoryUpdateText);
            pendingHistoryUpdateText = null;
          }
          return historyEntryId;
        }
        return null;
      })
      .catch(() => null);
  }

  function updateSubmittedHistory(text: string): void {
    if (!options.updateHistoryEntry) return;
    pendingHistoryUpdateText = text;
    if (historyEntryId) {
      updateHistoryEntryNow(historyEntryId, text);
      pendingHistoryUpdateText = null;
    }
    void historyEntryPromise;
  }

  function updateHistoryEntryNow(entryId: string, text: string): void {
    if (!options.updateHistoryEntry) return;
    void Promise.resolve(options.updateHistoryEntry(entryId, text)).catch(() => undefined);
  }
}

function buildEditableRange(
  insertion: MobileVoiceDraftInsertion,
  segmentIds: string[],
  userTouched: boolean,
): EditableRange {
  return {
    id: `mobile-voice-range-${segmentIds.join('-')}`,
    segmentIds,
    startOffset: insertion.start,
    endOffset: insertion.end,
    userTouched,
  };
}

function isInsertionIntact(draft: string, insertion: MobileVoiceDraftInsertion): boolean {
  return insertion.start >= 0
    && insertion.end >= insertion.start
    && insertion.end <= draft.length
    && draft.slice(insertion.start, insertion.end) === insertion.text;
}

function replaceInsertionText(
  draft: string,
  insertion: MobileVoiceDraftInsertion,
  text: string,
): string {
  return `${draft.slice(0, insertion.start)}${text}${draft.slice(insertion.end)}`;
}


function logMobileVoiceFailure(message: string, stage: 'microphone' | 'asr', error: unknown): void {
  logMobileVoice('warn', message, {
    stage,
    rateLimited: isMobileVoiceRateLimited(error),
    reason: classifyMobileVoiceFailure(error),
  });
}

function redactMobileVoiceError(error: unknown, credential: StoredMobileVoiceCredential): Error {
  // A recognized account limit gets the localized "wait and retry" text instead
  // of whatever the server or the last fallback candidate reported.
  const rateLimitMessage = mobileVoiceRateLimitMessage(error);
  if (rateLimitMessage) return new Error(rateLimitMessage);
  const message = redactMobileVoiceCredentialText(
    error instanceof Error ? error.message : String(error),
    credential,
  );
  const redacted = new Error(message);
  if (error instanceof Error) redacted.name = error.name;
  return redacted;
}

function mobileRefinerPromptCacheScope(credential: StoredMobileVoiceCredential): string {
  return `mobile-voice:${credential.hostDeviceId}`;
}

function createMobileRefiner(
  credential: StoredMobileVoiceCredential,
  options: {
    refinementContext?: DictationRefinementContext;
    localVoiceInputHistory?: MobileVoiceInputHistorySource;
    refinerTargetProvider?: (provider: string, options?: { refreshAccessToken?: boolean }) => Promise<{
      url: string;
      authorization: string;
    }>;
  },
): DictationRefiner | undefined {
  if (credential.settings?.refinementEnabled === false) return undefined;
  const refinerTargetProvider = options.refinerTargetProvider;
  if (!refinerTargetProvider) {
    throw new Error(i18n.t('composer.voice.missingRefinerTargetProvider'));
  }
  // 托管单请求:模型选择与 failover 在 voice-server(provider/model 都传 'auto',
  // 服务端按自己配置的模型链覆盖),客户端不再做多 provider fallback。
  return new DictationRefiner({
    client: new MobileLiteLlmTextModelClient({
      requestTargetProvider: (requestOptions) =>
        refinerTargetProvider(CINDY_MANAGED_REFINER_PROVIDER, requestOptions),
    }),
    model: CINDY_MANAGED_REFINER_PROVIDER,
    promptCacheScope: mobileRefinerPromptCacheScope(credential),
    contextProvider: () => buildMobileVoiceRefinementContext(credential, {
      refinementContext: options.refinementContext,
      localVoiceInputHistory: typeof options.localVoiceInputHistory === 'function'
        ? options.localVoiceInputHistory()
        : options.localVoiceInputHistory,
      // 词典来自被控桌面的只读快照缓存(拉取在开麦时异步触发)。桌面离线或还没
      // 拉到时是空数组 —— 润色照常进行,只是少了术语提示。
      dictionaryEntries: readCachedMobileVoiceDictionary(credential.hostDeviceId),
    }),
  });
}

function waitForDone(waiters: Array<() => void>): Promise<void> {
  return new Promise((resolve) => {
    waiters.push(resolve);
  });
}