Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

40.4 KB
Raw Blame
import type {
  AsrEvent,
  AsrProvider,
  AudioTrace,
  EditableRange,
  RefinementResult,
  SpeechSegment,
  VoiceInputCallbacks,
  VoiceInputErrorCode,
  VoiceInputDraftReason,
  VoiceInputDraftSource,
  VoiceInputState,
  VoiceInputTerminalOutcome,
} from './types';
import { VoiceTimelineLogger } from './VoiceTimelineLogger';
import type { AudioGate } from './SpeakerGate';
import { hasAdditionalSentence } from './pauseRefinement';

type DictationRefiner = {
  refine(input: {
    text: string;
    runId: string;
    segmentIds: string[];
    onPartial?: (text: string) => void;
  }): Promise<RefinementResult>;
};

type PendingRefinement = {
  text: string;
  segmentIds: string[];
  startedAt: number;
  latestPreview?: string;
  onPreview?: (text: string) => void;
  promise: Promise<{ result?: RefinementResult; error?: unknown }>;
  settled?: boolean;
  result?: RefinementResult;
};

type VoiceInputControllerOptions = {
  asr: AsrProvider;
  refiner?: DictationRefiner;
  logger: VoiceTimelineLogger;
  callbacks: VoiceInputCallbacks;
  stableWaitMs?: number;
  /** Host opt-in: refine during a speech pause and publish a live draft. */
  pauseRefinementEnabled?: boolean;
  /** Replay retained audio once after an audible recording ends without text. */
  retryEmptyTranscript?: boolean;
  /** Host opt-in for known recovery failures; undefined keeps the generic error. */
  recoveryErrorMessage?: (error: unknown) => string | undefined;
  /**
   * Optional filter in front of the ASR (e.g. the speaker gate). Activity and
   * stall accounting see the filtered audio, so a run where only bystanders
   * spoke ends as "no speech" instead of an empty-transcript failure.
   */
  audioGate?: AudioGate;
};

type CryptoWithUuid = {
  randomUUID?: () => string;
};

// ASR stall watchdog thresholds. This is diagnostic only: OpenAI realtime can
// legitimately pause partial deltas for 10+ seconds after a long pause, then
// still return a complete transcript on stop. Treating that partial gap as a
// recover trigger caused us to close a healthy session mid-dictation and lose
// words around the reconnect boundary.
//
// The audio-side gate counts ONLY voiced chunks (RMS above threshold), not
// raw audio duration. Counting raw duration meant a silent thinking pause
// looked identical to a real stall: any 6+ seconds of mic-open silence
// triggered false alarms. Voiced gating makes the warning meaningful while
// keeping it non-destructive; network recovery is only driven by explicit
// transport error/disconnected events.
const ASR_STALL_WARN_AFTER_WALL_MS = 4_000;
const ASR_STALL_WARN_AFTER_VOICED_MS = 2_000;
const ASR_STALL_CHECK_INTERVAL_MS = 1_000;
// Retry a few explicit transport failures (not partial gaps) so transient
// gateway/WebSocket interruptions do not abort an otherwise healthy dictation.
// Providers still own replay/merge safety; after the cap, surface the failure.
const ASR_NETWORK_RECOVERY_MAX_ATTEMPTS = 3;
// PCM16 RMS above this counts as "the user is talking right now". 300 is
// well above quiet-room background (~30) and typing/AC noise (~50-150)
// while being below normal speech (~800+) by enough margin to catch soft
// speakers. Hardcoded for now; a per-mic adaptive baseline would be the
// next refinement if false positives ever bite.
const ASR_VOICED_RMS_THRESHOLD = 300;
const EMPTY_TRANSCRIPT_ERROR = 'Voice input detected speech but returned no transcript. Please try again.';

/**
 * VoiceInputController owns click-to-dictate state.
 *
 * It deliberately treats ASR partials as the fast draft lane, Stop as the
 * commit boundary, and refinement as an optional downstream lane. Audio capture
 * stays outside this class so Electron renderer can own getUserMedia while
 * Electron main owns provider sessions and keys.
 */
export class VoiceInputController {
  private readonly asr: AsrProvider;
  private readonly refiner?: DictationRefiner;
  private readonly logger: VoiceTimelineLogger;
  private readonly callbacks: VoiceInputCallbacks;
  private readonly stableWaitMs: number;
  private readonly pauseRefinementEnabled: boolean;
  private readonly retryEmptyTranscript: boolean;
  private readonly recoveryErrorMessage?: (error: unknown) => string | undefined;
  private readonly audioGate?: AudioGate;
  private lastSoundAt = 0;
  private lastTranscriptChangeAt = 0;
  private pauseRefinement?: PendingRefinement;
  private liveRefinementPreview?: { basedOnText: string; text: string };
  private state: VoiceInputState = 'idle';
  private runId = '';
  private startedAt = 0;
  private latestPartial = '';
  private latestStable = '';
  // The most recent transcript regardless of lane, plus which lane produced it.
  // Salvage cannot prefer `latestStable`: every provider emits both lanes as the
  // full aggregate transcript, so a partial arriving after a stable (the next
  // utterance already in progress) is strictly more complete. Preferring the
  // stable lane would drop that tail — exactly the loss this salvage exists to
  // prevent. Stop waits for a new stable after a newer partial; an older stable
  // must never replace the tail while reusing a pause-time refinement.
  private latestTranscript = '';
  private latestTranscriptSource: 'partial' | 'stable' = 'partial';
  private firstPartialSeen = false;
  private firstAudioChunkSeen = false;
  // Run-level speech activity separates an intentional silent stop from an
  // ASR/provider failure that returned no transcript. This is deliberately
  // independent from the stall watchdog's rolling window counters.
  private speechActivitySeen = false;
  private pendingStableResolvers: Array<(text: string | undefined) => void> = [];
  // Set by cancel(); read by the in-flight refine() to short-circuit before
  // emitting more timeline events, calling applyRefinement, or clobbering the
  // 'done' state that cancel() already set. Cleared on each new start().
  private currentRunCancelled = false;
  // Stall watchdog state. lastAsrSignalAt is performance.now() of the most
  // recent partial/stable; voicedAudioMsSinceLastSignal accumulates only
  // trace.durationMs of chunks whose RMS clears the voiced threshold (so
  // silent thinking pauses don't tick toward the warning). audioMsSinceLastSignal
  // is kept as a diagnostic — useful in stall_warning logs to compare voiced
  // vs total audio when judging whether thresholds are tuned right.
  // stallWarned guards against repeated warn-level timeline events for the
  // same stall window — reset on every fresh ASR signal.
  private lastAsrSignalAt = 0;
  private audioMsSinceLastSignal = 0;
  private voicedAudioMsSinceLastSignal = 0;
  private stallWarned = false;
  private stallCheckTimer?: ReturnType<typeof setInterval>;
  private networkRecoveryInFlight = false;
  private networkRecoveryAttempts = 0;
  // Whether this run ever received a partial/stable. Used in the stall warning
  // event so the next reproduction immediately tells us "session never produced
  // anything" vs "stopped after producing some text".
  private everSawAsrSignal = false;
  // Whether this run already handed text to the host via onSubmitted. Guards the
  // failure-path salvage against double insertion: a recovery that rejects after
  // stop() already submitted would otherwise insert the same transcript twice.
  private transcriptEmitted = false;
  // Set when onSubmitted threw. A callback that throws part-way has already run
  // an unknown share of its side effects (mobile publishes the draft, then calls
  // an injected history recorder that can throw), so retrying it through salvage
  // would run those effects a second time. One attempt per run is the contract.
  private submitCallbackThrew = false;

  constructor(options: VoiceInputControllerOptions) {
    this.asr = options.asr;
    this.refiner = options.refiner;
    this.logger = options.logger;
    this.callbacks = options.callbacks;
    this.stableWaitMs = options.stableWaitMs ?? 500;
    this.pauseRefinementEnabled = options.pauseRefinementEnabled ?? false;
    this.retryEmptyTranscript = options.retryEmptyTranscript ?? false;
    this.recoveryErrorMessage = options.recoveryErrorMessage;
    this.audioGate = options.audioGate;

    this.asr.onEvent((event) => this.handleAsrEvent(event));
  }

  get id(): string {
    return this.runId;
  }

  get currentState(): VoiceInputState {
    return this.state;
  }

  async start(): Promise<string> {
    if (this.state !== 'idle' && this.state !== 'done' && this.state !== 'error') return this.runId;

    this.runId = createVoiceInputId();
    this.startedAt = performance.now();
    this.latestPartial = '';
    this.latestStable = '';
    this.latestTranscript = '';
    this.latestTranscriptSource = 'partial';
    this.firstPartialSeen = false;
    this.firstAudioChunkSeen = false;
    this.speechActivitySeen = false;
    this.currentRunCancelled = false;
    this.lastAsrSignalAt = performance.now();
    this.audioMsSinceLastSignal = 0;
    this.voicedAudioMsSinceLastSignal = 0;
    this.stallWarned = false;
    this.networkRecoveryAttempts = 0;
    this.networkRecoveryInFlight = false;
    this.everSawAsrSignal = false;
    this.transcriptEmitted = false;
    this.submitCallbackThrew = false;
    this.lastSoundAt = performance.now();
    this.lastTranscriptChangeAt = performance.now();
    this.pauseRefinement = undefined;
    this.liveRefinementPreview = undefined;
    const gateRunId = this.runId;
    this.audioGate?.begin((chunk, trace) => this.forwardAudio(gateRunId, chunk, trace));
    this.startStallWatchdog();
    this.setState('listening');
    this.logger.record({ type: 'start_clicked', runId: this.runId, at: Date.now() });

    try {
      await this.asr.start();
      return this.runId;
    } catch (error) {
      this.fail(error instanceof Error ? error.message : String(error));
      throw error;
    }
  }

  appendAudio(chunk: ArrayBuffer, trace?: AudioTrace): void {
    if (this.state !== 'listening') return;
    if (this.audioGate) {
      this.audioGate.push(chunk, trace);
      return;
    }
    this.forwardAudio(this.runId, chunk, trace);
  }

  /**
   * Hands audio that passed the optional gate to the ASR. The gate releases
   * held audio while stop() flushes it, so 'submitting' is still accepted.
   */
  private forwardAudio(runId: string, chunk: ArrayBuffer, trace?: AudioTrace): void {
    if (runId !== this.runId) return;
    if (this.state !== 'listening' && this.state !== 'submitting') return;
    if (!this.firstAudioChunkSeen) {
      this.firstAudioChunkSeen = true;
      this.logger.record({
        type: 'first_audio_chunk',
        runId: this.runId,
        at: Date.now(),
        elapsedMs: performance.now() - this.startedAt,
      });
    }
    const voiced = isChunkVoiced(chunk);
    if (voiced) {
      this.lastSoundAt = performance.now();
      this.speechActivitySeen = true;
    }
    if (typeof trace?.durationMs === 'number') {
      this.audioMsSinceLastSignal += trace.durationMs;
      if (voiced) {
        this.voicedAudioMsSinceLastSignal += trace.durationMs;
      }
    }
    this.asr.appendAudio(chunk, trace);
  }

  async stop(): Promise<void> {
    if (this.state !== 'listening') return;
    const runId = this.runId;
    this.stopStallWatchdog();
    this.setState('submitting');
    this.logger.record({ type: 'stop_clicked', runId, at: Date.now() });
    // Release audio still held by the gate before the provider flushes. The
    // gate fails open and bounds its own time, so this never rejects or hangs.
    // Without a gate, stop() must not gain an extra await: the optimistic
    // refinement below relies on starting in the same tick.
    if (this.audioGate) await this.audioGate.finish().catch(() => undefined);
    if (runId !== this.runId || this.leftSubmittingState()) return;
    const optimisticRefinement = this.startOptimisticRefinement(runId);
    const interrupted = (): boolean => {
      if (runId !== this.runId) return true;
      if (!this.currentRunCancelled) return false;
      this.discardRefinement(runId, optimisticRefinement, 'cancelled');
      return true;
    };

    let stable: string | undefined;
    try {
      await this.asr.flushAudio();
      if (interrupted()) return;
      if (!this.leftSubmittingState()) stable = await this.waitForStable(this.stableWaitMs);
      if (interrupted()) return;
      if (!this.leftSubmittingState() && !normalizeSubmittedText(stable || this.latestTranscript)
        && this.speechActivitySeen && this.retryEmptyTranscript && this.asr.recover
        && !this.networkRecoveryInFlight) {
        stable = await this.retryEmptyRecording(runId);
      }
    } catch (error) {
      if (interrupted()) return;
      this.fail(this.recoveryErrorMessage?.(error) ?? (error instanceof Error ? error.message : String(error)));
      await this.asr.stop();
      return;
    }

    if (interrupted()) return;
    const text = normalizeSubmittedText(stable || this.latestTranscript);
    const source = stable ? 'stable' : this.latestTranscriptSource;

    await this.asr.stop();

    // Deliberately placed after the LAST provider await in this method, so it
    // covers a failure landing during any of them. Providers hide such failures
    // from this path: both flushAudio() and stop() await an in-flight recover()
    // but swallow its rejection, so those awaits resolve normally on a run that
    // fail() has already finished (salvaging the transcript and setting the
    // terminal state). Continuing would submit the same transcript twice and
    // overwrite 'error' with 'refining'/'done', hiding the failure entirely.
    // transcriptEmitted is checked too: it is the direct invariant ("this run has
    // already given the host its text"), so it holds even if a future failure
    // path leaves the state alone.
    if (runId !== this.runId) return;
    if (this.leftSubmittingState() || this.transcriptEmitted) {
      // cancel() also lands here (it moves the run to 'done'); logging that as
      // a failure would misread the timeline for an intentional abort.
      this.discardRefinement(runId, optimisticRefinement, this.currentRunCancelled ? 'cancelled' : 'run_failed');
      return;
    }

    if (!text) {
      this.discardRefinement(runId, optimisticRefinement, 'no_submitted_text');
      if (this.speechActivitySeen) {
        this.fail(EMPTY_TRANSCRIPT_ERROR, 'empty_transcript');
      } else {
        this.setState('done', 'no_speech');
      }
      return;
    }

    const readyResult = this.pauseRefinementEnabled && optimisticRefinement?.text === text
      ? optimisticRefinement.result : undefined;
    const readyText = readyResult?.accepted && readyResult.refinedText
      && normalizeSubmittedText(readyResult.basedOnText) === text
      ? readyResult.refinedText : undefined;
    const segment: SpeechSegment = {
      id: optimisticRefinement?.text === text ? optimisticRefinement.segmentIds[0] : createVoiceInputId(),
      source: 'mic',
      status: 'submitted',
      text: readyText ?? text,
      ...(readyText ? { basedOnText: text } : {}),
      updatedAt: Date.now(),
    };
    let range: EditableRange | undefined;
    try {
      range = this.callbacks.onSubmitted(readyText ?? text, segment);
    } catch (error) {
      // salvageTranscript() already treats a throwing host as a real scenario
      // (destroyed window, torn-down editor); this path has to survive it too.
      // Letting the exception escape would reject stop() before any terminal
      // state is set, stranding the run in 'submitting' — after which start()
      // refuses to begin a new one and dictation is dead until reload.
      this.submitCallbackThrew = true;
      this.discardRefinement(runId, optimisticRefinement, 'run_failed');
      this.fail(error instanceof Error ? error.message : String(error));
      return;
    }
    // Same acceptance convention as salvageTranscript(): the returned range is
    // the host's "I took the text" signal, and a host may refuse (mobile stops
    // writing into the voice insertion once the user has edited it). Marking it
    // emitted regardless would let a later failure claim transcriptKept for
    // text that never landed.
    this.transcriptEmitted = Boolean(range);
    this.logger.record({ type: 'submitted', runId, at: Date.now(), text, source });

    if (this.refiner && range) {
      let refinement: PendingRefinement;
      if (optimisticRefinement?.text === text) {
        refinement = optimisticRefinement;
      } else {
        this.discardRefinement(runId, optimisticRefinement, 'final_text_changed');
        refinement = this.startRefinementRequest(runId, text, [segment.id]);
      }
      this.setState('refining');
      void this.finishRefinement(runId, segment, range, refinement);
    } else {
      this.discardRefinement(runId, optimisticRefinement, 'no_editable_range');
      this.setState('done');
    }
  }

  private async retryEmptyRecording(runId: string): Promise<string | undefined> {
    const startedAt = performance.now();
    this.logger.record({
      type: 'asr_recovery_attempted', runId, at: Date.now(), trigger: 'empty_transcript', attempt: 1,
      audioMsSinceLastSignal: this.audioMsSinceLastSignal,
      voicedAudioMsSinceLastSignal: this.voicedAudioMsSinceLastSignal,
      wallMsSinceLastSignal: startedAt - this.lastAsrSignalAt,
      everSawSignal: this.everSawAsrSignal,
    });
    try {
      // recover() owns replay of unconfirmed in-memory audio. stop() would
      // clear that audio, so only tear the provider down after this one attempt.
      await this.asr.recover!();
      if (runId !== this.runId || this.leftSubmittingState()) return;
      await this.asr.flushAudio();
      if (runId !== this.runId || this.leftSubmittingState()) return;
      const stable = await this.waitForStable(this.stableWaitMs);
      if (runId !== this.runId || this.leftSubmittingState()) return;
      const elapsedMs = performance.now() - startedAt;
      if (normalizeSubmittedText(stable || this.latestTranscript)) {
        this.logger.record({ type: 'asr_recovery_succeeded', runId, at: Date.now(), elapsedMs });
      } else {
        this.logger.record({ type: 'asr_recovery_failed', runId, at: Date.now(), elapsedMs, reason: 'empty_transcript' });
      }
      return stable;
    } catch (error) {
      if (runId !== this.runId || this.leftSubmittingState()) return;
      this.logger.record({
        type: 'asr_recovery_failed', runId, at: Date.now(), elapsedMs: performance.now() - startedAt,
        reason: error instanceof Error ? error.message : String(error),
      });
      throw error;
    }
  }

  /**
   * True when something moved this run out of the state stop() set on entry —
   * fail() from a rejected recovery, or cancel(). Deliberately a method: inside
   * stop(), the `state !== 'listening'` guard narrows this.state for the whole
   * body, so an inline comparison against 'submitting' is a type error.
   */
  private leftSubmittingState(): boolean {
    return this.state !== 'submitting';
  }

  async cancel(): Promise<void> {
    if (!this.runId) return;
    const runId = this.runId;
    this.currentRunCancelled = true;
    this.audioGate?.abort();
    this.stopStallWatchdog();
    this.pendingStableResolvers.splice(0).forEach((resolve) => resolve(undefined));
    this.logger.record({ type: 'cancelled', runId, at: Date.now() });
    // Enter the terminal state before closing the provider. Several realtime
    // transports emit a final `disconnected` while their socket is intentionally
    // closed; if we remain in `listening`, that normal teardown is indistinguishable
    // from a network drop and surfaces a false "connection interrupted" error.
    this.setState('done', 'cancelled');
    await this.asr.stop();
  }

  private handleAsrEvent(event: AsrEvent): void {
    if (!this.runId) return;

    switch (event.type) {
      case 'connected':
        this.logger.record({
          type: 'asr_connected',
          runId: this.runId,
          at: Date.now(),
          elapsedMs: performance.now() - this.startedAt,
        });
        break;
      case 'partial':
        if (this.state !== 'listening' && this.state !== 'submitting') return;
        if (event.text !== this.latestTranscript) this.lastTranscriptChangeAt = performance.now();
        this.latestPartial = event.text;
        this.latestTranscript = event.text;
        this.latestTranscriptSource = 'partial';
        this.resetStallCounters();
        if (this.state !== 'listening') return;
        if (!this.firstPartialSeen) {
          this.firstPartialSeen = true;
          this.logger.record({
            type: 'first_partial',
            runId: this.runId,
            at: Date.now(),
            elapsedMs: performance.now() - this.startedAt,
            text: event.text,
          });
        }
        this.publishDraft(event.text, 'partial');
        break;
      case 'stable':
        if (this.state !== 'listening' && this.state !== 'submitting') return;
        if (event.text !== this.latestTranscript) this.lastTranscriptChangeAt = performance.now();
        this.latestStable = event.text;
        this.latestTranscript = event.text;
        this.latestTranscriptSource = 'stable';
        this.resetStallCounters();
        this.logger.record({
          type: 'stable_received',
          runId: this.runId,
          at: Date.now(),
          text: event.text,
        });
        if (this.state === 'listening') {
          this.publishDraft(event.text, 'stable');
        }
        this.pendingStableResolvers.splice(0).forEach((resolve) => resolve(event.text));
        break;
      case 'error':
        if (this.currentRunCancelled) break;
        if (this.state === 'submitting' || this.state === 'refining') {
          const text = normalizeSubmittedText(this.latestStable || this.latestPartial);
          if (text) {
            this.logger.record({
              type: 'asr_stop_error_ignored',
              runId: this.runId,
              at: Date.now(),
              message: event.message,
              textChars: text.length,
            });
            break;
          }
        }
        // Try recover before failing: keepalive timeouts and transient
        // network errors arrive here as `error`. Partial stalls are only
        // diagnostic; this self-heal path is reserved for transport failures.
        if (this.state === 'listening' && this.tryRecover('error')) break;
        this.fail(event.message);
        break;
      case 'disconnected':
        if (this.currentRunCancelled) break;
        // Same as 'error' above. Without this hook, a keepalive-driven
        // socket terminate would emit `disconnected` and bypass recover
        // entirely, producing the exact failure mode the resilience pass
        // was supposed to prevent.
        if (this.state === 'listening' && this.tryRecover('disconnected')) break;
        if (this.state === 'listening') {
          this.fail(
            'Voice input connection was interrupted. Please try again.',
            'connection_interrupted',
          );
        }
        break;
    }
  }

  // Returns true if recover was kicked off (caller should NOT call fail()).
  // Returns false if recover is unavailable or already in flight from a
  // concurrent trigger; in the in-flight case the existing recovery will
  // either succeed and continue, or fail in its own catch handler.
  private tryRecover(trigger: 'error' | 'disconnected'): boolean {
    if (typeof this.asr.recover !== 'function') return false;
    if (this.networkRecoveryInFlight) return true;
    if (this.networkRecoveryAttempts >= ASR_NETWORK_RECOVERY_MAX_ATTEMPTS) return false;
    this.attemptNetworkRecovery(
      trigger,
      this.audioMsSinceLastSignal,
      this.voicedAudioMsSinceLastSignal,
      performance.now() - this.lastAsrSignalAt,
    );
    return true;
  }

  private publishDraft(text: string, source: VoiceInputDraftSource): void {
    let normalized = text.trim();
    if (!normalized) return;
    const preview = this.liveRefinementPreview;
    const rawText = normalizeSubmittedText(text);
    if (preview && rawText.startsWith(preview.basedOnText)) {
      normalized = preview.text + rawText.slice(preview.basedOnText.length);
      source = 'refinement';
    }
    const reason: VoiceInputDraftReason = source === 'refinement'
      ? 'refinement_preview'
      : source === 'stable' ? 'asr_stable' : 'asr_partial';
    const segment: SpeechSegment = {
      id: `draft-${this.runId}`,
      source: 'mic',
      status: 'draft',
      text: normalized,
      ...(source === 'refinement' ? { basedOnText: rawText } : {}),
      updatedAt: Date.now(),
    };
    this.callbacks.onDraftChanged(normalized, segment, source);
    this.logger.record({
      type: 'draft_changed',
      runId: this.runId,
      at: Date.now(),
      text: normalized,
      reason,
      source,
    });
  }

  private waitForStable(timeoutMs: number): Promise<string | undefined> {
    if (this.latestTranscriptSource === 'stable' && this.latestTranscript) {
      return Promise.resolve(this.latestTranscript);
    }
    return new Promise((resolve) => {
      const resolver = (text: string | undefined): void => {
        clearTimeout(timer);
        this.pendingStableResolvers = this.pendingStableResolvers.filter((item) => item !== resolver);
        resolve(text);
      };
      const timer = setTimeout(() => {
        this.pendingStableResolvers = this.pendingStableResolvers.filter((item) => item !== resolver);
        resolve(undefined);
      }, timeoutMs);
      this.pendingStableResolvers.push(resolver);
    });
  }

  private startOptimisticRefinement(runId: string): PendingRefinement | undefined {
    if (!this.refiner) return undefined;
    const text = normalizeSubmittedText(this.latestTranscript);
    if (!text) return undefined;
    if (this.pauseRefinement?.text === text) return this.pauseRefinement;
    // A pause request already became stale, or the visible tail is still
    // provisional. Wait for final ASR text instead of speculating again.
    if (this.pauseRefinementEnabled && (this.pauseRefinement || this.latestTranscriptSource !== 'stable')) return undefined;
    // Stop-time ASR finalization can take close to a second. Start refinement
    // against the visible draft immediately, but only apply it if the final
    // ASR text matches. If the provider corrects or extends the text, we
    // discard this request and refine the final transcript instead.
    return this.startRefinementRequest(runId, text, [createVoiceInputId()]);
  }

  private maybeRefineDuringPause(): void {
    if (!this.pauseRefinementEnabled || !this.refiner || this.state !== 'listening') return;
    const now = performance.now();
    if (now - this.lastSoundAt < 2_000 || now - this.lastTranscriptChangeAt < 2_000) return;
    if (this.pauseRefinement && !this.pauseRefinement.settled) return;
    const text = normalizeSubmittedText(this.latestTranscript);
    if (!hasAdditionalSentence(this.pauseRefinement?.text ?? '', text)) return;
    const runId = this.runId;
    const request = this.startRefinementRequest(runId, text, [createVoiceInputId()]);
    this.pauseRefinement = request;
    void request.promise.then(({ result }) => {
      if (runId !== this.runId || this.state !== 'listening' || this.currentRunCancelled) return;
      if (this.pauseRefinement !== request || !result?.accepted || !result.refinedText) return;
      if (normalizeSubmittedText(result.basedOnText) !== request.text) return;
      // A newer ASR revision may correct the beginning. Never overlay a result
      // on a different source; if only a suffix grew, keep that new suffix.
      if (!normalizeSubmittedText(this.latestTranscript).startsWith(request.text)) return;
      this.liveRefinementPreview = { basedOnText: request.text, text: result.refinedText };
      this.publishDraft(this.latestTranscript, this.latestTranscriptSource);
    });
  }

  private startRefinementRequest(runId: string, text: string, segmentIds: string[]): PendingRefinement {
    if (!this.refiner) throw new Error('Dictation refiner is not configured.');
    const refineStartedAt = performance.now();
    this.logger.record({ type: 'refine_requested', runId, at: Date.now(), text });

    const request: PendingRefinement = {
      text,
      segmentIds,
      startedAt: refineStartedAt,
      promise: Promise.resolve({}),
    };
    request.promise = this.refiner.refine({
      text,
      runId,
      segmentIds,
      onPartial: (partial) => {
        request.latestPreview = partial;
        request.onPreview?.(partial);
      },
    }).then(
      (result) => { request.settled = true; request.result = result; return { result }; },
      (error: unknown) => { request.settled = true; return { error }; },
    );
    return request;
  }

  private async finishRefinement(
    runId: string,
    segment: SpeechSegment,
    range: EditableRange,
    request: PendingRefinement,
  ): Promise<void> {
    // A completed result may already have been submitted atomically. Replaying
    // its earlier streaming preview would briefly replace it with older text.
    if (!request.settled) this.attachRefinementPreview(runId, segment, range, request);
    const { result, error } = await request.promise;
    if (runId !== this.runId) {
      this.discardRefinement(runId, request, 'stale_run');
      return;
    }
    if (this.currentRunCancelled) {
      this.discardRefinement(runId, request, 'cancelled');
      return;
    }
    // A failure can land while refinement is in flight (a recovery rejecting
    // after stop() already submitted). Every exit below writes a terminal state,
    // so continuing would overwrite 'error' with 'done' and erase the failure
    // the user was just told about — along with applying refined text to a run
    // whose host already reported it as failed.
    if (this.state === 'error') {
      this.discardRefinement(runId, request, 'run_failed');
      return;
    }

    if (error) {
      this.logger.record({
        type: 'refine_rejected',
        runId,
        at: Date.now(),
        reason: error instanceof Error ? error.message : String(error),
        elapsedMs: performance.now() - request.startedAt,
      });
      this.setState('done');
      return;
    }

    if (!result?.accepted || !result.refinedText) {
      this.logger.record({
        type: 'refine_rejected',
        runId,
        at: Date.now(),
        reason: result?.rejectionReason ?? 'unknown',
        elapsedMs: result?.elapsedMs,
        basedOnText: result?.basedOnText,
        refinedText: result?.refinedText,
      });
      this.setState('done');
      return;
    }

    this.logger.record({
      type: 'refine_accepted',
      runId,
      at: Date.now(),
      basedOnText: result.basedOnText,
      refinedText: result.refinedText,
      elapsedMs: result.elapsedMs,
    });

    if (this.callbacks.isRangeUserTouched?.(range) ?? range.userTouched) {
      this.logger.record({ type: 'refine_skipped_user_touched', runId, at: Date.now(), rangeId: range.id });
      this.setState('done');
      return;
    }

    const applied = this.callbacks.applyRefinement?.(range, result.refinedText) ?? false;
    if (applied) {
      this.logger.record({
        type: 'refine_applied',
        runId,
        at: Date.now(),
        rangeId: range.id,
        refinedText: result.refinedText,
      });
    }
    this.setState('done');
  }

  private attachRefinementPreview(
    runId: string,
    segment: SpeechSegment,
    range: EditableRange,
    request: PendingRefinement,
  ): void {
    const emitPreview = (text: string): void => {
      const normalized = text.replace(/\r\n?/g, '\n').trim();
      if (!normalized || normalized === request.text) return;
      // Same reason as finishRefinement's check: once the run has failed, its
      // host is showing an error — streaming refinement previews into it would
      // rewrite text the user was just told the session stopped producing.
      if (runId !== this.runId || this.currentRunCancelled || this.state === 'error') return;
      if (this.callbacks.isRangeUserTouched?.(range) ?? range.userTouched) return;
      this.callbacks.onRefinementPreview?.(normalized, {
        ...segment,
        status: 'draft',
        basedOnText: request.text,
        text: normalized,
        updatedAt: Date.now(),
      }, range);
    };
    request.onPreview = emitPreview;
    if (request.latestPreview) emitPreview(request.latestPreview);
  }

  private discardRefinement(
    runId: string,
    request: PendingRefinement | undefined,
    reason: 'final_text_changed' | 'stale_run' | 'cancelled' | 'no_submitted_text' | 'no_editable_range' | 'run_failed',
  ): void {
    if (!request) return;
    this.logger.record({
      type: 'refine_discarded',
      runId,
      at: Date.now(),
      reason,
      basedOnText: request.text,
    });
  }

  private setState(state: VoiceInputState, outcome?: VoiceInputTerminalOutcome): void {
    this.state = state;
    const terminalOutcome = outcome
      ?? (state === 'error' ? 'failed' : state === 'done' ? 'success' : undefined);
    this.callbacks.onStateChanged?.(state, terminalOutcome);
  }

  private fail(message: string, code?: VoiceInputErrorCode): void {
    // Idempotent: a recover() race can plausibly produce two fail() calls in
    // the same tick (one from a stale 'disconnected' event, one from
    // recovery_failed). Without this guard the renderer toasts both, which
    // shows the user a confusing second error after they already saw the first.
    if (this.state === 'error' || this.state === 'done') return;
    this.audioGate?.abort();
    if (this.runId) {
      this.logger.record({ type: 'error', runId: this.runId, at: Date.now(), message, code });
    }
    this.stopStallWatchdog();
    this.pendingStableResolvers.splice(0).forEach((resolve) => resolve(undefined));
    // Salvage before the terminal state: hosts tear the draft down when they see
    // 'error', so the submitted text has to reach them first.
    this.salvageTranscript();
    this.setState('error', 'failed');
    // Retention travels as a detail, never as the code: `message` may be the
    // only thing telling the user their credential expired or their quota ran
    // out. transcriptEmitted covers both salvage and a stop() that submitted
    // before the failure landed (a recovery rejecting during refinement).
    this.callbacks.onError?.(message, code, { transcriptKept: this.transcriptEmitted });
  }

  /**
   * Hand already-recognized text to the host before entering the error state.
   *
   * A socket that dies mid-dictation used to discard everything the user had
   * said so far: partials only ever existed as a host-side draft, and fail()
   * never offered them to onSubmitted. The transcript is worth more than the
   * dead session, so commit it raw — refinement needs the same network that
   * just failed — and let the host report the failure with transcriptKept set.
   *
   * The host's returned range is the acceptance signal. A host can legitimately
   * refuse: mobile stops writing into the voice insertion once the user has
   * edited it, so the text never lands. Claiming retention there would tell the
   * user their words were kept when the tail was actually dropped.
   */
  private salvageTranscript(): boolean {
    if (this.transcriptEmitted || this.submitCallbackThrew) return false;
    const text = normalizeSubmittedText(this.latestTranscript);
    if (!text) return false;
    const segment: SpeechSegment = {
      id: createVoiceInputId(),
      source: 'mic',
      status: 'submitted',
      text,
      updatedAt: Date.now(),
    };
    let accepted = false;
    try {
      accepted = Boolean(this.callbacks.onSubmitted(text, segment));
    } catch {
      // A host that cannot take the text (destroyed window, torn-down editor)
      // must not also swallow the error report the user is waiting for.
      accepted = false;
    }
    this.logger.record({
      type: 'transcript_salvaged',
      runId: this.runId,
      at: Date.now(),
      text,
      source: this.latestTranscriptSource,
      accepted,
    });
    if (!accepted) return false;
    this.transcriptEmitted = true;
    return true;
  }

  private resetStallCounters(): void {
    this.lastAsrSignalAt = performance.now();
    this.audioMsSinceLastSignal = 0;
    this.voicedAudioMsSinceLastSignal = 0;
    this.stallWarned = false;
    this.everSawAsrSignal = true;
  }

  private startStallWatchdog(): void {
    this.stopStallWatchdog();
    this.stallCheckTimer = setInterval(() => this.checkAsrStall(), ASR_STALL_CHECK_INTERVAL_MS);
  }

  private stopStallWatchdog(): void {
    if (this.stallCheckTimer === undefined) return;
    clearInterval(this.stallCheckTimer);
    this.stallCheckTimer = undefined;
  }

  private checkAsrStall(): void {
    if (this.state !== 'listening') {
      this.stopStallWatchdog();
      return;
    }
    if (this.networkRecoveryInFlight) return;
    this.maybeRefineDuringPause();

    const wallMs = performance.now() - this.lastAsrSignalAt;
    const audioMs = this.audioMsSinceLastSignal;
    const voicedMs = this.voicedAudioMsSinceLastSignal;

    if (
      !this.stallWarned &&
      voicedMs >= ASR_STALL_WARN_AFTER_VOICED_MS &&
      wallMs >= ASR_STALL_WARN_AFTER_WALL_MS
    ) {
      this.stallWarned = true;
      this.logger.record({
        type: 'asr_stall_warning',
        runId: this.runId,
        at: Date.now(),
        audioMsSinceLastSignal: Math.round(audioMs),
        voicedAudioMsSinceLastSignal: Math.round(voicedMs),
        wallMsSinceLastSignal: Math.round(wallMs),
        everSawSignal: this.everSawAsrSignal,
      });
    }
  }

  private attemptNetworkRecovery(
    trigger: 'error' | 'disconnected',
    audioMs: number,
    voicedMs: number,
    wallMs: number,
  ): void {
    if (!this.asr.recover) return;
    const runId = this.runId;
    this.networkRecoveryInFlight = true;
    this.networkRecoveryAttempts += 1;
    const startedAt = performance.now();
    this.logger.record({
      type: 'asr_recovery_attempted',
      runId,
      at: Date.now(),
      trigger,
      attempt: this.networkRecoveryAttempts,
      audioMsSinceLastSignal: Math.round(audioMs),
      voicedAudioMsSinceLastSignal: Math.round(voicedMs),
      wallMsSinceLastSignal: Math.round(wallMs),
      everSawSignal: this.everSawAsrSignal,
    });
    void this.asr
      .recover()
      .then(() => {
        if (runId !== this.runId) return;
        // Reset stall counters so the next watchdog window starts fresh.
        this.lastAsrSignalAt = performance.now();
        this.audioMsSinceLastSignal = 0;
        this.voicedAudioMsSinceLastSignal = 0;
        this.stallWarned = false;
        this.logger.record({
          type: 'asr_recovery_succeeded',
          runId,
          at: Date.now(),
          elapsedMs: performance.now() - startedAt,
        });
      })
      .catch((error: unknown) => {
        if (runId !== this.runId) return;
        const reason = error instanceof Error ? error.message : String(error);
        this.logger.record({
          type: 'asr_recovery_failed',
          runId,
          at: Date.now(),
          elapsedMs: performance.now() - startedAt,
          reason,
        });
        this.stopStallWatchdog();
        const message = this.recoveryErrorMessage?.(error);
        this.fail(
          message ?? 'Voice input stopped receiving recognition. Please try again.',
          message === undefined ? 'recognition_stalled' : undefined,
        );
      })
      .finally(() => {
        if (runId !== this.runId) return;
        this.networkRecoveryInFlight = false;
      });
  }
}

function normalizeSubmittedText(text: string): string {
  return text.replace(/\s+/g, ' ').trim();
}

/**
 * True when the chunk's PCM16 RMS is loud enough to plausibly contain speech.
 * Used by the stall watchdog to distinguish "user is silent" from "user is
 * speaking but server is quiet": only voiced chunks count toward the warning
 * threshold, so a 30-second thinking pause never creates noisy diagnostics.
 *
 * Pure silence in a quiet room is RMS ~10-30; typing/AC/fan noise ~50-150;
 * normal speech ~800+. The threshold (300) sits in the gap between background
 * noise and soft speech — well below typical voice levels but high enough to
 * reject room hum.
 */
function isChunkVoiced(chunk: ArrayBuffer): boolean {
  if (chunk.byteLength < 2) return false;
  const sampleCount = Math.floor(chunk.byteLength / 2);
  if (sampleCount === 0) return false;
  const samples = new Int16Array(chunk, 0, sampleCount);
  let sumSquares = 0;
  for (let i = 0; i < sampleCount; i += 1) {
    const s = samples[i];
    sumSquares += s * s;
  }
  const rms = Math.sqrt(sumSquares / sampleCount);
  return rms >= ASR_VOICED_RMS_THRESHOLD;
}

function createVoiceInputId(): string {
  const cryptoLike = (globalThis as { crypto?: CryptoWithUuid }).crypto;
  if (typeof cryptoLike?.randomUUID === 'function') {
    return cryptoLike.randomUUID();
  }
  return `voice-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`;
}