Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

20.9 KB
Raw Blame
import type { AsrEndpointing, AsrProvider, AsrSegment, AudioTrace } from "./types.js";
import { ConversationActivity } from "./conversationActivity.js";
import type { AudioGate } from "./SpeakerGate.js";

// Endpointing only: this input layer never ends voice mode for prolonged silence.
// The host owns idle/intent/lifecycle exits using task and playback activity too;
// ASR transport rotation is separate from that policy.
const AUTO_SEND_QUIET_MS = 1_250;
const FINAL_STABLE_MS = 350;
const BLOCKED_DIAGNOSTIC_INTERVAL_MS = 5_000;
const FINAL_TIMEOUT_MS = 8_000;

type InputFailureReason =
  | "final_timeout"
  | "upstream_disconnected"
  | "upstream_error"
  | "audio_buffer_overflow"
  | "transcript_limit"
  | "rotation_unfinalized"
  | "rotation_failed";

type SettlementBlocker =
  | "speaking"
  | "transport-rotation"
  | "asr-final"
  | "filtered-silence"
  | "audio-silence"
  | "acoustic-silence"
  | "text-silence"
  | "text-stability";

type BlockedDiagnostic = {
  reasons: SettlementBlocker[];
  pendingMs: number;
  speaking: boolean;
  manual: boolean;
  rotating: boolean;
  segments: number;
  pendingFinalSegments: number;
  characters: number;
  speakerFilterActive: boolean;
  acousticQuietRemainingMs: number;
  textQuietRemainingMs: number;
  finalStableRemainingMs: number;
  filteredQuietRemainingMs: number;
  audioQuietRemainingMs: number;
  captureAgeMs: number | null;
  forwardedAudioAgeMs: number | null;
  asrUpdateAgeMs: number | null;
  rms: number | null;
  threshold: number | null;
  noiseFloor: number | null;
};

export type ConversationInputEvent =
  | { type: "ready" }
  | { type: "speech-started" }
  | { type: "speech-idle" }
  | { type: "draft"; text: string }
  | { type: "utterance"; id: string; text: string }
  | { type: "error"; text: string };

/**
 * Continuous ASR ownership and endpointing. Provider IDs, never string-prefix
 * subtraction, identify submitted speech. Connections rotate below the managed
 * service's default lifetime while capture is buffered; the product call lives on.
 */
export class VoiceConversationInput {
  private provider: AsrProvider | null = null;
  private generation = 0;
  private sequence = 0;
  private closed = false;
  private rotating = false;
  private activity = new ConversationActivity();
  private speaking = false;
  private manual = false;
  private segments = new Map<string, AsrSegment & {
    generation: number;
    // Global released-audio clock, surviving provider rotation.
    audioEndPositionMs?: number;
    textQuietAfter: number;
  }>();
  private consumed = new Set<string>();
  private pending: Array<{ pcm: ArrayBuffer; trace?: AudioTrace }> = [];
  private pendingBytes = 0;
  private settleTimer?: ReturnType<typeof setTimeout>;
  private rotationTimer?: ReturnType<typeof setTimeout>;
  private finalTimer?: ReturnType<typeof setTimeout>;
  private finalWaitStartedAt: number | null = null;
  private blockedDiagnosticTimer?: ReturnType<typeof setInterval>;
  private pendingSince = 0;
  private lastCaptureAt: number | null = null;
  private lastForwardedAt: number | null = null;
  private lastAsrUpdateAt: number | null = null;
  private lastActivity: ReturnType<ConversationActivity["push"]> | null = null;
  private awaitingSettlement = false;
  private acousticQuietAfter = 0;
  private audioPositionMs = 0;
  private providerAudioMs = 0;
  private stableAfter = 0;
  private filteredQuietMs = 0;

  constructor(
    private readonly options: {
      createProvider: () => Promise<AsrProvider>;
      audioGate?: AudioGate;
      // Recheck ownership when asynchronously filtered audio is released.
      isCurrent?: () => boolean;
      emit: (event: ConversationInputEvent) => void;
      rotationMs?: number;
      // Numeric endpointing evidence only; the host owns logging and storage.
      diagnostic?: (event: {
        trigger: "manual" | "silence";
        inactiveMs: number;
        pauseBasis: "audio" | "text-fallback";
        endpointing: AsrEndpointing;
        endSignal: AsrEndpointing;
        finalStableMs: number | null;
        segments: number;
        characters: number;
        filteredQuietMs?: number;
      }) => void;
      blockedDiagnostic?: (event: BlockedDiagnostic) => void;
      failureDiagnostic?: (event: {
        reason: InputFailureReason;
        endpointing: AsrEndpointing;
        generation: number;
        manual: boolean;
        rotating: boolean;
        speaking: boolean;
        segments: number;
        pendingFinalSegments: number;
        characters: number;
        pendingMs: number;
        finalWaitMs: number | null;
        asrUpdateAgeMs: number | null;
        bufferedAudioMs: number;
      }) => void;
    },
  ) {}

  async start(): Promise<void> {
    if (this.closed) return;
    // One gate per microphone run, not per ASR transport. Rotation must retain
    // held onsets and accepted short-phrase context. No second debounce here:
    // accepted audio immediately reaches activity detection and ASR together.
    this.options.audioGate?.begin((pcm, trace) =>
      this.forwardAudio(pcm, trace),
    );
    await this.open();
  }

  appendAudio(pcm: ArrayBuffer, trace?: AudioTrace): void {
    if (this.closed) return;
    this.lastCaptureAt = Date.now();
    if (this.options.audioGate) {
      this.options.audioGate.push(pcm, trace);
      return;
    }
    this.forwardAudio(pcm, trace);
  }

  private forwardAudio(pcm: ArrayBuffer, trace?: AudioTrace): void {
    if (this.closed || this.options.isCurrent?.() === false) return;
    this.audioPositionMs += pcm.byteLength / 32;
    const activity = this.activity.push(pcm);
    this.lastActivity = activity;
    this.lastForwardedAt = Date.now();
    this.speaking = activity.active;
    // Wall-clock silence while the gate is still deciding a speaker is not a
    // pause. Require actual released quiet audio too, without adding another
    // 1.25s timer after it. Rejected speech arrives as silence from AudioGate.
    this.filteredQuietMs = activity.sound
      ? 0
      : Math.min(
          AUTO_SEND_QUIET_MS,
          this.filteredQuietMs + pcm.byteLength / 32,
        );
    // Acoustic silence and transcript stability have independent clocks. The
    // 900ms release commits manual ASR buffers, never submits a user message.
    if (activity.sound) {
      this.acousticQuietAfter = Date.now() + AUTO_SEND_QUIET_MS;
      if (!this.manual) clearTimeout(this.settleTimer);
    }
    if (activity.started) {
      this.markAwaitingSettlement();
      this.clearFinalWait();
      if (!this.manual) clearTimeout(this.settleTimer);
      this.options.emit({ type: "speech-started" });
    }
    if (!this.provider || this.rotating) {
      this.pending.push({ pcm, trace });
      this.pendingBytes += pcm.byteLength;
      // Five seconds of 16 kHz PCM. Never silently discard a user's speech.
      if (this.pendingBytes > 160_000) this.fail("audio_buffer_overflow");
      return;
    }
    this.appendToProvider(pcm, trace);
    if (activity.ended) this.finishUtterance();
    else if (!activity.active) this.scheduleSettlement();
  }

  private appendToProvider(pcm: ArrayBuffer, trace?: AudioTrace): void {
    this.providerAudioMs += pcm.byteLength / 32;
    this.provider?.appendAudio(pcm, trace);
  }

  finishUtterance(force = false): void {
    if (this.closed) return;
    this.manual ||= force;
    // Send commits the already recognized text. Do not finish()/bypass the
    // continuous gate: audio awaiting verification must still be judged.
    this.provider?.commitUtterance?.();
    this.scheduleSettlement();
    this.startFinalWait();
  }

  private startFinalWait(quietUntil = Date.now()): void {
    this.clearFinalWait();
    this.finalWaitStartedAt = quietUntil;
    this.finalTimer = setTimeout(() => {
      if (
        this.segments.size &&
        [...this.segments.values()].some((part) => !part.final)
      )
        this.fail("final_timeout");
      else this.clearFinalWait();
    }, Math.max(0, quietUntil - Date.now()) + FINAL_TIMEOUT_MS);
  }

  private clearFinalWait(): void {
    clearTimeout(this.finalTimer);
    this.finalTimer = undefined;
    this.finalWaitStartedAt = null;
  }

  async stop(): Promise<void> {
    if (this.closed) return;
    this.closed = true;
    this.options.audioGate?.abort();
    ++this.generation;
    clearTimeout(this.settleTimer);
    clearTimeout(this.rotationTimer);
    this.clearFinalWait();
    clearInterval(this.blockedDiagnosticTimer);
    this.pending = [];
    this.pendingBytes = 0;
    const provider = this.provider;
    this.provider = null;
    if (provider) {
      try {
        await provider.stop();
      } finally {
        await provider.dispose?.();
      }
    }
  }

  private async open(): Promise<void> {
    const generation = ++this.generation;
    const provider = await this.options.createProvider();
    if (this.closed || generation !== this.generation) {
      await provider.stop();
      await provider.dispose?.();
      return;
    }
    this.provider = provider;
    const audioOffsetMs = this.audioPositionMs - this.pendingBytes / 32;
    this.providerAudioMs = 0;
    if (!provider.onSegment)
      throw new Error("Continuous transcription is unavailable.");
    provider.onSegment((segment) => {
      if (this.closed || generation !== this.generation) return;
      const key = `${generation}:${segment.id}`;
      if (this.consumed.has(key)) return;
      const prior = this.segments.get(key);
      if (prior?.final) return;
      this.markAwaitingSettlement();
      this.lastAsrUpdateAt = Date.now();
      const text = segment.text.trim();
      // Only provider audio offsets can prove that delayed/revised text belongs
      // to speech whose quiet window has already elapsed. Without them, keep the
      // conservative text-arrival fallback for soft speech missed by local VAD.
      const audioEndPositionMs = segment.final
        && typeof segment.audioEndMs === "number"
        && Number.isFinite(segment.audioEndMs)
        && segment.audioEndMs >= 0
        && segment.audioEndMs <= this.providerAudioMs
        ? audioOffsetMs + segment.audioEndMs
        : undefined;
      const textChanged = (text && text !== prior?.text.trim()) || !prior;
      const textQuietAfter = textChanged
        ? Date.now() + AUTO_SEND_QUIET_MS
        : prior.textQuietAfter;
      this.stableAfter = Date.now() + FINAL_STABLE_MS;
      if (!prior?.text && segment.text.trim() && !this.speaking)
        this.options.emit({ type: "speech-started" });
      this.segments.set(key, {
        ...segment, generation, audioEndPositionMs, textQuietAfter,
      });
      // ASR can hear soft speech below local VAD's threshold. Such a partial
      // has no activity.ended to arm finalization: use its quiet boundary.
      // New words extend that boundary; repeated partials do not. Real acoustic
      // speech clears this timer above, and a final still follows normal settling.
      if (!segment.final && !this.speaking && !this.manual
        && (textChanged || this.finalWaitStartedAt === null))
        this.startFinalWait(Math.max(textQuietAfter, this.acousticQuietAfter, Date.now()));
      this.options.emit({ type: "draft", text: this.draft() });
      if (this.draft().length > 16_000) {
        this.fail("transcript_limit");
        return;
      }
      this.scheduleSettlement();
    });
    provider.onEvent((event) => {
      if (this.closed || generation !== this.generation || this.rotating)
        return;
      if (event.type === "error") this.fail("upstream_error");
      else if (event.type === "disconnected") this.fail("upstream_disconnected");
    });
    await provider.start();
    if (this.closed || generation !== this.generation) return;
    this.rotating = false;
    const hadBufferedAudio = this.pending.length > 0;
    for (const chunk of this.pending.splice(0))
      this.appendToProvider(chunk.pcm, chunk.trace);
    this.pendingBytes = 0;
    if (hadBufferedAudio && !this.speaking) provider.commitUtterance?.();
    this.rotationTimer = setTimeout(() => {
      void this.rotate().catch(() => this.fail("rotation_failed"));
    }, this.options.rotationMs ?? 150_000);
    this.options.emit({ type: "ready" });
  }

  private draft(): string {
    return [...this.segments.values()]
      .sort(
        (a, b) =>
          a.generation - b.generation || (a.order ?? 0) - (b.order ?? 0),
      )
      .map((segment) => segment.text.trim())
      .filter(Boolean)
      .join(" ")
      .trim();
  }

  private settlementWait(): number | null {
    if ((this.speaking && !this.manual) || this.rotating || !this.awaitingSettlement)
      return null;
    const segments = [...this.segments.values()];
    if (segments.some((segment) => !segment.final)) return null;
    if (this.manual) return 0;
    // No wall-clock timeout may stand in for audio still held by the speaker
    // filter or still needed to establish silence after ASR-only soft speech.
    if (this.options.audioGate && this.filteredQuietMs < AUTO_SEND_QUIET_MS)
      return null;
    if (segments.some((segment) => segment.audioEndPositionMs !== undefined
      && this.audioPositionMs - segment.audioEndPositionMs < AUTO_SEND_QUIET_MS))
      return null;
    const textQuietAfter = Math.max(0, ...segments
      .filter((segment) => segment.audioEndPositionMs === undefined)
      .map((segment) => segment.textQuietAfter));
    // Capability and boundary evidence are deliberately separate from `final`.
    // This phase keeps 1.25s even for a semantic-capable provider. Enabling eager
    // semantic endpointing requires a separately validated policy, not a flag.
    return Math.max(0, this.acousticQuietAfter - Date.now(),
      textQuietAfter - Date.now(), this.stableAfter - Date.now());
  }

  private markAwaitingSettlement(): void {
    if (this.awaitingSettlement) return;
    this.awaitingSettlement = true;
    this.pendingSince = Date.now();
    // Independent of audio/ASR callbacks: stalled capture must remain observable.
    // This timer only reports evidence; settlementWait remains the send policy.
    if (this.options.blockedDiagnostic) {
      this.blockedDiagnosticTimer = setInterval(
        () => this.reportSettlementBlockers(),
        BLOCKED_DIAGNOSTIC_INTERVAL_MS,
      );
    }
  }

  private reportSettlementBlockers(): void {
    if (this.closed || this.options.isCurrent?.() === false
      || !this.awaitingSettlement || this.settlementWait() === 0) return;
    const now = Date.now();
    const remaining = (deadline: number) => Math.max(0, deadline - now);
    const age = (at: number | null) => at === null ? null : Math.max(0, now - at);
    const segments = [...this.segments.values()];
    const pendingFinalSegments = segments.filter((segment) => !segment.final).length;
    const acousticQuietRemainingMs = remaining(this.acousticQuietAfter);
    const textQuietRemainingMs = remaining(Math.max(0, ...segments
      .filter((segment) => segment.audioEndPositionMs === undefined)
      .map((segment) => segment.textQuietAfter)));
    const finalStableRemainingMs = remaining(this.stableAfter);
    const filteredQuietRemainingMs = this.options.audioGate
      ? Math.max(0, AUTO_SEND_QUIET_MS - this.filteredQuietMs) : 0;
    const audioQuietRemainingMs = Math.max(0, ...segments
      .filter((segment) => segment.audioEndPositionMs !== undefined)
      .map((segment) => AUTO_SEND_QUIET_MS
        - (this.audioPositionMs - segment.audioEndPositionMs!)));
    const reasons: SettlementBlocker[] = [];
    if (this.speaking && !this.manual) reasons.push("speaking");
    if (this.rotating) reasons.push("transport-rotation");
    if (pendingFinalSegments) reasons.push("asr-final");
    if (!this.manual) {
      if (filteredQuietRemainingMs) reasons.push("filtered-silence");
      if (audioQuietRemainingMs) reasons.push("audio-silence");
      if (acousticQuietRemainingMs) reasons.push("acoustic-silence");
      if (textQuietRemainingMs) reasons.push("text-silence");
      if (finalStableRemainingMs) reasons.push("text-stability");
    }
    this.options.blockedDiagnostic?.({
      reasons,
      pendingMs: now - this.pendingSince,
      speaking: this.speaking,
      manual: this.manual,
      rotating: this.rotating,
      segments: segments.length,
      pendingFinalSegments,
      characters: this.draft().length,
      speakerFilterActive: Boolean(this.options.audioGate),
      acousticQuietRemainingMs,
      textQuietRemainingMs,
      finalStableRemainingMs,
      filteredQuietRemainingMs,
      audioQuietRemainingMs,
      captureAgeMs: age(this.lastCaptureAt),
      forwardedAudioAgeMs: age(this.lastForwardedAt),
      asrUpdateAgeMs: age(this.lastAsrUpdateAt),
      rms: this.lastActivity ? Math.round(this.lastActivity.rms) : null,
      threshold: this.lastActivity ? Math.round(this.lastActivity.threshold) : null,
      noiseFloor: this.lastActivity ? Math.round(this.lastActivity.noiseFloor) : null,
    });
  }

  private scheduleSettlement(): void {
    clearTimeout(this.settleTimer);
    const wait = this.settlementWait();
    if (wait === null) return;
    this.settleTimer = setTimeout(
      () => {
        if (
          this.closed ||
          this.options.isCurrent?.() === false ||
          this.settlementWait() !== 0
        )
          return;
        const text = this.draft();
        const segments = [...this.segments.values()];
        const usesTextFallback = segments.some((segment) => segment.audioEndPositionMs === undefined);
        const endpointing = this.provider?.endpointing ?? "segment-final";
        const lastSegment = segments.at(-1);
        const inactivity = Math.min(
          this.acousticQuietAfter ? Date.now() - this.acousticQuietAfter + AUTO_SEND_QUIET_MS : Infinity,
          ...segments.map((segment) => segment.audioEndPositionMs !== undefined
            ? this.audioPositionMs - segment.audioEndPositionMs
            : Date.now() - segment.textQuietAfter + AUTO_SEND_QUIET_MS),
        );
        this.options.diagnostic?.({
          trigger: this.manual ? "manual" : "silence",
          inactiveMs: Number.isFinite(inactivity) ? Math.max(0, inactivity) : 0,
          pauseBasis: usesTextFallback ? "text-fallback" : "audio",
          endpointing,
          endSignal: lastSegment?.turnEnd === endpointing ? endpointing : "segment-final",
          finalStableMs: this.stableAfter
            ? Date.now() - this.stableAfter + FINAL_STABLE_MS
            : null,
          segments: this.segments.size,
          characters: text.length,
          ...(this.options.audioGate
            ? { filteredQuietMs: this.filteredQuietMs }
            : {}),
        });
        for (const key of this.segments.keys()) this.consumed.add(key);
        this.segments.clear();
        this.awaitingSettlement = false;
        clearInterval(this.blockedDiagnosticTimer);
        this.acousticQuietAfter = 0;
        this.stableAfter = 0;
        this.manual = false;
        this.clearFinalWait();
        if (text)
          this.options.emit({
            type: "utterance",
            id: String(++this.sequence),
            text,
          });
        this.options.emit({ type: "draft", text: "" });
        if (!text) this.options.emit({ type: "speech-idle" });
      },
      wait,
    );
  }

  private async rotate(): Promise<void> {
    if (this.closed || !this.provider) return;
    this.rotating = true;
    clearTimeout(this.settleTimer);
    const previous = this.provider;
    // Finalize the previous transport before accepting its last segments. New
    // audio is already buffered for the replacement, so boundaries cannot overlap.
    await previous.flushAudio();
    if (this.closed) return;
    if ([...this.segments.values()].some((segment) => !segment.final)) {
      this.fail("rotation_unfinalized");
      return;
    }
    ++this.generation;
    await previous.stop();
    await previous.dispose?.();
    this.provider = null;
    this.consumed.clear();
    await this.open();
    this.scheduleSettlement();
  }

  private fail(reason: InputFailureReason): void {
    if (this.closed) return;
    const now = Date.now();
    const segments = [...this.segments.values()];
    // Stable, content-free reason codes. Never log the upstream error message,
    // segment IDs, recognized words or audio; the host owns the existing logger.
    this.options.failureDiagnostic?.({
      reason,
      endpointing: this.provider?.endpointing ?? "segment-final",
      generation: this.generation,
      manual: this.manual,
      rotating: this.rotating,
      speaking: this.speaking,
      segments: segments.length,
      pendingFinalSegments: segments.filter((segment) => !segment.final).length,
      characters: this.draft().length,
      pendingMs: this.awaitingSettlement ? now - this.pendingSince : 0,
      // A soft-speech timer includes the quiet window. Until that ends, an
      // upstream failure is not time spent waiting for finalization.
      finalWaitMs: this.finalWaitStartedAt === null || now < this.finalWaitStartedAt
        ? null : now - this.finalWaitStartedAt,
      asrUpdateAgeMs: this.lastAsrUpdateAt === null ? null : now - this.lastAsrUpdateAt,
      bufferedAudioMs: this.pendingBytes / 32,
    });
    this.options.emit({ type: "error", text: this.draft() });
    void this.stop().catch(() => undefined);
  }
}