Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(desktop): preserve voice replies across false speech onsets
JiangJiahao12d68e24a39f
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/service.test.tsindex b4b0b82130..19715189ab 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts@@ -1,4 +1,5 @@import { afterEach, describe, expect, it, vi } from 'vitest';+import type { AsrSegment } from '@cindy/voice-input-core';import { VoiceConversationService } from '../service.js';import { hasVoiceConversationLease } from '../lease.js';@@ -108,7 +109,7 @@ function fixture() {dispose: vi.fn(async () => {}),appendAudio: vi.fn(),flushAudio: vi.fn(async () => {}),- onSegment: vi.fn(),+ onSegment: vi.fn<(callback: (segment: AsrSegment) => void) => void>(),onEvent: vi.fn(),};const deps = {@@ -127,6 +128,7 @@ function fixture() {new Response(new Uint8Array([1, 2]), { headers: { 'Content-Type': 'audio/pcm' } }),),emit: vi.fn(),+ log: { info: vi.fn(), warn: vi.fn() },};const service = new VoiceConversationService(deps);active.push(service);@@ -134,6 +136,44 @@ function fixture() {}describe('local companion call ownership', () => {+ it('logs send timing and speech stages through the injected logger without content or credentials', async () => {+ vi.useFakeTimers();+ const { service, deps, provider } = fixture();+ await service.start(1, selection);+ provider.onSegment.mock.calls[0][0]({ id: 'segment', text: 'private utterance', final: true });+ await vi.advanceTimersByTimeAsync(2_000);+ expect(deps.log.info).toHaveBeenCalledWith('input settled', {+ callId: selection.callId,+ trigger: 'silence',+ inactiveMs: 2_000,+ finalStableMs: 2_000,+ segments: 1,+ characters: 17,+ });+ await service.speak(1, selection.callId, 'request-1', 'private reply');+ await service.readSpeech(1, selection.callId, 'request-1');+ await service.readSpeech(1, selection.callId, 'request-1');+ expect(deps.log.info).toHaveBeenCalledWith(+ 'speech transport',+ expect.objectContaining({+ callId: selection.callId,+ requestId: 'request-1',+ protocol: 'openai',+ stage: 'complete',+ bytes: 2,+ }),+ );+ const logs = JSON.stringify([...deps.log.info.mock.calls, ...deps.log.warn.mock.calls]);+ for (const privateValue of [+ 'private utterance',+ 'private reply',+ 'test-key',+ 'gateway.test',+ 'alloy',+ ])+ expect(logs).not.toContain(privateValue);+ });+it('requires a real available speech model and the canonical companion task before allocating ASR', async () => {const { service, deps } = fixture();await expect(service.start(1, { ...selection, modelId: 'invented' })).rejects.toThrow(diff --git a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.tsindex a8664c2064..cfa253301c 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts@@ -7,6 +7,7 @@ const make = (),) => {const assertCurrent = vi.fn();+ const diagnostic = vi.fn();const speech = new SpeechSynthesisSession({connection: () => ({baseUrl: 'https://gateway.test/prefix/v1/',@@ -15,14 +16,15 @@ const make = (}),fetch: fetcher,assertCurrent,+ diagnostic,});- return { speech, fetcher, assertCurrent };+ return { speech, fetcher, assertCurrent, diagnostic };};const input = { requestId: 'one', modelId: 'speech-model', voice: 'alloy', text: 'hello' };describe('speech transport', () => {it('preserves the server-provided endpoint prefix, sends the selected model, and pulls binary PCM', async () => {- const { speech, fetcher } = make();+ const { speech, fetcher, diagnostic } = make();expect(await speech.start(input)).toEqual({ sampleRate: 24000 });expect(fetcher).toHaveBeenCalledWith(new URL('https://gateway.test/prefix/v1/audio/speech'),@@ -40,6 +42,13 @@ describe('speech transport', () => {expect(new Uint8Array((await speech.read('one')).pcm)).toEqual(new Uint8Array([1, 2]));expect((await speech.read('one')).done).toBe(true);speech.cancel();+ expect(diagnostic.mock.calls.map(([event]) => event.stage)).toEqual([+ 'request',+ 'headers',+ 'first_pcm',+ 'complete',+ ]);+ expect(JSON.stringify(diagnostic.mock.calls)).not.toMatch(/hello|test-key|gateway\.test|alloy/);});it.each(['audio/mpeg', 'application/json'])(@@ -53,10 +62,19 @@ describe('speech transport', () => {);it('does not pass upstream error bodies to the renderer', async () => {- const { speech } = make(+ const { speech, diagnostic } = make(vi.fn(async () => new Response('upstream-sensitive-error', { status: 401 })),);await expect(speech.start(input)).rejects.toThrow(/^request$/);+ expect(diagnostic).toHaveBeenCalledWith(+ expect.objectContaining({ stage: 'headers', httpStatus: 401 }),+ );+ expect(diagnostic).toHaveBeenCalledWith(+ expect.objectContaining({ stage: 'error', code: 'request' }),+ );+ expect(JSON.stringify(diagnostic.mock.calls)).not.toMatch(+ /upstream-sensitive-error|hello|test-key/,+ );});it.each([new Uint8Array(), new Uint8Array([1])])(diff --git a/apps/desktop/src/main/voice-conversation/index.ts b/apps/desktop/src/main/voice-conversation/index.tsindex 72ba61cf89..fb05a51077 100644--- a/apps/desktop/src/main/voice-conversation/index.ts+++ b/apps/desktop/src/main/voice-conversation/index.ts@@ -19,6 +19,9 @@ import { VoiceConversationService } from './service.js';import { resolveSpeechConnection, speechModels, type SpeechCatalogDeps } from './models.js';import { companionVoicePreference, updateCompanionVoicePreference } from './preferences.js';import type { VoiceInputSettings } from '../../shared/voiceInputData.js';+import { createLogger } from '../logger.js';++const log = createLogger('voice-conversation');function boundedString(value: unknown, name: string, limit: number): string {const text = requireString(value, name);@@ -48,6 +51,7 @@ export function registerVoiceConversationIpc(deps: {};const service = new VoiceConversationService({...deps,+ log,owner: () => (isAppSessionBoundaryPending() ? 'changing-account' : activeOwnerScopeKey()),assertCompanion: async (botId, sessionId) => {const bot = await getBotRemoteResourceSource(botId);@@ -81,6 +85,8 @@ export function registerVoiceConversationIpc(deps: {try {return await action(event, ...args);} catch {+ // Do not log args or upstream errors: either can include text or secrets.+ log.warn('IPC operation failed', { operation: name });throwIpcError('PRECONDITION_FAILED', 'Voice conversation is unavailable.');}});diff --git a/apps/desktop/src/main/voice-conversation/service.ts b/apps/desktop/src/main/voice-conversation/service.tsindex a6e1ca4988..29e9b884d1 100644--- a/apps/desktop/src/main/voice-conversation/service.ts+++ b/apps/desktop/src/main/voice-conversation/service.ts@@ -13,6 +13,7 @@ import type {import { acquireVoiceConversationLease, releaseVoiceConversationLease } from './lease.js';import { SpeechSynthesisSession, type SpeechConnection } from './speech.js';import { loadSpeechVoices } from './voices.js';+import type { Logger } from '../logger.js';type Call = {id: string;@@ -45,6 +46,7 @@ export class VoiceConversationService {connection(selection: SpeechModelSelection): SpeechConnection;fetch: typeof fetch;emit(sender: number, event: VoiceConversationEvent): void;+ log?: Pick<Logger, 'info' | 'warn'>;},) {}@@ -82,6 +84,11 @@ export class VoiceConversationService {sender,owner,speech: new SpeechSynthesisSession({+ diagnostic: (event) =>+ this.deps.log?.[event.stage === 'error' ? 'warn' : 'info']('speech transport', {+ previewId: selection.previewId,+ ...event,+ }),connection: () => binding,fetch: this.deps.fetch,assertCurrent: () => {@@ -146,6 +153,7 @@ export class VoiceConversationService {if (this.call?.id !== id || this.deps.owner() !== owner) throw new Error('call_ended');};const input = new VoiceConversationInput({+ diagnostic: (event) => this.deps.log?.info('input settled', { callId: id, ...event }),createProvider: async () => {assertCurrent();return this.deps.createProvider();@@ -156,11 +164,24 @@ export class VoiceConversationService {void this.end(sender, id);return;}+ if (event.type !== 'draft')+ this.deps.log?.[event.type === 'error' ? 'warn' : 'info']('input event', {+ callId: id,+ event: event.type,+ ...(event.type === 'utterance'+ ? { utteranceId: event.id, characters: event.text.length }+ : {}),+ });this.deps.emit(sender, { callId: id, event });if (event.type === 'error') void this.end(sender, id);},});const speech = new SpeechSynthesisSession({+ diagnostic: (event) =>+ this.deps.log?.[event.stage === 'error' ? 'warn' : 'info']('speech transport', {+ callId: id,+ ...event,+ }),connection: () => binding,fetch: this.deps.fetch,assertCurrent: () => {@@ -171,13 +192,17 @@ export class VoiceConversationService {},});this.call = { id, owner, sender, selection, input, speech };+ const startedAt = Date.now();+ this.deps.log?.info('call starting', { callId: id, protocol: binding.protocol });try {await this.deps.assertCompanion(selection.botId, selection.sessionId);assertCurrent();await input.start();assertCurrent();+ this.deps.log?.info('call ready', { callId: id, elapsedMs: Date.now() - startedAt });return { callId: id };} catch (error) {+ this.deps.log?.warn('call start failed', { callId: id, elapsedMs: Date.now() - startedAt });await this.end(sender, id);throw error;}@@ -223,6 +248,7 @@ export class VoiceConversationService {const call = this.call;if (!call || call.sender !== sender || (id && call.id !== id)) return;this.call = null;+ this.deps.log?.info('call ended', { callId: call.id });this.deps.emit(call.sender, { callId: call.id, event: { type: 'ended' } });call.speech.cancel();try {diff --git a/apps/desktop/src/main/voice-conversation/speech.ts b/apps/desktop/src/main/voice-conversation/speech.tsindex dc81beefa4..39c3a5ceae 100644--- a/apps/desktop/src/main/voice-conversation/speech.ts+++ b/apps/desktop/src/main/voice-conversation/speech.ts@@ -1,6 +1,15 @@import { miniMaxPcmStream, speechRequest, type SpeechConnection } from './speechProtocols.js';import { SpeechError } from './speechError.js';export type { SpeechConnection } from './speechProtocols.js';+export type SpeechDiagnostic = {+ stage: 'request' | 'headers' | 'first_pcm' | 'complete' | 'cancelled' | 'error';+ requestId: string;+ protocol: SpeechConnection['protocol'];+ elapsedMs: number;+ bytes: number;+ httpStatus?: number;+ code?: SpeechError['code'] | 'headers_timeout' | 'read_timeout';+};/** PCM speech transport with provider adapters. Reads are pulled by playback for bounded buffering. */export class SpeechSynthesisSession {private epoch = 0;@@ -9,12 +18,17 @@ export class SpeechSynthesisSession {private requestId: string | null = null;private totalBytes = 0;private reading = false;+ private diagnostic?: (+ stage: SpeechDiagnostic['stage'],+ details?: Pick<SpeechDiagnostic, 'httpStatus' | 'code'>,+ ) => void;constructor(private readonly deps: {connection(): SpeechConnection;assertCurrent(): void;fetch: typeof fetch;+ diagnostic?(event: SpeechDiagnostic): void;},) {}@@ -32,7 +46,22 @@ export class SpeechSynthesisSession {const controller = new AbortController();this.controller = controller;this.requestId = input.requestId;- const timer = setTimeout(() => controller.abort(), 30_000);+ const startedAt = Date.now();+ this.diagnostic = (stage, details = {}) =>+ this.deps.diagnostic?.({+ stage,+ requestId: input.requestId,+ protocol: connection.protocol,+ elapsedMs: Date.now() - startedAt,+ bytes: this.totalBytes,+ ...details,+ });+ this.diagnostic('request');+ let timedOut = false;+ const timer = setTimeout(() => {+ timedOut = true;+ controller.abort();+ }, 30_000);let response: Response | undefined;try {response = await this.deps.fetch(url, {@@ -47,6 +76,7 @@ export class SpeechSynthesisSession {await response.body?.cancel();throw new SpeechError('cancelled');}+ this.diagnostic('headers', { httpStatus: response.status });if (!response.ok || !response.body) {await response.body?.cancel();throw new SpeechError('request');@@ -71,7 +101,16 @@ export class SpeechSynthesisSession {} catch (error) {if (response?.body && !response.body.locked)void response.body.cancel().catch(() => undefined);- if (epoch === this.epoch) this.cancel();+ if (epoch === this.epoch) {+ this.diagnostic?.('error', {+ code: timedOut+ ? 'headers_timeout'+ : error instanceof SpeechError+ ? error.code+ : 'request',+ });+ this.cancel('failed');+ }throw error instanceof SpeechError ? error : new SpeechError('request');} finally {clearTimeout(timer);@@ -91,27 +130,36 @@ export class SpeechSynthesisSession {const epoch = this.epoch;this.reading = true;const timer = setTimeout(() => {- if (epoch === this.epoch) this.cancel();+ if (epoch === this.epoch) {+ this.diagnostic?.('error', { code: 'read_timeout' });+ this.cancel('failed');+ }}, 15_000);try {const result = await reader.read();this.deps.assertCurrent();if (epoch !== this.epoch) throw new SpeechError('cancelled');const value = result.value ?? new Uint8Array();+ const firstPcm = this.totalBytes === 0 && value.byteLength > 0;this.totalBytes += value.byteLength;if (value.byteLength > 1_048_576 || this.totalBytes > 24_000 * 2 * 600) {- this.cancel();throw new SpeechError('format');}+ if (firstPcm) this.diagnostic?.('first_pcm');if (result.done) {if (this.totalBytes === 0 || this.totalBytes % 2 !== 0) throw new SpeechError('format');reader.releaseLock();this.reader = null;this.requestId = null;+ this.diagnostic?.('complete');+ this.diagnostic = undefined;}return { pcm: Uint8Array.from(value).buffer, done: result.done };} catch (error) {- if (epoch === this.epoch) this.cancel();+ if (epoch === this.epoch) {+ this.diagnostic?.('error', { code: error instanceof SpeechError ? error.code : 'request' });+ this.cancel('failed');+ }throw error instanceof SpeechError ? error : new SpeechError('request');} finally {clearTimeout(timer);@@ -119,7 +167,9 @@ export class SpeechSynthesisSession {}}- cancel(): void {+ cancel(reason: 'interrupted' | 'failed' = 'interrupted'): void {+ if (reason === 'interrupted') this.diagnostic?.('cancelled');+ this.diagnostic = undefined;++this.epoch;this.controller?.abort();this.controller = null;diff --git a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.tsindex a371005591..4b9e2cfac5 100644--- a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts+++ b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts@@ -18,7 +18,13 @@ export class Pcm16Decoder {}export interface SpeechPlayer {- play(read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>, sampleRate: number): Promise<void>;+ play(+ read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,+ sampleRate: number,+ onStarted?: () => void,+ ): Promise<void>;+ pause(): void;+ resume(): void;cancel(): void;close(): void;}@@ -29,35 +35,37 @@ export class PcmSpeechPlayer implements SpeechPlayer {private generation = 0;private sources = new Set<AudioBufferSourceNode>();private timers = new Set<ReturnType<typeof setTimeout>>();+ private paused = false;+ private contextTransition: Promise<void>;+ private resumeWaiters = new Set<() => void>();constructor(private readonly onLevel: (level: number) => void) {// Construct/resume in the explicit Start gesture, before any network await.this.context = new AudioContext();- void this.context.resume().catch(() => undefined);+ this.contextTransition = this.context.resume();+ void this.contextTransition.catch(() => undefined);}async play(read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,sampleRate: number,+ onStarted?: () => void,): Promise<void> {this.cancel();const generation = this.generation;const decoder = new Pcm16Decoder();let nextAt = this.context.currentTime;- const assertActive = () => {- if (generation !== this.generation || this.context.state !== 'running')- throw new Error('Playback interrupted');- };- await this.context.resume();+ let started = false;+ this.updateContext();for (;;) {- assertActive();+ await this.waitUntilRunning(generation);const chunk = await read();- assertActive();+ await this.waitUntilRunning(generation);const samples = decoder.decode(chunk.pcm);for (let offset = 0; offset < samples.length; offset += 2_400) {while (nextAt - this.context.currentTime > 0.3) {await new Promise((resolve) => setTimeout(resolve, 25));- assertActive();+ await this.waitUntilRunning(generation);}const part = samples.subarray(offset, offset + 2_400);const buffer = this.context.createBuffer(1, part.length, sampleRate);@@ -78,12 +86,16 @@ export class PcmSpeechPlayer implements SpeechPlayer {const timer = setTimeout(() => {this.timers.delete(timer);- if (generation === this.generation) this.onLevel(level);+ if (generation === this.generation && !this.paused) this.onLevel(level);},Math.max(0, nextAt - this.context.currentTime) * 1000,);this.timers.add(timer);source.start(nextAt);+ if (!started) {+ started = true;+ onStarted?.();+ }nextAt += part.length / sampleRate;}if (chunk.done) break;@@ -91,13 +103,64 @@ export class PcmSpeechPlayer implements SpeechPlayer {decoder.finish();while (nextAt > this.context.currentTime) {await new Promise((resolve) => setTimeout(resolve, 25));- assertActive();+ await this.waitUntilRunning(generation);}this.onLevel(0);}+ // Suspend the audio clock as well as reads: queued samples resume from their+ // exact position after a false onset, without another paid synthesis request.+ pause(): void {+ if (this.paused) return;+ this.paused = true;+ this.onLevel(0);+ this.updateContext();+ }++ resume(): void {+ if (!this.paused) return;+ this.paused = false;+ this.updateContext();+ this.wakePlayback();+ }++ private updateContext(): void {+ // Serialize native transitions, consulting current intent so a late suspend+ // cannot overtake Resume or reopen a context closed at an account boundary.+ this.contextTransition = this.contextTransition+ .catch(() => undefined)+ .then(async () => {+ if (this.context.state === 'closed') return;+ if (this.paused) await this.context.suspend();+ else await this.context.resume();+ });+ void this.contextTransition.catch(() => undefined);+ }++ private async waitUntilRunning(generation: number): Promise<void> {+ for (;;) {+ if (generation !== this.generation || this.context.state === 'closed')+ throw new Error('Playback interrupted');+ if (this.paused) {+ await new Promise<void>((resolve) => this.resumeWaiters.add(resolve));+ continue;+ }+ await this.contextTransition;+ if (generation !== this.generation) throw new Error('Playback interrupted');+ if (this.paused) continue;+ if (this.context.state !== 'running') throw new Error('Playback interrupted');+ return;+ }+ }++ private wakePlayback(): void {+ for (const resolve of this.resumeWaiters) resolve();+ this.resumeWaiters.clear();+ }+cancel(): void {++this.generation;+ this.wakePlayback();for (const timer of this.timers) clearTimeout(timer);this.timers.clear();for (const source of this.sources) {diff --git a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.tsindex 93d6e5fa33..0f244f5fbe 100644--- a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts+++ b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts@@ -3,10 +3,13 @@ import type {VoiceConversationSelection,} from '@/../shared/voiceConversation';import type { ChatMessage } from '@/lib/makerChatStore';+import { createLogger } from '@/lib/logger';import type { PcmChunk } from '@/voice-input/WebMicAudioEngine';import type { SpeechPlayer } from './PcmSpeechPlayer';import { speechChunks, VoiceReplyTracker } from './VoiceReplyTracker';+const log = createLogger('voice-conversation');+export type VoicePhase ='idle' | 'connecting' | 'listening' | 'thinking' | 'preparing' | 'speaking' | 'paused' | 'error';export type VoiceSnapshot = {@@ -51,6 +54,8 @@ export class VoiceConversationController {private queued: string[] = [];private pendingStart: Promise<unknown> = Promise.resolve();private teardown: Promise<unknown> = Promise.resolve();+ private speechCandidate = false;+ private replyPhase: 'thinking' | 'preparing' | 'speaking' | null = null;constructor(private readonly deps: {@@ -73,6 +78,8 @@ export class VoiceConversationController {return () => this.listeners.delete(listener);};private patch(patch: Partial<VoiceSnapshot>): void {+ if (patch.phase && patch.phase !== this.state.phase)+ log.info('phase changed', { callId: this.callId, from: this.state.phase, to: patch.phase });this.state = { ...this.state, ...patch };for (const listener of this.listeners) listener();}@@ -93,8 +100,25 @@ export class VoiceConversationController {this.unsubscribe = this.deps.api.onEvent(({ callId, event }) => {if (!current() || callId !== this.callId) return;if (event.type === 'speech-started') {- this.interrupt();- this.patch({ phase: 'listening' });+ // An acoustic onset can be noise. Pause playback without consuming+ // the reply; only a finalized, nonempty utterance replaces its owner.+ if (!this.speechCandidate) {+ this.speechCandidate = true;+ this.player?.pause();+ log.info('reply held for possible speech', { callId, pendingInput: this.pendingInput });+ this.patch({ phase: 'listening', level: 0 });+ }+ } else if (event.type === 'speech-idle') {+ if (this.speechCandidate) {+ this.speechCandidate = false;+ this.player?.resume();+ log.info('reply resumed after empty speech', {+ callId,+ pendingInput: this.pendingInput,+ });+ this.setReplyPhase(this.replyPhase);+ this.updateMessages(this.latestMessages);+ }} else if (event.type === 'draft') {this.patch({ transcript: event.text });} else if (event.type === 'utterance') {@@ -151,19 +175,28 @@ export class VoiceConversationController {await mic.stop();return;}- this.patch({ phase: this.pendingInput ? 'thinking' : 'listening' });+ this.setReplyPhase(this.pendingInput ? 'thinking' : null);this.updateMessages(this.latestMessages);} catch {if (current()) this.fail('connection');}}- /** Speech onset fences old replies and audio before a new transcript is committed. */- interrupt(): void {+ private setReplyPhase(phase: typeof this.replyPhase): void {+ this.replyPhase = phase;+ this.patch({ phase: this.speechCandidate ? 'listening' : (phase ?? 'listening'), level: 0 });+ }++ /** Explicit stop or a committed new utterance permanently relinquishes the old reply. */+ interrupt(reason: 'user_stop' | 'new_utterance' = 'user_stop'): void {+ log.info('reply interrupted', { callId: this.callId, reason, pendingInput: this.pendingInput });++this.speechEpoch;+ this.speechCandidate = false;+ this.replyPhase = null;this.tracker.interrupt();this.pendingInput = null;this.player?.cancel();+ this.player?.resume();if (this.callId) void this.deps.api.interrupt(this.callId).catch(() => undefined);if (this.state.phase === 'speaking' ||@@ -177,8 +210,9 @@ export class VoiceConversationController {const current = () =>this.enabled && epoch === this.epoch && !this.blocked && !this.state.muted;if (!current()) return;- this.interrupt();- this.patch({ phase: 'thinking', transcript: '', unsent: '' });+ this.interrupt('new_utterance');+ this.setReplyPhase('thinking');+ this.patch({ transcript: '', unsent: '' });const replyEpoch = this.speechEpoch;let clientId: string | null = null;try {@@ -201,6 +235,7 @@ export class VoiceConversationController {return;}if (replyEpoch !== this.speechEpoch) return;+ log.info('voice input accepted', { callId: this.callId, clientId, characters: text.length });this.pendingInput = clientId;this.tracker.expect(clientId);this.updateMessages(this.latestMessages);@@ -228,17 +263,33 @@ export class VoiceConversationController {updateMessages(messages: readonly ChatMessage[]): void {this.latestMessages = messages;- if (!this.enabled || !this.callId || this.blocked || this.state.muted || !this.pendingInput)+ if (+ !this.enabled ||+ !this.callId ||+ this.blocked ||+ this.state.muted ||+ !this.pendingInput ||+ this.speechCandidate+ )return;const reply = this.tracker.take(messages);if (!reply) {if (!this.tracker.hasPendingReply) {+ log.info('reply ownership superseded', {+ callId: this.callId,+ pendingInput: this.pendingInput,+ });this.pendingInput = null;- this.patch({ phase: 'listening' });+ this.setReplyPhase(null);}return;}this.pendingInput = null;+ log.info('final reply selected', {+ callId: this.callId,+ replyId: reply.id,+ characters: reply.text.length,+ });void this.speak(reply.text);}@@ -250,17 +301,25 @@ export class VoiceConversationController {if (!callId || !player) return;const current = () =>epoch === this.epoch && speechEpoch === this.speechEpoch && this.enabled && !this.blocked;- this.patch({ phase: 'preparing', level: 0 });+ this.setReplyPhase('preparing');try {for (const [index, chunk] of speechChunks(text).entries()) {if (!current()) return;const requestId = `${speechEpoch}:${index}`;+ log.info('speech requested', { callId, requestId, characters: chunk.length });const { sampleRate } = await this.deps.api.speak({ callId, requestId, text: chunk });if (!current()) return;- this.patch({ phase: 'speaking' });- await player.play(() => this.deps.api.readSpeech(callId, requestId), sampleRate);+ this.setReplyPhase('speaking');+ await player.play(+ () => this.deps.api.readSpeech(callId, requestId),+ sampleRate,+ () => {+ log.info('playback scheduled', { callId, requestId, sampleRate });+ },+ );+ if (current()) log.info('playback completed', { callId, requestId });}- if (current()) this.patch({ phase: 'listening', level: 0 });+ if (current()) this.setReplyPhase(null);} catch {if (current()) this.fail('speech');}@@ -307,6 +366,7 @@ export class VoiceConversationController {}private fail(error: VoiceSnapshot['error'], text?: string): void {+ log.warn('voice mode failed', { callId: this.callId, phase: this.state.phase, reason: error });const unsent = [text || this.state.transcript || this.state.unsent, ...this.queued].filter(Boolean).join(' ');@@ -317,6 +377,8 @@ export class VoiceConversationController {private release(): void {++this.epoch;++this.speechEpoch;+ this.speechCandidate = false;+ this.replyPhase = null;this.queued = [];const callId = this.callId;const microphone = this.microphone;diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsxindex c684ee0f82..9d22504eb8 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx@@ -26,6 +26,8 @@ vi.mock('@/voice-input/WebMicAudioEngine', () => ({vi.mock('../PcmSpeechPlayer', () => ({PcmSpeechPlayer: class {close = audio.close;+ pause() {}+ resume() {}cancel() {}async play() {}},diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsxindex 7bd7ca8d96..eeb0e7f6bb 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx@@ -59,6 +59,8 @@ vi.mock('@/voice-input/WebMicAudioEngine', () => ({vi.mock('../PcmSpeechPlayer', () => ({PcmSpeechPlayer: class {close = audio.close;+ pause() {}+ resume() {}cancel() {}async play() {}},diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.tsindex cb12461721..8e9684fe18 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts@@ -17,13 +17,23 @@ class TestAudioContext {state = 'running';destination = {};sources: ReturnType<typeof createTestSource>[] = [];+ private elapsed = 0;+ private runningAt = Date.now();get currentTime() {- return Date.now() / 1000;+ return this.elapsed + (this.state === 'running' ? (Date.now() - this.runningAt) / 1000 : 0);}constructor() {TestAudioContext.latest = this;}- resume = vi.fn(async () => {});+ resume = vi.fn(async () => {+ if (this.state === 'running') return;+ this.runningAt = Date.now();+ this.state = 'running';+ });+ suspend = vi.fn(async () => {+ this.elapsed = this.currentTime;+ this.state = 'suspended';+ });close = vi.fn(async () => {this.state = 'closed';});@@ -85,3 +95,43 @@ it('fences an IPC read that finishes after cancellation', async () => {expect(TestAudioContext.latest.sources).toHaveLength(0);player.close();});++it('freezes the audio clock and queued samples during a false onset, then continues without rereading', async () => {+ const player = new PcmSpeechPlayer(vi.fn());+ const read = vi.fn(async () => ({ pcm: new Int16Array(24_000).buffer, done: true }));+ const started = vi.fn();+ const playing = player.play(read, 24_000, started);+ await vi.advanceTimersByTimeAsync(100);+ const context = TestAudioContext.latest;+ const sourcesBefore = [...context.sources];+ player.pause();+ await vi.advanceTimersByTimeAsync(0);+ const position = context.currentTime;+ await vi.advanceTimersByTimeAsync(2_000);+ expect(context.currentTime).toBe(position);+ expect(context.sources).toEqual(sourcesBefore);+ expect(sourcesBefore.every((source) => source.stop.mock.calls.length === 0)).toBe(true);+ player.resume();+ await vi.advanceTimersByTimeAsync(1_100);+ await playing;+ expect(read).toHaveBeenCalledOnce();+ expect(started).toHaveBeenCalledOnce();+ expect(context.sources.slice(0, sourcesBefore.length)).toEqual(sourcesBefore);+ player.close();+});++it('holds a synthesis response that arrives during noise and cancels a suspended player on teardown', async () => {+ const player = new PcmSpeechPlayer(vi.fn());+ player.pause();+ const read = vi.fn(async () => ({ pcm: new Int16Array(2_400).buffer, done: true }));+ const playing = player.play(read, 24_000);+ const rejected = expect(playing).rejects.toThrow('Playback interrupted');+ await vi.advanceTimersByTimeAsync(1_000);+ expect(read).not.toHaveBeenCalled();+ expect(TestAudioContext.latest.sources).toHaveLength(0);+ player.close();+ await rejected;+ player.resume();+ await vi.advanceTimersByTimeAsync(0);+ expect(TestAudioContext.latest.state).toBe('closed');+});diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.tsindex 82bd4c7778..4f4327a4c0 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts@@ -1,8 +1,12 @@-import { describe, expect, it, vi } from 'vitest';+import { beforeEach, describe, expect, it, vi } from 'vitest';import type { VoiceConversationApi, VoiceConversationEvent } from '@/../shared/voiceConversation';import type { ChatMessage } from '@/lib/makerChatStore';import { VoiceConversationController } from '../VoiceConversationController';+const log = vi.hoisted(() => ({ info: vi.fn(), warn: vi.fn() }));+vi.mock('@/lib/logger', () => ({ createLogger: () => log }));+beforeEach(() => vi.clearAllMocks());+const selection = { modelId: 'tts', voice: 'alloy' };const user = (id = 'user-1'): ChatMessage =>({ clientId: id, role: 'user', content: 'hello' }) as ChatMessage;@@ -39,7 +43,13 @@ function fixture() {stop: vi.fn(async () => {}),onPcm16k: vi.fn(),};- const player = { play: vi.fn(async () => {}), cancel: vi.fn(), close: vi.fn() };+ const player = {+ play: vi.fn(async () => {}),+ pause: vi.fn(),+ resume: vi.fn(),+ cancel: vi.fn(),+ close: vi.fn(),+ };const send = vi.fn(async (_text: string, created: (id: string) => void, current: () => boolean) => {if (!current()) return false;@@ -89,7 +99,7 @@ describe('companion voice loop', () => {expect(f.microphone.stop).toHaveBeenCalledOnce();});- it('interrupts pending TTS immediately and discards its late response without stopping the task', async () => {+ it('pauses on acoustic onset and cancels pending TTS only when a new utterance commits', async () => {const f = fixture();let finish!: (value: { sampleRate: number }) => void;f.api.speak.mockImplementationOnce(@@ -104,14 +114,16 @@ describe('companion voice loop', () => {await Promise.resolve();f.controller.updateMessages([user(), reply()]);expect(f.api.speak).toHaveBeenCalledOnce();+ const interruptions = f.api.interrupt.mock.calls.length;f.emit({ type: 'speech-started' });- expect(f.player.cancel).toHaveBeenCalled();- expect(f.api.interrupt).toHaveBeenCalledWith('call-1');+ expect(f.player.pause).toHaveBeenCalledOnce();+ expect(f.api.interrupt).toHaveBeenCalledTimes(interruptions);+ f.utterance('actually, another question');+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));+ expect(f.api.interrupt).toHaveBeenCalledTimes(interruptions + 1);finish({ sampleRate: 24000 });await Promise.resolve();expect(f.player.play).not.toHaveBeenCalled();- f.utterance('actually, another question');- await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));expect(f.api.end).not.toHaveBeenCalled();f.controller.end();});@@ -180,7 +192,7 @@ describe('companion voice loop', () => {expect(f.controller.getSnapshot().phase).toBe('idle');});- it('an interruption before a delayed send acknowledgement does not resurrect its old reply', async () => {+ it('keeps a delayed send acknowledgement through a false onset and reads its final reply after quiet', async () => {const f = fixture();let done!: () => void;f.send.mockImplementationOnce(async (_text, created) => {@@ -198,6 +210,105 @@ describe('companion voice loop', () => {await Promise.resolve();f.controller.updateMessages([user('old-user'), reply()]);expect(f.api.speak).not.toHaveBeenCalled();+ f.emit({ type: 'speech-idle' });+ await vi.waitFor(() => expect(f.api.speak).toHaveBeenCalledOnce());+ expect(f.player.resume).toHaveBeenCalled();+ f.controller.end();+ });++ it('reads the first final reply after a noise-only onset and still reads the next turn', async () => {+ const f = fixture();+ await f.controller.start(selection);+ f.utterance();+ await Promise.resolve();+ await Promise.resolve();+ f.emit({ type: 'speech-started' });+ f.controller.updateMessages([user(), reply()]);+ expect(f.api.speak).not.toHaveBeenCalled();+ f.emit({ type: 'draft', text: '' });+ f.emit({ type: 'speech-idle' });+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());+ f.emit({ type: 'speech-idle' });+ expect(f.api.speak).toHaveBeenCalledOnce();+ f.utterance('next question');+ await Promise.resolve();+ await Promise.resolve();+ f.controller.updateMessages([user(), reply(), user('user-2'), reply('reply-2')]);+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledTimes(2));+ expect(f.controller.getSnapshot().error).toBeNull();+ // Existing logger receives identifiers and counts, never voice/reply text.+ expect(JSON.stringify(log.info.mock.calls)).not.toContain('next question');+ expect(JSON.stringify(log.info.mock.calls)).not.toContain('Hello back');+ f.controller.end();+ });++ it('resumes an already playing reply after empty speech without requesting synthesis again', async () => {+ const f = fixture();+ let played!: () => void;+ f.player.play.mockImplementationOnce(+ () =>+ new Promise<void>((resolve) => {+ played = resolve;+ }),+ );+ await f.controller.start(selection);+ f.utterance();+ await Promise.resolve();+ await Promise.resolve();+ f.controller.updateMessages([user(), reply()]);+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());+ f.emit({ type: 'speech-started' });+ expect(f.controller.getSnapshot().phase).toBe('listening');+ f.emit({ type: 'speech-idle' });+ expect(f.player.resume).toHaveBeenCalled();+ expect(f.controller.getSnapshot().phase).toBe('speaking');+ expect(f.api.speak).toHaveBeenCalledOnce();+ played();+ await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('listening'));+ f.controller.end();+ });++ it.each(['permission', 'mute', 'end'] as const)(+ 'does not resume a held reply across the %s boundary',+ async (boundary) => {+ const f = fixture();+ await f.controller.start(selection);+ f.utterance();+ await Promise.resolve();+ await Promise.resolve();+ f.emit({ type: 'speech-started' });+ if (boundary === 'permission') f.controller.setBlocked(true);+ else if (boundary === 'mute') f.controller.toggleMute();+ else f.controller.end();+ f.emit({ type: 'speech-idle' });+ f.controller.updateMessages([user(), reply()]);+ expect(f.api.speak).not.toHaveBeenCalled();+ expect(f.player.close).toHaveBeenCalled();+ f.controller.end();+ },+ );++ it('does not restore reply ownership after an explicit stop before send acknowledgement', async () => {+ const f = fixture();+ let done!: () => void;+ f.send.mockImplementationOnce(async (_text, created) => {+ created('old-user');+ await new Promise<void>((resolve) => {+ done = resolve;+ });+ return true;+ });+ await f.controller.start(selection);+ f.utterance();+ f.emit({ type: 'speech-started' });+ f.controller.interrupt();+ done();+ await Promise.resolve();+ await Promise.resolve();+ f.emit({ type: 'speech-idle' });+ f.controller.updateMessages([user('old-user'), reply()]);+ expect(f.api.speak).not.toHaveBeenCalled();+ expect(f.controller.getSnapshot().phase).toBe('listening');f.controller.end();});diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.tsindex 3d87b1b964..70a1041ff6 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts@@ -11,6 +11,8 @@ function fixture() {play: vi.fn(async (read: () => Promise<unknown>) => {await read();}),+ pause: vi.fn(),+ resume: vi.fn(),cancel: vi.fn(),close: vi.fn(),};diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.mdindex 41ff8d1f46..a1ca406b17 100644--- a/docs/product-rules/companion-voice-mode.md+++ b/docs/product-rules/companion-voice-mode.md@@ -4,7 +4,7 @@- 入口位于本机伙伴一对一输入框的听写按钮与发送按钮之间,使用声波图标和“开始语音对话”提示。标题栏不再提供语音入口。沿用伙伴 canonical 任务、模型、上下文、工作目录和工具权限,不创建第二条任务或切换聊天模型。- 用户点击输入框的语音对话按钮后,重新读取当前目录和该伙伴偏好;配置有效即开始连接并采集麦克风,无需再点击“开始”。模型缺失、音色未选或接口不明时直接打开“伙伴设置 → 语音”,不采集麦克风;设置完成不会自动开始,需返回输入框明确点击。使用现有语音输入的 ASR 设置和当前账号的服务模式;不启用听写润色,不修改原有听写行为。-- 停顿约 900ms 后等待 ASR 的最终文字,再用短暂的稳定窗口合并同一段发言。只有最终、非空文字能自动发送;声音活动检测本身不能产生消息。+- 最后一次声音活动或识别文字更新后,至少等待 2 秒再自动发送;期间继续说话会合并到同一段发言,只有 ASR 检出的弱声也遵守相同门槛。900ms 的声音释放只负责提前提交 ASR 缓冲,不触发发送;还须等待全部分段为最终文字及最后一次 ASR 更新后的 350ms 稳定窗口,延迟返回的识别结果可能延长等待。手动发送可跳过停顿等待,但不能发送未完成的识别文字。只有最终、非空文字能发送;声音活动检测本身不能产生消息。- 原任务空闲时正常发送,仍在运行时沿既有插话路径投递。语音是本机用户本人输入,不盖自动化或远程设备来源标签,不增加系统提示词。识别结果按普通文字发送,不执行桌面斜杠命令。- 最终回复经过持久化完成标记确认后才朗读。按本次语音发送的 clientId 关联,不朗读历史、过程文字、工具、子代理、内部协调或群来源私信。文本消息始终是对话记录的正本。- 在“伙伴设置 → 语音”中独立选择朗读模型和音色,按当前账号下的伙伴保存,不改变聊天模型。来源包括有权限的 Cindy AI,以及已连接的本地 BYOK 供应商;候选必须是 `audio_speech`、未停用且具备可用的 API key 或明确的本机免鉴权连接。同名模型按连接 ID、runtime 和上游模型 ID 区分,不能借另一连接的密钥。无候选时显示空状态,不采集麦克风。@@ -13,7 +13,7 @@- 更换连接、runtime、型号或接口时,不沿用旧音色;目标型号有明确参考默认值才填入,否则等待用户选择或输入。同一连接下的不同型号也不视为音色兼容。已有保存音色不因刷新失败、列表未收录而被覆盖;刷新失败保留已知名称,成功获取后仍未收录的 ID 显示在手动输入中。- 旧偏好中的接口与官方地址冲突时,面板采用官方接口,接口选择仍隐藏,在音色区域上方提示重新选择音色,重新选择前不试听或开始;正确接口下的已有音色继续保留。读取偏好时不写回推导默认值,用户选择音色后通过既有保存流程移除不适用的接口覆盖。自定义/中转地址的显式接口与音色偏好不受影响。- 刷新重新读取当前有效目录,并尝试刷新 Cindy AI;Gateway 刷新失败不阻止本机 BYOK 目录显示。新增第三方型号仍沿设置页已有的获取列表、手动添加和启用流程。-- 朗读结束继续监听。用户开口时立即取消待生成及正在播放的声音;新的文字仍投递到原任务。停止朗读不会取消 Agent 工具、清空任务队列或代答权限确认。+- 朗读结束继续监听。检测到可能开口时先暂停播放并保留待回复的关联;只有最终、非空的新发言提交时才永久取消旧朗读,新的文字仍投递到原任务。仅有噪声或最终为空时,停顿窗口结束后恢复等待或从原播放位置继续,不再次请求合成。用户明确停止朗读仍立即取消;停止朗读不会取消 Agent 工具、清空任务队列或代答权限确认。- 需要确认权限、问题或计划,或任务队列暂停时,关闭麦克风并暂停语音模式;用户完成确认后显式继续。静音、锁屏、休眠、切走任务、关面板、账号切换同样释放采集和播放资源,不在后台重新开始。- 未发送的文字在可恢复错误或暂停后留在面板内;用户可选中复制或明确重新连接并发送。TTS 失败保留任务中的文字回复,不自动重发用户消息。@@ -55,6 +55,12 @@- 用户点击试听后,以当前连接、型号、接口和音色合成短示例文本,并明确提示可能产生用量。复用既有 PCM 合成、解码和播放;不采集麦克风、不发送伙伴消息、不调用聊天模型,不使用供应商返回的任意 `preview_url`。试听结果表示这次调用结果,不持久标记整个账号或型号已验证。- 试听与语音模式/听写互斥,绑定可信窗口、账号和伙伴 canonical 任务。生成过程中也可停止;改选、隐藏页面、关面板、账号变化、系统挂起或权限阻塞时取消;开始语音前等待试听停止。窗口销毁与 60 秒总时限均释放 Main 的语音占用。晚到的结果不恢复播放、不覆盖新选择。+## 诊断日志++复用现有 Main / Renderer `createLogger`,使用 `voice-conversation` / `r:voice-conversation` scope,统一写入既有 `main-YYYY-MM-DD.log`,沿用按天轮换和 30 天保留规则。开发实例保存在 `apps/desktop/logs`,安装版在当前 `userData/logs`;不创建另一套日志文件或音频存储,不改变现有上传来源白名单。++记录通话开始、ASR 就绪、发送判定、消息关联、可能插话时的暂停与恢复、明确取消、TTS 请求 / 响应状态 / 首段 PCM / 完成,以及播放调度和失败。用 callId、clientId、requestId 关联各阶段,只附接口枚举、耗时、字符数、字节数、HTTP 状态和受控错误码;发送判定包含手动或停顿触发、距离最后一次声音或文字更新的毫秒数、ASR 稳定时长。噪声归零也留下判定记录,便于区分“没有触发朗读”“上游失败”和“播放被暂停或取消”。不记录识别文字、回复正文、音频、音色 ID、供应商地址、请求头、凭证或上游原始错误,不逐音频帧输出日志。新增 scope 沿用现有默认不上传规则,诊断完整记录保留在本机。+## 后续扩展计划以下为后续演进方向,本 PR 尚未实现适配器拆分、统一参数描述或公共 Registry schema 扩展,也不包含 Qwen / CosyVoice 原生执行器。@@ -68,6 +74,8 @@## 验证+停顿与插话回归覆盖 2 秒等待、弱声识别、思考停顿后继续说话、空识别和无 ASR 结果的噪声、首轮回复在噪声期间完成、播放暂停后原位置继续,以及权限、静音、退出或明确停止后的迟到结果。日志测试确认输出只包含诊断元数据,不包含输入、回复或凭证。+自动化覆盖分段去重、乱序完成、空转写、连接轮换、权限与账号边界、取消启动、插话抢占、迟到音频、PCM 字节边界、回复关联、无模型空状态及现有 ASR 回归。BYOK 回归覆盖接口请求、SSE 断帧与结束聚合包、连接/密钥隔离、停用与账号切换、伙伴偏好重开与重置、未知接口须用户选择,以及合成失败不换供应商、不重发消息。音色与试听回归包含真实 MiniMax ID、OpenAI 型号差异、ElevenLabs 分页、账号私有列表边界、手动输入与刷新失败、改选清除旧音色、试听取消与迟到结果、窗口销毁,以及试听不启动 ASR。入口回归覆盖设置深链、输入框按钮顺序与草稿保留、缺配置转到设置、直接启动、StrictMode 不重复采集、关闭后的迟到配置结果,以及设置关闭前保存失败不离开。没有真实 Gateway / BYOK TTS 账号可用于验收时,自动化和组件预览使用明确的测试替身;产品目录不添加演示模型。真实模型开放后,还需验证其音色和 PCM 合同,并在 macOS/Windows 的耳机、扬声器、蓝牙设备上验证连续多轮、回声与插话延迟。测试替身不构成这些项目已通过的证据。diff --git a/packages/voice-input-core/src/VoiceConversationInput.ts b/packages/voice-input-core/src/VoiceConversationInput.tsindex 43db464b61..e70bddb781 100644--- a/packages/voice-input-core/src/VoiceConversationInput.ts+++ b/packages/voice-input-core/src/VoiceConversationInput.ts@@ -1,9 +1,13 @@import type { AsrProvider, AsrSegment, AudioTrace } from "./types.js";import { ConversationActivity } from "./conversationActivity.js";+const AUTO_SEND_QUIET_MS = 2_000;+const FINAL_STABLE_MS = 350;+export type ConversationInputEvent =| { type: "ready" }| { type: "speech-started" }+ | { type: "speech-idle" }| { type: "draft"; text: string }| { type: "utterance"; id: string; text: string }| { type: "error"; text: string };@@ -29,12 +33,23 @@ export class VoiceConversationInput {private settleTimer?: ReturnType<typeof setTimeout>;private rotationTimer?: ReturnType<typeof setTimeout>;private finalTimer?: ReturnType<typeof setTimeout>;+ private awaitingSettlement = false;+ private quietAfter = 0;+ private stableAfter = 0;constructor(private readonly options: {createProvider: () => Promise<AsrProvider>;emit: (event: ConversationInputEvent) => void;rotationMs?: number;+ // Numeric endpointing evidence only; the host owns logging and storage.+ diagnostic?: (event: {+ trigger: "manual" | "silence";+ inactiveMs: number;+ finalStableMs: number | null;+ segments: number;+ characters: number;+ }) => void;},) {}@@ -47,7 +62,14 @@ export class VoiceConversationInput {if (this.closed) return;const activity = this.activity.push(pcm);this.speaking = activity.active;+ // The 900ms acoustic release only commits ASR buffers. Sending always uses+ // the same two-second quiet window, including soft speech seen only by ASR.+ if (activity.sound) {+ this.quietAfter = Date.now() + AUTO_SEND_QUIET_MS;+ if (!this.manual) clearTimeout(this.settleTimer);+ }if (activity.started) {+ this.awaitingSettlement = true;clearTimeout(this.finalTimer);if (!this.manual) clearTimeout(this.settleTimer);this.options.emit({ type: "speech-started" });@@ -61,6 +83,7 @@ export class VoiceConversationInput {}this.provider.appendAudio(pcm, trace);if (activity.ended) this.finishUtterance();+ else if (activity.sound && !activity.active) this.scheduleSettlement();}finishUtterance(force = false): void {@@ -115,6 +138,11 @@ export class VoiceConversationInput {if (this.consumed.has(key)) return;const prior = this.segments.get(key);if (prior?.final) return;+ this.awaitingSettlement = true;+ const text = segment.text.trim();+ if ((text && text !== prior?.text.trim()) || !this.quietAfter)+ this.quietAfter = Date.now() + AUTO_SEND_QUIET_MS;+ this.stableAfter = Date.now() + FINAL_STABLE_MS;if (!prior?.text && segment.text.trim() && !this.speaking)this.options.emit({ type: "speech-started" });this.segments.set(key, { ...segment, generation });@@ -157,7 +185,11 @@ export class VoiceConversationInput {private scheduleSettlement(): void {clearTimeout(this.settleTimer);- if ((this.speaking && !this.manual) || this.rotating || !this.segments.size)+ if (+ (this.speaking && !this.manual) ||+ this.rotating ||+ !this.awaitingSettlement+ )return;if ([...this.segments.values()].some((segment) => !segment.final)) return;this.settleTimer = setTimeout(@@ -165,8 +197,20 @@ export class VoiceConversationInput {if (this.closed || (this.speaking && !this.manual) || this.rotating)return;const text = this.draft();+ this.options.diagnostic?.({+ trigger: this.manual ? "manual" : "silence",+ inactiveMs: Date.now() - this.quietAfter + AUTO_SEND_QUIET_MS,+ finalStableMs: this.stableAfter+ ? Date.now() - this.stableAfter + FINAL_STABLE_MS+ : null,+ segments: this.segments.size,+ characters: text.length,+ });for (const key of this.segments.keys()) this.consumed.add(key);this.segments.clear();+ this.awaitingSettlement = false;+ this.quietAfter = 0;+ this.stableAfter = 0;this.manual = false;clearTimeout(this.finalTimer);if (text)@@ -176,8 +220,15 @@ export class VoiceConversationInput {text,});this.options.emit({ type: "draft", text: "" });+ if (!text) this.options.emit({ type: "speech-idle" });},- this.manual ? 0 : 350,+ this.manual+ ? 0+ : Math.max(+ 0,+ this.quietAfter - Date.now(),+ this.stableAfter - Date.now(),+ ),);}diff --git a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.tsindex e1ce118106..693444fe36 100644--- a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts+++ b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts@@ -50,10 +50,10 @@ describe("continuous speech input", () => {await vi.advanceTimersByTimeAsync(400);expect(events.filter((e) => e.type === "utterance")).toHaveLength(0);provider.segment({ id: "one", text: "你好。", final: true });- await vi.advanceTimersByTimeAsync(350);+ await vi.advanceTimersByTimeAsync(2000);provider.segment({ id: "one", text: "你好。", final: true });provider.segment({ id: "two", text: "你好。", final: true });- await vi.advanceTimersByTimeAsync(350);+ await vi.advanceTimersByTimeAsync(2000);expect(events.filter((e) => e.type === "utterance")).toEqual([{ type: "utterance", id: "1", text: "你好。" },{ type: "utterance", id: "2", text: "你好。" },@@ -61,6 +61,95 @@ describe("continuous speech input", () => {await input.stop();});+ it("requires two quiet seconds for ASR-only soft speech and merges a thinking pause", async () => {+ const provider = new Provider();+ const events: ConversationInputEvent[] = [];+ const diagnostic = vi.fn();+ const input = new VoiceConversationInput({+ createProvider: async () => provider,+ emit: (event) => events.push(event),+ diagnostic,+ });+ await input.start();+ provider.segment({ id: "soft", text: "first part", final: true });+ await vi.advanceTimersByTimeAsync(1_500);+ expect(events.some((event) => event.type === "utterance")).toBe(false);+ // Continue within the thinking pause. Even a provider-finalized first part+ // must remain in this utterance until the new activity has also settled.+ input.appendAudio(pcm(2000, 200));+ provider.segment({ id: "next", text: "second part", final: true });+ await vi.advanceTimersByTimeAsync(900);+ input.appendAudio(pcm(0, 900));+ expect(provider.commitUtterance).toHaveBeenCalledOnce();+ await vi.advanceTimersByTimeAsync(1_099);+ expect(events.some((event) => event.type === "utterance")).toBe(false);+ await vi.advanceTimersByTimeAsync(1);+ expect(events.filter((event) => event.type === "utterance")).toEqual([+ { type: "utterance", id: "1", text: "first part second part" },+ ]);+ expect(diagnostic).toHaveBeenCalledExactlyOnceWith({+ trigger: "silence",+ inactiveMs: 2_000,+ finalStableMs: 2_000,+ segments: 2,+ characters: 22,+ });+ await input.stop();+ });++ it("releases a noise-only onset after the quiet window without sending or waiting for an ASR segment", async () => {+ const provider = new Provider();+ const emit = vi.fn();+ const input = new VoiceConversationInput({+ createProvider: async () => provider,+ emit,+ });+ await input.start();+ input.appendAudio(pcm(800, 200));+ expect(emit).toHaveBeenCalledWith({ type: "speech-started" });+ await vi.advanceTimersByTimeAsync(900);+ input.appendAudio(pcm(0, 900));+ await vi.advanceTimersByTimeAsync(1_099);+ expect(emit).not.toHaveBeenCalledWith({ type: "speech-idle" });+ await vi.advanceTimersByTimeAsync(1);+ expect(emit).toHaveBeenCalledWith({ type: "speech-idle" });+ expect(emit).not.toHaveBeenCalledWith(+ expect.objectContaining({ type: "utterance" }),+ );+ await input.stop();+ });++ it("does not let an unchanged final bypass the remaining quiet window or retain an empty draft", async () => {+ const provider = new Provider();+ const emit = vi.fn();+ const input = new VoiceConversationInput({+ createProvider: async () => provider,+ emit,+ });+ await input.start();+ provider.segment({ id: "one", text: "soft speech", final: false });+ await vi.advanceTimersByTimeAsync(1_500);+ provider.segment({ id: "one", text: "soft speech", final: true });+ await vi.advanceTimersByTimeAsync(499);+ expect(emit).not.toHaveBeenCalledWith(+ expect.objectContaining({ type: "utterance" }),+ );+ await vi.advanceTimersByTimeAsync(1);+ expect(emit).toHaveBeenCalledWith({+ type: "utterance",+ id: "1",+ text: "soft speech",+ });+ provider.segment({ id: "empty", text: "discarded partial", final: false });+ provider.segment({ id: "empty", text: "", final: true });+ await vi.advanceTimersByTimeAsync(2_000);+ expect(emit).toHaveBeenCalledWith({ type: "speech-idle" });+ expect(+ emit.mock.calls.filter(([event]) => event.type === "utterance"),+ ).toHaveLength(1);+ await input.stop();+ });+it("does not mistake an out-of-order final for the complete utterance", async () => {const provider = new Provider();const emit = vi.fn();@@ -76,7 +165,7 @@ describe("continuous speech input", () => {expect.objectContaining({ type: "utterance" }),);provider.segment({ id: "a", text: "first", final: true, order: 0 });- await vi.advanceTimersByTimeAsync(350);+ await vi.advanceTimersByTimeAsync(2000);expect(emit).toHaveBeenCalledWith({type: "utterance",id: "1",@@ -128,12 +217,12 @@ describe("continuous speech input", () => {const input = new VoiceConversationInput({createProvider,emit,- rotationMs: 1000,+ rotationMs: 10_000,});await input.start();a.segment({ id: "same", text: "before", final: true });- await vi.advanceTimersByTimeAsync(350);- await vi.advanceTimersByTimeAsync(650);+ await vi.advanceTimersByTimeAsync(2000);+ await vi.advanceTimersByTimeAsync(8000);const buffered = pcm(1000, 200);input.appendAudio(buffered);finish();@@ -142,7 +231,7 @@ describe("continuous speech input", () => {a.segment({ id: "late", text: "ignore", final: true });b.segment({ id: "same", text: "after", final: true });input.appendAudio(pcm(0, 900));- await vi.advanceTimersByTimeAsync(350);+ await vi.advanceTimersByTimeAsync(2000);expect(emit.mock.calls.filter(([e]) => e.type === "utterance")@@ -199,9 +288,9 @@ describe("continuous speech input", () => {await input.start();p.segment({ id: "noise", text: "", final: false });p.segment({ id: "noise", text: "", final: true });- await vi.advanceTimersByTimeAsync(350);+ await vi.advanceTimersByTimeAsync(2000);p.segment({ id: "speech", text: "hello", final: true });- await vi.advanceTimersByTimeAsync(350);+ await vi.advanceTimersByTimeAsync(2000);expect(emit.mock.calls.filter(([e]) => e.type === "utterance")).toEqual([[{ type: "utterance", id: "1", text: "hello" }],]);diff --git a/packages/voice-input-core/src/conversationActivity.ts b/packages/voice-input-core/src/conversationActivity.tsindex c0bdc65008..b91b7b116a 100644--- a/packages/voice-input-core/src/conversationActivity.ts+++ b/packages/voice-input-core/src/conversationActivity.ts@@ -13,7 +13,13 @@ export class ConversationActivity {push(pcm: ArrayBuffer,sampleRate = 16_000,- ): { level: number; started: boolean; ended: boolean; active: boolean } {+ ): {+ level: number;+ started: boolean;+ ended: boolean;+ active: boolean;+ sound: boolean;+ } {const samples = new Int16Array(pcm);let energy = 0;for (const sample of samples) energy += sample * sample;@@ -48,6 +54,7 @@ export class ConversationActivity {started,ended,active: this.active,+ sound,};}}