Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(desktop): preserve voice replies across false speech onsets

JiangJiahao12d68e24a39f
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
index b4b0b82130..19715189ab 100644
--- a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
+++ b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
@@ -1,4 +1,5 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
+import type { AsrSegment } from '@cindy/voice-input-core';
import { VoiceConversationService } from '../service.js';
import { hasVoiceConversationLease } from '../lease.js';
@@ -108,7 +109,7 @@ function fixture() {
dispose: vi.fn(async () => {}),
appendAudio: vi.fn(),
flushAudio: vi.fn(async () => {}),
- onSegment: vi.fn(),
+ onSegment: vi.fn<(callback: (segment: AsrSegment) => void) => void>(),
onEvent: vi.fn(),
};
const deps = {
@@ -127,6 +128,7 @@ function fixture() {
new Response(new Uint8Array([1, 2]), { headers: { 'Content-Type': 'audio/pcm' } }),
),
emit: vi.fn(),
+ log: { info: vi.fn(), warn: vi.fn() },
};
const service = new VoiceConversationService(deps);
active.push(service);
@@ -134,6 +136,44 @@ function fixture() {
}
describe('local companion call ownership', () => {
+ it('logs send timing and speech stages through the injected logger without content or credentials', async () => {
+ vi.useFakeTimers();
+ const { service, deps, provider } = fixture();
+ await service.start(1, selection);
+ provider.onSegment.mock.calls[0][0]({ id: 'segment', text: 'private utterance', final: true });
+ await vi.advanceTimersByTimeAsync(2_000);
+ expect(deps.log.info).toHaveBeenCalledWith('input settled', {
+ callId: selection.callId,
+ trigger: 'silence',
+ inactiveMs: 2_000,
+ finalStableMs: 2_000,
+ segments: 1,
+ characters: 17,
+ });
+ await service.speak(1, selection.callId, 'request-1', 'private reply');
+ await service.readSpeech(1, selection.callId, 'request-1');
+ await service.readSpeech(1, selection.callId, 'request-1');
+ expect(deps.log.info).toHaveBeenCalledWith(
+ 'speech transport',
+ expect.objectContaining({
+ callId: selection.callId,
+ requestId: 'request-1',
+ protocol: 'openai',
+ stage: 'complete',
+ bytes: 2,
+ }),
+ );
+ const logs = JSON.stringify([...deps.log.info.mock.calls, ...deps.log.warn.mock.calls]);
+ for (const privateValue of [
+ 'private utterance',
+ 'private reply',
+ 'test-key',
+ 'gateway.test',
+ 'alloy',
+ ])
+ expect(logs).not.toContain(privateValue);
+ });
+
it('requires a real available speech model and the canonical companion task before allocating ASR', async () => {
const { service, deps } = fixture();
await expect(service.start(1, { ...selection, modelId: 'invented' })).rejects.toThrow(
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
index a8664c2064..cfa253301c 100644
--- a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
+++ b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
@@ -7,6 +7,7 @@ const make = (
),
) => {
const assertCurrent = vi.fn();
+ const diagnostic = vi.fn();
const speech = new SpeechSynthesisSession({
connection: () => ({
baseUrl: 'https://gateway.test/prefix/v1/',
@@ -15,14 +16,15 @@ const make = (
}),
fetch: fetcher,
assertCurrent,
+ diagnostic,
});
- return { speech, fetcher, assertCurrent };
+ return { speech, fetcher, assertCurrent, diagnostic };
};
const input = { requestId: 'one', modelId: 'speech-model', voice: 'alloy', text: 'hello' };
describe('speech transport', () => {
it('preserves the server-provided endpoint prefix, sends the selected model, and pulls binary PCM', async () => {
- const { speech, fetcher } = make();
+ const { speech, fetcher, diagnostic } = make();
expect(await speech.start(input)).toEqual({ sampleRate: 24000 });
expect(fetcher).toHaveBeenCalledWith(
new URL('https://gateway.test/prefix/v1/audio/speech'),
@@ -40,6 +42,13 @@ describe('speech transport', () => {
expect(new Uint8Array((await speech.read('one')).pcm)).toEqual(new Uint8Array([1, 2]));
expect((await speech.read('one')).done).toBe(true);
speech.cancel();
+ expect(diagnostic.mock.calls.map(([event]) => event.stage)).toEqual([
+ 'request',
+ 'headers',
+ 'first_pcm',
+ 'complete',
+ ]);
+ expect(JSON.stringify(diagnostic.mock.calls)).not.toMatch(/hello|test-key|gateway\.test|alloy/);
});
it.each(['audio/mpeg', 'application/json'])(
@@ -53,10 +62,19 @@ describe('speech transport', () => {
);
it('does not pass upstream error bodies to the renderer', async () => {
- const { speech } = make(
+ const { speech, diagnostic } = make(
vi.fn(async () => new Response('upstream-sensitive-error', { status: 401 })),
);
await expect(speech.start(input)).rejects.toThrow(/^request$/);
+ expect(diagnostic).toHaveBeenCalledWith(
+ expect.objectContaining({ stage: 'headers', httpStatus: 401 }),
+ );
+ expect(diagnostic).toHaveBeenCalledWith(
+ expect.objectContaining({ stage: 'error', code: 'request' }),
+ );
+ expect(JSON.stringify(diagnostic.mock.calls)).not.toMatch(
+ /upstream-sensitive-error|hello|test-key/,
+ );
});
it.each([new Uint8Array(), new Uint8Array([1])])(
diff --git a/apps/desktop/src/main/voice-conversation/index.ts b/apps/desktop/src/main/voice-conversation/index.ts
index 72ba61cf89..fb05a51077 100644
--- a/apps/desktop/src/main/voice-conversation/index.ts
+++ b/apps/desktop/src/main/voice-conversation/index.ts
@@ -19,6 +19,9 @@ import { VoiceConversationService } from './service.js';
import { resolveSpeechConnection, speechModels, type SpeechCatalogDeps } from './models.js';
import { companionVoicePreference, updateCompanionVoicePreference } from './preferences.js';
import type { VoiceInputSettings } from '../../shared/voiceInputData.js';
+import { createLogger } from '../logger.js';
+
+const log = createLogger('voice-conversation');
function boundedString(value: unknown, name: string, limit: number): string {
const text = requireString(value, name);
@@ -48,6 +51,7 @@ export function registerVoiceConversationIpc(deps: {
};
const service = new VoiceConversationService({
...deps,
+ log,
owner: () => (isAppSessionBoundaryPending() ? 'changing-account' : activeOwnerScopeKey()),
assertCompanion: async (botId, sessionId) => {
const bot = await getBotRemoteResourceSource(botId);
@@ -81,6 +85,8 @@ export function registerVoiceConversationIpc(deps: {
try {
return await action(event, ...args);
} catch {
+ // Do not log args or upstream errors: either can include text or secrets.
+ log.warn('IPC operation failed', { operation: name });
throwIpcError('PRECONDITION_FAILED', 'Voice conversation is unavailable.');
}
});
diff --git a/apps/desktop/src/main/voice-conversation/service.ts b/apps/desktop/src/main/voice-conversation/service.ts
index a6e1ca4988..29e9b884d1 100644
--- a/apps/desktop/src/main/voice-conversation/service.ts
+++ b/apps/desktop/src/main/voice-conversation/service.ts
@@ -13,6 +13,7 @@ import type {
import { acquireVoiceConversationLease, releaseVoiceConversationLease } from './lease.js';
import { SpeechSynthesisSession, type SpeechConnection } from './speech.js';
import { loadSpeechVoices } from './voices.js';
+import type { Logger } from '../logger.js';
type Call = {
id: string;
@@ -45,6 +46,7 @@ export class VoiceConversationService {
connection(selection: SpeechModelSelection): SpeechConnection;
fetch: typeof fetch;
emit(sender: number, event: VoiceConversationEvent): void;
+ log?: Pick<Logger, 'info' | 'warn'>;
},
) {}
@@ -82,6 +84,11 @@ export class VoiceConversationService {
sender,
owner,
speech: new SpeechSynthesisSession({
+ diagnostic: (event) =>
+ this.deps.log?.[event.stage === 'error' ? 'warn' : 'info']('speech transport', {
+ previewId: selection.previewId,
+ ...event,
+ }),
connection: () => binding,
fetch: this.deps.fetch,
assertCurrent: () => {
@@ -146,6 +153,7 @@ export class VoiceConversationService {
if (this.call?.id !== id || this.deps.owner() !== owner) throw new Error('call_ended');
};
const input = new VoiceConversationInput({
+ diagnostic: (event) => this.deps.log?.info('input settled', { callId: id, ...event }),
createProvider: async () => {
assertCurrent();
return this.deps.createProvider();
@@ -156,11 +164,24 @@ export class VoiceConversationService {
void this.end(sender, id);
return;
}
+ if (event.type !== 'draft')
+ this.deps.log?.[event.type === 'error' ? 'warn' : 'info']('input event', {
+ callId: id,
+ event: event.type,
+ ...(event.type === 'utterance'
+ ? { utteranceId: event.id, characters: event.text.length }
+ : {}),
+ });
this.deps.emit(sender, { callId: id, event });
if (event.type === 'error') void this.end(sender, id);
},
});
const speech = new SpeechSynthesisSession({
+ diagnostic: (event) =>
+ this.deps.log?.[event.stage === 'error' ? 'warn' : 'info']('speech transport', {
+ callId: id,
+ ...event,
+ }),
connection: () => binding,
fetch: this.deps.fetch,
assertCurrent: () => {
@@ -171,13 +192,17 @@ export class VoiceConversationService {
},
});
this.call = { id, owner, sender, selection, input, speech };
+ const startedAt = Date.now();
+ this.deps.log?.info('call starting', { callId: id, protocol: binding.protocol });
try {
await this.deps.assertCompanion(selection.botId, selection.sessionId);
assertCurrent();
await input.start();
assertCurrent();
+ this.deps.log?.info('call ready', { callId: id, elapsedMs: Date.now() - startedAt });
return { callId: id };
} catch (error) {
+ this.deps.log?.warn('call start failed', { callId: id, elapsedMs: Date.now() - startedAt });
await this.end(sender, id);
throw error;
}
@@ -223,6 +248,7 @@ export class VoiceConversationService {
const call = this.call;
if (!call || call.sender !== sender || (id && call.id !== id)) return;
this.call = null;
+ this.deps.log?.info('call ended', { callId: call.id });
this.deps.emit(call.sender, { callId: call.id, event: { type: 'ended' } });
call.speech.cancel();
try {
diff --git a/apps/desktop/src/main/voice-conversation/speech.ts b/apps/desktop/src/main/voice-conversation/speech.ts
index dc81beefa4..39c3a5ceae 100644
--- a/apps/desktop/src/main/voice-conversation/speech.ts
+++ b/apps/desktop/src/main/voice-conversation/speech.ts
@@ -1,6 +1,15 @@
import { miniMaxPcmStream, speechRequest, type SpeechConnection } from './speechProtocols.js';
import { SpeechError } from './speechError.js';
export type { SpeechConnection } from './speechProtocols.js';
+export type SpeechDiagnostic = {
+ stage: 'request' | 'headers' | 'first_pcm' | 'complete' | 'cancelled' | 'error';
+ requestId: string;
+ protocol: SpeechConnection['protocol'];
+ elapsedMs: number;
+ bytes: number;
+ httpStatus?: number;
+ code?: SpeechError['code'] | 'headers_timeout' | 'read_timeout';
+};
/** PCM speech transport with provider adapters. Reads are pulled by playback for bounded buffering. */
export class SpeechSynthesisSession {
private epoch = 0;
@@ -9,12 +18,17 @@ export class SpeechSynthesisSession {
private requestId: string | null = null;
private totalBytes = 0;
private reading = false;
+ private diagnostic?: (
+ stage: SpeechDiagnostic['stage'],
+ details?: Pick<SpeechDiagnostic, 'httpStatus' | 'code'>,
+ ) => void;
constructor(
private readonly deps: {
connection(): SpeechConnection;
assertCurrent(): void;
fetch: typeof fetch;
+ diagnostic?(event: SpeechDiagnostic): void;
},
) {}
@@ -32,7 +46,22 @@ export class SpeechSynthesisSession {
const controller = new AbortController();
this.controller = controller;
this.requestId = input.requestId;
- const timer = setTimeout(() => controller.abort(), 30_000);
+ const startedAt = Date.now();
+ this.diagnostic = (stage, details = {}) =>
+ this.deps.diagnostic?.({
+ stage,
+ requestId: input.requestId,
+ protocol: connection.protocol,
+ elapsedMs: Date.now() - startedAt,
+ bytes: this.totalBytes,
+ ...details,
+ });
+ this.diagnostic('request');
+ let timedOut = false;
+ const timer = setTimeout(() => {
+ timedOut = true;
+ controller.abort();
+ }, 30_000);
let response: Response | undefined;
try {
response = await this.deps.fetch(url, {
@@ -47,6 +76,7 @@ export class SpeechSynthesisSession {
await response.body?.cancel();
throw new SpeechError('cancelled');
}
+ this.diagnostic('headers', { httpStatus: response.status });
if (!response.ok || !response.body) {
await response.body?.cancel();
throw new SpeechError('request');
@@ -71,7 +101,16 @@ export class SpeechSynthesisSession {
} catch (error) {
if (response?.body && !response.body.locked)
void response.body.cancel().catch(() => undefined);
- if (epoch === this.epoch) this.cancel();
+ if (epoch === this.epoch) {
+ this.diagnostic?.('error', {
+ code: timedOut
+ ? 'headers_timeout'
+ : error instanceof SpeechError
+ ? error.code
+ : 'request',
+ });
+ this.cancel('failed');
+ }
throw error instanceof SpeechError ? error : new SpeechError('request');
} finally {
clearTimeout(timer);
@@ -91,27 +130,36 @@ export class SpeechSynthesisSession {
const epoch = this.epoch;
this.reading = true;
const timer = setTimeout(() => {
- if (epoch === this.epoch) this.cancel();
+ if (epoch === this.epoch) {
+ this.diagnostic?.('error', { code: 'read_timeout' });
+ this.cancel('failed');
+ }
}, 15_000);
try {
const result = await reader.read();
this.deps.assertCurrent();
if (epoch !== this.epoch) throw new SpeechError('cancelled');
const value = result.value ?? new Uint8Array();
+ const firstPcm = this.totalBytes === 0 && value.byteLength > 0;
this.totalBytes += value.byteLength;
if (value.byteLength > 1_048_576 || this.totalBytes > 24_000 * 2 * 600) {
- this.cancel();
throw new SpeechError('format');
}
+ if (firstPcm) this.diagnostic?.('first_pcm');
if (result.done) {
if (this.totalBytes === 0 || this.totalBytes % 2 !== 0) throw new SpeechError('format');
reader.releaseLock();
this.reader = null;
this.requestId = null;
+ this.diagnostic?.('complete');
+ this.diagnostic = undefined;
}
return { pcm: Uint8Array.from(value).buffer, done: result.done };
} catch (error) {
- if (epoch === this.epoch) this.cancel();
+ if (epoch === this.epoch) {
+ this.diagnostic?.('error', { code: error instanceof SpeechError ? error.code : 'request' });
+ this.cancel('failed');
+ }
throw error instanceof SpeechError ? error : new SpeechError('request');
} finally {
clearTimeout(timer);
@@ -119,7 +167,9 @@ export class SpeechSynthesisSession {
}
}
- cancel(): void {
+ cancel(reason: 'interrupted' | 'failed' = 'interrupted'): void {
+ if (reason === 'interrupted') this.diagnostic?.('cancelled');
+ this.diagnostic = undefined;
++this.epoch;
this.controller?.abort();
this.controller = null;
diff --git a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
index a371005591..4b9e2cfac5 100644
--- a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
@@ -18,7 +18,13 @@ export class Pcm16Decoder {
}
export interface SpeechPlayer {
- play(read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>, sampleRate: number): Promise<void>;
+ play(
+ read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,
+ sampleRate: number,
+ onStarted?: () => void,
+ ): Promise<void>;
+ pause(): void;
+ resume(): void;
cancel(): void;
close(): void;
}
@@ -29,35 +35,37 @@ export class PcmSpeechPlayer implements SpeechPlayer {
private generation = 0;
private sources = new Set<AudioBufferSourceNode>();
private timers = new Set<ReturnType<typeof setTimeout>>();
+ private paused = false;
+ private contextTransition: Promise<void>;
+ private resumeWaiters = new Set<() => void>();
constructor(private readonly onLevel: (level: number) => void) {
// Construct/resume in the explicit Start gesture, before any network await.
this.context = new AudioContext();
- void this.context.resume().catch(() => undefined);
+ this.contextTransition = this.context.resume();
+ void this.contextTransition.catch(() => undefined);
}
async play(
read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,
sampleRate: number,
+ onStarted?: () => void,
): Promise<void> {
this.cancel();
const generation = this.generation;
const decoder = new Pcm16Decoder();
let nextAt = this.context.currentTime;
- const assertActive = () => {
- if (generation !== this.generation || this.context.state !== 'running')
- throw new Error('Playback interrupted');
- };
- await this.context.resume();
+ let started = false;
+ this.updateContext();
for (;;) {
- assertActive();
+ await this.waitUntilRunning(generation);
const chunk = await read();
- assertActive();
+ await this.waitUntilRunning(generation);
const samples = decoder.decode(chunk.pcm);
for (let offset = 0; offset < samples.length; offset += 2_400) {
while (nextAt - this.context.currentTime > 0.3) {
await new Promise((resolve) => setTimeout(resolve, 25));
- assertActive();
+ await this.waitUntilRunning(generation);
}
const part = samples.subarray(offset, offset + 2_400);
const buffer = this.context.createBuffer(1, part.length, sampleRate);
@@ -78,12 +86,16 @@ export class PcmSpeechPlayer implements SpeechPlayer {
const timer = setTimeout(
() => {
this.timers.delete(timer);
- if (generation === this.generation) this.onLevel(level);
+ if (generation === this.generation && !this.paused) this.onLevel(level);
},
Math.max(0, nextAt - this.context.currentTime) * 1000,
);
this.timers.add(timer);
source.start(nextAt);
+ if (!started) {
+ started = true;
+ onStarted?.();
+ }
nextAt += part.length / sampleRate;
}
if (chunk.done) break;
@@ -91,13 +103,64 @@ export class PcmSpeechPlayer implements SpeechPlayer {
decoder.finish();
while (nextAt > this.context.currentTime) {
await new Promise((resolve) => setTimeout(resolve, 25));
- assertActive();
+ await this.waitUntilRunning(generation);
}
this.onLevel(0);
}
+ // Suspend the audio clock as well as reads: queued samples resume from their
+ // exact position after a false onset, without another paid synthesis request.
+ pause(): void {
+ if (this.paused) return;
+ this.paused = true;
+ this.onLevel(0);
+ this.updateContext();
+ }
+
+ resume(): void {
+ if (!this.paused) return;
+ this.paused = false;
+ this.updateContext();
+ this.wakePlayback();
+ }
+
+ private updateContext(): void {
+ // Serialize native transitions, consulting current intent so a late suspend
+ // cannot overtake Resume or reopen a context closed at an account boundary.
+ this.contextTransition = this.contextTransition
+ .catch(() => undefined)
+ .then(async () => {
+ if (this.context.state === 'closed') return;
+ if (this.paused) await this.context.suspend();
+ else await this.context.resume();
+ });
+ void this.contextTransition.catch(() => undefined);
+ }
+
+ private async waitUntilRunning(generation: number): Promise<void> {
+ for (;;) {
+ if (generation !== this.generation || this.context.state === 'closed')
+ throw new Error('Playback interrupted');
+ if (this.paused) {
+ await new Promise<void>((resolve) => this.resumeWaiters.add(resolve));
+ continue;
+ }
+ await this.contextTransition;
+ if (generation !== this.generation) throw new Error('Playback interrupted');
+ if (this.paused) continue;
+ if (this.context.state !== 'running') throw new Error('Playback interrupted');
+ return;
+ }
+ }
+
+ private wakePlayback(): void {
+ for (const resolve of this.resumeWaiters) resolve();
+ this.resumeWaiters.clear();
+ }
+
cancel(): void {
++this.generation;
+ this.wakePlayback();
for (const timer of this.timers) clearTimeout(timer);
this.timers.clear();
for (const source of this.sources) {
diff --git a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
index 93d6e5fa33..0f244f5fbe 100644
--- a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
@@ -3,10 +3,13 @@ import type {
VoiceConversationSelection,
} from '@/../shared/voiceConversation';
import type { ChatMessage } from '@/lib/makerChatStore';
+import { createLogger } from '@/lib/logger';
import type { PcmChunk } from '@/voice-input/WebMicAudioEngine';
import type { SpeechPlayer } from './PcmSpeechPlayer';
import { speechChunks, VoiceReplyTracker } from './VoiceReplyTracker';
+const log = createLogger('voice-conversation');
+
export type VoicePhase =
'idle' | 'connecting' | 'listening' | 'thinking' | 'preparing' | 'speaking' | 'paused' | 'error';
export type VoiceSnapshot = {
@@ -51,6 +54,8 @@ export class VoiceConversationController {
private queued: string[] = [];
private pendingStart: Promise<unknown> = Promise.resolve();
private teardown: Promise<unknown> = Promise.resolve();
+ private speechCandidate = false;
+ private replyPhase: 'thinking' | 'preparing' | 'speaking' | null = null;
constructor(
private readonly deps: {
@@ -73,6 +78,8 @@ export class VoiceConversationController {
return () => this.listeners.delete(listener);
};
private patch(patch: Partial<VoiceSnapshot>): void {
+ if (patch.phase && patch.phase !== this.state.phase)
+ log.info('phase changed', { callId: this.callId, from: this.state.phase, to: patch.phase });
this.state = { ...this.state, ...patch };
for (const listener of this.listeners) listener();
}
@@ -93,8 +100,25 @@ export class VoiceConversationController {
this.unsubscribe = this.deps.api.onEvent(({ callId, event }) => {
if (!current() || callId !== this.callId) return;
if (event.type === 'speech-started') {
- this.interrupt();
- this.patch({ phase: 'listening' });
+ // An acoustic onset can be noise. Pause playback without consuming
+ // the reply; only a finalized, nonempty utterance replaces its owner.
+ if (!this.speechCandidate) {
+ this.speechCandidate = true;
+ this.player?.pause();
+ log.info('reply held for possible speech', { callId, pendingInput: this.pendingInput });
+ this.patch({ phase: 'listening', level: 0 });
+ }
+ } else if (event.type === 'speech-idle') {
+ if (this.speechCandidate) {
+ this.speechCandidate = false;
+ this.player?.resume();
+ log.info('reply resumed after empty speech', {
+ callId,
+ pendingInput: this.pendingInput,
+ });
+ this.setReplyPhase(this.replyPhase);
+ this.updateMessages(this.latestMessages);
+ }
} else if (event.type === 'draft') {
this.patch({ transcript: event.text });
} else if (event.type === 'utterance') {
@@ -151,19 +175,28 @@ export class VoiceConversationController {
await mic.stop();
return;
}
- this.patch({ phase: this.pendingInput ? 'thinking' : 'listening' });
+ this.setReplyPhase(this.pendingInput ? 'thinking' : null);
this.updateMessages(this.latestMessages);
} catch {
if (current()) this.fail('connection');
}
}
- /** Speech onset fences old replies and audio before a new transcript is committed. */
- interrupt(): void {
+ private setReplyPhase(phase: typeof this.replyPhase): void {
+ this.replyPhase = phase;
+ this.patch({ phase: this.speechCandidate ? 'listening' : (phase ?? 'listening'), level: 0 });
+ }
+
+ /** Explicit stop or a committed new utterance permanently relinquishes the old reply. */
+ interrupt(reason: 'user_stop' | 'new_utterance' = 'user_stop'): void {
+ log.info('reply interrupted', { callId: this.callId, reason, pendingInput: this.pendingInput });
++this.speechEpoch;
+ this.speechCandidate = false;
+ this.replyPhase = null;
this.tracker.interrupt();
this.pendingInput = null;
this.player?.cancel();
+ this.player?.resume();
if (this.callId) void this.deps.api.interrupt(this.callId).catch(() => undefined);
if (
this.state.phase === 'speaking' ||
@@ -177,8 +210,9 @@ export class VoiceConversationController {
const current = () =>
this.enabled && epoch === this.epoch && !this.blocked && !this.state.muted;
if (!current()) return;
- this.interrupt();
- this.patch({ phase: 'thinking', transcript: '', unsent: '' });
+ this.interrupt('new_utterance');
+ this.setReplyPhase('thinking');
+ this.patch({ transcript: '', unsent: '' });
const replyEpoch = this.speechEpoch;
let clientId: string | null = null;
try {
@@ -201,6 +235,7 @@ export class VoiceConversationController {
return;
}
if (replyEpoch !== this.speechEpoch) return;
+ log.info('voice input accepted', { callId: this.callId, clientId, characters: text.length });
this.pendingInput = clientId;
this.tracker.expect(clientId);
this.updateMessages(this.latestMessages);
@@ -228,17 +263,33 @@ export class VoiceConversationController {
updateMessages(messages: readonly ChatMessage[]): void {
this.latestMessages = messages;
- if (!this.enabled || !this.callId || this.blocked || this.state.muted || !this.pendingInput)
+ if (
+ !this.enabled ||
+ !this.callId ||
+ this.blocked ||
+ this.state.muted ||
+ !this.pendingInput ||
+ this.speechCandidate
+ )
return;
const reply = this.tracker.take(messages);
if (!reply) {
if (!this.tracker.hasPendingReply) {
+ log.info('reply ownership superseded', {
+ callId: this.callId,
+ pendingInput: this.pendingInput,
+ });
this.pendingInput = null;
- this.patch({ phase: 'listening' });
+ this.setReplyPhase(null);
}
return;
}
this.pendingInput = null;
+ log.info('final reply selected', {
+ callId: this.callId,
+ replyId: reply.id,
+ characters: reply.text.length,
+ });
void this.speak(reply.text);
}
@@ -250,17 +301,25 @@ export class VoiceConversationController {
if (!callId || !player) return;
const current = () =>
epoch === this.epoch && speechEpoch === this.speechEpoch && this.enabled && !this.blocked;
- this.patch({ phase: 'preparing', level: 0 });
+ this.setReplyPhase('preparing');
try {
for (const [index, chunk] of speechChunks(text).entries()) {
if (!current()) return;
const requestId = `${speechEpoch}:${index}`;
+ log.info('speech requested', { callId, requestId, characters: chunk.length });
const { sampleRate } = await this.deps.api.speak({ callId, requestId, text: chunk });
if (!current()) return;
- this.patch({ phase: 'speaking' });
- await player.play(() => this.deps.api.readSpeech(callId, requestId), sampleRate);
+ this.setReplyPhase('speaking');
+ await player.play(
+ () => this.deps.api.readSpeech(callId, requestId),
+ sampleRate,
+ () => {
+ log.info('playback scheduled', { callId, requestId, sampleRate });
+ },
+ );
+ if (current()) log.info('playback completed', { callId, requestId });
}
- if (current()) this.patch({ phase: 'listening', level: 0 });
+ if (current()) this.setReplyPhase(null);
} catch {
if (current()) this.fail('speech');
}
@@ -307,6 +366,7 @@ export class VoiceConversationController {
}
private fail(error: VoiceSnapshot['error'], text?: string): void {
+ log.warn('voice mode failed', { callId: this.callId, phase: this.state.phase, reason: error });
const unsent = [text || this.state.transcript || this.state.unsent, ...this.queued]
.filter(Boolean)
.join(' ');
@@ -317,6 +377,8 @@ export class VoiceConversationController {
private release(): void {
++this.epoch;
++this.speechEpoch;
+ this.speechCandidate = false;
+ this.replyPhase = null;
this.queued = [];
const callId = this.callId;
const microphone = this.microphone;
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
index c684ee0f82..9d22504eb8 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
@@ -26,6 +26,8 @@ vi.mock('@/voice-input/WebMicAudioEngine', () => ({
vi.mock('../PcmSpeechPlayer', () => ({
PcmSpeechPlayer: class {
close = audio.close;
+ pause() {}
+ resume() {}
cancel() {}
async play() {}
},
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx
index 7bd7ca8d96..eeb0e7f6bb 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx
@@ -59,6 +59,8 @@ vi.mock('@/voice-input/WebMicAudioEngine', () => ({
vi.mock('../PcmSpeechPlayer', () => ({
PcmSpeechPlayer: class {
close = audio.close;
+ pause() {}
+ resume() {}
cancel() {}
async play() {}
},
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
index cb12461721..8e9684fe18 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
@@ -17,13 +17,23 @@ class TestAudioContext {
state = 'running';
destination = {};
sources: ReturnType<typeof createTestSource>[] = [];
+ private elapsed = 0;
+ private runningAt = Date.now();
get currentTime() {
- return Date.now() / 1000;
+ return this.elapsed + (this.state === 'running' ? (Date.now() - this.runningAt) / 1000 : 0);
}
constructor() {
TestAudioContext.latest = this;
}
- resume = vi.fn(async () => {});
+ resume = vi.fn(async () => {
+ if (this.state === 'running') return;
+ this.runningAt = Date.now();
+ this.state = 'running';
+ });
+ suspend = vi.fn(async () => {
+ this.elapsed = this.currentTime;
+ this.state = 'suspended';
+ });
close = vi.fn(async () => {
this.state = 'closed';
});
@@ -85,3 +95,43 @@ it('fences an IPC read that finishes after cancellation', async () => {
expect(TestAudioContext.latest.sources).toHaveLength(0);
player.close();
});
+
+it('freezes the audio clock and queued samples during a false onset, then continues without rereading', async () => {
+ const player = new PcmSpeechPlayer(vi.fn());
+ const read = vi.fn(async () => ({ pcm: new Int16Array(24_000).buffer, done: true }));
+ const started = vi.fn();
+ const playing = player.play(read, 24_000, started);
+ await vi.advanceTimersByTimeAsync(100);
+ const context = TestAudioContext.latest;
+ const sourcesBefore = [...context.sources];
+ player.pause();
+ await vi.advanceTimersByTimeAsync(0);
+ const position = context.currentTime;
+ await vi.advanceTimersByTimeAsync(2_000);
+ expect(context.currentTime).toBe(position);
+ expect(context.sources).toEqual(sourcesBefore);
+ expect(sourcesBefore.every((source) => source.stop.mock.calls.length === 0)).toBe(true);
+ player.resume();
+ await vi.advanceTimersByTimeAsync(1_100);
+ await playing;
+ expect(read).toHaveBeenCalledOnce();
+ expect(started).toHaveBeenCalledOnce();
+ expect(context.sources.slice(0, sourcesBefore.length)).toEqual(sourcesBefore);
+ player.close();
+});
+
+it('holds a synthesis response that arrives during noise and cancels a suspended player on teardown', async () => {
+ const player = new PcmSpeechPlayer(vi.fn());
+ player.pause();
+ const read = vi.fn(async () => ({ pcm: new Int16Array(2_400).buffer, done: true }));
+ const playing = player.play(read, 24_000);
+ const rejected = expect(playing).rejects.toThrow('Playback interrupted');
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(read).not.toHaveBeenCalled();
+ expect(TestAudioContext.latest.sources).toHaveLength(0);
+ player.close();
+ await rejected;
+ player.resume();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(TestAudioContext.latest.state).toBe('closed');
+});
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
index 82bd4c7778..4f4327a4c0 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
@@ -1,8 +1,12 @@
-import { describe, expect, it, vi } from 'vitest';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { VoiceConversationApi, VoiceConversationEvent } from '@/../shared/voiceConversation';
import type { ChatMessage } from '@/lib/makerChatStore';
import { VoiceConversationController } from '../VoiceConversationController';
+const log = vi.hoisted(() => ({ info: vi.fn(), warn: vi.fn() }));
+vi.mock('@/lib/logger', () => ({ createLogger: () => log }));
+beforeEach(() => vi.clearAllMocks());
+
const selection = { modelId: 'tts', voice: 'alloy' };
const user = (id = 'user-1'): ChatMessage =>
({ clientId: id, role: 'user', content: 'hello' }) as ChatMessage;
@@ -39,7 +43,13 @@ function fixture() {
stop: vi.fn(async () => {}),
onPcm16k: vi.fn(),
};
- const player = { play: vi.fn(async () => {}), cancel: vi.fn(), close: vi.fn() };
+ const player = {
+ play: vi.fn(async () => {}),
+ pause: vi.fn(),
+ resume: vi.fn(),
+ cancel: vi.fn(),
+ close: vi.fn(),
+ };
const send = vi.fn(
async (_text: string, created: (id: string) => void, current: () => boolean) => {
if (!current()) return false;
@@ -89,7 +99,7 @@ describe('companion voice loop', () => {
expect(f.microphone.stop).toHaveBeenCalledOnce();
});
- it('interrupts pending TTS immediately and discards its late response without stopping the task', async () => {
+ it('pauses on acoustic onset and cancels pending TTS only when a new utterance commits', async () => {
const f = fixture();
let finish!: (value: { sampleRate: number }) => void;
f.api.speak.mockImplementationOnce(
@@ -104,14 +114,16 @@ describe('companion voice loop', () => {
await Promise.resolve();
f.controller.updateMessages([user(), reply()]);
expect(f.api.speak).toHaveBeenCalledOnce();
+ const interruptions = f.api.interrupt.mock.calls.length;
f.emit({ type: 'speech-started' });
- expect(f.player.cancel).toHaveBeenCalled();
- expect(f.api.interrupt).toHaveBeenCalledWith('call-1');
+ expect(f.player.pause).toHaveBeenCalledOnce();
+ expect(f.api.interrupt).toHaveBeenCalledTimes(interruptions);
+ f.utterance('actually, another question');
+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));
+ expect(f.api.interrupt).toHaveBeenCalledTimes(interruptions + 1);
finish({ sampleRate: 24000 });
await Promise.resolve();
expect(f.player.play).not.toHaveBeenCalled();
- f.utterance('actually, another question');
- await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));
expect(f.api.end).not.toHaveBeenCalled();
f.controller.end();
});
@@ -180,7 +192,7 @@ describe('companion voice loop', () => {
expect(f.controller.getSnapshot().phase).toBe('idle');
});
- it('an interruption before a delayed send acknowledgement does not resurrect its old reply', async () => {
+ it('keeps a delayed send acknowledgement through a false onset and reads its final reply after quiet', async () => {
const f = fixture();
let done!: () => void;
f.send.mockImplementationOnce(async (_text, created) => {
@@ -198,6 +210,105 @@ describe('companion voice loop', () => {
await Promise.resolve();
f.controller.updateMessages([user('old-user'), reply()]);
expect(f.api.speak).not.toHaveBeenCalled();
+ f.emit({ type: 'speech-idle' });
+ await vi.waitFor(() => expect(f.api.speak).toHaveBeenCalledOnce());
+ expect(f.player.resume).toHaveBeenCalled();
+ f.controller.end();
+ });
+
+ it('reads the first final reply after a noise-only onset and still reads the next turn', async () => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.utterance();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.emit({ type: 'speech-started' });
+ f.controller.updateMessages([user(), reply()]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ f.emit({ type: 'draft', text: '' });
+ f.emit({ type: 'speech-idle' });
+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());
+ f.emit({ type: 'speech-idle' });
+ expect(f.api.speak).toHaveBeenCalledOnce();
+ f.utterance('next question');
+ await Promise.resolve();
+ await Promise.resolve();
+ f.controller.updateMessages([user(), reply(), user('user-2'), reply('reply-2')]);
+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledTimes(2));
+ expect(f.controller.getSnapshot().error).toBeNull();
+ // Existing logger receives identifiers and counts, never voice/reply text.
+ expect(JSON.stringify(log.info.mock.calls)).not.toContain('next question');
+ expect(JSON.stringify(log.info.mock.calls)).not.toContain('Hello back');
+ f.controller.end();
+ });
+
+ it('resumes an already playing reply after empty speech without requesting synthesis again', async () => {
+ const f = fixture();
+ let played!: () => void;
+ f.player.play.mockImplementationOnce(
+ () =>
+ new Promise<void>((resolve) => {
+ played = resolve;
+ }),
+ );
+ await f.controller.start(selection);
+ f.utterance();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.controller.updateMessages([user(), reply()]);
+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());
+ f.emit({ type: 'speech-started' });
+ expect(f.controller.getSnapshot().phase).toBe('listening');
+ f.emit({ type: 'speech-idle' });
+ expect(f.player.resume).toHaveBeenCalled();
+ expect(f.controller.getSnapshot().phase).toBe('speaking');
+ expect(f.api.speak).toHaveBeenCalledOnce();
+ played();
+ await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('listening'));
+ f.controller.end();
+ });
+
+ it.each(['permission', 'mute', 'end'] as const)(
+ 'does not resume a held reply across the %s boundary',
+ async (boundary) => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.utterance();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.emit({ type: 'speech-started' });
+ if (boundary === 'permission') f.controller.setBlocked(true);
+ else if (boundary === 'mute') f.controller.toggleMute();
+ else f.controller.end();
+ f.emit({ type: 'speech-idle' });
+ f.controller.updateMessages([user(), reply()]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ expect(f.player.close).toHaveBeenCalled();
+ f.controller.end();
+ },
+ );
+
+ it('does not restore reply ownership after an explicit stop before send acknowledgement', async () => {
+ const f = fixture();
+ let done!: () => void;
+ f.send.mockImplementationOnce(async (_text, created) => {
+ created('old-user');
+ await new Promise<void>((resolve) => {
+ done = resolve;
+ });
+ return true;
+ });
+ await f.controller.start(selection);
+ f.utterance();
+ f.emit({ type: 'speech-started' });
+ f.controller.interrupt();
+ done();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.emit({ type: 'speech-idle' });
+ f.controller.updateMessages([user('old-user'), reply()]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ expect(f.controller.getSnapshot().phase).toBe('listening');
f.controller.end();
});
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts
index 3d87b1b964..70a1041ff6 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts
@@ -11,6 +11,8 @@ function fixture() {
play: vi.fn(async (read: () => Promise<unknown>) => {
await read();
}),
+ pause: vi.fn(),
+ resume: vi.fn(),
cancel: vi.fn(),
close: vi.fn(),
};
diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.md
index 41ff8d1f46..a1ca406b17 100644
--- a/docs/product-rules/companion-voice-mode.md
+++ b/docs/product-rules/companion-voice-mode.md
@@ -4,7 +4,7 @@
- 入口位于本机伙伴一对一输入框的听写按钮与发送按钮之间,使用声波图标和“开始语音对话”提示。标题栏不再提供语音入口。沿用伙伴 canonical 任务、模型、上下文、工作目录和工具权限,不创建第二条任务或切换聊天模型。
- 用户点击输入框的语音对话按钮后,重新读取当前目录和该伙伴偏好;配置有效即开始连接并采集麦克风,无需再点击“开始”。模型缺失、音色未选或接口不明时直接打开“伙伴设置 → 语音”,不采集麦克风;设置完成不会自动开始,需返回输入框明确点击。使用现有语音输入的 ASR 设置和当前账号的服务模式;不启用听写润色,不修改原有听写行为。
-- 停顿约 900ms 后等待 ASR 的最终文字,再用短暂的稳定窗口合并同一段发言。只有最终、非空文字能自动发送;声音活动检测本身不能产生消息。
+- 最后一次声音活动或识别文字更新后,至少等待 2 秒再自动发送;期间继续说话会合并到同一段发言,只有 ASR 检出的弱声也遵守相同门槛。900ms 的声音释放只负责提前提交 ASR 缓冲,不触发发送;还须等待全部分段为最终文字及最后一次 ASR 更新后的 350ms 稳定窗口,延迟返回的识别结果可能延长等待。手动发送可跳过停顿等待,但不能发送未完成的识别文字。只有最终、非空文字能发送;声音活动检测本身不能产生消息。
- 原任务空闲时正常发送,仍在运行时沿既有插话路径投递。语音是本机用户本人输入,不盖自动化或远程设备来源标签,不增加系统提示词。识别结果按普通文字发送,不执行桌面斜杠命令。
- 最终回复经过持久化完成标记确认后才朗读。按本次语音发送的 clientId 关联,不朗读历史、过程文字、工具、子代理、内部协调或群来源私信。文本消息始终是对话记录的正本。
- 在“伙伴设置 → 语音”中独立选择朗读模型和音色,按当前账号下的伙伴保存,不改变聊天模型。来源包括有权限的 Cindy AI,以及已连接的本地 BYOK 供应商;候选必须是 `audio_speech`、未停用且具备可用的 API key 或明确的本机免鉴权连接。同名模型按连接 ID、runtime 和上游模型 ID 区分,不能借另一连接的密钥。无候选时显示空状态,不采集麦克风。
@@ -13,7 +13,7 @@
- 更换连接、runtime、型号或接口时,不沿用旧音色;目标型号有明确参考默认值才填入,否则等待用户选择或输入。同一连接下的不同型号也不视为音色兼容。已有保存音色不因刷新失败、列表未收录而被覆盖;刷新失败保留已知名称,成功获取后仍未收录的 ID 显示在手动输入中。
- 旧偏好中的接口与官方地址冲突时,面板采用官方接口,接口选择仍隐藏,在音色区域上方提示重新选择音色,重新选择前不试听或开始;正确接口下的已有音色继续保留。读取偏好时不写回推导默认值,用户选择音色后通过既有保存流程移除不适用的接口覆盖。自定义/中转地址的显式接口与音色偏好不受影响。
- 刷新重新读取当前有效目录,并尝试刷新 Cindy AI;Gateway 刷新失败不阻止本机 BYOK 目录显示。新增第三方型号仍沿设置页已有的获取列表、手动添加和启用流程。
-- 朗读结束继续监听。用户开口时立即取消待生成及正在播放的声音;新的文字仍投递到原任务。停止朗读不会取消 Agent 工具、清空任务队列或代答权限确认。
+- 朗读结束继续监听。检测到可能开口时先暂停播放并保留待回复的关联;只有最终、非空的新发言提交时才永久取消旧朗读,新的文字仍投递到原任务。仅有噪声或最终为空时,停顿窗口结束后恢复等待或从原播放位置继续,不再次请求合成。用户明确停止朗读仍立即取消;停止朗读不会取消 Agent 工具、清空任务队列或代答权限确认。
- 需要确认权限、问题或计划,或任务队列暂停时,关闭麦克风并暂停语音模式;用户完成确认后显式继续。静音、锁屏、休眠、切走任务、关面板、账号切换同样释放采集和播放资源,不在后台重新开始。
- 未发送的文字在可恢复错误或暂停后留在面板内;用户可选中复制或明确重新连接并发送。TTS 失败保留任务中的文字回复,不自动重发用户消息。
@@ -55,6 +55,12 @@
- 用户点击试听后,以当前连接、型号、接口和音色合成短示例文本,并明确提示可能产生用量。复用既有 PCM 合成、解码和播放;不采集麦克风、不发送伙伴消息、不调用聊天模型,不使用供应商返回的任意 `preview_url`。试听结果表示这次调用结果,不持久标记整个账号或型号已验证。
- 试听与语音模式/听写互斥,绑定可信窗口、账号和伙伴 canonical 任务。生成过程中也可停止;改选、隐藏页面、关面板、账号变化、系统挂起或权限阻塞时取消;开始语音前等待试听停止。窗口销毁与 60 秒总时限均释放 Main 的语音占用。晚到的结果不恢复播放、不覆盖新选择。
+## 诊断日志
+
+复用现有 Main / Renderer `createLogger`,使用 `voice-conversation` / `r:voice-conversation` scope,统一写入既有 `main-YYYY-MM-DD.log`,沿用按天轮换和 30 天保留规则。开发实例保存在 `apps/desktop/logs`,安装版在当前 `userData/logs`;不创建另一套日志文件或音频存储,不改变现有上传来源白名单。
+
+记录通话开始、ASR 就绪、发送判定、消息关联、可能插话时的暂停与恢复、明确取消、TTS 请求 / 响应状态 / 首段 PCM / 完成,以及播放调度和失败。用 callId、clientId、requestId 关联各阶段,只附接口枚举、耗时、字符数、字节数、HTTP 状态和受控错误码;发送判定包含手动或停顿触发、距离最后一次声音或文字更新的毫秒数、ASR 稳定时长。噪声归零也留下判定记录,便于区分“没有触发朗读”“上游失败”和“播放被暂停或取消”。不记录识别文字、回复正文、音频、音色 ID、供应商地址、请求头、凭证或上游原始错误,不逐音频帧输出日志。新增 scope 沿用现有默认不上传规则,诊断完整记录保留在本机。
+
## 后续扩展计划
以下为后续演进方向,本 PR 尚未实现适配器拆分、统一参数描述或公共 Registry schema 扩展,也不包含 Qwen / CosyVoice 原生执行器。
@@ -68,6 +74,8 @@
## 验证
+停顿与插话回归覆盖 2 秒等待、弱声识别、思考停顿后继续说话、空识别和无 ASR 结果的噪声、首轮回复在噪声期间完成、播放暂停后原位置继续,以及权限、静音、退出或明确停止后的迟到结果。日志测试确认输出只包含诊断元数据,不包含输入、回复或凭证。
+
自动化覆盖分段去重、乱序完成、空转写、连接轮换、权限与账号边界、取消启动、插话抢占、迟到音频、PCM 字节边界、回复关联、无模型空状态及现有 ASR 回归。BYOK 回归覆盖接口请求、SSE 断帧与结束聚合包、连接/密钥隔离、停用与账号切换、伙伴偏好重开与重置、未知接口须用户选择,以及合成失败不换供应商、不重发消息。音色与试听回归包含真实 MiniMax ID、OpenAI 型号差异、ElevenLabs 分页、账号私有列表边界、手动输入与刷新失败、改选清除旧音色、试听取消与迟到结果、窗口销毁,以及试听不启动 ASR。入口回归覆盖设置深链、输入框按钮顺序与草稿保留、缺配置转到设置、直接启动、StrictMode 不重复采集、关闭后的迟到配置结果,以及设置关闭前保存失败不离开。
没有真实 Gateway / BYOK TTS 账号可用于验收时,自动化和组件预览使用明确的测试替身;产品目录不添加演示模型。真实模型开放后,还需验证其音色和 PCM 合同,并在 macOS/Windows 的耳机、扬声器、蓝牙设备上验证连续多轮、回声与插话延迟。测试替身不构成这些项目已通过的证据。
diff --git a/packages/voice-input-core/src/VoiceConversationInput.ts b/packages/voice-input-core/src/VoiceConversationInput.ts
index 43db464b61..e70bddb781 100644
--- a/packages/voice-input-core/src/VoiceConversationInput.ts
+++ b/packages/voice-input-core/src/VoiceConversationInput.ts
@@ -1,9 +1,13 @@
import type { AsrProvider, AsrSegment, AudioTrace } from "./types.js";
import { ConversationActivity } from "./conversationActivity.js";
+const AUTO_SEND_QUIET_MS = 2_000;
+const FINAL_STABLE_MS = 350;
+
export type ConversationInputEvent =
| { type: "ready" }
| { type: "speech-started" }
+ | { type: "speech-idle" }
| { type: "draft"; text: string }
| { type: "utterance"; id: string; text: string }
| { type: "error"; text: string };
@@ -29,12 +33,23 @@ export class VoiceConversationInput {
private settleTimer?: ReturnType<typeof setTimeout>;
private rotationTimer?: ReturnType<typeof setTimeout>;
private finalTimer?: ReturnType<typeof setTimeout>;
+ private awaitingSettlement = false;
+ private quietAfter = 0;
+ private stableAfter = 0;
constructor(
private readonly options: {
createProvider: () => Promise<AsrProvider>;
emit: (event: ConversationInputEvent) => void;
rotationMs?: number;
+ // Numeric endpointing evidence only; the host owns logging and storage.
+ diagnostic?: (event: {
+ trigger: "manual" | "silence";
+ inactiveMs: number;
+ finalStableMs: number | null;
+ segments: number;
+ characters: number;
+ }) => void;
},
) {}
@@ -47,7 +62,14 @@ export class VoiceConversationInput {
if (this.closed) return;
const activity = this.activity.push(pcm);
this.speaking = activity.active;
+ // The 900ms acoustic release only commits ASR buffers. Sending always uses
+ // the same two-second quiet window, including soft speech seen only by ASR.
+ if (activity.sound) {
+ this.quietAfter = Date.now() + AUTO_SEND_QUIET_MS;
+ if (!this.manual) clearTimeout(this.settleTimer);
+ }
if (activity.started) {
+ this.awaitingSettlement = true;
clearTimeout(this.finalTimer);
if (!this.manual) clearTimeout(this.settleTimer);
this.options.emit({ type: "speech-started" });
@@ -61,6 +83,7 @@ export class VoiceConversationInput {
}
this.provider.appendAudio(pcm, trace);
if (activity.ended) this.finishUtterance();
+ else if (activity.sound && !activity.active) this.scheduleSettlement();
}
finishUtterance(force = false): void {
@@ -115,6 +138,11 @@ export class VoiceConversationInput {
if (this.consumed.has(key)) return;
const prior = this.segments.get(key);
if (prior?.final) return;
+ this.awaitingSettlement = true;
+ const text = segment.text.trim();
+ if ((text && text !== prior?.text.trim()) || !this.quietAfter)
+ this.quietAfter = Date.now() + AUTO_SEND_QUIET_MS;
+ this.stableAfter = Date.now() + FINAL_STABLE_MS;
if (!prior?.text && segment.text.trim() && !this.speaking)
this.options.emit({ type: "speech-started" });
this.segments.set(key, { ...segment, generation });
@@ -157,7 +185,11 @@ export class VoiceConversationInput {
private scheduleSettlement(): void {
clearTimeout(this.settleTimer);
- if ((this.speaking && !this.manual) || this.rotating || !this.segments.size)
+ if (
+ (this.speaking && !this.manual) ||
+ this.rotating ||
+ !this.awaitingSettlement
+ )
return;
if ([...this.segments.values()].some((segment) => !segment.final)) return;
this.settleTimer = setTimeout(
@@ -165,8 +197,20 @@ export class VoiceConversationInput {
if (this.closed || (this.speaking && !this.manual) || this.rotating)
return;
const text = this.draft();
+ this.options.diagnostic?.({
+ trigger: this.manual ? "manual" : "silence",
+ inactiveMs: Date.now() - this.quietAfter + AUTO_SEND_QUIET_MS,
+ finalStableMs: this.stableAfter
+ ? Date.now() - this.stableAfter + FINAL_STABLE_MS
+ : null,
+ segments: this.segments.size,
+ characters: text.length,
+ });
for (const key of this.segments.keys()) this.consumed.add(key);
this.segments.clear();
+ this.awaitingSettlement = false;
+ this.quietAfter = 0;
+ this.stableAfter = 0;
this.manual = false;
clearTimeout(this.finalTimer);
if (text)
@@ -176,8 +220,15 @@ export class VoiceConversationInput {
text,
});
this.options.emit({ type: "draft", text: "" });
+ if (!text) this.options.emit({ type: "speech-idle" });
},
- this.manual ? 0 : 350,
+ this.manual
+ ? 0
+ : Math.max(
+ 0,
+ this.quietAfter - Date.now(),
+ this.stableAfter - Date.now(),
+ ),
);
}
diff --git a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
index e1ce118106..693444fe36 100644
--- a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
+++ b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
@@ -50,10 +50,10 @@ describe("continuous speech input", () => {
await vi.advanceTimersByTimeAsync(400);
expect(events.filter((e) => e.type === "utterance")).toHaveLength(0);
provider.segment({ id: "one", text: "你好。", final: true });
- await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(2000);
provider.segment({ id: "one", text: "你好。", final: true });
provider.segment({ id: "two", text: "你好。", final: true });
- await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(2000);
expect(events.filter((e) => e.type === "utterance")).toEqual([
{ type: "utterance", id: "1", text: "你好。" },
{ type: "utterance", id: "2", text: "你好。" },
@@ -61,6 +61,95 @@ describe("continuous speech input", () => {
await input.stop();
});
+ it("requires two quiet seconds for ASR-only soft speech and merges a thinking pause", async () => {
+ const provider = new Provider();
+ const events: ConversationInputEvent[] = [];
+ const diagnostic = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => provider,
+ emit: (event) => events.push(event),
+ diagnostic,
+ });
+ await input.start();
+ provider.segment({ id: "soft", text: "first part", final: true });
+ await vi.advanceTimersByTimeAsync(1_500);
+ expect(events.some((event) => event.type === "utterance")).toBe(false);
+ // Continue within the thinking pause. Even a provider-finalized first part
+ // must remain in this utterance until the new activity has also settled.
+ input.appendAudio(pcm(2000, 200));
+ provider.segment({ id: "next", text: "second part", final: true });
+ await vi.advanceTimersByTimeAsync(900);
+ input.appendAudio(pcm(0, 900));
+ expect(provider.commitUtterance).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(1_099);
+ expect(events.some((event) => event.type === "utterance")).toBe(false);
+ await vi.advanceTimersByTimeAsync(1);
+ expect(events.filter((event) => event.type === "utterance")).toEqual([
+ { type: "utterance", id: "1", text: "first part second part" },
+ ]);
+ expect(diagnostic).toHaveBeenCalledExactlyOnceWith({
+ trigger: "silence",
+ inactiveMs: 2_000,
+ finalStableMs: 2_000,
+ segments: 2,
+ characters: 22,
+ });
+ await input.stop();
+ });
+
+ it("releases a noise-only onset after the quiet window without sending or waiting for an ASR segment", async () => {
+ const provider = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => provider,
+ emit,
+ });
+ await input.start();
+ input.appendAudio(pcm(800, 200));
+ expect(emit).toHaveBeenCalledWith({ type: "speech-started" });
+ await vi.advanceTimersByTimeAsync(900);
+ input.appendAudio(pcm(0, 900));
+ await vi.advanceTimersByTimeAsync(1_099);
+ expect(emit).not.toHaveBeenCalledWith({ type: "speech-idle" });
+ await vi.advanceTimersByTimeAsync(1);
+ expect(emit).toHaveBeenCalledWith({ type: "speech-idle" });
+ expect(emit).not.toHaveBeenCalledWith(
+ expect.objectContaining({ type: "utterance" }),
+ );
+ await input.stop();
+ });
+
+ it("does not let an unchanged final bypass the remaining quiet window or retain an empty draft", async () => {
+ const provider = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => provider,
+ emit,
+ });
+ await input.start();
+ provider.segment({ id: "one", text: "soft speech", final: false });
+ await vi.advanceTimersByTimeAsync(1_500);
+ provider.segment({ id: "one", text: "soft speech", final: true });
+ await vi.advanceTimersByTimeAsync(499);
+ expect(emit).not.toHaveBeenCalledWith(
+ expect.objectContaining({ type: "utterance" }),
+ );
+ await vi.advanceTimersByTimeAsync(1);
+ expect(emit).toHaveBeenCalledWith({
+ type: "utterance",
+ id: "1",
+ text: "soft speech",
+ });
+ provider.segment({ id: "empty", text: "discarded partial", final: false });
+ provider.segment({ id: "empty", text: "", final: true });
+ await vi.advanceTimersByTimeAsync(2_000);
+ expect(emit).toHaveBeenCalledWith({ type: "speech-idle" });
+ expect(
+ emit.mock.calls.filter(([event]) => event.type === "utterance"),
+ ).toHaveLength(1);
+ await input.stop();
+ });
+
it("does not mistake an out-of-order final for the complete utterance", async () => {
const provider = new Provider();
const emit = vi.fn();
@@ -76,7 +165,7 @@ describe("continuous speech input", () => {
expect.objectContaining({ type: "utterance" }),
);
provider.segment({ id: "a", text: "first", final: true, order: 0 });
- await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(2000);
expect(emit).toHaveBeenCalledWith({
type: "utterance",
id: "1",
@@ -128,12 +217,12 @@ describe("continuous speech input", () => {
const input = new VoiceConversationInput({
createProvider,
emit,
- rotationMs: 1000,
+ rotationMs: 10_000,
});
await input.start();
a.segment({ id: "same", text: "before", final: true });
- await vi.advanceTimersByTimeAsync(350);
- await vi.advanceTimersByTimeAsync(650);
+ await vi.advanceTimersByTimeAsync(2000);
+ await vi.advanceTimersByTimeAsync(8000);
const buffered = pcm(1000, 200);
input.appendAudio(buffered);
finish();
@@ -142,7 +231,7 @@ describe("continuous speech input", () => {
a.segment({ id: "late", text: "ignore", final: true });
b.segment({ id: "same", text: "after", final: true });
input.appendAudio(pcm(0, 900));
- await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(2000);
expect(
emit.mock.calls
.filter(([e]) => e.type === "utterance")
@@ -199,9 +288,9 @@ describe("continuous speech input", () => {
await input.start();
p.segment({ id: "noise", text: "", final: false });
p.segment({ id: "noise", text: "", final: true });
- await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(2000);
p.segment({ id: "speech", text: "hello", final: true });
- await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(2000);
expect(emit.mock.calls.filter(([e]) => e.type === "utterance")).toEqual([
[{ type: "utterance", id: "1", text: "hello" }],
]);
diff --git a/packages/voice-input-core/src/conversationActivity.ts b/packages/voice-input-core/src/conversationActivity.ts
index c0bdc65008..b91b7b116a 100644
--- a/packages/voice-input-core/src/conversationActivity.ts
+++ b/packages/voice-input-core/src/conversationActivity.ts
@@ -13,7 +13,13 @@ export class ConversationActivity {
push(
pcm: ArrayBuffer,
sampleRate = 16_000,
- ): { level: number; started: boolean; ended: boolean; active: boolean } {
+ ): {
+ level: number;
+ started: boolean;
+ ended: boolean;
+ active: boolean;
+ sound: boolean;
+ } {
const samples = new Int16Array(pcm);
let energy = 0;
for (const sample of samples) energy += sample * sample;
@@ -48,6 +54,7 @@ export class ConversationActivity {
started,
ended,
active: this.active,
+ sound,
};
}
}