Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(voice): preserve drafts and release failed reply waits

JiangJiahao9c4a70deac24
diff --git a/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx b/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx
index cec1d4a81e..a09313dab1 100644
--- a/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx
+++ b/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx
@@ -10,6 +10,7 @@ import { VoiceConversationController } from './VoiceConversationController';
import { PcmSpeechPlayer } from './PcmSpeechPlayer';
import { resolveVoiceConfiguration } from './voiceConfiguration';
import { CompanionVoiceToolbar } from './CompanionVoiceToolbar';
+import type { VoiceTaskFailure } from './VoiceReplyTracker';
type Props = {
open: boolean;
@@ -18,6 +19,7 @@ type Props = {
messages: readonly ChatMessage[];
blocked: boolean;
taskBusy: boolean;
+ taskFailure?: VoiceTaskFailure | null;
onClose(): void;
onConfigure(): void;
onSend(text: string, onCreated: (id: string) => void, isCurrent: () => boolean): Promise<boolean>;
@@ -122,8 +124,8 @@ function CompanionVoiceCall(props: Props) {
}
}, [controller, props.blocked, props.open]);
useEffect(() => {
- if (props.open) controller.updateMessages(props.messages);
- }, [controller, props.messages, props.open]);
+ if (props.open) controller.updateMessages(props.messages, props.taskFailure ?? null);
+ }, [controller, props.messages, props.taskFailure, props.open]);
useEffect(() => {
if (props.open) controller.refreshActivity();
}, [controller, props.taskBusy, props.open]);
diff --git a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
index f49065217c..aad592ab3c 100644
--- a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
@@ -7,7 +7,7 @@ import type { ChatMessage } from '@/lib/makerChatStore';
import { createLogger } from '@/lib/logger';
import type { PcmChunk } from '@/voice-input/WebMicAudioEngine';
import type { SpeechPlayer } from './PcmSpeechPlayer';
-import { speechChunks, VoiceReplyTracker } from './VoiceReplyTracker';
+import { speechChunks, VoiceReplyTracker, type VoiceTaskFailure } from './VoiceReplyTracker';
import { SpeechPlaybackQueue } from './SpeechPlaybackQueue';
import { microphoneDiagnostic } from './microphoneDiagnostics';
import { voiceExitIntent } from './voiceExitIntent';
@@ -89,6 +89,7 @@ export class VoiceConversationController {
private tracker = new VoiceReplyTracker();
private pendingInput: string | null = null;
private latestMessages: readonly ChatMessage[] = [];
+ private latestTaskFailure: VoiceTaskFailure | null = null;
private sending = false;
private inFlight: Promise<void> = Promise.resolve();
private queued: QueuedUtterance[] = [];
@@ -419,6 +420,9 @@ export class VoiceConversationController {
const utterance = { text: event.text, id: event.id };
this.queued.push(utterance);
this.latestUtterance = utterance;
+ // The finalized draft now belongs to the queue. Do not retain it
+ // again if a send/queue-limit failure runs before ASR's empty draft.
+ this.patch({ transcript: '' });
if (
[this.state.unsent, ...this.queued.map((item) => item.text)].join(' ').length > 16_000
)
@@ -533,7 +537,7 @@ export class VoiceConversationController {
}
/** Explicit stop or a committed new utterance permanently relinquishes the old reply. */
- interrupt(reason: 'user_stop' | 'new_utterance' = 'user_stop'): void {
+ interrupt(reason: 'user_stop' | 'new_utterance' | 'task_error' | 'task_cancelled' = 'user_stop'): void {
if (reason === 'user_stop') this.cancelCountdown('control');
log.info('reply interrupted', {
callId: this.callId,
@@ -698,16 +702,27 @@ export class VoiceConversationController {
}
}
- updateMessages(messages: readonly ChatMessage[]): void {
+ updateMessages(
+ messages: readonly ChatMessage[],
+ failure: VoiceTaskFailure | null = this.latestTaskFailure,
+ ): void {
if (messages !== this.latestMessages && !this.hasIntentCountdown())
this.cancelCountdown('activity');
this.latestMessages = messages;
+ // A terminal projection can arrive before send acknowledgement. Keep it
+ // alongside the messages so submit() can reconcile the accepted input.
+ this.latestTaskFailure = failure;
this.refreshActivity();
+ if (!this.enabled || !this.callId || this.blocked || !this.pendingInput) return;
+ if (failure?.clientId === this.pendingInput && !this.replyComplete) {
+ const speechCandidate = this.speechCandidate;
+ this.interrupt(failure.reason === 'error' ? 'task_error' : 'task_cancelled');
+ // Relinquish only the failed reply, preserving an in-progress new draft.
+ this.speechCandidate = speechCandidate;
+ this.setReplyPhase(null);
+ return;
+ }
if (
- !this.enabled ||
- !this.callId ||
- this.blocked ||
- !this.pendingInput ||
this.speechCandidate ||
this.hasIntentCountdown() ||
this.deferringSpeech
@@ -957,8 +972,11 @@ export class VoiceConversationController {
log.warn('voice mode failed', { callId: this.callId, phase: this.state.phase, reason: error });
const unsent = [
this.state.unsent,
- text || this.state.transcript,
+ // Send failure text owns the in-flight utterance; connection failure
+ // text is the ASR's copy of the current draft, not a second utterance.
+ error === 'send' ? text : '',
...this.queued.map((item) => item.text),
+ error === 'send' ? this.state.transcript : text || this.state.transcript,
]
.filter(Boolean)
.join(' ');
diff --git a/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts b/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts
index 53425bf1e1..7a46c126f3 100644
--- a/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts
@@ -1,7 +1,26 @@
-import type { ChatMessage } from '@/lib/makerChatStore';
+import type { ChatMessage, SessionChatState } from '@/lib/makerChatStore';
import { completeSpeechPrefix, streamingSpeechText } from './SpeechTextBuffer';
export type VoiceReply = { id: string; text: string; complete: boolean; invalidated?: true };
+export type VoiceTaskFailure = { clientId: string; reason: 'error' | 'cancelled' };
+
+/** Use the host's terminal state; idle alone can mean queued or retrying. */
+export function voiceTaskFailure(state: Pick<SessionChatState,
+ 'messages' | 'agentStatus' | 'isStreaming' | 'error' | 'inputRecovery'
+ | 'usageLimitWait' | 'turnStoppedByUser' | 'lastStopWasSideTask'
+>): VoiceTaskFailure | null {
+ if (state.agentStatus.isRunning || state.isStreaming || state.usageLimitWait
+ || state.lastStopWasSideTask) return null;
+ // Failed input ownership comes from the canonical recovery projection, not
+ // the latest bubble (which may belong to a different queued input).
+ if (state.error && state.inputRecovery?.kind === 'active-turn')
+ return { clientId: state.inputRecovery.item.clientId, reason: 'error' };
+ if (state.turnStoppedByUser) {
+ const input = state.messages.findLast((message) => message.role === 'user');
+ if (input) return { clientId: input.clientId, reason: 'cancelled' };
+ }
+ return null;
+}
/** Correlate one voice input with its final answer, emitting each stable prefix once. */
export class VoiceReplyTracker {
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
index 22f98c326f..62fe738f95 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
@@ -44,6 +44,7 @@ vi.mock('../PcmSpeechPlayer', () => ({
}));
import { CompanionVoicePanel } from '../CompanionVoicePanel';
+import type { VoiceTaskFailure } from '../VoiceReplyTracker';
let models: VoiceConversationModel[];
let inputEvent: (event: VoiceConversationEvent) => void;
const start = vi.fn(async () => ({ callId: 'call' }));
@@ -85,7 +86,7 @@ afterEach(cleanup);
afterEach(() => vi.useRealTimers());
const configure = vi.fn();
const close = vi.fn();
-const panel = (blocked = false, taskBusy = false, open = true) => (
+const panel = (blocked = false, taskBusy = false, open = true, taskFailure?: VoiceTaskFailure) => (
<CompanionVoicePanel
open={open}
bot={{ id: 'bot', name: 'Partner' }}
@@ -93,6 +94,7 @@ const panel = (blocked = false, taskBusy = false, open = true) => (
messages={[]}
blocked={blocked}
taskBusy={taskBusy}
+ taskFailure={taskFailure}
onSend={vi.fn(async (_text, created) => {
created('input');
return true;
@@ -102,6 +104,27 @@ const panel = (blocked = false, taskBusy = false, open = true) => (
/>
);
+// #5718 review: real panel/controller wiring must observe a terminal failure
+// even without a final assistant message. Earlier cases only used busy/idle.
+// Unit/jsdom, fake audio/IPC: accepted input -> matching error -> listening.
+// Standalone file validation: 1.39s wall time.
+it('passes terminal input failure to the live call without stopping capture', async () => {
+ vi.useFakeTimers();
+ models = [{ id: 'tts', name: 'Speech', defaultVoice: 'alloy' }];
+ const view = render(panel());
+ await act(async () => {});
+ await act(async () => {
+ inputEvent({ callId: 'call', event: { type: 'utterance', id: 'utterance', text: 'Hello' } });
+ });
+ expect(screen.getByRole('status').textContent).toBe('bots.voiceMode.status.thinking');
+ view.rerender(panel(false, false, true, { clientId: 'input', reason: 'error' }));
+ expect(screen.getByRole('status').textContent).toBe('bots.voiceMode.status.listening');
+ expect(audio.stop).not.toHaveBeenCalled();
+ expect(window.electronAPI.voiceConversation.end).not.toHaveBeenCalled();
+ await act(() => vi.advanceTimersByTimeAsync(60_000));
+ expect(screen.getByRole('timer')).toBeTruthy();
+});
+
// Exit animation retains the real panel briefly. The old unmount-only cleanup
// would keep recording/playing until that animation ended. Existing cases only
// unmount immediately; these unit cases close a still-mounted panel and reopen
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
index bf4e57aebf..be6ac67de2 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
@@ -655,6 +655,55 @@ describe('automatic voice exit', () => {
expect(vi.getTimerCount()).toBe(0);
});
+ // #5718 review: ordinary reply waiting is covered, but accepted input with
+ // no reply after a terminal failure was stuck forever. Unit/fake time:
+ // correlated terminal input resumes listening without resending or ending work.
+ // Standalone file validation including both review fixes: 2.72s wall time.
+ it.each(['error', 'cancelled'] as const)('releases reply ownership on task %s and restores idle exit', async (reason) => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.utterance();
+ await vi.advanceTimersByTimeAsync(0);
+ f.taskBusy.mockReturnValue(true); // Independent background work stays busy.
+ f.controller.updateMessages([user()], { clientId: 'user-1', reason });
+ expect(f.controller.getSnapshot().phase).toBe('listening');
+ await vi.advanceTimersByTimeAsync(70_000);
+ expect(f.api.end).not.toHaveBeenCalled();
+ expect(f.send).toHaveBeenCalledOnce();
+ f.taskBusy.mockReturnValue(false);
+ f.controller.refreshActivity();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(f.controller.getSnapshot().exitCountdown?.reason).toBe('idle');
+ f.controller.end();
+ });
+
+ it('ignores another input failure and handles a matching failure received before send acknowledgement', async () => {
+ const f = fixture();
+ let accept!: () => void;
+ f.send.mockImplementationOnce(async (_text, created) => {
+ created('user-1');
+ await new Promise<void>((resolve) => { accept = resolve; });
+ return true;
+ });
+ await f.controller.start(selection);
+ f.utterance();
+ f.controller.updateMessages([user()], { clientId: 'older', reason: 'error' });
+ expect(f.controller.getSnapshot().phase).toBe('thinking');
+ f.controller.updateMessages([user()], { clientId: 'user-1', reason: 'error' });
+ f.emit({ type: 'speech-started' });
+ f.emit({ type: 'draft', text: 'still speaking' });
+ accept();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(f.controller.getSnapshot()).toMatchObject({ phase: 'listening', transcript: 'still speaking' });
+ f.emit({ type: 'draft', text: '' });
+ f.emit({ type: 'speech-idle' });
+ expect(f.controller.getSnapshot().phase).toBe('listening');
+ f.controller.updateMessages([user(), reply()]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ expect(f.send).toHaveBeenCalledOnce();
+ f.controller.end();
+ });
+
it('waits through send preflight, reply generation, synthesis, playback and gaps between sentences', async () => {
const f = fixture();
let accept!: () => void;
@@ -1209,6 +1258,45 @@ describe('companion voice loop', () => {
f.controller.end();
});
+ // #5718 review: queued finals were covered, but a newer partial was lost on
+ // send rejection. Unit: real controller, deferred send + ASR events; preserve
+ // chronological input once, including deliberate repetition, through retry.
+ it.each(['second draft', 'first'])('retains a newer partial after send failure (%s)', async (draft) => {
+ const f = fixture();
+ let reject!: (value: boolean) => void;
+ f.send.mockImplementationOnce(() => new Promise((resolve) => { reject = resolve; }));
+ await f.controller.start(selection);
+ f.utterance('first');
+ f.emit({ type: 'draft', text: 'queued' });
+ f.utterance('queued');
+ f.emit({ type: 'draft', text: '' });
+ f.emit({ type: 'speech-started' });
+ f.emit({ type: 'draft', text: draft });
+ reject(false);
+ await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('error'));
+ expect(f.controller.getSnapshot()).toMatchObject({
+ unsent: `first queued ${draft}`,
+ transcript: '',
+ });
+ await f.controller.start(selection);
+ f.controller.retryUnsent();
+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));
+ expect(f.send.mock.calls[1][0]).toBe(`first queued ${draft}`);
+ f.controller.end();
+ });
+
+ it('retains a finalized draft only once when it exceeds the send queue limit', async () => {
+ const f = fixture();
+ await f.controller.start(selection);
+ const text = 'a'.repeat(16_001);
+ f.emit({ type: 'draft', text });
+ // Queue rejection runs synchronously, before the ASR's next empty draft.
+ f.utterance(text);
+ expect(f.controller.getSnapshot()).toMatchObject({ phase: 'error', unsent: text, transcript: '' });
+ expect(f.send).not.toHaveBeenCalled();
+ f.controller.end();
+ });
+
it('cancels a pending start, then waits for teardown before opening another call', async () => {
const f = fixture();
let resolve!: (value: { callId: string }) => void;
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts
index cce7cbb8c1..80d785c626 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts
@@ -1,10 +1,41 @@
import { describe, expect, it } from 'vitest';
import type { ChatMessage } from '@/lib/makerChatStore';
-import { speechChunks, speechText, VoiceReplyTracker } from '../VoiceReplyTracker';
+import { speechChunks, speechText, voiceTaskFailure, VoiceReplyTracker } from '../VoiceReplyTracker';
import { Pcm16Decoder } from '../PcmSpeechPlayer';
const message = (clientId: string, role: ChatMessage['role'], extra: Partial<ChatMessage> = {}) =>
({ clientId, role, content: 'Hello', ...extra }) as ChatMessage;
+
+// #5718 review: the host projection must distinguish a failed input from idle,
+// queue errors and automatic recovery. Earlier reply tests only inspected
+// messages. Unit: canonical active-turn failure with a different latest bubble.
+// Standalone file validation: 0.26s wall time.
+it('projects only terminal foreground failures with their input owner', () => {
+ type State = Parameters<typeof voiceTaskFailure>[0];
+ const state: State = {
+ messages: [message('mine', 'user'), message('other', 'user')],
+ agentStatus: { isRunning: false } as State['agentStatus'],
+ isStreaming: false,
+ error: 'failed',
+ inputRecovery: {
+ kind: 'active-turn',
+ item: { clientId: 'mine' } as Extract<NonNullable<State['inputRecovery']>, { kind: 'active-turn' }>['item'],
+ },
+ usageLimitWait: null,
+ turnStoppedByUser: false,
+ lastStopWasSideTask: false,
+ };
+ expect(voiceTaskFailure(state)).toEqual({ clientId: 'mine', reason: 'error' });
+ expect(voiceTaskFailure({ ...state, error: null })).toBeNull();
+ expect(voiceTaskFailure({ ...state, inputRecovery: { kind: 'queue-head', clientId: 'queued' } })).toBeNull();
+ expect(voiceTaskFailure({ ...state, agentStatus: { ...state.agentStatus, isRunning: true } })).toBeNull();
+ expect(voiceTaskFailure({ ...state, isStreaming: true })).toBeNull();
+ expect(voiceTaskFailure({ ...state, usageLimitWait: { resumeAt: 123 } })).toBeNull();
+ expect(voiceTaskFailure({ ...state, lastStopWasSideTask: true })).toBeNull();
+ expect(voiceTaskFailure({ ...state, error: null, inputRecovery: null, turnStoppedByUser: true }))
+ .toEqual({ clientId: 'other', reason: 'cancelled' });
+});
+
describe('voice reply correlation', () => {
it('excludes history, commentary, tool/subagent/group replies and failed turns', () => {
const tracker = new VoiceReplyTracker();
diff --git a/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx b/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
index dc3bdb2791..e9e0d522cc 100644
--- a/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
+++ b/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
@@ -1,4 +1,5 @@
import { CompanionVoicePanel } from '../bots/voice/CompanionVoicePanel';
+import { voiceTaskFailure } from '../bots/voice/VoiceReplyTracker';
import { useMessageReadAloud } from '../bots/voice/useMessageReadAloud';
import { MessageReadAloudContext } from '@/components/chat/MessageReadAloudContext';
import { formatSessionDuration } from '@/lib/sessionDurationFormat';
@@ -4187,6 +4188,7 @@ function CCAgentSessionContent({
const voiceTaskBusy = !historyLoaded || isAgentBusy || agentStatus.isRunning || isStreaming
|| pendingQueue.length > 0 || continuationInFlightClientId !== null
|| backgroundModelActive || backgroundBashTasks.length > 0 || runningWorkflow !== null;
+ const voiceFailure = sessionId ? voiceTaskFailure(makerChatStore.getSnapshot(sessionId)) : null;
const messageReadAloud = useMessageReadAloud({
botId: botChatIdentity?.id,
sessionId: sessionId ?? '',
@@ -5744,7 +5746,7 @@ function CCAgentSessionContent({
permission cards reserve space instead of covering the latest reply. */}
{voiceModeAvailable && botChatIdentity && sessionId ? (
<CompanionVoicePanel key={sessionId} open={voiceModeOpen} bot={botChatIdentity} sessionId={sessionId} messages={messages}
- blocked={voiceModeBlocked} taskBusy={voiceTaskBusy} onSend={sendVoiceMessage} onClose={closeVoiceMode} onConfigure={openVoiceSettings} />
+ blocked={voiceModeBlocked} taskBusy={voiceTaskBusy} taskFailure={voiceFailure} onSend={sendVoiceMessage} onClose={closeVoiceMode} onConfigure={openVoiceSettings} />
) : null}
{/* 互斥:控制端能终结的 pending interaction 会接管 composer;
Desktop-only 只读确认只能提示等待,必须保留 ChatInput,避免控制端
diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.md
index 51d3dba537..ab74106d4f 100644
--- a/docs/product-rules/companion-voice-mode.md
+++ b/docs/product-rules/companion-voice-mode.md
@@ -87,6 +87,8 @@
ASR 输入故障统一通过现有 WARN `input failed` 记录受控原因:`final_timeout`(等待定稿满 8 秒)、`upstream_disconnected`、`upstream_error`,以及缓冲上限、文字上限和连接轮换失败。附通话/输入代次、等待时长、待定稿分段数量、字符数及缓冲音频时长;不记录供应商原始错误、分段 ID 或正文。每次失败只记一次,继续沿用原有保留未发送文字和清理流程,不把临时识别文字当作最终结果自动提交。
+轻声未触发本地发声检测、但 ASR 已返回临时分段时,同样从文字停顿边界起等待最多 8 秒定稿;新文字延后边界,重复的相同临时结果不延长等待。重新检测到发声时取消旧计时,真实最终结果仍走现有稳定与静音条件。超时保留草稿供恢复,不自动提交临时识别结果。
+
SAUC 使用 `result_type=full`,只有 `definite=true` 的分句才不可变;未定稿分句的时间坐标不能充当永久 ID([协议依据](https://docs.volcengine.com/docs/DoubaoVoice/bidirectional-streaming-automatic-speech-recognition-websocket?lang=zh))。适配器按有效全量快照撤掉已被替换的临时分句,避免旧草稿残留并一直等待定稿;保留已确定分句的去重,ACK/缺失或非法分句列表不视为撤稿。修订时用既有 DEBUG `conversation snapshot reconciled` 记录撤掉的临时分句数和当前待定稿数,不记录文本或时间坐标。1.25 秒停顿、文字稳定窗口和真实定稿超时预算保持原值,不通过缩短等待或提交未定稿结果掩盖故障。
办公室噪声排查增加 DEBUG 级 `input settlement blocked`:有待完成的发言且发送判定持续受阻时,每 5 秒至多一条;正常空闲、发送完成、输入停止后不输出。记录发声活动、ASR 未定稿数量、连接轮换、各静音/文字稳定窗口剩余毫秒数,以及最近一帧的 RMS/判定阈值/噪声基线、最近采集/过滤后音频/ASR 更新的间隔。用于区分持续声音、识别未定稿和音频尚未释放,不把“超阈值”当成真人发声结论,也不记录正文或原始 PCM。该诊断不参与发送判定、不更改停顿时长,复用当前日志等级与保存方式;问题确认后复核并移除临时诊断。
@@ -99,6 +101,7 @@ SAUC 使用 `result_type=full`,只有 `definite=true` 的分句才不可变;
- 点击时重新读取并校验配置,复用现有 Main TTS 队列、PCM 播放器、连接凭证与诊断日志。完整回复分段播放,不截断长文;不启动 ASR/麦克风,不新建任务、不重发消息、不写语音偏好。朗读失败不自动切换供应商或重试。
- 离开伙伴页面、打开设置、账号切换、系统挂起或窗口销毁时释放播放;进入连续语音模式前先等待消息朗读释放。连续语音期间不并行朗读历史消息,避免声音重叠。消息朗读与试听一样,在用户点击后可能产生所选 TTS 的用量。
- **连续空闲 1 分钟后显示 10 秒退出倒计时**。只有输入已就绪,且没有用户发声、识别草稿/未发送文字、发送中的输入、待回复关联、任务/队列/后台子任务活动、待合成/播放中的声音时才计为空闲;不能只看上次 ASR 文字的时间。等待设置、连接、麦克风开启、权限确认或错误暂停期间不计时。任务活动复用当前任务的运行、历史加载、队列、后台模型/Bash 与工作流状态,不因其他无关任务运行而一直保留通话。
+- 已接收输入在最终回复前失败或被取消时,按现有任务状态及输入 ID 结束对应的朗读等待,恢复监听和正常空闲判断;保留用户正在说的新草稿,不重发已接收消息、不停止后台任务。普通空闲、排队、自动重试及用量恢复等待不等于任务失败;其他输入的错误不能解除当前回复关联。
- 倒计时期间继续说话或点击“取消退出”即可保留语音;活动结束后重新累计完整 1 分钟。倒计时每次更新及到期前复查任务和语音活动,并校验当前通话代次。任务在倒计时中启动也会取消空闲退出。静音且没有其他活动时同样可空闲退出;取消退出不会偷偷打开已静音的麦克风。声音音量帧本身不重置空闲时间,使用现有声音活动事件,避免底噪永久保留通话。
- 语音退出只检查 ASR 汇总后的完整发言,不检查临时识别文字。首版保守整句匹配:简/繁体“结束语音”“退出语音模式”“关闭语音对话”等明确指令(可带“请”)直接退出;独立“再见/再見/拜拜”及 `bye` / `bye bye` / `goodbye` 显示 **3 秒可取消退出**。英语明确指令支持 `end` / `exit` / `close` + `voice mode` / `voice chat`。只忽略首尾空白和末尾句号/感叹号,保留问号、引号、否定与额外子句;“再见用英语怎么说”“他说了拜拜”“不要结束语音”“暂停一下”不命中本地退出规则,不做子串匹配。已有保留文字或排队发言时不把后来的短语单独当成退出指令。
- 本地退出指令/告别只控制当前语音,不发送成伙伴消息、不新建任务、不调用额外模型。告别倒计时暂时暂停朗读,保留位置、回复关联及队列;取消后恢复,新的用户发言仍走原有插话流程。所有退出复用原有资源清理,关闭麦克风、ASR、TTS 及面板,恢复输入框焦点,保留历史与伙伴上下文,**不停止已接受的伙伴任务或后台工作**。迟到回调不能恢复旧通话或结束新通话。
@@ -106,6 +109,7 @@ SAUC 使用 `result_type=full`,只有 `definite=true` 的分句才不可变;
- 本地规则未命中时,可用**独立临时语义判断**补充自然表达:只发送当前 ASR 最终完整发言,不读取或发送伙伴身份、历史、长期记忆、工作目录、工具和上一条响应;调用现有无工具 HTTP 文本通道,固定使用 canonical 任务保存的供应商、引擎与型号,分类请求本身不创建/唤醒 Agent,不写入伙伴消息或学习链;原始发言作为正式消息按原有伙伴流程处理。只接受严格 `EXIT` / `CONTINUE` / `SUBMIT_THEN_EXIT`。`EXIT` 只用于没有工作或实质消息的独立退出。否定、引用、讨论退出功能、暂时暂停/静音/停止朗读,以及“等报告做完再挂断”等以未来结果为退出条件的要求仍按普通文字处理,保持语音;不确定时同样继续。
- “帮我查资料,然后退出语音模式”等明确交代工作后离开语音的复合发言使用 `SUBMIT_THEN_EXIT`:把 ASR 原句完整提交到原伙伴任务,不让分类模型提取、改写或删减指令。判断先返回、发送尚未确认时显示“正在提交,成功后退出语音模式…”,保留取消退出入口;只有判断确认退出且发送返回已接受并关联到输入 clientId 后,才**立即退出,不再倒计时**,不等待后台执行完成。判断晚于接收回执时直接退出,不再发一次消息。等待接收期间取消退出只取消关闭,不撤回或重复发送工作;独立告别和纯语义退出仍保留 3 秒可取消倒计时。
- 真实模型可能误判;纯语义 `EXIT` 也先确认原句已接受,再开始 3 秒可取消倒计时。点击取消、重新开口、静音或权限暂停只取消退出,不将已经发送的消息恢复成草稿、不撤回或重复提交。发送失败则保留尚未发送的原句,取消自动退出并显示原有错误/重试入口;重试不重新分类或恢复已取消的退出意图。
+- 前一句发送失败时,按发言顺序保留失败原句、排队发言和随后尚未定稿的新草稿。定稿文字进入队列后不再同时作为草稿拼接;用户实际重复说出的相同内容仍须保留。
- 前一句已经出队、仍等待发送前检查或接收回执时,后来的明确退出指令与告别倒计时不能使该发送失效:等待确认已接受后再关闭。新的发声、静音、权限暂停或明确取消会撤销等待中的退出,迟到的接收回执不能重启退出。手动结束及账号/页面生命周期清理仍立即释放资源,不推迟强制关闭;后台已经接受的工作不受影响。
- 内部开关在 `apps/desktop/src/main/voice-conversation/semanticExitConfig.ts` 的 `SEMANTIC_VOICE_EXIT_ENABLED`,暂设 `true` 供本地验收;改为 `false` 并重启 Desktop Main 后,完全跳过语义模型请求,本地指令、告别及空闲退出仍生效。没有 UI 开关或持久化偏好,不改用户已有设置。开启时未命中规则的短发言会多一次后台模型调用及相应用量;整条判断(含路由与凭证读取)最多等待 2 秒,该等待不在消息发送链路中。超时/失败/不支持直连/所选路由不可用时保留语音模式,不影响已开始的原句发送,不换模型兜底。超出 512 字符的完整发言直接继续,不截断后判断;带有未发送/排队上下文或重试的输入不单独取后半句判断。
- 语义 IPC 仅限可信本机顶层 Renderer,绑定 sender、callId、ASR 代次和最后一次最终识别 ID;文字由 Main 读取自己的临时 ASR 结果,同一结果最多判断一次。新发声、静音、权限暂停和结束会使在途判断失效,结束后新通话不继承旧判断或文字。派发前与结果返回后复核伙伴、账号及模型路由;Main 仅返回本地决定(沿用 true/false,并增加 `submit_then_exit`),UI 再复核当前输入和接收结果:纯退出在接收成功后开启倒计时,复合指令接收成功后立即关闭。没有新增 SSH、device-link、Mobile 或插件执行入口,不改跨设备 wire 协议或权限。分类提示与结果不进 Cindy 历史、长期记忆或正文日志;上游供应商仍按自身请求计费/留存规则处理。
diff --git a/packages/voice-input-core/src/VoiceConversationInput.ts b/packages/voice-input-core/src/VoiceConversationInput.ts
index 169e958955..9a086b7c99 100644
--- a/packages/voice-input-core/src/VoiceConversationInput.ts
+++ b/packages/voice-input-core/src/VoiceConversationInput.ts
@@ -212,8 +212,12 @@ export class VoiceConversationInput {
// continuous gate: audio awaiting verification must still be judged.
this.provider?.commitUtterance?.();
this.scheduleSettlement();
+ this.startFinalWait();
+ }
+
+ private startFinalWait(quietUntil = Date.now()): void {
this.clearFinalWait();
- this.finalWaitStartedAt = Date.now();
+ this.finalWaitStartedAt = quietUntil;
this.finalTimer = setTimeout(() => {
if (
this.segments.size &&
@@ -221,7 +225,7 @@ export class VoiceConversationInput {
)
this.fail("final_timeout");
else this.clearFinalWait();
- }, FINAL_TIMEOUT_MS);
+ }, Math.max(0, quietUntil - Date.now()) + FINAL_TIMEOUT_MS);
}
private clearFinalWait(): void {
@@ -284,7 +288,8 @@ export class VoiceConversationInput {
&& segment.audioEndMs <= this.providerAudioMs
? audioOffsetMs + segment.audioEndMs
: undefined;
- const textQuietAfter = (text && text !== prior?.text.trim()) || !prior
+ const textChanged = (text && text !== prior?.text.trim()) || !prior;
+ const textQuietAfter = textChanged
? Date.now() + AUTO_SEND_QUIET_MS
: prior.textQuietAfter;
this.stableAfter = Date.now() + FINAL_STABLE_MS;
@@ -293,6 +298,13 @@ export class VoiceConversationInput {
this.segments.set(key, {
...segment, generation, audioEndPositionMs, textQuietAfter,
});
+ // ASR can hear soft speech below local VAD's threshold. Such a partial
+ // has no activity.ended to arm finalization: use its quiet boundary.
+ // New words extend that boundary; repeated partials do not. Real acoustic
+ // speech clears this timer above, and a final still follows normal settling.
+ if (!segment.final && !this.speaking && !this.manual
+ && (textChanged || this.finalWaitStartedAt === null))
+ this.startFinalWait(Math.max(textQuietAfter, this.acousticQuietAfter, Date.now()));
this.options.emit({ type: "draft", text: this.draft() });
if (this.draft().length > 16_000) {
this.fail("transcript_limit");
@@ -521,7 +533,10 @@ export class VoiceConversationInput {
pendingFinalSegments: segments.filter((segment) => !segment.final).length,
characters: this.draft().length,
pendingMs: this.awaitingSettlement ? now - this.pendingSince : 0,
- finalWaitMs: this.finalWaitStartedAt === null ? null : now - this.finalWaitStartedAt,
+ // A soft-speech timer includes the quiet window. Until that ends, an
+ // upstream failure is not time spent waiting for finalization.
+ finalWaitMs: this.finalWaitStartedAt === null || now < this.finalWaitStartedAt
+ ? null : now - this.finalWaitStartedAt,
asrUpdateAgeMs: this.lastAsrUpdateAt === null ? null : now - this.lastAsrUpdateAt,
bufferedAudioMs: this.pendingBytes / 32,
});
diff --git a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
index 4a5519cfc4..791394672d 100644
--- a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
+++ b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
@@ -336,6 +336,53 @@ describe("continuous speech input", () => {
await input.stop();
});
+ // #5718 review: existing soft-speech cases deliver a final, and timeout
+ // coverage manually finishes input. Unit/fake time through the production
+ // input: a partial without local VAD must time out after quiet, retaining text.
+ // Standalone file validation: 0.40s wall time.
+ it("bounds finalization for ASR-only speech and preserves the partial", async () => {
+ const provider = new Provider();
+ const emit = vi.fn();
+ const failureDiagnostic = vi.fn();
+ const input = new VoiceConversationInput({ createProvider: async () => provider, emit, failureDiagnostic });
+ await input.start();
+ input.appendAudio(pcm(80, 200));
+ provider.segment({ id: "soft", text: "keep this", final: false });
+ await vi.advanceTimersByTimeAsync(4_000);
+ provider.segment({ id: "soft", text: "keep this", final: false });
+ await vi.advanceTimersByTimeAsync(5_249);
+ expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: "error" }));
+ await vi.advanceTimersByTimeAsync(1);
+ expect(emit).toHaveBeenCalledWith({ type: "error", text: "keep this" });
+ expect(failureDiagnostic).toHaveBeenCalledWith(expect.objectContaining({
+ reason: "final_timeout", finalWaitMs: 8_000,
+ }));
+ expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: "utterance" }));
+ await input.stop();
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it("extends the soft-speech deadline on new words and cancels it on real speech or a final", async () => {
+ const provider = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({ createProvider: async () => provider, emit });
+ await input.start();
+ provider.segment({ id: "soft", text: "first", final: false });
+ await vi.advanceTimersByTimeAsync(8_000);
+ provider.segment({ id: "soft", text: "first second", final: false });
+ await vi.advanceTimersByTimeAsync(2_000);
+ expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: "error" }));
+ input.appendAudio(pcm(2_000, 200));
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: "error" }));
+ input.appendAudio(pcm(0, 900));
+ provider.segment({ id: "soft", text: "first second third", final: true });
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(emit).toHaveBeenCalledWith({ type: "utterance", id: "1", text: "first second third" });
+ expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: "error" }));
+ await input.stop();
+ });
+
it("releases a noise-only onset after the quiet window without sending or waiting for an ASR segment", async () => {
const provider = new Provider();
const emit = vi.fn();