Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat(desktop): add continuous companion voice mode

JiangJiahao9101ebff3fd1
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.ts
new file mode 100644
index 0000000000..882a339f4d
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.ts
@@ -0,0 +1,127 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+
+type Handler = (...args: unknown[]) => unknown;
+const h = vi.hoisted(() => ({
+ handlers: new Map<string, Handler>(),
+ listeners: new Map<string, Handler>(),
+ trusted: vi.fn(),
+ emit: vi.fn(),
+ append: vi.fn(),
+ boundary: false,
+}));
+vi.mock('electron', () => ({
+ ipcMain: {
+ handle: (name: string, fn: Handler) => h.handlers.set(name, fn),
+ on: (name: string, fn: Handler) => h.listeners.set(name, fn),
+ },
+ webContents: { fromId: () => ({ isDestroyed: () => false, send: h.emit }) },
+}));
+vi.mock('../../security/trustedAppRenderer.js', () => ({
+ assertTrustedAppRendererEvent: h.trusted,
+}));
+vi.mock('../../maker-host/active-catalog.js', () => ({
+ getActiveCatalog: () => ({
+ providers: [
+ {
+ id: 'xd',
+ audioModels: [
+ { id: 'voice-test', name: 'Speech', mode: 'audio_speech', availability: 'available' },
+ ],
+ },
+ ],
+ }),
+}));
+vi.mock('../../secrets/providerSecretStore.js', () => ({
+ getProviderSecretStore: () => ({ get: () => 'test-key' }),
+}));
+vi.mock('../../model-access/effectiveEndpoint.js', () => ({
+ effectiveXdGatewayBaseUrl: () => 'https://gateway.test/v1',
+}));
+vi.mock('../../maker-host/outbound-fetch.js', () => ({ outboundFetch: vi.fn() }));
+vi.mock('../../appCapabilities.js', () => ({
+ getAppCapabilities: () => ({ canUseCindyGateway: true }),
+}));
+vi.mock('../../appSessionState.js', () => ({
+ activeOwnerScopeKey: () => 'test-owner',
+ isAppSessionBoundaryPending: () => h.boundary,
+}));
+vi.mock('../../localDb/ipc/bots.js', () => ({
+ getBotRemoteResourceSource: async () => ({ hiddenAt: null, canonicalSessionId: 'canonical' }),
+}));
+vi.mock('../../utility-model/oneShotCandidates.js', () => ({
+ isProviderModelRouteDisabled: () => false,
+}));
+
+import { registerVoiceConversationIpc } from '../index.js';
+const sender = (id = 1) => ({ sender: { id, once: vi.fn() } });
+const call = (name: string, ...args: unknown[]) =>
+ h.handlers.get(`voice-conversation:${name}`)!(sender(), ...args);
+beforeEach(() => {
+ h.handlers.clear();
+ h.listeners.clear();
+ h.trusted.mockReset();
+ h.append.mockReset();
+ h.boundary = false;
+ registerVoiceConversationIpc({
+ hasDictation: () => false,
+ createProvider: async () => ({
+ start: async () => {},
+ stop: async () => {},
+ appendAudio: h.append,
+ flushAudio: async () => {},
+ onEvent: () => {},
+ onSegment: () => {},
+ }),
+ });
+});
+afterEach(async () => {
+ h.trusted.mockReset();
+ await call('end', 'call');
+});
+const start = () =>
+ call('start', {
+ callId: 'call',
+ botId: 'bot',
+ sessionId: 'canonical',
+ modelId: 'voice-test',
+ voice: 'alloy',
+ });
+
+it('rejects an untrusted sender before returning models or opening a connection', async () => {
+ h.trusted.mockImplementation(() => {
+ throw new Error('untrusted frame');
+ });
+ await expect(call('models')).rejects.toThrow('untrusted frame');
+ await expect(start()).rejects.toThrow('untrusted frame');
+});
+
+it('enforces sender ownership and PCM size/alignment on the non-awaitable capture channel', async () => {
+ await start();
+ const audio = h.listeners.get('voice-conversation:audio')!;
+ audio(sender(2), { callId: 'call', pcm: new ArrayBuffer(2) });
+ audio(sender(), { callId: 'call', pcm: new ArrayBuffer(16_002) });
+ audio(sender(), { callId: 'call', pcm: new ArrayBuffer(3) });
+ audio(sender(), { callId: 'call', pcm: 'invalid' });
+ expect(h.append).not.toHaveBeenCalled();
+ audio(sender(), { callId: 'call', pcm: new ArrayBuffer(1280) });
+ expect(h.append).toHaveBeenCalledOnce();
+});
+
+it('refuses a start during an account transition, malformed voice parameters and oversized TTS text', async () => {
+ h.boundary = true;
+ await expect(start()).rejects.toThrow();
+ h.boundary = false;
+ await expect(
+ call('start', {
+ callId: 'call',
+ botId: 'bot',
+ sessionId: 'canonical',
+ modelId: 'voice-test',
+ voice: '../bad',
+ }),
+ ).rejects.toThrow();
+ await start();
+ await expect(
+ call('speak', { callId: 'call', requestId: 'x', text: 'x'.repeat(4097) }),
+ ).rejects.toThrow();
+});
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/models.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/models.test.ts
new file mode 100644
index 0000000000..c217ae3777
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/__tests__/models.test.ts
@@ -0,0 +1,51 @@
+import { expect, it } from 'vitest';
+import type { Catalog, ProviderMediaModel } from '@cindy/model-providers';
+import { speechModels } from '../models.js';
+import { normalizeVoiceInputSettings } from '../../../shared/voiceInputData.js';
+
+const model = (id: string, extra: Partial<ProviderMediaModel> = {}): ProviderMediaModel => ({
+ id,
+ name: id,
+ mode: 'audio_speech',
+ availability: 'available',
+ ...extra,
+});
+it('offers only available, enabled Gateway speech models with a supported protocol', () => {
+ const catalog = {
+ providers: [
+ {
+ id: 'xd',
+ audioModels: [
+ model('new-speech'),
+ model('paid', { availability: 'requires_payment' }),
+ model('registry-only', { availability: undefined }),
+ model('disabled', { disabled: true }),
+ model('override'),
+ model('asr', { mode: 'audio_transcription' }),
+ model('wrong-protocol', { nativeApi: 'openai-responses' }),
+ model('unverified', { nativeApi: null }),
+ ],
+ },
+ { id: 'other', audioModels: [model('other-speech')] },
+ ],
+ } as unknown as Catalog;
+ expect(speechModels(catalog, true, (id) => id === 'override')).toEqual([
+ { id: 'new-speech', name: 'new-speech' },
+ ]);
+ expect(speechModels(catalog, false, () => false)).toEqual([]);
+});
+
+it('does not pin a default speech model and lets null reset an explicit preference', () => {
+ expect(normalizeVoiceInputSettings({}).conversationSelection).toBeUndefined();
+ expect(
+ normalizeVoiceInputSettings({ conversationSelection: { modelId: 'tts', voice: 'alloy' } })
+ .conversationSelection,
+ ).toEqual({ modelId: 'tts', voice: 'alloy' });
+ expect(
+ normalizeVoiceInputSettings({ conversationSelection: null }).conversationSelection,
+ ).toBeUndefined();
+ expect(
+ normalizeVoiceInputSettings({ conversationSelection: { modelId: 'tts', voice: 'bad\nvoice' } })
+ .conversationSelection,
+ ).toBeUndefined();
+});
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
new file mode 100644
index 0000000000..2902b075ef
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
@@ -0,0 +1,103 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { VoiceConversationService } from '../service.js';
+import { hasVoiceConversationLease } from '../lease.js';
+
+const active: VoiceConversationService[] = [];
+afterEach(async () => {
+ await Promise.all(active.splice(0).map((service) => service.end(1)));
+});
+const selection = {
+ callId: 'call-1',
+ botId: 'bot',
+ sessionId: 'canonical',
+ modelId: 'tts',
+ voice: 'alloy',
+};
+function fixture() {
+ const provider = {
+ start: vi.fn(async () => {}),
+ stop: vi.fn(async () => {}),
+ dispose: vi.fn(async () => {}),
+ appendAudio: vi.fn(),
+ flushAudio: vi.fn(async () => {}),
+ onSegment: vi.fn(),
+ onEvent: vi.fn(),
+ };
+ const deps = {
+ owner: vi.fn(() => 'owner-a'),
+ hasDictation: vi.fn(() => false),
+ assertCompanion: vi.fn(async () => {}),
+ models: vi.fn(() => [{ id: 'tts', name: 'Speech' }]),
+ createProvider: vi.fn(async () => provider),
+ connection: () => ({ baseUrl: 'https://gateway.test/v1', apiKey: 'test-key' }),
+ fetch: vi.fn(
+ async () =>
+ new Response(new Uint8Array([1, 2]), { headers: { 'Content-Type': 'audio/pcm' } }),
+ ),
+ emit: vi.fn(),
+ };
+ const service = new VoiceConversationService(deps);
+ active.push(service);
+ return { service, deps, provider };
+}
+
+describe('local companion call ownership', () => {
+ it('requires a real available speech model and the canonical companion task before allocating ASR', async () => {
+ const { service, deps } = fixture();
+ await expect(service.start(1, { ...selection, modelId: 'invented' })).rejects.toThrow(
+ 'model_unavailable',
+ );
+ deps.assertCompanion.mockRejectedValueOnce(new Error('not canonical'));
+ await expect(service.start(1, selection)).rejects.toThrow('not canonical');
+ expect(deps.createProvider).not.toHaveBeenCalled();
+ expect(hasVoiceConversationLease()).toBe(false);
+ });
+
+ it('blocks concurrent dictation/calls and binds all operations to the initiating renderer', async () => {
+ const { service, deps, provider } = fixture();
+ deps.hasDictation.mockReturnValueOnce(true);
+ await expect(service.start(1, selection)).rejects.toThrow('busy');
+ const { callId } = await service.start(1, selection);
+ await expect(service.start(1, selection)).rejects.toThrow('busy');
+ expect(() => service.audio(2, callId, new ArrayBuffer(2))).toThrow('call_ended');
+ await service.end(2, callId);
+ expect(provider.stop).not.toHaveBeenCalled();
+ await service.end(1, callId);
+ expect(provider.stop).toHaveBeenCalledOnce();
+ });
+
+ it('invalidates the old call and releases the microphone on account change', async () => {
+ const { service, deps, provider } = fixture();
+ const { callId } = await service.start(1, selection);
+ deps.owner.mockReturnValue('owner-b');
+ expect(() => service.audio(1, callId, new ArrayBuffer(2))).toThrow('call_ended');
+ await vi.waitFor(() => expect(provider.stop).toHaveBeenCalledOnce());
+ expect(deps.emit).toHaveBeenCalledWith(1, { callId, event: { type: 'ended' } });
+ });
+
+ it('rechecks catalog eligibility before speech dispatch', async () => {
+ const { service, deps } = fixture();
+ const { callId } = await service.start(1, selection);
+ deps.models.mockReturnValue([]);
+ await expect(service.speak(1, callId, 'request', 'hello')).rejects.toThrow('model_unavailable');
+ expect(deps.fetch).not.toHaveBeenCalled();
+ });
+
+ it('cancels a pending provider allocation without starting a stale microphone session', async () => {
+ const { service, deps, provider } = fixture();
+ let resolve!: (value: typeof provider) => void;
+ deps.createProvider.mockImplementation(
+ () =>
+ new Promise((done) => {
+ resolve = done;
+ }),
+ );
+ const starting = service.start(1, selection);
+ await vi.waitFor(() => expect(deps.createProvider).toHaveBeenCalledOnce());
+ await service.end(1);
+ resolve(provider);
+ await expect(starting).rejects.toThrow('call_ended');
+ expect(provider.start).not.toHaveBeenCalled();
+ expect(provider.dispose).toHaveBeenCalledOnce();
+ });
+});
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
new file mode 100644
index 0000000000..20b42a672b
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
@@ -0,0 +1,107 @@
+import { describe, expect, it, vi } from 'vitest';
+import { SpeechSynthesisSession } from '../speech.js';
+
+const make = (
+ fetcher: typeof fetch = vi.fn(
+ async () => new Response(new Uint8Array([1, 2]), { headers: { 'Content-Type': 'audio/pcm' } }),
+ ),
+) => {
+ const assertCurrent = vi.fn();
+ const speech = new SpeechSynthesisSession({
+ connection: () => ({ baseUrl: 'https://gateway.test/prefix/v1/', apiKey: 'test-key' }),
+ fetch: fetcher,
+ assertCurrent,
+ });
+ return { speech, fetcher, assertCurrent };
+};
+const input = { requestId: 'one', modelId: 'speech-model', voice: 'alloy', text: 'hello' };
+
+describe('speech transport', () => {
+ it('preserves the server-provided endpoint prefix, sends the selected model, and pulls binary PCM', async () => {
+ const { speech, fetcher } = make();
+ expect(await speech.start(input)).toEqual({ sampleRate: 24000 });
+ expect(fetcher).toHaveBeenCalledWith(
+ new URL('https://gateway.test/prefix/v1/audio/speech'),
+ expect.objectContaining({
+ method: 'POST',
+ redirect: 'error',
+ body: JSON.stringify({
+ model: 'speech-model',
+ voice: 'alloy',
+ input: 'hello',
+ response_format: 'pcm',
+ }),
+ }),
+ );
+ expect(new Uint8Array((await speech.read('one')).pcm)).toEqual(new Uint8Array([1, 2]));
+ expect((await speech.read('one')).done).toBe(true);
+ speech.cancel();
+ });
+
+ it.each(['audio/mpeg', 'application/json'])(
+ 'rejects an incompatible %s body instead of playing it as PCM',
+ async (contentType) => {
+ const { speech } = make(
+ vi.fn(async () => new Response('not PCM', { headers: { 'Content-Type': contentType } })),
+ );
+ await expect(speech.start(input)).rejects.toThrow('format');
+ },
+ );
+
+ it('does not pass upstream error bodies to the renderer', async () => {
+ const { speech } = make(
+ vi.fn(async () => new Response('upstream-sensitive-error', { status: 401 })),
+ );
+ await expect(speech.start(input)).rejects.toThrow(/^request$/);
+ });
+
+ it.each([new Uint8Array(), new Uint8Array([1])])(
+ 'rejects empty or truncated PCM at EOF',
+ async (bytes) => {
+ const { speech } = make(
+ vi.fn(async () => new Response(bytes, { headers: { 'Content-Type': 'audio/pcm' } })),
+ );
+ await speech.start(input);
+ if (bytes.length) await speech.read('one');
+ await expect(speech.read('one')).rejects.toThrow('format');
+ },
+ );
+
+ it('cancels pending headers and refuses a late body after interruption', async () => {
+ let resolve!: (response: Response) => void;
+ const fetcher = vi.fn(
+ () =>
+ new Promise<Response>((done) => {
+ resolve = done;
+ }),
+ );
+ const { speech } = make(fetcher);
+ const starting = speech.start(input);
+ speech.cancel();
+ resolve(new Response(new Uint8Array([1, 2]), { headers: { 'Content-Type': 'audio/pcm' } }));
+ await expect(starting).rejects.toThrow('cancelled');
+ await expect(speech.read('one')).rejects.toThrow('cancelled');
+ });
+
+ it('allows only one read and invalidates a blocked read on interruption', async () => {
+ const stream = new ReadableStream<Uint8Array>();
+ const { speech } = make(
+ vi.fn(async () => new Response(stream, { headers: { 'Content-Type': 'audio/pcm' } })),
+ );
+ await speech.start(input);
+ const reading = speech.read('one');
+ await expect(speech.read('one')).rejects.toThrow('cancelled');
+ speech.cancel();
+ await expect(reading).rejects.toThrow('cancelled');
+ });
+
+ it('rechecks the owner/model before each read', async () => {
+ const { speech, assertCurrent } = make();
+ await speech.start(input);
+ assertCurrent.mockImplementation(() => {
+ throw new Error('owner changed');
+ });
+ await expect(speech.read('one')).rejects.toThrow('owner changed');
+ speech.cancel();
+ });
+});
diff --git a/apps/desktop/src/main/voice-conversation/index.ts b/apps/desktop/src/main/voice-conversation/index.ts
new file mode 100644
index 0000000000..838d46bb07
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/index.ts
@@ -0,0 +1,127 @@
+import { ipcMain, webContents, type IpcMainInvokeEvent } from 'electron';
+import type { AsrProvider } from '@cindy/voice-input-core';
+import { normalizeVoiceConversationSelection } from '../../shared/voiceConversation.js';
+import { getActiveCatalog } from '../maker-host/active-catalog.js';
+import { getProviderSecretStore } from '../secrets/providerSecretStore.js';
+import { effectiveXdGatewayBaseUrl } from '../model-access/effectiveEndpoint.js';
+import { outboundFetch } from '../maker-host/outbound-fetch.js';
+import { getAppCapabilities } from '../appCapabilities.js';
+import { activeOwnerScopeKey, isAppSessionBoundaryPending } from '../appSessionState.js';
+import { getBotRemoteResourceSource } from '../localDb/ipc/bots.js';
+import { isProviderModelRouteDisabled } from '../utility-model/oneShotCandidates.js';
+import { assertTrustedAppRendererEvent } from '../security/trustedAppRenderer.js';
+import { requireObject, requireString, throwIpcError } from '../utils/ipcValidate.js';
+import { VoiceConversationService } from './service.js';
+import { speechModels } from './models.js';
+
+function boundedString(value: unknown, name: string, limit: number): string {
+ const text = requireString(value, name);
+ if (!text.trim() || text.length > limit) throwIpcError('INVALID_PARAMS', `Invalid ${name}`);
+ return text;
+}
+
+/** Local top-level renderer only; deliberately absent from device-link and plugin allowlists. */
+export function registerVoiceConversationIpc(deps: {
+ createProvider(): Promise<AsrProvider>;
+ hasDictation(): boolean;
+}): void {
+ const service = new VoiceConversationService({
+ ...deps,
+ owner: () => (isAppSessionBoundaryPending() ? 'changing-account' : activeOwnerScopeKey()),
+ assertCompanion: async (botId, sessionId) => {
+ const bot = await getBotRemoteResourceSource(botId);
+ if (bot.hiddenAt !== null || bot.canonicalSessionId !== sessionId)
+ throw new Error('invalid_companion');
+ },
+ models: () =>
+ speechModels(getActiveCatalog(), getAppCapabilities().canUseCindyGateway, (id) =>
+ isProviderModelRouteDisabled('xd', id),
+ ),
+ connection: () => {
+ if (!getAppCapabilities().canUseCindyGateway) throw new Error('unavailable');
+ return {
+ baseUrl: effectiveXdGatewayBaseUrl(),
+ apiKey: getProviderSecretStore().get('xd') ?? '',
+ };
+ },
+ fetch: outboundFetch,
+ emit: (sender, event) => {
+ const target = webContents.fromId(sender);
+ if (target && !target.isDestroyed()) target.send('voice-conversation:event', event);
+ },
+ });
+ const observed = new Set<number>();
+ const handle = (
+ name: string,
+ action: (event: IpcMainInvokeEvent, ...args: unknown[]) => unknown,
+ ): void => {
+ ipcMain.handle(`voice-conversation:${name}`, async (event, ...args: unknown[]) => {
+ assertTrustedAppRendererEvent(event);
+ try {
+ return await action(event, ...args);
+ } catch {
+ throwIpcError('PRECONDITION_FAILED', 'Voice conversation is unavailable.');
+ }
+ });
+ };
+ handle('models', () => service.models());
+ handle('start', async (event, raw) => {
+ if (isAppSessionBoundaryPending()) throw new Error('changing_account');
+ const value = requireObject(raw);
+ const selection = normalizeVoiceConversationSelection(value);
+ if (!selection) throwIpcError('INVALID_PARAMS', 'Invalid speech selection');
+ if (!observed.has(event.sender.id)) {
+ observed.add(event.sender.id);
+ const sender = event.sender.id;
+ event.sender.once('destroyed', () => {
+ observed.delete(sender);
+ void service.end(sender);
+ });
+ }
+ return service.start(event.sender.id, {
+ ...selection,
+ callId: boundedString(value.callId, 'callId', 64),
+ botId: boundedString(value.botId, 'botId', 256),
+ sessionId: boundedString(value.sessionId, 'sessionId', 256),
+ });
+ });
+ ipcMain.on('voice-conversation:audio', (event, raw: unknown) => {
+ try {
+ assertTrustedAppRendererEvent(event);
+ const value = requireObject(raw);
+ if (
+ !(value.pcm instanceof ArrayBuffer) ||
+ !value.pcm.byteLength ||
+ value.pcm.byteLength > 16_000 ||
+ value.pcm.byteLength % 2
+ )
+ return;
+ service.audio(event.sender.id, boundedString(value.callId, 'callId', 64), value.pcm);
+ } catch {
+ /* Untrusted or late frames are ignored; no payload is logged. */
+ }
+ });
+ handle('finish-utterance', (event, id) =>
+ service.finishUtterance(event.sender.id, boundedString(id, 'callId', 64)),
+ );
+ handle('end', (event, id) => service.end(event.sender.id, boundedString(id, 'callId', 64)));
+ handle('interrupt', (event, id) =>
+ service.interrupt(event.sender.id, boundedString(id, 'callId', 64)),
+ );
+ handle('speak', (event, raw) => {
+ const value = requireObject(raw);
+ return service.speak(
+ event.sender.id,
+ boundedString(value.callId, 'callId', 64),
+ boundedString(value.requestId, 'requestId', 128),
+ boundedString(value.text, 'text', 4_096),
+ );
+ });
+ handle('read-speech', (event, id, requestId) =>
+ service.readSpeech(
+ event.sender.id,
+ boundedString(id, 'callId', 64),
+ boundedString(requestId, 'requestId', 128),
+ ),
+ );
+}
diff --git a/apps/desktop/src/main/voice-conversation/lease.ts b/apps/desktop/src/main/voice-conversation/lease.ts
new file mode 100644
index 0000000000..d560fc9024
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/lease.ts
@@ -0,0 +1,13 @@
+/** Device-local microphone ownership shared with dictation, including its global shortcut. */
+let owner: string | null = null;
+export function hasVoiceConversationLease(): boolean {
+ return owner !== null;
+}
+export function acquireVoiceConversationLease(callId: string): boolean {
+ if (owner !== null) return false;
+ owner = callId;
+ return true;
+}
+export function releaseVoiceConversationLease(callId: string): void {
+ if (owner === callId) owner = null;
+}
diff --git a/apps/desktop/src/main/voice-conversation/models.ts b/apps/desktop/src/main/voice-conversation/models.ts
new file mode 100644
index 0000000000..65c9380b6f
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/models.ts
@@ -0,0 +1,33 @@
+import { BUNDLED_CATALOG, resolveModelNativeApi, type Catalog } from '@cindy/model-providers';
+import type { VoiceConversationModel } from '../../shared/voiceConversation.js';
+
+/** Registry metadata explains the protocol; only the account catalog grants access. */
+export function speechModels(
+ catalog: Catalog,
+ gatewayAllowed: boolean,
+ disabled: (modelId: string) => boolean,
+): VoiceConversationModel[] {
+ if (!gatewayAllowed) return [];
+ return (catalog.providers.find((provider) => provider.id === 'xd')?.audioModels ?? [])
+ .filter((model) => {
+ if (
+ model.mode !== 'audio_speech' ||
+ model.availability !== 'available' ||
+ model.disabled ||
+ disabled(model.id)
+ )
+ return false;
+ const declared =
+ model.nativeApi !== undefined
+ ? model.nativeApi
+ : resolveModelNativeApi(catalog.modelRegistry, 'xd', model.id);
+ const protocol =
+ declared !== undefined
+ ? declared
+ : resolveModelNativeApi(BUNDLED_CATALOG.modelRegistry, 'xd', model.id);
+ // Unknown Gateway entries use its OpenAI speech contract. Explicitly unverified
+ // (null) or another native API must not be silently treated as that protocol.
+ return protocol === undefined || protocol === 'openai-audio-speech';
+ })
+ .map((model) => ({ id: model.id, name: model.name }));
+}
diff --git a/apps/desktop/src/main/voice-conversation/service.ts b/apps/desktop/src/main/voice-conversation/service.ts
new file mode 100644
index 0000000000..08d0c5ba9c
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/service.ts
@@ -0,0 +1,146 @@
+import { VoiceConversationInput, type AsrProvider, type AudioTrace } from '@cindy/voice-input-core';
+import type {
+ VoiceConversationEvent,
+ VoiceConversationModel,
+ VoiceConversationStart,
+} from '../../shared/voiceConversation.js';
+import { acquireVoiceConversationLease, releaseVoiceConversationLease } from './lease.js';
+import { SpeechSynthesisSession, type SpeechConnection } from './speech.js';
+
+type Call = {
+ id: string;
+ sender: number;
+ owner: string;
+ input: VoiceConversationInput;
+ speech: SpeechSynthesisSession;
+ selection: VoiceConversationStart;
+};
+
+/** One local call owns the microphone. Cancellation invalidates all pending starts and audio reads. */
+export class VoiceConversationService {
+ private call: Call | null = null;
+
+ constructor(
+ private readonly deps: {
+ owner(): string;
+ hasDictation(): boolean;
+ assertCompanion(botId: string, sessionId: string): Promise<void>;
+ models(): VoiceConversationModel[];
+ createProvider(): Promise<AsrProvider>;
+ connection(): SpeechConnection;
+ fetch: typeof fetch;
+ emit(sender: number, event: VoiceConversationEvent): void;
+ },
+ ) {}
+
+ models(): VoiceConversationModel[] {
+ return this.deps.models();
+ }
+
+ async start(sender: number, selection: VoiceConversationStart): Promise<{ callId: string }> {
+ if (this.call || this.deps.hasDictation()) throw new Error('busy');
+ if (!this.models().some((model) => model.id === selection.modelId))
+ throw new Error('model_unavailable');
+ const id = selection.callId;
+ const owner = this.deps.owner();
+ if (!acquireVoiceConversationLease(id)) throw new Error('busy');
+ const assertCurrent = (): void => {
+ if (this.call?.id !== id || this.deps.owner() !== owner) throw new Error('call_ended');
+ };
+ const input = new VoiceConversationInput({
+ createProvider: async () => {
+ assertCurrent();
+ return this.deps.createProvider();
+ },
+ emit: (event) => {
+ if (this.call?.id !== id) return;
+ if (this.deps.owner() !== owner) {
+ void this.end(sender, id);
+ return;
+ }
+ this.deps.emit(sender, { callId: id, event });
+ if (event.type === 'error') void this.end(sender, id);
+ },
+ });
+ const speech = new SpeechSynthesisSession({
+ connection: this.deps.connection,
+ fetch: this.deps.fetch,
+ assertCurrent: () => {
+ assertCurrent();
+ if (!this.models().some((model) => model.id === selection.modelId))
+ throw new Error('model_unavailable');
+ },
+ });
+ this.call = { id, owner, sender, selection, input, speech };
+ try {
+ await this.deps.assertCompanion(selection.botId, selection.sessionId);
+ assertCurrent();
+ await input.start();
+ assertCurrent();
+ return { callId: id };
+ } catch (error) {
+ await this.end(sender, id);
+ throw error;
+ }
+ }
+
+ audio(sender: number, id: string, pcm: ArrayBuffer, trace?: AudioTrace): void {
+ this.requireCall(sender, id).input.appendAudio(pcm, trace);
+ }
+
+ finishUtterance(sender: number, id: string): void {
+ this.requireCall(sender, id).input.finishUtterance(true);
+ }
+
+ async speak(
+ sender: number,
+ id: string,
+ requestId: string,
+ text: string,
+ ): Promise<{ sampleRate: number }> {
+ const call = this.requireCall(sender, id);
+ return call.speech.start({
+ requestId,
+ text,
+ modelId: call.selection.modelId,
+ voice: call.selection.voice,
+ });
+ }
+
+ readSpeech(
+ sender: number,
+ id: string,
+ requestId: string,
+ ): Promise<{ pcm: ArrayBuffer; done: boolean }> {
+ return this.requireCall(sender, id).speech.read(requestId);
+ }
+
+ interrupt(sender: number, id: string): void {
+ this.requireCall(sender, id).speech.cancel();
+ }
+
+ async end(sender: number, id?: string): Promise<void> {
+ const call = this.call;
+ if (!call || call.sender !== sender || (id && call.id !== id)) return;
+ this.call = null;
+ this.deps.emit(call.sender, { callId: call.id, event: { type: 'ended' } });
+ call.speech.cancel();
+ try {
+ await call.input.stop();
+ } catch {
+ /* Ownership release must survive provider teardown. */
+ } finally {
+ releaseVoiceConversationLease(call.id);
+ }
+ }
+
+ private requireCall(sender: number, id: string): Call {
+ const call = this.call;
+ if (!call || call.sender !== sender || call.id !== id) throw new Error('call_ended');
+ if (call.owner !== this.deps.owner()) {
+ void this.end(sender, id);
+ throw new Error('call_ended');
+ }
+ return call;
+ }
+}
diff --git a/apps/desktop/src/main/voice-conversation/speech.ts b/apps/desktop/src/main/voice-conversation/speech.ts
new file mode 100644
index 0000000000..787f4cfea6
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/speech.ts
@@ -0,0 +1,131 @@
+export type SpeechConnection = { baseUrl: string; apiKey: string };
+
+/** Safe, fixed-protocol errors: upstream bodies and credentials never cross IPC. */
+export class SpeechError extends Error {
+ constructor(readonly code: 'unavailable' | 'request' | 'format' | 'cancelled') {
+ super(code);
+ }
+}
+
+/** OpenAI-compatible speech transport. Reads are pulled by playback for bounded buffering. */
+export class SpeechSynthesisSession {
+ private epoch = 0;
+ private controller: AbortController | null = null;
+ private reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
+ private requestId: string | null = null;
+ private totalBytes = 0;
+ private reading = false;
+
+ constructor(
+ private readonly deps: {
+ connection(): SpeechConnection;
+ assertCurrent(): void;
+ fetch: typeof fetch;
+ },
+ ) {}
+
+ async start(input: {
+ requestId: string;
+ modelId: string;
+ voice: string;
+ text: string;
+ }): Promise<{ sampleRate: number }> {
+ this.cancel();
+ const epoch = this.epoch;
+ this.deps.assertCurrent();
+ const { baseUrl, apiKey } = this.deps.connection();
+ const url = new URL(baseUrl.replace(/\/+$/, '') + '/audio/speech');
+ if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || !apiKey)
+ throw new SpeechError('unavailable');
+ const controller = new AbortController();
+ this.controller = controller;
+ this.requestId = input.requestId;
+ const timer = setTimeout(() => controller.abort(), 30_000);
+ try {
+ const response = await this.deps.fetch(url, {
+ method: 'POST',
+ redirect: 'error',
+ signal: controller.signal,
+ headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ model: input.modelId,
+ voice: input.voice,
+ input: input.text,
+ response_format: 'pcm',
+ }),
+ });
+ this.deps.assertCurrent();
+ if (epoch !== this.epoch || controller.signal.aborted) {
+ await response.body?.cancel();
+ throw new SpeechError('cancelled');
+ }
+ if (!response.ok || !response.body) {
+ await response.body?.cancel();
+ throw new SpeechError('request');
+ }
+ const contentType = response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase();
+ if (
+ !contentType ||
+ !['audio/pcm', 'audio/x-pcm', 'application/octet-stream'].includes(contentType)
+ ) {
+ await response.body.cancel();
+ throw new SpeechError('format');
+ }
+ this.reader = response.body.getReader();
+ // This sample rate is the OpenAI PCM protocol contract, not guessed from model mode.
+ return { sampleRate: 24_000 };
+ } catch (error) {
+ if (epoch === this.epoch) this.cancel();
+ throw error instanceof SpeechError ? error : new SpeechError('request');
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+
+ async read(requestId: string): Promise<{ pcm: ArrayBuffer; done: boolean }> {
+ this.deps.assertCurrent();
+ if (requestId !== this.requestId || !this.reader || this.reading)
+ throw new SpeechError('cancelled');
+ const reader = this.reader;
+ const epoch = this.epoch;
+ this.reading = true;
+ const timer = setTimeout(() => {
+ if (epoch === this.epoch) this.cancel();
+ }, 15_000);
+ try {
+ const result = await reader.read();
+ this.deps.assertCurrent();
+ if (epoch !== this.epoch) throw new SpeechError('cancelled');
+ const value = result.value ?? new Uint8Array();
+ this.totalBytes += value.byteLength;
+ if (value.byteLength > 1_048_576 || this.totalBytes > 24_000 * 2 * 600) {
+ this.cancel();
+ throw new SpeechError('format');
+ }
+ if (result.done) {
+ if (this.totalBytes === 0 || this.totalBytes % 2 !== 0) throw new SpeechError('format');
+ reader.releaseLock();
+ this.reader = null;
+ this.requestId = null;
+ }
+ return { pcm: Uint8Array.from(value).buffer, done: result.done };
+ } catch (error) {
+ if (epoch === this.epoch) this.cancel();
+ throw error instanceof SpeechError ? error : new SpeechError('request');
+ } finally {
+ clearTimeout(timer);
+ if (epoch === this.epoch) this.reading = false;
+ }
+ }
+
+ cancel(): void {
+ ++this.epoch;
+ this.controller?.abort();
+ this.controller = null;
+ void this.reader?.cancel().catch(() => undefined);
+ this.reader = null;
+ this.requestId = null;
+ this.reading = false;
+ this.totalBytes = 0;
+ }
+}
diff --git a/apps/desktop/src/main/voice-input/ElevenLabsScribeProvider.ts b/apps/desktop/src/main/voice-input/ElevenLabsScribeProvider.ts
index 376a550769..9dbc33bec5 100644
--- a/apps/desktop/src/main/voice-input/ElevenLabsScribeProvider.ts
+++ b/apps/desktop/src/main/voice-input/ElevenLabsScribeProvider.ts
@@ -75,6 +75,12 @@ export class ElevenLabsScribeProvider implements AsrProvider {
this.connectTimeoutMs = options.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
}
+ private segmentCallback?: (segment: import('@cindy/voice-input-core').AsrSegment) => void;
+
+ onSegment(callback: (segment: import('@cindy/voice-input-core').AsrSegment) => void): void {
+ this.segmentCallback = callback;
+ }
+
onEvent(callback: (event: AsrEvent) => void): void {
this.callback = callback;
}
@@ -421,6 +427,7 @@ export class ElevenLabsScribeProvider implements AsrProvider {
changedFromPrevious: previous !== partial,
draftChars: draft.length,
});
+ this.segmentCallback?.({ id: String(this.committedSegments.length), text: partial, final: false });
this.callback({ type: 'partial', text: draft, at: Date.now() });
}
@@ -428,6 +435,7 @@ export class ElevenLabsScribeProvider implements AsrProvider {
const committed = mergeRecoveredTranscript(this.recoveryPartialPrefix, text);
if (!hasSpeechContent(committed) || this.isDuplicateStable(committed)) return;
+ this.segmentCallback?.({ id: String(this.committedSegments.length), text: committed, final: true });
this.committedSegments.push(committed);
this.currentPartial = '';
this.recoveryPartialPrefix = '';
diff --git a/apps/desktop/src/main/voice-input/FallbackAsrProvider.ts b/apps/desktop/src/main/voice-input/FallbackAsrProvider.ts
index c044015087..8ea20bd09d 100644
--- a/apps/desktop/src/main/voice-input/FallbackAsrProvider.ts
+++ b/apps/desktop/src/main/voice-input/FallbackAsrProvider.ts
@@ -255,6 +255,14 @@ export class FallbackAsrProvider implements AsrProvider {
await this.active.flushAudio();
}
+ private segmentCallback?: (segment: import('@cindy/voice-input-core').AsrSegment) => void;
+
+ onSegment(callback: (segment: import('@cindy/voice-input-core').AsrSegment) => void): void {
+ this.segmentCallback = callback;
+ }
+
+ commitUtterance(): void { this.active?.commitUtterance?.(); }
+
onEvent(callback: (event: AsrEvent) => void): void {
this.eventCallbacks.push(callback);
}
@@ -327,6 +335,9 @@ export class FallbackAsrProvider implements AsrProvider {
await this.cleanupAttempt(attempt);
return { index, status: 'cancelled' };
}
+ provider.onSegment?.((segment) => {
+ if (this.active === provider) this.segmentCallback?.(segment);
+ });
provider.onEvent((event) => {
if (this.active === provider) {
for (const callback of this.eventCallbacks) callback(event);
diff --git a/apps/desktop/src/main/voice-input/RealtimeAsrWebSocketProvider.ts b/apps/desktop/src/main/voice-input/RealtimeAsrWebSocketProvider.ts
index 0f7ace4653..2974503d8e 100644
--- a/apps/desktop/src/main/voice-input/RealtimeAsrWebSocketProvider.ts
+++ b/apps/desktop/src/main/voice-input/RealtimeAsrWebSocketProvider.ts
@@ -718,6 +718,16 @@ export class RealtimeAsrWebSocketProvider implements AsrProvider {
}
}
+ private segmentCallback?: (segment: import('@cindy/voice-input-core').AsrSegment) => void;
+
+ onSegment(callback: (segment: import('@cindy/voice-input-core').AsrSegment) => void): void {
+ this.segmentCallback = callback;
+ }
+
+ commitUtterance(): void {
+ if (this.protocolProfile === 'openai-transcription-manual') this.commitBufferedAudio('utterance');
+ }
+
onEvent(callback: (event: AsrEvent) => void): void {
this.callback = callback;
}
@@ -1240,6 +1250,7 @@ export class RealtimeAsrWebSocketProvider implements AsrProvider {
this.registerItem(event.item_id);
const next = `${this.partialsByItem.get(event.item_id) ?? ''}${event.delta}`;
this.partialsByItem.set(event.item_id, next);
+ this.segmentCallback?.({ id: event.item_id, order: this.itemOrder.indexOf(event.item_id), text: next, final: false });
// Trim — not clear — the replay buffer. The partial event reflects audio
// that the server's transcription pipeline has already consumed, but we
// don't know exactly which frames; the safe assumption is "anything older
@@ -1263,6 +1274,7 @@ export class RealtimeAsrWebSocketProvider implements AsrProvider {
// final source of truth.
const preview = `${event.text}${typeof event.stash === 'string' ? event.stash : ''}`;
this.partialsByItem.set(event.item_id, preview);
+ this.segmentCallback?.({ id: event.item_id, order: this.itemOrder.indexOf(event.item_id), text: preview, final: false });
this.trimUnconfirmedAudioOlderThan(Date.now() - PARTIAL_CONFIRMATION_LATENCY_MS);
this.callback({ type: 'partial', text: this.aggregateTranscript(), at: Date.now() });
}
@@ -1291,6 +1303,7 @@ export class RealtimeAsrWebSocketProvider implements AsrProvider {
const aggregateBefore = this.aggregateTranscript();
this.registerItem(event.item_id);
this.finalsByItem.set(event.item_id, itemTranscript);
+ this.segmentCallback?.({ id: event.item_id, order: this.itemOrder.indexOf(event.item_id), text: itemTranscript, final: true });
this.partialsByItem.delete(event.item_id);
if (this.pendingCommitCount > 0) this.pendingCommitCount -= 1;
// A completed item means the server consumed that utterance, not
@@ -1315,6 +1328,7 @@ export class RealtimeAsrWebSocketProvider implements AsrProvider {
private registerItem(itemId: string): void {
if (this.itemOrder.includes(itemId)) return;
this.itemOrder.push(itemId);
+ this.segmentCallback?.({ id: itemId, order: this.itemOrder.length - 1, text: '', final: false });
}
private aggregateTranscript(): string {
diff --git a/apps/desktop/src/main/voice-input/SystemAudioMuteGuard.ts b/apps/desktop/src/main/voice-input/SystemAudioMuteGuard.ts
index 569741b173..9e67e55254 100644
--- a/apps/desktop/src/main/voice-input/SystemAudioMuteGuard.ts
+++ b/apps/desktop/src/main/voice-input/SystemAudioMuteGuard.ts
@@ -58,22 +58,33 @@ const SUPPORTS_MUTE = process.platform === 'darwin' || process.platform === 'win
export class SystemAudioMuteGuard {
private readonly owners = new Set<AudioMuteOwner>();
private snapshot: AudioSnapshot | null = null;
+ private pendingMutes = 0;
private tail: Promise<void> = Promise.resolve();
+ /** Includes the async OS mutation so speaker playback cannot race a starting dictation. */
+ get hasPendingOrActiveMute(): boolean {
+ return this.pendingMutes > 0 || this.owners.size > 0 || this.snapshot !== null;
+ }
+
async mute(ownerId: AudioMuteOwner): Promise<void> {
if (!SUPPORTS_MUTE) return;
- await this.enqueue(async () => {
- if (this.owners.has(ownerId)) return;
- if (this.snapshot === null) {
- this.snapshot = await muteOutputAndReadSnapshot();
- log.info('muted for voice input', { ownerId, wasMuted: this.snapshot.outputMuted });
- } else if (this.owners.size === 0) {
- // A rejected restore may still have reached the OS. Reassert mute for
- // the new owner without replacing the outstanding original snapshot.
- await setOutputMuted(true);
- }
- this.owners.add(ownerId);
- });
+ this.pendingMutes++;
+ try {
+ await this.enqueue(async () => {
+ if (this.owners.has(ownerId)) return;
+ if (this.snapshot === null) {
+ this.snapshot = await muteOutputAndReadSnapshot();
+ log.info('muted for voice input', { ownerId, wasMuted: this.snapshot.outputMuted });
+ } else if (this.owners.size === 0) {
+ // A rejected restore may still have reached the OS. Reassert mute for
+ // the new owner without replacing the outstanding original snapshot.
+ await setOutputMuted(true);
+ }
+ this.owners.add(ownerId);
+ });
+ } finally {
+ this.pendingMutes--;
+ }
}
async restore(ownerId: AudioMuteOwner): Promise<void> {
diff --git a/apps/desktop/src/main/voice-input/VolcengineSaucAsrProvider.ts b/apps/desktop/src/main/voice-input/VolcengineSaucAsrProvider.ts
index ad8c1eb328..69593762ee 100644
--- a/apps/desktop/src/main/voice-input/VolcengineSaucAsrProvider.ts
+++ b/apps/desktop/src/main/voice-input/VolcengineSaucAsrProvider.ts
@@ -124,6 +124,12 @@ export class VolcengineSaucAsrProvider implements AsrProvider {
this.errorFallbackMessage = options.errorFallbackMessage ?? 'Volcengine SAUC transcription failed.';
}
+ private segmentCallback?: (segment: import('@cindy/voice-input-core').AsrSegment) => void;
+
+ onSegment(callback: (segment: import('@cindy/voice-input-core').AsrSegment) => void): void {
+ this.segmentCallback = callback;
+ }
+
onEvent(callback: (event: AsrEvent) => void): void {
this.callback = callback;
}
@@ -552,6 +558,7 @@ export class VolcengineSaucAsrProvider implements AsrProvider {
}
this.startResolve?.();
+ for (const segment of extractConversationSegments(message.payload)) this.segmentCallback?.(segment);
const rawTranscript = extractTranscript(message.payload);
const transcript = mergeRecoveredTranscript(this.sessionTranscriptPrefix, rawTranscript);
const confirmation = getTranscriptConfirmation(message.payload, rawTranscript);
@@ -872,3 +879,17 @@ function looksLikeJson(text: string): boolean {
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
+
+/** SAUC utterance timestamps are stable within one transport; text can be revised. */
+export function extractConversationSegments(payload: unknown): import('@cindy/voice-input-core').AsrSegment[] {
+ if (!isRecord(payload)) return [];
+ const results = Array.isArray(payload.result) ? payload.result : [payload.result];
+ return results.flatMap((result) => {
+ if (!isRecord(result) || !Array.isArray(result.utterances)) return [];
+ return result.utterances.flatMap((utterance) => {
+ if (!isRecord(utterance) || typeof utterance.text !== 'string'
+ || typeof utterance.start_time !== 'number' || !Number.isFinite(utterance.start_time)) return [];
+ return [{ id: String(utterance.start_time), order: utterance.start_time, text: utterance.text, final: utterance.definite === true }];
+ });
+ });
+}
diff --git a/apps/desktop/src/main/voice-input/__tests__/RealtimeAsrWebSocketProvider.test.ts b/apps/desktop/src/main/voice-input/__tests__/RealtimeAsrWebSocketProvider.test.ts
index cee7add429..cfcbbbc7f2 100644
--- a/apps/desktop/src/main/voice-input/__tests__/RealtimeAsrWebSocketProvider.test.ts
+++ b/apps/desktop/src/main/voice-input/__tests__/RealtimeAsrWebSocketProvider.test.ts
@@ -38,6 +38,42 @@ import {
} from '../../../shared/voiceInputRefinerProfiles.js';
describe('RealtimeAsrWebSocketProvider helpers', () => {
+ it('exposes item identities and commit order even when final transcriptions arrive out of order', async () => {
+ const server = new WebSocketServer({ port: 0 });
+ let socket: WebSocket | undefined;
+ let provider: RealtimeAsrWebSocketProvider | undefined;
+ server.on('connection', (connected) => {
+ socket = connected;
+ connected.on('message', (data) => {
+ if (JSON.parse(data.toString()).type === 'session.update') connected.send(JSON.stringify({ type: 'session.updated' }));
+ });
+ });
+ try {
+ await waitFor(() => server.address() !== null);
+ const address = server.address();
+ if (!address || typeof address === 'string') throw new Error('Expected local test server address.');
+ provider = new RealtimeAsrWebSocketProvider({ accessTokenProvider: async () => 'test-key', realtimeUrl: `ws://127.0.0.1:${address.port}/v1/realtime`, model: 'gpt-realtime-whisper' });
+ const segment = vi.fn(); provider.onSegment(segment); provider.onEvent(() => {});
+ await provider.start();
+ for (const event of [
+ { type: 'input_audio_buffer.committed', item_id: 'a' },
+ { type: 'input_audio_buffer.committed', item_id: 'b' },
+ { type: 'conversation.item.input_audio_transcription.completed', item_id: 'b', transcript: 'second' },
+ { type: 'conversation.item.input_audio_transcription.completed', item_id: 'a', transcript: 'first' },
+ ]) socket!.send(JSON.stringify(event));
+ await waitFor(() => segment.mock.calls.length === 4);
+ expect(segment.mock.calls.map(([value]) => value)).toEqual([
+ { id: 'a', order: 0, text: '', final: false },
+ { id: 'b', order: 1, text: '', final: false },
+ { id: 'b', order: 1, text: 'second', final: true },
+ { id: 'a', order: 0, text: 'first', final: true },
+ ]);
+ } finally {
+ await provider?.stop(); socket?.terminate();
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ }
+ });
+
it('resamples 16 kHz PCM16 audio to OpenAI realtime 24 kHz PCM16', () => {
const input = Buffer.alloc(4);
input.writeInt16LE(0, 0);
diff --git a/apps/desktop/src/main/voice-input/__tests__/SystemAudioMuteGuard.test.ts b/apps/desktop/src/main/voice-input/__tests__/SystemAudioMuteGuard.test.ts
index 91bc1df2e4..a28b50904f 100644
--- a/apps/desktop/src/main/voice-input/__tests__/SystemAudioMuteGuard.test.ts
+++ b/apps/desktop/src/main/voice-input/__tests__/SystemAudioMuteGuard.test.ts
@@ -13,6 +13,21 @@ beforeEach(() => {
});
afterEach(() => vi.restoreAllMocks());
+it('reserves output while the OS mute is pending, and releases only after restoration', async () => {
+ const { SystemAudioMuteGuard } = await import('../SystemAudioMuteGuard');
+ let finish!: (value: boolean) => void;
+ audio.getMuted.mockImplementationOnce(() => new Promise((resolve) => { finish = resolve; }));
+ const guard = new SystemAudioMuteGuard();
+ expect(guard.hasPendingOrActiveMute).toBe(false);
+ const muting = guard.mute(1);
+ expect(guard.hasPendingOrActiveMute).toBe(true);
+ await vi.waitFor(() => expect(audio.getMuted).toHaveBeenCalledOnce());
+ finish(false); await muting;
+ expect(guard.hasPendingOrActiveMute).toBe(true);
+ await guard.restore(1);
+ expect(guard.hasPendingOrActiveMute).toBe(false);
+});
+
it.each(['restore', 'restoreAll'] as const)(
'retains original audio state after failed %s through a new owner',
async (method) => {
diff --git a/apps/desktop/src/main/voice-input/__tests__/VolcengineSaucAsrProvider.test.ts b/apps/desktop/src/main/voice-input/__tests__/VolcengineSaucAsrProvider.test.ts
index 62d417a06d..aa33211f50 100644
--- a/apps/desktop/src/main/voice-input/__tests__/VolcengineSaucAsrProvider.test.ts
+++ b/apps/desktop/src/main/voice-input/__tests__/VolcengineSaucAsrProvider.test.ts
@@ -10,12 +10,23 @@ import {
encodeAudioOnlyRequest,
encodeFullClientRequest,
getTranscriptConfirmation,
+ extractConversationSegments,
VolcengineSaucAsrProvider,
} from '../VolcengineSaucAsrProvider.js';
import { volcengineSaucLanguageCode } from '../language.js';
import { mergeRecoveredTranscript } from '../transcriptMerge.js';
describe('VolcengineSaucAsrProvider protocol helpers', () => {
+ it('keeps stable utterance identities for continuous speech, including repeated text and provisional segments', () => {
+ expect(extractConversationSegments({ result: { utterances: [
+ { start_time: 0, text: '你好', definite: true },
+ { start_time: 1500, text: '你好', definite: false },
+ { text: 'missing timestamp', definite: true },
+ ] } })).toEqual([
+ { id: '0', order: 0, text: '你好', final: true },
+ { id: '1500', order: 1500, text: '你好', final: false },
+ ]);
+ });
it('requires all utterances to cover the aggregate transcript', () => {
const check = (utterances: unknown[]) => getTranscriptConfirmation({ result: { text: '一句。二句。', utterances } }, '一句。二句。');
expect(check([{ text: '一句。', definite: true }, { text: '二句。', definite: false }]).confirmed).toBe(false);
diff --git a/apps/desktop/src/main/voice-input/index.ts b/apps/desktop/src/main/voice-input/index.ts
index b774b5b932..baa1e28ad5 100644
--- a/apps/desktop/src/main/voice-input/index.ts
+++ b/apps/desktop/src/main/voice-input/index.ts
@@ -1,3 +1,6 @@
+import { registerVoiceConversationIpc } from '../voice-conversation/index.js';
+import { hasVoiceConversationLease } from '../voice-conversation/lease.js';
+import { VOICE_CONVERSATION_ACTIVE_MESSAGE } from '../../shared/voiceInputErrors.js';
import { app, ipcMain, shell, systemPreferences, type WebContents } from 'electron';
import fs from 'node:fs/promises';
@@ -1876,6 +1879,32 @@ export async function transcribeVoiceInputAudioFile(
*/
export function registerVoiceInputIpc(): void {
registerVoiceInputDataStoreIpc();
+ let pendingDictationStarts = 0;
+ registerVoiceConversationIpc({
+ hasDictation: () => activeByWebContentsId.size > 0 || pendingDictationStarts > 0 || systemAudioMuteGuard.hasPendingOrActiveMute,
+ createProvider: async () => {
+ const owner = activeOwnerScopeKey();
+ const snapshot = getEffectiveAuxiliaryModelChainSnapshot();
+ assertVoiceInputOwnerScopeCurrent(owner, snapshot);
+ const language = resolveAsrLanguageHint(voiceInputDataStore.getSettings().language);
+ const managed = !isVoiceInputByokMode();
+ const chain = (await resolveStartableAsrChain()).filter(
+ (kind) => getVoiceInputAsrProfile(kind).mode !== 'batch-http',
+ );
+ assertVoiceInputOwnerScopeCurrent(owner, snapshot);
+ if (!chain.length || (managed && !isCindyVoiceServiceReady())) throw new Error('ASR unavailable');
+ // A fresh managed connection for each rotation; never cross into BYOK implicitly.
+ const context = managed ? new CindyVoiceRunContext(language, undefined) : undefined;
+ return new FallbackAsrProvider(chain.map((kind) => ({
+ kind,
+ create: () => {
+ assertVoiceInputOwnerScopeCurrent(owner, snapshot);
+ if (!context && isAsrProfileRouteDisabled(getVoiceInputAsrProfile(kind))) throw new Error('ASR disabled');
+ return createVoiceInputProvider(kind, language, context);
+ },
+ })), { hedgeDelayMs: context ? null : undefined, sharedAccountRateLimit: Boolean(context) });
+ },
+ });
if (!appRestoreRegistered) {
appRestoreRegistered = true;
app.once('before-quit', () => {
@@ -2066,6 +2095,7 @@ export function registerVoiceInputIpc(): void {
);
ipcMain.handle('voice-input:mute-system-audio', async (event): Promise<VoiceInputActionResult> => {
+ if (hasVoiceConversationLease()) return { ok: false, error: VOICE_CONVERSATION_ACTIVE_MESSAGE };
try {
await systemAudioMuteGuard.mute(event.sender.id);
return { ok: true };
@@ -2087,7 +2117,7 @@ export function registerVoiceInputIpc(): void {
}
});
- ipcMain.handle('voice-input:start', async (event, payload: StartPayload | undefined): Promise<StartResult> => {
+ const startVoiceInput = async (event: Electron.IpcMainInvokeEvent, payload: StartPayload | undefined): Promise<StartResult> => {
// Bind this run to the owner that initiated it before any cancellation,
// readiness, or credential await. A later account switch must fail closed
// at the final refiner dispatch instead of using the new owner's route.
@@ -2447,6 +2477,13 @@ export function registerVoiceInputIpc(): void {
: message,
};
}
+ };
+ ipcMain.handle('voice-input:start', async (event, payload: StartPayload | undefined): Promise<StartResult> => {
+ if (hasVoiceConversationLease()) return { ok: false, error: VOICE_CONVERSATION_ACTIVE_MESSAGE };
+ // Reserve before the first await, including readiness/auth and provider connection.
+ pendingDictationStarts++;
+ try { return await startVoiceInput(event, payload); }
+ finally { pendingDictationStarts--; }
});
ipcMain.on('voice-input:audio', (event, payload: AudioPayload | undefined) => {
diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts
index 37d8f9a40a..41465da4f0 100644
--- a/apps/desktop/src/preload/preload.ts
+++ b/apps/desktop/src/preload/preload.ts
@@ -1,3 +1,4 @@
+import type { VoiceConversationApi } from '../shared/voiceConversation';
import { createAccessibilitySupportBridge } from './accessibilitySupport';
import type { CompanionImportApi, CompanionImportSelection } from '@cindy/maker-shared/companion-import';
import { TASK_MIGRATION_LOCAL_CHANNEL } from '@cindy/device-link';
@@ -1587,6 +1588,21 @@ contextBridge.exposeInMainWorld('electronAPI', {
resolveConfirm: (requestId: string, confirmed: boolean): Promise<{ handled: boolean }> =>
ipcRenderer.invoke('plugin-publisher:resolve-confirm', { requestId, confirmed }),
},
+ voiceConversation: {
+ models: () => ipcRenderer.invoke('voice-conversation:models'),
+ start: (input) => ipcRenderer.invoke('voice-conversation:start', input),
+ audio: (input) => ipcRenderer.send('voice-conversation:audio', input),
+ finishUtterance: (callId) => ipcRenderer.invoke('voice-conversation:finish-utterance', callId),
+ end: (callId) => ipcRenderer.invoke('voice-conversation:end', callId),
+ speak: (input) => ipcRenderer.invoke('voice-conversation:speak', input),
+ readSpeech: (callId, requestId) => ipcRenderer.invoke('voice-conversation:read-speech', callId, requestId),
+ interrupt: (callId) => ipcRenderer.invoke('voice-conversation:interrupt', callId),
+ onEvent: (callback) => {
+ const listener = (_event: Electron.IpcRendererEvent, payload: Parameters<typeof callback>[0]) => callback(payload);
+ ipcRenderer.on('voice-conversation:event', listener);
+ return () => ipcRenderer.removeListener('voice-conversation:event', listener);
+ },
+ } satisfies VoiceConversationApi,
voiceInput: {
prewarm: (payload?: {
sourceLanguage?: string;
diff --git a/apps/desktop/src/renderer/__tests__/chatInputSessionFocus.test.ts b/apps/desktop/src/renderer/__tests__/chatInputSessionFocus.test.ts
index ad7b930eae..3a8f47253c 100644
--- a/apps/desktop/src/renderer/__tests__/chatInputSessionFocus.test.ts
+++ b/apps/desktop/src/renderer/__tests__/chatInputSessionFocus.test.ts
@@ -83,7 +83,7 @@ describe('ChatInput session switch focus contract', () => {
);
expect(sessionViewSource).toContain('focusOnStorageKeyChange={ownsRoute}');
expect(sessionViewSource).toContain(
- 'ownsHardwareComposerActions={ownsHardwareTaskActions}',
+ 'ownsHardwareComposerActions={ownsHardwareTaskActions && !voiceModeOpen}',
);
expect(chatInputSource).toContain('workLouderVoiceGestureRef.current?.cancelHeldPress();');
expect(sessionViewSource).toContain(
diff --git a/apps/desktop/src/renderer/__tests__/makerChatStoreTextDeltaBatching.test.ts b/apps/desktop/src/renderer/__tests__/makerChatStoreTextDeltaBatching.test.ts
index cd10158499..4bf5bde35a 100644
--- a/apps/desktop/src/renderer/__tests__/makerChatStoreTextDeltaBatching.test.ts
+++ b/apps/desktop/src/renderer/__tests__/makerChatStoreTextDeltaBatching.test.ts
@@ -3521,6 +3521,26 @@ describe('makerChatStore text delta batching', () => {
expect(snap.messages.some((m) => m.role === 'user' && m.content === 'queued')).toBe(false);
});
+ it.each(['sendMessage', 'steerMessage'] as const)('correlates voice input through %s without serializing the renderer callback', async (method) => {
+ const created = vi.fn<(id: string) => void>();
+ await expect(makerChatStore[method](SESSION_ID, 'spoken sentence', MODEL, EFFORT, PERMISSION_MODE, WORKING_DIR,
+ undefined, undefined, { onInputCreated: created, beforeEnqueue: async () => true, slashCommandRanges: [] })).resolves.toBe(true);
+ expect(created).toHaveBeenCalledOnce();
+ const transport = method === 'sendMessage' ? input.enqueue : input.steer;
+ expect(transport).toHaveBeenCalledWith(SESSION_ID, expect.objectContaining({
+ clientId: created.mock.calls[0][0],
+ chatMessage: expect.objectContaining({ clientId: created.mock.calls[0][0], content: 'spoken sentence', slashCommandRanges: [] }),
+ }), expect.any(Object));
+ expect(JSON.stringify(transport.mock.calls)).not.toContain('onInputCreated');
+ });
+
+ it.each(['sendMessage', 'steerMessage'] as const)('cancels a retired voice call before %s reaches the main queue', async (method) => {
+ await expect(makerChatStore[method](SESSION_ID, 'cancelled speech', MODEL, EFFORT, PERMISSION_MODE, WORKING_DIR,
+ undefined, undefined, { beforeEnqueue: async () => false })).resolves.toBe(false);
+ expect(input.enqueue).not.toHaveBeenCalled(); expect(input.steer).not.toHaveBeenCalled();
+ expect(makerChatStore.getSnapshot(SESSION_ID).messages.some(m => m.content === 'cancelled speech')).toBe(false);
+ });
+
it('shows a local busy send before the enqueue projection settles', async () => {
makerChatStore.__applyStatusUpdateForTest(SESSION_ID, {
sessionId: SESSION_ID,
diff --git a/apps/desktop/src/renderer/features/bots/BotSessionContentHeader.tsx b/apps/desktop/src/renderer/features/bots/BotSessionContentHeader.tsx
index 9258fe494c..e17f1085af 100644
--- a/apps/desktop/src/renderer/features/bots/BotSessionContentHeader.tsx
+++ b/apps/desktop/src/renderer/features/bots/BotSessionContentHeader.tsx
@@ -11,7 +11,7 @@
* nothing else reopens it.
*/
import { useMemo } from 'react';
-import { LayoutGrid, Settings2 } from 'lucide-react';
+import { AudioLines, LayoutGrid, Settings2 } from 'lucide-react';
import { useLocation, useNavigate } from 'react-router-dom';
import { useTranslation } from 'react-i18next';
@@ -34,7 +34,7 @@ export interface BotChatIdentity {
avatarColor?: string | null;
}
-export function BotSessionContentHeader({ bot }: { bot: BotChatIdentity }) {
+export function BotSessionContentHeader({ bot, onVoice, voiceActive }: { bot: BotChatIdentity; onVoice?: () => void; voiceActive?: boolean }) {
const { t } = useTranslation();
const navigate = useNavigate();
const location = useLocation();
@@ -67,6 +67,12 @@ export function BotSessionContentHeader({ bot }: { bot: BotChatIdentity }) {
{bot.deviceName}
</span>
) : null}
+ {onVoice ? (
+ <button type="button" onClick={onVoice} aria-label={t('bots.voiceMode.title')}
+ title={t('bots.voiceMode.title')} aria-pressed={voiceActive ?? false}
+ className="flex h-8 w-8 items-center justify-center rounded-full text-[var(--text-secondary)] hover:bg-[var(--surface-hover)] aria-pressed:bg-[var(--surface-hover)]"
+ style={WINDOW_NO_DRAG_STYLE}><AudioLines size={16} /></button>
+ ) : null}
{!bot.deviceId && bot.sessionId ? (
<button
type="button"
@@ -98,7 +104,8 @@ export function BotSessionContentHeader({ bot }: { bot: BotChatIdentity }) {
* mounting registers, unmounting clears, and only the route-owning chat instance
* renders it.
*/
-export function BotSessionContentHeaderRegistration({ bot }: { bot: BotChatIdentity }) {
- useRegisterContentHeader(useMemo(() => <BotSessionContentHeader bot={bot} />, [bot]));
+export function BotSessionContentHeaderRegistration(props: { bot: BotChatIdentity; onVoice?: () => void; voiceActive?: boolean }) {
+ const { bot, onVoice, voiceActive } = props;
+ useRegisterContentHeader(useMemo(() => <BotSessionContentHeader bot={bot} onVoice={onVoice} voiceActive={voiceActive} />, [bot, onVoice, voiceActive]));
return null;
}
diff --git a/apps/desktop/src/renderer/features/bots/__tests__/botChatSurface.test.ts b/apps/desktop/src/renderer/features/bots/__tests__/botChatSurface.test.ts
index 66307d8995..dfd579e59c 100644
--- a/apps/desktop/src/renderer/features/bots/__tests__/botChatSurface.test.ts
+++ b/apps/desktop/src/renderer/features/bots/__tests__/botChatSurface.test.ts
@@ -33,9 +33,8 @@ describe('Bot 对话的判定条件', () => {
});
it('伙伴对话换掉任务顶栏,而不是在它旁边再加一个', () => {
- expect(sessionView).toContain(
- '<BotSessionContentHeaderRegistration bot={botChatIdentity} />',
- );
+ expect(sessionView).toMatch(/botChatIdentity\s*\?\s*\(\s*<BotSessionContentHeaderRegistration bot=\{botChatIdentity\}/);
+ expect(sessionView).toContain('onVoice={voiceModeAvailable ? toggleVoiceMode : undefined}');
expect(sessionView).toContain('<SessionContentHeaderRegistration');
});
});
diff --git a/apps/desktop/src/renderer/features/bots/__tests__/botSessionContentHeader.test.tsx b/apps/desktop/src/renderer/features/bots/__tests__/botSessionContentHeader.test.tsx
index 9bf200621c..07c5f6808a 100644
--- a/apps/desktop/src/renderer/features/bots/__tests__/botSessionContentHeader.test.tsx
+++ b/apps/desktop/src/renderer/features/bots/__tests__/botSessionContentHeader.test.tsx
@@ -53,6 +53,17 @@ afterEach(() => {
});
describe('BotSessionContentHeader', () => {
+ it('opens voice mode only when its route supplies the local capability and reflects the active state', () => {
+ const onVoice = vi.fn();
+ const view = render(<BotSessionContentHeader bot={bot} />);
+ expect(screen.queryByRole('button', { name: 'bots.voiceMode.title' })).toBeNull();
+ view.rerender(<BotSessionContentHeader bot={bot} onVoice={onVoice} voiceActive />);
+ const button = screen.getByRole('button', { name: 'bots.voiceMode.title' });
+ expect(button.getAttribute('aria-pressed')).toBe('true');
+ expect(appRegionOf(button)).toBe('no-drag');
+ fireEvent.click(button); expect(onVoice).toHaveBeenCalledOnce();
+ });
+
it('preserves the local header until a second Cindy is available and restores it when removed', () => {
deviceData.local = [localCindy];
const view = render(<BotSessionContentHeader bot={localCindy} />);
diff --git a/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx b/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx
new file mode 100644
index 0000000000..4bb9df956a
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx
@@ -0,0 +1,325 @@
+import {
+ useCallback,
+ useEffect,
+ useId,
+ useMemo,
+ useRef,
+ useState,
+ useSyncExternalStore,
+} from 'react';
+import { AudioLines, Mic, MicOff, PhoneOff, RefreshCw, Send, VolumeX, X } from 'lucide-react';
+import { useTranslation } from 'react-i18next';
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { Select } from '@/components/ui/select';
+import { getDataOwnerGeneration } from '@/contexts/dataOwnerGeneration';
+import { getVoiceInputSettings } from '@/hooks/useVoiceInputSettings';
+import type { ChatMessage } from '@/lib/makerChatStore';
+import { WebMicAudioEngine } from '@/voice-input/WebMicAudioEngine';
+import { getVoiceInputWorkletUrl } from '@/voice-input/workletUrl';
+import type {
+ VoiceConversationModel,
+ VoiceConversationSelection,
+} from '@/../shared/voiceConversation';
+import { normalizeVoiceConversationSelection } from '@/../shared/voiceConversation';
+import { BotAvatar } from '../BotAvatar';
+import type { BotChatIdentity } from '../BotSessionContentHeader';
+import { VoiceConversationController } from './VoiceConversationController';
+import { PcmSpeechPlayer } from './PcmSpeechPlayer';
+import './voiceConversation.css';
+
+type Props = {
+ bot: BotChatIdentity;
+ sessionId: string;
+ messages: readonly ChatMessage[];
+ blocked: boolean;
+ onClose(): void;
+ onSend(text: string, onCreated: (id: string) => void, isCurrent: () => boolean): Promise<boolean>;
+};
+
+export function CompanionVoicePanel(props: Props) {
+ const { t } = useTranslation();
+ const labelId = useId();
+ const voiceId = useId();
+ const latest = useRef(props);
+ latest.current = props;
+ const titleRef = useRef<HTMLHeadingElement>(null);
+ const [models, setModels] = useState<VoiceConversationModel[]>([]);
+ const [loading, setLoading] = useState(true);
+ const [catalogError, setCatalogError] = useState(false);
+ const [preference, setPreference] = useState<VoiceConversationSelection | null>(
+ () => getVoiceInputSettings().conversationSelection ?? null,
+ );
+ const [voice, setVoice] = useState(preference?.voice ?? 'alloy');
+ const modelId = preference?.modelId ?? models[0]?.id ?? '';
+ const validModel = models.some((model) => model.id === modelId);
+ const selection = normalizeVoiceConversationSelection({ modelId, voice });
+ const controller = useMemo(
+ () =>
+ new VoiceConversationController({
+ api: window.electronAPI.voiceConversation,
+ botId: props.bot.id,
+ sessionId: props.sessionId,
+ send: (...args) => latest.current.onSend(...args),
+ microphone: (onInterrupted) =>
+ new WebMicAudioEngine({
+ workletUrl: getVoiceInputWorkletUrl(),
+ deviceId: getVoiceInputSettings().microphoneDeviceId ?? undefined,
+ chunkMs: 40,
+ latencyMs: 10,
+ keepAlive: false,
+ audioProcessing: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: false,
+ },
+ onInterrupted,
+ }),
+ player: (onLevel) => new PcmSpeechPlayer(onLevel),
+ }),
+ [props.bot.id, props.sessionId],
+ );
+ const state = useSyncExternalStore(controller.subscribe, controller.getSnapshot);
+ const active = ['connecting', 'listening', 'thinking', 'preparing', 'speaking'].includes(
+ state.phase,
+ );
+ const refreshVersion = useRef(0);
+ const refresh = useCallback(async (fetchCatalog = false) => {
+ const version = ++refreshVersion.current;
+ setLoading(true);
+ setCatalogError(false);
+ try {
+ if (fetchCatalog) await window.electronAPI.maker.refreshBuiltinProviderModels('xd');
+ const next = await window.electronAPI.voiceConversation.models();
+ if (version === refreshVersion.current) setModels(next);
+ } catch {
+ if (version === refreshVersion.current) setCatalogError(true);
+ } finally {
+ if (version === refreshVersion.current) setLoading(false);
+ }
+ }, []);
+
+ useEffect(() => {
+ void refresh();
+ titleRef.current?.focus();
+ return () => {
+ ++refreshVersion.current;
+ controller.end();
+ };
+ }, [controller, refresh]);
+ useEffect(() => controller.setBlocked(props.blocked), [controller, props.blocked]);
+ useEffect(() => controller.updateMessages(props.messages), [controller, props.messages]);
+ useEffect(() => {
+ const initialOwner = getDataOwnerGeneration();
+ let owner = `${initialOwner.dataOwnerId}:${initialOwner.generation}`;
+ const removeAuth = window.electronAPI.onAuthStateChange((next) => {
+ const key = `${next.dataOwnerId}:${next.ownerGeneration}`;
+ if (next.ownerBoundaryPending || key !== owner) {
+ controller.end();
+ latest.current.onClose();
+ }
+ owner = key;
+ });
+ const removePower = window.electronAPI.voiceInput.onPowerStateChange(() =>
+ controller.setBlocked(true),
+ );
+ return () => {
+ removeAuth();
+ removePower();
+ };
+ }, [controller]);
+
+ const persist = (next: VoiceConversationSelection) => {
+ setPreference(next);
+ void window.electronAPI.voiceInput
+ .updateSettings({ conversationSelection: next })
+ .catch(() => setCatalogError(true));
+ };
+ const start = () => {
+ if (!selection || !validModel || props.blocked) return;
+ controller.setBlocked(false);
+ void controller.start(selection).then(() => controller.retryUnsent());
+ };
+ const status = props.blocked ? 'confirmation' : state.phase;
+
+ return (
+ <section
+ className="companion-voice-panel flex max-h-[min(36rem,65vh)] w-full flex-col rounded-xl border border-[var(--border-default)] bg-[var(--surface-elevated)] p-4 text-[var(--text-primary)]"
+ aria-labelledby={labelId}
+ >
+ <div className="flex shrink-0 items-center gap-2">
+ <AudioLines size={16} aria-hidden="true" />
+ <h2 ref={titleRef} tabIndex={-1} id={labelId} className="text-13 font-medium outline-none">
+ {t('bots.voiceMode.title')}
+ </h2>
+ <Button
+ variant="secondary"
+ size="sm"
+ className="ml-auto w-8 px-0"
+ aria-label={t('bots.voiceMode.close')}
+ onClick={props.onClose}
+ >
+ <X size={16} />
+ </Button>
+ </div>
+
+ <div className="min-h-0 flex-1 overflow-y-auto overscroll-contain">
+ <div className="flex flex-col items-center gap-3 py-5">
+ <div className="flex items-center gap-5">
+ <BotAvatar bot={props.bot} size="xl" />
+ <div
+ className="companion-voice-meter flex h-10 items-center gap-1"
+ aria-hidden="true"
+ data-active={active}
+ >
+ {[0.4, 0.7, 1, 0.65, 0.9, 0.5, 0.75].map((weight, index) => (
+ <span
+ key={index}
+ className="h-8 w-1 rounded-full bg-[var(--text-secondary)]"
+ style={{ transform: `scaleY(${0.12 + state.level * weight * 0.88})` }}
+ />
+ ))}
+ </div>
+ </div>
+ <div className="text-center">
+ <p className="text-15 font-medium">{props.bot.name}</p>
+ <p
+ className="mt-1 text-13 text-[var(--text-secondary)]"
+ role="status"
+ aria-live="polite"
+ >
+ {t(`bots.voiceMode.status.${status}`)}
+ </p>
+ </div>
+ </div>
+
+ {state.transcript ? (
+ <p className="max-h-24 overflow-y-auto whitespace-pre-wrap break-words text-center text-13">
+ {state.transcript}
+ </p>
+ ) : null}
+ {state.unsent ? (
+ <div className="my-3 rounded-xl border border-[var(--border-default)] p-3">
+ <p className="mb-1 text-12 text-[var(--text-secondary)]">
+ {t('bots.voiceMode.unsent')}
+ </p>
+ <p className="max-h-24 select-text overflow-y-auto whitespace-pre-wrap break-words text-13">
+ {state.unsent}
+ </p>
+ </div>
+ ) : null}
+ {state.error ? (
+ <p role="alert" className="mb-3 text-13 text-[var(--error-fg)]">
+ {t(`bots.voiceMode.errors.${state.error}`)}
+ </p>
+ ) : null}
+
+ {!active ? (
+ <div className="space-y-3">
+ {catalogError || (!loading && models.length === 0) ? (
+ <p className="text-13 text-[var(--text-secondary)]">
+ {t(catalogError ? 'bots.voiceMode.catalogError' : 'bots.voiceMode.noModels')}
+ </p>
+ ) : null}
+ <div className="flex flex-wrap items-center gap-2">
+ <Select
+ label={t('bots.voiceMode.model')}
+ value={validModel ? modelId : ''}
+ options={models.map((model) => ({ value: model.id, label: model.name }))}
+ onValueChange={(id) => persist({ modelId: id, voice: selection?.voice ?? 'alloy' })}
+ disabled={loading || models.length === 0}
+ className="min-w-40 flex-1"
+ />
+ <Button
+ variant="secondary"
+ size="md"
+ disabled={loading}
+ aria-label={t('bots.voiceMode.refresh')}
+ title={t('bots.voiceMode.refresh')}
+ onClick={() => void refresh(true)}
+ >
+ <RefreshCw size={15} />
+ </Button>
+ </div>
+ {!validModel && preference && models.length > 0 ? (
+ <p className="text-12 text-[var(--text-secondary)]">
+ {t('bots.voiceMode.modelUnavailable')}
+ </p>
+ ) : null}
+ {models.length > 0 ? (
+ <div className="flex flex-wrap items-center gap-2">
+ <label htmlFor={voiceId} className="text-13 text-[var(--text-secondary)]">
+ {t('bots.voiceMode.voice')}
+ </label>
+ <Input
+ id={voiceId}
+ value={voice}
+ maxLength={128}
+ onChange={setVoice}
+ onBlur={() => {
+ if (selection && validModel) persist(selection);
+ }}
+ className="min-w-24 flex-1"
+ aria-describedby={`${voiceId}-hint`}
+ />
+ <p id={`${voiceId}-hint`} className="w-full text-12 text-[var(--text-tertiary)]">
+ {t('bots.voiceMode.voiceHint')}
+ </p>
+ </div>
+ ) : null}
+ </div>
+ ) : null}
+ </div>
+ <div className="mt-4 flex shrink-0 flex-wrap items-center justify-center gap-2">
+ {!active ? (
+ <Button
+ variant="primary"
+ size="lg"
+ onClick={start}
+ disabled={!validModel || !selection || props.blocked || loading}
+ >
+ <Mic size={16} />
+ {t(
+ state.unsent
+ ? 'bots.voiceMode.retrySend'
+ : state.phase === 'idle'
+ ? 'bots.voiceMode.start'
+ : 'bots.voiceMode.resume',
+ )}
+ </Button>
+ ) : (
+ <>
+ <Button
+ variant="secondary"
+ size="lg"
+ onClick={() => controller.toggleMute()}
+ disabled={state.phase === 'connecting'}
+ >
+ <MicOff size={16} />
+ {t('bots.voiceMode.mute')}
+ </Button>
+ {state.phase === 'speaking' || state.phase === 'preparing' ? (
+ <Button variant="secondary" size="lg" onClick={() => controller.interrupt()}>
+ <VolumeX size={16} />
+ {t('bots.voiceMode.interrupt')}
+ </Button>
+ ) : null}
+ {state.transcript ? (
+ <Button variant="secondary" size="lg" onClick={() => controller.finishUtterance()}>
+ <Send size={16} />
+ {t('bots.voiceMode.send')}
+ </Button>
+ ) : null}
+ <Button variant="primary" size="lg" onClick={() => controller.end()}>
+ <PhoneOff size={16} />
+ {t('bots.voiceMode.end')}
+ </Button>
+ </>
+ )}
+ </div>
+ <p className="mt-3 shrink-0 text-center text-12 leading-relaxed text-[var(--text-tertiary)]">
+ {t('bots.voiceMode.hint')}
+ </p>
+ </section>
+ );
+}
diff --git a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
new file mode 100644
index 0000000000..a371005591
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
@@ -0,0 +1,120 @@
+/** PCM may cross arbitrary HTTP chunk boundaries, including the middle of a 16-bit sample. */
+export class Pcm16Decoder {
+ private carry: number | undefined;
+ decode(buffer: ArrayBuffer): Float32Array<ArrayBuffer> {
+ const incoming = new Uint8Array(buffer);
+ const bytes = new Uint8Array(incoming.length + (this.carry === undefined ? 0 : 1));
+ if (this.carry !== undefined) bytes[0] = this.carry;
+ bytes.set(incoming, this.carry === undefined ? 0 : 1);
+ this.carry = bytes.length % 2 ? bytes[bytes.length - 1] : undefined;
+ const view = new DataView(bytes.buffer);
+ const samples = new Float32Array(Math.floor(bytes.length / 2));
+ for (let i = 0; i < samples.length; i++) samples[i] = view.getInt16(i * 2, true) / 32768;
+ return samples;
+ }
+ finish(): void {
+ if (this.carry !== undefined) throw new Error('Incomplete PCM sample');
+ }
+}
+
+export interface SpeechPlayer {
+ play(read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>, sampleRate: number): Promise<void>;
+ cancel(): void;
+ close(): void;
+}
+
+/** A short playback horizon bounds memory and makes an interruption stop immediately. */
+export class PcmSpeechPlayer implements SpeechPlayer {
+ private readonly context: AudioContext;
+ private generation = 0;
+ private sources = new Set<AudioBufferSourceNode>();
+ private timers = new Set<ReturnType<typeof setTimeout>>();
+
+ constructor(private readonly onLevel: (level: number) => void) {
+ // Construct/resume in the explicit Start gesture, before any network await.
+ this.context = new AudioContext();
+ void this.context.resume().catch(() => undefined);
+ }
+
+ async play(
+ read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,
+ sampleRate: number,
+ ): Promise<void> {
+ this.cancel();
+ const generation = this.generation;
+ const decoder = new Pcm16Decoder();
+ let nextAt = this.context.currentTime;
+ const assertActive = () => {
+ if (generation !== this.generation || this.context.state !== 'running')
+ throw new Error('Playback interrupted');
+ };
+ await this.context.resume();
+ for (;;) {
+ assertActive();
+ const chunk = await read();
+ assertActive();
+ const samples = decoder.decode(chunk.pcm);
+ for (let offset = 0; offset < samples.length; offset += 2_400) {
+ while (nextAt - this.context.currentTime > 0.3) {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ assertActive();
+ }
+ const part = samples.subarray(offset, offset + 2_400);
+ const buffer = this.context.createBuffer(1, part.length, sampleRate);
+ buffer.copyToChannel(part, 0);
+ const source = this.context.createBufferSource();
+ source.buffer = buffer;
+ source.connect(this.context.destination);
+ this.sources.add(source);
+ source.onended = () => {
+ this.sources.delete(source);
+ source.disconnect();
+ };
+ nextAt = Math.max(nextAt, this.context.currentTime + 0.015);
+ const level = Math.min(
+ 1,
+ Math.sqrt(part.reduce((sum, value) => sum + value * value, 0) / part.length) * 5,
+ );
+ const timer = setTimeout(
+ () => {
+ this.timers.delete(timer);
+ if (generation === this.generation) this.onLevel(level);
+ },
+ Math.max(0, nextAt - this.context.currentTime) * 1000,
+ );
+ this.timers.add(timer);
+ source.start(nextAt);
+ nextAt += part.length / sampleRate;
+ }
+ if (chunk.done) break;
+ }
+ decoder.finish();
+ while (nextAt > this.context.currentTime) {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ assertActive();
+ }
+ this.onLevel(0);
+ }
+
+ cancel(): void {
+ ++this.generation;
+ for (const timer of this.timers) clearTimeout(timer);
+ this.timers.clear();
+ for (const source of this.sources) {
+ source.onended = null;
+ try {
+ source.stop();
+ } catch {
+ /* Already ended. */
+ }
+ source.disconnect();
+ }
+ this.sources.clear();
+ this.onLevel(0);
+ }
+
+ close(): void {
+ this.cancel();
+ void this.context.close().catch(() => undefined);
+ }
+}
diff --git a/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
new file mode 100644
index 0000000000..93d6e5fa33
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts
@@ -0,0 +1,345 @@
+import type {
+ VoiceConversationApi,
+ VoiceConversationSelection,
+} from '@/../shared/voiceConversation';
+import type { ChatMessage } from '@/lib/makerChatStore';
+import type { PcmChunk } from '@/voice-input/WebMicAudioEngine';
+import type { SpeechPlayer } from './PcmSpeechPlayer';
+import { speechChunks, VoiceReplyTracker } from './VoiceReplyTracker';
+
+export type VoicePhase =
+ 'idle' | 'connecting' | 'listening' | 'thinking' | 'preparing' | 'speaking' | 'paused' | 'error';
+export type VoiceSnapshot = {
+ phase: VoicePhase;
+ transcript: string;
+ unsent: string;
+ level: number;
+ error: 'connection' | 'microphone' | 'send' | 'speech' | null;
+ muted: boolean;
+};
+interface Microphone {
+ onPcm16k(callback: (chunk: PcmChunk) => void): void;
+ start(): Promise<void>;
+ stop(): Promise<unknown>;
+}
+
+/** Call-local orchestration. It never stops the companion task or answers permission prompts. */
+export class VoiceConversationController {
+ private state: VoiceSnapshot = {
+ phase: 'idle',
+ transcript: '',
+ unsent: '',
+ level: 0,
+ error: null,
+ muted: false,
+ };
+ private listeners = new Set<() => void>();
+ private epoch = 0;
+ private speechEpoch = 0;
+ private callId: string | null = null;
+ private microphone: Microphone | null = null;
+ private player: SpeechPlayer | null = null;
+ private unsubscribe?: () => void;
+ private selection?: VoiceConversationSelection;
+ private blocked = false;
+ private enabled = false;
+ private tracker = new VoiceReplyTracker();
+ private pendingInput: string | null = null;
+ private latestMessages: readonly ChatMessage[] = [];
+ private sending = false;
+ private inFlight: Promise<void> = Promise.resolve();
+ private queued: string[] = [];
+ private pendingStart: Promise<unknown> = Promise.resolve();
+ private teardown: Promise<unknown> = Promise.resolve();
+
+ constructor(
+ private readonly deps: {
+ api: VoiceConversationApi;
+ botId: string;
+ sessionId: string;
+ microphone(onInterrupted: () => void): Microphone;
+ player(onLevel: (level: number) => void): SpeechPlayer;
+ send(
+ text: string,
+ onCreated: (id: string) => void,
+ isCurrent: () => boolean,
+ ): Promise<boolean>;
+ },
+ ) {}
+
+ getSnapshot = (): VoiceSnapshot => this.state;
+ subscribe = (listener: () => void): (() => void) => {
+ this.listeners.add(listener);
+ return () => this.listeners.delete(listener);
+ };
+ private patch(patch: Partial<VoiceSnapshot>): void {
+ this.state = { ...this.state, ...patch };
+ for (const listener of this.listeners) listener();
+ }
+
+ async start(selection: VoiceConversationSelection): Promise<void> {
+ if (this.blocked || this.callId || this.state.phase === 'connecting') return;
+ this.selection = selection;
+ this.enabled = true;
+ const epoch = ++this.epoch;
+ this.patch({ phase: 'connecting', error: null, muted: false });
+ const current = () => this.enabled && epoch === this.epoch && !this.blocked;
+ try {
+ this.player = this.deps.player((level) => {
+ if (current() && this.state.phase === 'speaking') this.patch({ level });
+ });
+ await this.teardown;
+ if (!current()) return;
+ this.unsubscribe = this.deps.api.onEvent(({ callId, event }) => {
+ if (!current() || callId !== this.callId) return;
+ if (event.type === 'speech-started') {
+ this.interrupt();
+ this.patch({ phase: 'listening' });
+ } else if (event.type === 'draft') {
+ this.patch({ transcript: event.text });
+ } else if (event.type === 'utterance') {
+ this.queued.push(event.text);
+ if (this.queued.join(' ').length > 16_000) this.fail('send');
+ else void this.drain(epoch);
+ } else if (event.type === 'ended') {
+ this.end();
+ } else if (event.type === 'error') {
+ this.fail('connection', event.text);
+ }
+ });
+ // Reserve a call id before IPC so End can cancel an in-flight ASR dial.
+ this.callId = crypto.randomUUID();
+ const opening = this.deps.api.start({
+ ...selection,
+ callId: this.callId,
+ botId: this.deps.botId,
+ sessionId: this.deps.sessionId,
+ });
+ this.pendingStart = opening
+ .then(async (result) => {
+ if (!current()) await this.deps.api.end(result.callId);
+ })
+ .catch(() => undefined);
+ const result = await opening;
+ if (!current()) {
+ await this.pendingStart;
+ return;
+ }
+ this.callId = result.callId;
+ const mic = this.deps.microphone(() => {
+ if (current()) this.fail('microphone');
+ });
+ this.microphone = mic;
+ mic.onPcm16k(({ pcm16k }) => {
+ if (!current() || !this.callId) return;
+ this.deps.api.audio({ callId: this.callId, pcm: pcm16k });
+ if (this.state.phase !== 'speaking') {
+ const samples = new Int16Array(pcm16k);
+ const rms = Math.sqrt(
+ samples.reduce((sum, sample) => sum + sample * sample, 0) / Math.max(1, samples.length),
+ );
+ this.patch({ level: Math.min(1, rms / 5000) });
+ }
+ });
+ try {
+ await mic.start();
+ } catch {
+ if (current()) this.fail('microphone');
+ return;
+ }
+ if (!current()) {
+ await mic.stop();
+ return;
+ }
+ this.patch({ phase: this.pendingInput ? 'thinking' : 'listening' });
+ this.updateMessages(this.latestMessages);
+ } catch {
+ if (current()) this.fail('connection');
+ }
+ }
+
+ /** Speech onset fences old replies and audio before a new transcript is committed. */
+ interrupt(): void {
+ ++this.speechEpoch;
+ this.tracker.interrupt();
+ this.pendingInput = null;
+ this.player?.cancel();
+ if (this.callId) void this.deps.api.interrupt(this.callId).catch(() => undefined);
+ if (
+ this.state.phase === 'speaking' ||
+ this.state.phase === 'preparing' ||
+ this.state.phase === 'thinking'
+ )
+ this.patch({ phase: 'listening', level: 0 });
+ }
+
+ private async submit(text: string, epoch: number): Promise<void> {
+ const current = () =>
+ this.enabled && epoch === this.epoch && !this.blocked && !this.state.muted;
+ if (!current()) return;
+ this.interrupt();
+ this.patch({ phase: 'thinking', transcript: '', unsent: '' });
+ const replyEpoch = this.speechEpoch;
+ let clientId: string | null = null;
+ try {
+ const accepted = await this.deps.send(
+ text,
+ (id) => {
+ clientId = id;
+ },
+ current,
+ );
+ if (!current()) {
+ // A permission/mute boundary can cancel preflight before enqueue. Wait
+ // for its outcome before resuming, retaining only genuinely unsent text.
+ if (!accepted && this.enabled)
+ this.patch({ unsent: [text, this.state.unsent].filter(Boolean).join(' ') });
+ return;
+ }
+ if (!accepted || !clientId) {
+ this.fail('send', text);
+ return;
+ }
+ if (replyEpoch !== this.speechEpoch) return;
+ this.pendingInput = clientId;
+ this.tracker.expect(clientId);
+ this.updateMessages(this.latestMessages);
+ } catch {
+ if (current()) this.fail('send', text);
+ else if (this.enabled)
+ this.patch({ unsent: [text, this.state.unsent].filter(Boolean).join(' ') });
+ }
+ }
+
+ private async drain(epoch: number): Promise<void> {
+ if (this.sending) return;
+ this.sending = true;
+ try {
+ while (this.queued.length && this.enabled && epoch === this.epoch && !this.blocked) {
+ this.inFlight = this.submit(this.queued.shift()!, epoch);
+ await this.inFlight;
+ }
+ } finally {
+ this.sending = false;
+ if (this.queued.length && this.callId && this.enabled && !this.blocked)
+ void this.drain(this.epoch);
+ }
+ }
+
+ updateMessages(messages: readonly ChatMessage[]): void {
+ this.latestMessages = messages;
+ if (!this.enabled || !this.callId || this.blocked || this.state.muted || !this.pendingInput)
+ return;
+ const reply = this.tracker.take(messages);
+ if (!reply) {
+ if (!this.tracker.hasPendingReply) {
+ this.pendingInput = null;
+ this.patch({ phase: 'listening' });
+ }
+ return;
+ }
+ this.pendingInput = null;
+ void this.speak(reply.text);
+ }
+
+ private async speak(text: string): Promise<void> {
+ const epoch = this.epoch;
+ const speechEpoch = ++this.speechEpoch;
+ const callId = this.callId;
+ const player = this.player;
+ if (!callId || !player) return;
+ const current = () =>
+ epoch === this.epoch && speechEpoch === this.speechEpoch && this.enabled && !this.blocked;
+ this.patch({ phase: 'preparing', level: 0 });
+ try {
+ for (const [index, chunk] of speechChunks(text).entries()) {
+ if (!current()) return;
+ const requestId = `${speechEpoch}:${index}`;
+ const { sampleRate } = await this.deps.api.speak({ callId, requestId, text: chunk });
+ if (!current()) return;
+ this.patch({ phase: 'speaking' });
+ await player.play(() => this.deps.api.readSpeech(callId, requestId), sampleRate);
+ }
+ if (current()) this.patch({ phase: 'listening', level: 0 });
+ } catch {
+ if (current()) this.fail('speech');
+ }
+ }
+
+ finishUtterance(): void {
+ const epoch = this.epoch;
+ if (this.callId)
+ void this.deps.api.finishUtterance(this.callId).catch(() => {
+ if (epoch === this.epoch) this.fail('connection');
+ });
+ }
+
+ setBlocked(blocked: boolean): void {
+ if (blocked === this.blocked) return;
+ this.blocked = blocked;
+ if (blocked && this.enabled) {
+ const unsent = [this.state.unsent, ...this.queued, this.state.transcript]
+ .filter(Boolean)
+ .join(' ');
+ this.release();
+ this.patch({ phase: 'paused', unsent, transcript: '', level: 0 });
+ }
+ // Confirmation completion does not silently restart a microphone.
+ }
+
+ toggleMute(): void {
+ if (this.state.muted) {
+ if (this.selection) void this.start(this.selection);
+ return;
+ }
+ const unsent = [this.state.unsent, ...this.queued, this.state.transcript]
+ .filter(Boolean)
+ .join(' ');
+ this.release();
+ this.patch({ muted: true, phase: 'paused', transcript: '', unsent, level: 0 });
+ }
+
+ retryUnsent(): void {
+ if (!this.state.unsent || !this.callId || this.blocked) return;
+ const text = this.state.unsent;
+ this.queued.push(text);
+ void this.drain(this.epoch);
+ }
+
+ private fail(error: VoiceSnapshot['error'], text?: string): void {
+ const unsent = [text || this.state.transcript || this.state.unsent, ...this.queued]
+ .filter(Boolean)
+ .join(' ');
+ this.release();
+ this.patch({ phase: 'error', error, unsent, transcript: '', level: 0 });
+ }
+
+ private release(): void {
+ ++this.epoch;
+ ++this.speechEpoch;
+ this.queued = [];
+ const callId = this.callId;
+ const microphone = this.microphone;
+ this.callId = null;
+ this.microphone = null;
+ this.unsubscribe?.();
+ this.unsubscribe = undefined;
+ this.player?.close();
+ this.player = null;
+ this.teardown = Promise.allSettled([
+ this.teardown,
+ this.pendingStart,
+ this.inFlight,
+ microphone?.stop(),
+ callId ? this.deps.api.end(callId) : undefined,
+ ]);
+ }
+
+ end(): void {
+ this.enabled = false;
+ this.release();
+ this.tracker.interrupt();
+ this.pendingInput = null;
+ this.patch({ phase: 'idle', transcript: '', level: 0, muted: false });
+ }
+}
diff --git a/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts b/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts
new file mode 100644
index 0000000000..f9ec23b65a
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts
@@ -0,0 +1,87 @@
+import type { ChatMessage } from '@/lib/makerChatStore';
+
+/** Only the completed, top-level answer to a voice input belongs to this call. */
+export class VoiceReplyTracker {
+ private waitingFor: string | null = null;
+ private consumed = new Set<string>();
+
+ expect(clientId: string): void {
+ this.waitingFor = clientId;
+ }
+ interrupt(): void {
+ this.waitingFor = null;
+ }
+ get hasPendingReply(): boolean {
+ return this.waitingFor !== null;
+ }
+
+ take(messages: readonly ChatMessage[]): { id: string; text: string } | null {
+ if (!this.waitingFor) return null;
+ const index = messages.findIndex((message) => message.clientId === this.waitingFor);
+ if (index < 0) return null; // A queued input has not entered the task yet.
+ if (messages[index].blockedByGhost) {
+ this.interrupt();
+ return null;
+ }
+ for (const message of messages.slice(index + 1)) {
+ // An input from another surface/automation owns the subsequent answer.
+ if (message.role === 'user') {
+ this.interrupt();
+ return null;
+ }
+ if (
+ message.role !== 'assistant' ||
+ message.parentToolUseId ||
+ message.explicitDelivery ||
+ message.sourceGroup ||
+ message.botPrivateReply ||
+ message.assistantPhase === 'commentary' ||
+ message.isStreaming ||
+ message.ghostReplyPending ||
+ message.turnCompleted !== true
+ )
+ continue;
+ if (this.consumed.has(message.clientId)) continue;
+ const text = speechText(message.content);
+ this.consumed.add(message.clientId);
+ this.interrupt();
+ return { id: message.clientId, text };
+ }
+ return null;
+ }
+}
+
+/** Read prose, not code blocks, image URLs or Markdown punctuation. The visible reply is unchanged. */
+export function speechText(markdown: string): string {
+ return markdown
+ .replace(/```[^]*?```|~~~[^]*?~~~/g, '')
+ .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+ .replace(/<[^>]+>/g, '')
+ .replace(/^\s{0,3}(?:#{1,6}\s+|>\s*|[-*+]\s+)/gm, '')
+ .replace(/[`*_~]/g, '')
+ .replace(/https?:\/\/\S+/g, '')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+/** Sequential requests preserve the entire final reply without exceeding the speech endpoint limit. */
+export function speechChunks(text: string, limit = 3_500): string[] {
+ const chunks: string[] = [];
+ while (text.length > limit) {
+ const prefix = text.slice(0, limit);
+ const boundary = Math.max(
+ prefix.lastIndexOf('。'),
+ prefix.lastIndexOf('. '),
+ prefix.lastIndexOf('!'),
+ prefix.lastIndexOf('?'),
+ prefix.lastIndexOf('\n'),
+ );
+ let end = boundary > limit / 2 ? boundary + 1 : limit;
+ if (/^[\uDC00-\uDFFF]/.test(text.slice(end))) end--;
+ chunks.push(text.slice(0, end).trim());
+ text = text.slice(end).trim();
+ }
+ if (text) chunks.push(text);
+ return chunks;
+}
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
new file mode 100644
index 0000000000..92540bcfba
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx
@@ -0,0 +1,112 @@
+// @vitest-environment jsdom
+import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+
+vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }));
+vi.mock('@/hooks/useVoiceInputSettings', () => ({
+ getVoiceInputSettings: () => ({ microphoneDeviceId: null }),
+}));
+vi.mock('../../BotAvatar', () => ({ BotAvatar: () => <span>Avatar</span> }));
+vi.mock('@/voice-input/workletUrl', () => ({ getVoiceInputWorkletUrl: () => 'worklet.js' }));
+const audio = vi.hoisted(() => ({
+ start: vi.fn(async () => {}),
+ stop: vi.fn(async () => {}),
+ onPcm16k: vi.fn(),
+ close: vi.fn(),
+}));
+vi.mock('@/voice-input/WebMicAudioEngine', () => ({
+ WebMicAudioEngine: class {
+ start = audio.start;
+ stop = audio.stop;
+ onPcm16k = audio.onPcm16k;
+ },
+}));
+vi.mock('../PcmSpeechPlayer', () => ({
+ PcmSpeechPlayer: class {
+ close = audio.close;
+ cancel() {}
+ async play() {}
+ },
+}));
+
+import { CompanionVoicePanel } from '../CompanionVoicePanel';
+let models: Array<{ id: string; name: string }>;
+const start = vi.fn(async () => ({ callId: 'call' }));
+const refresh = vi.fn(async () => undefined);
+beforeEach(() => {
+ vi.clearAllMocks();
+ models = [];
+ Object.defineProperty(window, 'electronAPI', {
+ configurable: true,
+ value: {
+ voiceConversation: {
+ models: async () => models,
+ start,
+ end: vi.fn(async () => {}),
+ onEvent: () => () => {},
+ },
+ maker: { refreshBuiltinProviderModels: refresh },
+ voiceInput: { updateSettings: vi.fn(async () => ({})), onPowerStateChange: () => () => {} },
+ onAuthStateChange: () => () => {},
+ },
+ });
+});
+afterEach(cleanup);
+const renderPanel = (blocked = false) =>
+ render(
+ <CompanionVoicePanel
+ bot={{ id: 'bot', name: 'Partner' }}
+ sessionId="canonical"
+ messages={[]}
+ blocked={blocked}
+ onSend={vi.fn(async () => true)}
+ onClose={vi.fn()}
+ />,
+ );
+
+it('shows a useful empty state without requesting a microphone or inventing models', async () => {
+ renderPanel();
+ await screen.findByText('bots.voiceMode.noModels');
+ const button = screen.getByRole('button', { name: 'bots.voiceMode.start' });
+ expect(button.hasAttribute('disabled')).toBe(true);
+ fireEvent.click(button);
+ expect(start).not.toHaveBeenCalled();
+ expect(audio.start).not.toHaveBeenCalled();
+ fireEvent.click(screen.getByRole('button', { name: 'bots.voiceMode.refresh' }));
+ await waitFor(() => expect(refresh).toHaveBeenCalledWith('xd'));
+});
+
+it('opens the chosen voice route and releases capture when the panel unmounts', async () => {
+ models = [{ id: 'gateway-speech', name: 'Speech' }];
+ const view = renderPanel();
+ await waitFor(() =>
+ expect(
+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),
+ ).toBe(false),
+ );
+ fireEvent.change(screen.getByLabelText('bots.voiceMode.voice'), { target: { value: 'nova' } });
+ fireEvent.click(screen.getByRole('button', { name: 'bots.voiceMode.start' }));
+ await waitFor(() => expect(audio.start).toHaveBeenCalledOnce());
+ expect(start).toHaveBeenCalledWith({
+ callId: expect.any(String),
+ botId: 'bot',
+ sessionId: 'canonical',
+ modelId: 'gateway-speech',
+ voice: 'nova',
+ });
+ expect(screen.getByRole('status').textContent).toBe('bots.voiceMode.status.listening');
+ view.unmount();
+ expect(audio.stop).toHaveBeenCalledOnce();
+ expect(audio.close).toHaveBeenCalledOnce();
+});
+
+it('keeps permission cards authoritative and never starts through a pending confirmation', async () => {
+ models = [{ id: 'tts', name: 'Speech' }];
+ renderPanel(true);
+ await waitFor(() =>
+ expect(screen.getByRole('status').textContent).toBe('bots.voiceMode.status.confirmation'),
+ );
+ expect(
+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),
+ ).toBe(true);
+});
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
new file mode 100644
index 0000000000..cb12461721
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
@@ -0,0 +1,87 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { PcmSpeechPlayer } from '../PcmSpeechPlayer';
+
+function createTestSource() {
+ return {
+ buffer: null,
+ onended: null,
+ connect: vi.fn(),
+ disconnect: vi.fn(),
+ start: vi.fn(),
+ stop: vi.fn(),
+ };
+}
+
+class TestAudioContext {
+ static latest: TestAudioContext;
+ state = 'running';
+ destination = {};
+ sources: ReturnType<typeof createTestSource>[] = [];
+ get currentTime() {
+ return Date.now() / 1000;
+ }
+ constructor() {
+ TestAudioContext.latest = this;
+ }
+ resume = vi.fn(async () => {});
+ close = vi.fn(async () => {
+ this.state = 'closed';
+ });
+ createBuffer = vi.fn(() => ({ copyToChannel: vi.fn() }));
+ createBufferSource() {
+ const source = createTestSource();
+ this.sources.push(source);
+ return source;
+ }
+}
+beforeEach(() => {
+ vi.useFakeTimers();
+ vi.stubGlobal('AudioContext', TestAudioContext);
+});
+afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+});
+
+it('bounds scheduled playback and immediately stops every source when the user interrupts', async () => {
+ const level = vi.fn();
+ const player = new PcmSpeechPlayer(level);
+ const pcm = new Int16Array(24_000).fill(1000).buffer;
+ const playing = player.play(
+ vi.fn(async () => ({ pcm, done: true })),
+ 24000,
+ );
+ const rejected = expect(playing).rejects.toThrow('Playback interrupted');
+ await vi.advanceTimersByTimeAsync(0);
+ const context = TestAudioContext.latest;
+ expect(context.sources.length).toBeGreaterThan(0);
+ expect(context.sources.length).toBeLessThanOrEqual(4);
+ player.cancel();
+ expect(context.sources.every((source) => source.stop.mock.calls.length === 1)).toBe(true);
+ const count = context.sources.length;
+ await vi.advanceTimersByTimeAsync(1000);
+ await rejected;
+ expect(context.sources).toHaveLength(count);
+ expect(level).toHaveBeenLastCalledWith(0);
+ player.close();
+ expect(context.close).toHaveBeenCalledOnce();
+});
+
+it('fences an IPC read that finishes after cancellation', async () => {
+ const player = new PcmSpeechPlayer(vi.fn());
+ let resolve!: (value: { pcm: ArrayBuffer; done: boolean }) => void;
+ const playing = player.play(
+ () =>
+ new Promise((done) => {
+ resolve = done;
+ }),
+ 24000,
+ );
+ const rejected = expect(playing).rejects.toThrow('Playback interrupted');
+ await vi.advanceTimersByTimeAsync(0);
+ player.cancel();
+ resolve({ pcm: new Int16Array(2400).buffer, done: true });
+ await rejected;
+ expect(TestAudioContext.latest.sources).toHaveLength(0);
+ player.close();
+});
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
new file mode 100644
index 0000000000..bfa22ff4fd
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
@@ -0,0 +1,241 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { VoiceConversationApi, VoiceConversationEvent } from '@/../shared/voiceConversation';
+import type { ChatMessage } from '@/lib/makerChatStore';
+import { VoiceConversationController } from '../VoiceConversationController';
+
+const selection = { modelId: 'tts', voice: 'alloy' };
+const user = (id = 'user-1'): ChatMessage =>
+ ({ clientId: id, role: 'user', content: 'hello' }) as ChatMessage;
+const reply = (id = 'reply-1'): ChatMessage =>
+ ({ clientId: id, role: 'assistant', content: 'Hello back', turnCompleted: true }) as ChatMessage;
+function fixture() {
+ let listener: (event: VoiceConversationEvent) => void = () => {};
+ let inputs = 0;
+ let calls = 0;
+ const api = {
+ models: vi.fn(async () => []),
+ start: vi.fn(async () => ({ callId: `call-${++calls}` })),
+ audio: vi.fn(),
+ finishUtterance: vi.fn(async () => {}),
+ end: vi.fn(async () => {}),
+ interrupt: vi.fn(async () => {}),
+ speak: vi.fn(async () => ({ sampleRate: 24000 })),
+ readSpeech: vi.fn(async () => ({ pcm: new ArrayBuffer(0), done: true })),
+ onEvent: vi.fn((callback) => {
+ listener = callback;
+ return () => {
+ listener = () => {};
+ };
+ }),
+ } satisfies VoiceConversationApi;
+ const microphone = {
+ start: vi.fn(async () => {}),
+ stop: vi.fn(async () => {}),
+ onPcm16k: vi.fn(),
+ };
+ const player = { play: vi.fn(async () => {}), cancel: vi.fn(), close: vi.fn() };
+ const send = vi.fn(
+ async (_text: string, created: (id: string) => void, current: () => boolean) => {
+ if (!current()) return false;
+ created(`user-${++inputs}`);
+ return true;
+ },
+ );
+ const controller = new VoiceConversationController({
+ api,
+ botId: 'bot',
+ sessionId: 'task',
+ microphone: () => microphone,
+ player: () => player,
+ send,
+ });
+ const emit = (event: VoiceConversationEvent['event'], callId = `call-${calls}`) =>
+ listener({ callId, event });
+ const utterance = (text = 'hello') => emit({ type: 'utterance', id: crypto.randomUUID(), text });
+ return { controller, api, microphone, player, send, emit, utterance };
+}
+
+describe('companion voice loop', () => {
+ it('sends speech through the existing task, reads only its final reply, then listens again', async () => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.controller.updateMessages([reply('history')]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ f.utterance();
+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledOnce());
+ await Promise.resolve();
+ f.controller.updateMessages([user(), { ...reply(), isStreaming: true }]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ f.controller.updateMessages([user(), reply()]);
+ await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());
+ expect(f.api.speak).toHaveBeenCalledWith({
+ callId: 'call-1',
+ requestId: expect.any(String),
+ text: 'Hello back',
+ });
+ expect(f.controller.getSnapshot().phase).toBe('listening');
+ f.controller.updateMessages([user(), reply()]);
+ expect(f.api.speak).toHaveBeenCalledOnce();
+ f.utterance('next');
+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));
+ expect(f.api.start).toHaveBeenCalledOnce();
+ f.controller.end();
+ expect(f.microphone.stop).toHaveBeenCalledOnce();
+ });
+
+ it('interrupts pending TTS immediately and discards its late response without stopping the task', async () => {
+ const f = fixture();
+ let finish!: (value: { sampleRate: number }) => void;
+ f.api.speak.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ }),
+ );
+ await f.controller.start(selection);
+ f.utterance();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.controller.updateMessages([user(), reply()]);
+ expect(f.api.speak).toHaveBeenCalledOnce();
+ f.emit({ type: 'speech-started' });
+ expect(f.player.cancel).toHaveBeenCalled();
+ expect(f.api.interrupt).toHaveBeenCalledWith('call-1');
+ finish({ sampleRate: 24000 });
+ await Promise.resolve();
+ expect(f.player.play).not.toHaveBeenCalled();
+ f.utterance('actually, another question');
+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));
+ expect(f.api.end).not.toHaveBeenCalled();
+ f.controller.end();
+ });
+
+ it('stops the microphone for a permission prompt and requires explicit resume', async () => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.emit({ type: 'draft', text: 'not sent yet' });
+ f.controller.setBlocked(true);
+ f.utterance();
+ expect(f.send).not.toHaveBeenCalled();
+ expect(f.microphone.stop).toHaveBeenCalledOnce();
+ expect(f.controller.getSnapshot()).toMatchObject({ phase: 'paused', unsent: 'not sent yet' });
+ f.controller.setBlocked(false);
+ expect(f.api.start).toHaveBeenCalledOnce();
+ await f.controller.start(selection);
+ expect(f.api.start).toHaveBeenCalledTimes(2);
+ f.controller.end();
+ });
+
+ it('stops listening and preserves rejected input plus unsent queued speech', async () => {
+ const f = fixture();
+ let reject!: (value: boolean) => void;
+ f.send.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ reject = resolve;
+ }),
+ );
+ await f.controller.start(selection);
+ f.utterance('first');
+ f.utterance('second');
+ reject(false);
+ await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('error'));
+ expect(f.controller.getSnapshot().unsent).toBe('first second');
+ expect(f.send).toHaveBeenCalledOnce();
+ f.controller.end();
+ });
+
+ it('cancels a pending start, then waits for teardown before opening another call', async () => {
+ const f = fixture();
+ let resolve!: (value: { callId: string }) => void;
+ f.api.start.mockImplementationOnce(
+ () =>
+ new Promise((done) => {
+ resolve = done;
+ }),
+ );
+ const first = f.controller.start(selection);
+ await vi.waitFor(() => expect(f.api.start).toHaveBeenCalledOnce());
+ f.controller.end();
+ const second = f.controller.start(selection);
+ resolve({ callId: 'cancelled-call' });
+ await first;
+ await second;
+ expect(f.api.end).toHaveBeenCalledWith('cancelled-call');
+ expect(f.microphone.start).toHaveBeenCalledOnce();
+ f.controller.end();
+ });
+
+ it('ends capture when the main process revokes call ownership', async () => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.emit({ type: 'ended' });
+ expect(f.microphone.stop).toHaveBeenCalledOnce();
+ expect(f.controller.getSnapshot().phase).toBe('idle');
+ });
+
+ it('an interruption before a delayed send acknowledgement does not resurrect its old reply', async () => {
+ const f = fixture();
+ let done!: () => void;
+ f.send.mockImplementationOnce(async (_text, created) => {
+ created('old-user');
+ await new Promise<void>((resolve) => {
+ done = resolve;
+ });
+ return true;
+ });
+ await f.controller.start(selection);
+ f.utterance();
+ f.emit({ type: 'speech-started' });
+ done();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.controller.updateMessages([user('old-user'), reply()]);
+ expect(f.api.speak).not.toHaveBeenCalled();
+ f.controller.end();
+ });
+
+ it('preserves speech rejected at a permission boundary and waits for preflight before resuming', async () => {
+ const f = fixture();
+ let finish!: () => void;
+ f.send.mockImplementationOnce(async (_text, _created, current) => {
+ await new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ return current();
+ });
+ await f.controller.start(selection);
+ f.utterance('not enqueued');
+ f.controller.setBlocked(true);
+ f.controller.setBlocked(false);
+ const resumed = f.controller.start(selection);
+ await Promise.resolve();
+ expect(f.api.start).toHaveBeenCalledOnce();
+ finish();
+ await resumed;
+ expect(f.controller.getSnapshot().unsent).toBe('not enqueued');
+ f.controller.retryUnsent();
+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledTimes(2));
+ expect(f.send.mock.calls[1][0]).toBe('not enqueued');
+ expect(f.controller.getSnapshot().unsent).toBe('');
+ f.controller.end();
+ });
+
+ it('resumes listening for an answer with no spoken prose or an intervening input from another surface', async () => {
+ const f = fixture();
+ await f.controller.start(selection);
+ f.utterance();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.controller.updateMessages([user(), { ...reply(), content: '```ts\nconst a = 1;\n```' }]);
+ expect(f.controller.getSnapshot().phase).toBe('listening');
+ expect(f.api.speak).not.toHaveBeenCalled();
+ f.utterance();
+ await Promise.resolve();
+ await Promise.resolve();
+ f.controller.updateMessages([user('user-2'), user('external'), reply()]);
+ expect(f.controller.getSnapshot().phase).toBe('listening');
+ expect(f.api.speak).not.toHaveBeenCalled();
+ f.controller.end();
+ });
+});
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts
new file mode 100644
index 0000000000..314e228f85
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceReplyTracker.test.ts
@@ -0,0 +1,75 @@
+import { describe, expect, it } from 'vitest';
+import type { ChatMessage } from '@/lib/makerChatStore';
+import { speechChunks, speechText, VoiceReplyTracker } from '../VoiceReplyTracker';
+import { Pcm16Decoder } from '../PcmSpeechPlayer';
+
+const message = (clientId: string, role: ChatMessage['role'], extra: Partial<ChatMessage> = {}) =>
+ ({ clientId, role, content: 'Hello', ...extra }) as ChatMessage;
+describe('voice reply correlation', () => {
+ it('excludes history, commentary, tool/subagent/group replies and failed turns', () => {
+ const tracker = new VoiceReplyTracker();
+ expect(tracker.take([message('old', 'assistant', { turnCompleted: true })])).toBeNull();
+ tracker.expect('mine');
+ const messages = [
+ message('mine', 'user'),
+ message('progress', 'assistant', { assistantPhase: 'commentary', turnCompleted: true }),
+ message('child', 'assistant', { parentToolUseId: 'tool', turnCompleted: true }),
+ message('group', 'assistant', { explicitDelivery: true, turnCompleted: true }),
+ message('failed', 'assistant', { turnCompleted: false }),
+ ];
+ expect(tracker.take(messages)).toBeNull();
+ messages.push(message('answer', 'assistant', { content: '**Done**', turnCompleted: true }));
+ expect(tracker.take(messages)).toEqual({ id: 'answer', text: 'Done' });
+ expect(tracker.take(messages)).toBeNull();
+ });
+ it('waits for a queued input and abandons speech ownership when another input takes over', () => {
+ const tracker = new VoiceReplyTracker();
+ tracker.expect('queued');
+ expect(tracker.take([message('old', 'assistant', { turnCompleted: true })])).toBeNull();
+ expect(
+ tracker.take([
+ message('queued', 'user'),
+ message('other', 'user'),
+ message('answer', 'assistant', { turnCompleted: true }),
+ ]),
+ ).toBeNull();
+ });
+ it('waits for an outgoing reply hook to finish before reading its final text', () => {
+ const tracker = new VoiceReplyTracker();
+ tracker.expect('mine');
+ const messages = [
+ message('mine', 'user'),
+ message('reply', 'assistant', { turnCompleted: true, ghostReplyPending: true }),
+ ];
+ expect(tracker.take(messages)).toBeNull();
+ messages[1].ghostReplyPending = false;
+ expect(tracker.take(messages)?.id).toBe('reply');
+ });
+});
+
+it('reads prose and link labels without executing markup or speaking code/image URLs', () => {
+ expect(
+ speechText(
+ '## **Hello**\n[world](https://example.test) ![image](https://image.test)\n```js\nsecretCode()\n```',
+ ),
+ ).toBe('Hello world');
+});
+
+it('chunks the whole reply without splitting surrogate pairs or silently truncating it', () => {
+ const text = '🙂'.repeat(4000);
+ const chunks = speechChunks(text);
+ expect(chunks.join('')).toBe(text);
+ expect(chunks.every((part) => part.length <= 3500 && !/^[\uDC00-\uDFFF]/.test(part))).toBe(true);
+});
+
+it('decodes signed PCM across odd byte boundaries and refuses a truncated last sample', () => {
+ const decoder = new Pcm16Decoder();
+ expect([...decoder.decode(new Uint8Array([0]).buffer)]).toEqual([]);
+ expect([...decoder.decode(new Uint8Array([128, 255, 127, 0]).buffer)]).toEqual([
+ -1,
+ 32767 / 32768,
+ ]);
+ expect(() => decoder.finish()).toThrow('Incomplete');
+ expect([...decoder.decode(new Uint8Array([0]).buffer)]).toEqual([0]);
+ expect(() => decoder.finish()).not.toThrow();
+});
diff --git a/apps/desktop/src/renderer/features/bots/voice/voiceConversation.css b/apps/desktop/src/renderer/features/bots/voice/voiceConversation.css
new file mode 100644
index 0000000000..214be5ec6e
--- /dev/null
+++ b/apps/desktop/src/renderer/features/bots/voice/voiceConversation.css
@@ -0,0 +1,33 @@
+.companion-voice-panel {
+ animation: companion-voice-enter var(--motion-enter) var(--motion-ease-out) both;
+}
+
+.companion-voice-meter > span {
+ transform-origin: center;
+ transition: transform var(--motion-instant) linear;
+}
+
+.companion-voice-meter[data-active='false'] {
+ opacity: 0.45;
+}
+
+@keyframes companion-voice-enter {
+ from {
+ opacity: 0;
+ transform: translateY(6px);
+ }
+ to {
+ opacity: 1;
+ transform: translateY(0);
+ }
+}
+
+@media (prefers-reduced-motion: reduce) {
+ .companion-voice-panel {
+ animation: none;
+ }
+ .companion-voice-meter > span {
+ transition: none;
+ transform: scaleY(0.2) !important;
+ }
+}
diff --git a/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx b/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
index 622f069324..be5e662917 100644
--- a/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
+++ b/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
@@ -1,3 +1,4 @@
+import { CompanionVoicePanel } from '../bots/voice/CompanionVoicePanel';
import { formatSessionDuration } from '@/lib/sessionDurationFormat';
import { shouldShowOpenPathError } from '../../../shared/openPathResult';
import { shouldShowFailedScheduleNotice } from '@cindy/maker-shared/schedule-model';
@@ -982,6 +983,9 @@ export function CCAgentSessionView({
// Bot route gates have already checked durable ownership. The async runtime
// snapshot may be absent during load/reconnect; it must never change the skin.
+ const [voiceModeOpen, setVoiceModeOpen] = useState(false);
+ const toggleVoiceMode = useCallback(() => setVoiceModeOpen((open) => !open), []);
+ const closeVoiceMode = useCallback(() => setVoiceModeOpen(false), []);
const botChatIdentity = resolveBotChatIdentity(botIdentity, sessionId);
// 伙伴没有 RunningStatusBar,折叠呼吸灯继续留在输入框上方,不能随状态行一起消失。
const showCenteredControlledBanner =
@@ -3701,6 +3705,9 @@ export function CCAgentSessionView({
mentions?: MentionedResource[],
opts?: {
deliveryMode?: MessageDeliveryMode;
+ literalText?: boolean;
+ beforeEnqueue?: () => Promise<boolean>;
+ onInputCreated?: (clientId: string) => void;
quotesEncoded?: boolean;
agentReferences?: AgentInputReference[];
pastedTextRanges?: PastedTextRange[];
@@ -3715,11 +3722,11 @@ export function CCAgentSessionView({
const deliveryMode = opts?.deliveryMode ?? 'queue';
const originalMessage = message;
const navigationRequestVersion =
- deliveryMode !== 'steer' && matchNavigationCommandName(message)
+ !opts?.literalText && deliveryMode !== 'steer' && matchNavigationCommandName(message)
? ++sessionNavigationVersionRef.current
: null;
if (
- deliveryMode !== 'steer' &&
+ !opts?.literalText && deliveryMode !== 'steer' &&
(await tryHandleNavigationCommand(message, {
navigate,
t,
@@ -3734,7 +3741,7 @@ export function CCAgentSessionView({
return;
}
- if (deliveryMode !== 'steer' && (await maybeShowContextUsage(message))) {
+ if (!opts?.literalText && deliveryMode !== 'steer' && (await maybeShowContextUsage(message))) {
return;
}
@@ -3745,7 +3752,7 @@ export function CCAgentSessionView({
// Codex only supports upstream automatic compaction, so keep the task
// intact and explain that it manages compaction automatically.
const botNewMatch =
- deliveryMode !== 'steer' && sessionRef.current?.source === 'bot'
+ !opts?.literalText && deliveryMode !== 'steer' && sessionRef.current?.source === 'bot'
? message.match(/^\/new(?:\s+(.*))?$/s)
: null;
if (botNewMatch) {
@@ -3785,7 +3792,7 @@ export function CCAgentSessionView({
// - agent-builtin / agent-skill / 没命中任何已知命令 → 走默认 send,
// 原文(含前导 `/`)直接送 agent, 由 SDK 自己识别 (/compact 等)。
const slashDispatch =
- deliveryMode === 'steer'
+ opts?.literalText ? { handled: false, message, accepted: false } : deliveryMode === 'steer'
? await maybeDispatchDesktopSlashCommand(message, files, {
allowDesktopDispatch: false,
piRuntimeRetryDelaysMs: PI_RUNTIME_SKILL_RETRY_DELAYS_MS,
@@ -3859,6 +3866,7 @@ export function CCAgentSessionView({
if (!proceed) return false;
}
+ if (opts?.beforeEnqueue && !(await opts.beforeEnqueue())) return false;
// Popover open → prevent re-entry
if (folderPickerOpen) return false;
// 会话交接尚未完成(建 worktree / 远程开协同)时不放行:否则新输入会插到
@@ -3921,6 +3929,8 @@ export function CCAgentSessionView({
// ④ Execute send — effort + permissionMode came straight from ChatInput (fresh value)
const sendOptions = {
+ beforeEnqueue: opts?.beforeEnqueue,
+ onInputCreated: opts?.onInputCreated,
...orcaLeadVendorOptions,
...(opts?.quotesEncoded ? { quotesEncoded: true } : {}),
...(opts?.agentReferences?.length ? { agentReferences: opts.agentReferences } : {}),
@@ -4002,6 +4012,25 @@ export function CCAgentSessionView({
],
);
+ const voiceModeAvailable = Boolean(botChatIdentity && !botChatIdentity.deviceId && sessionId
+ && !remoteDeviceId && !session?.remoteHostId && !readOnly && ownsRoute && viewVisible);
+ useEffect(() => { if (!voiceModeAvailable) setVoiceModeOpen(false); }, [voiceModeAvailable]);
+ const voiceModeBlocked = Boolean(pendingPermission || pendingAskUser || pendingPlanReview
+ || pendingPluginSetup || pendingIssueConfirm || pendingRenameSessionsConfirm
+ || pendingGhostGrantConfirm || pendingRemoteDesktopConfirmation || sessionBinding.attached
+ || remoteHandoffPreparing || cindyMakeInputLocked || queuePaused);
+ const sendVoiceMessage = useCallback(async (text: string, onCreated: (id: string) => void, isCurrent: () => boolean) => {
+ if (!session?.workingDir || !session.model || voiceModeBlocked || !isCurrent()) return false;
+ const accepted = await handleSend(text, session.model, session.effort ?? '', session.permissionMode ?? 'default', undefined, undefined, {
+ deliveryMode: isRunningRef.current ? 'steer' : 'queue',
+ literalText: true,
+ slashCommandRanges: [],
+ beforeEnqueue: async () => isCurrent(),
+ onInputCreated: onCreated,
+ });
+ return accepted === true;
+ }, [session, voiceModeBlocked, handleSend]);
+
const handleStopSession = useCallback(() => {
if (remoteSessionUnavailable) {
toast.warning(t('ccAgent.remoteSession.actionsUnavailable'));
@@ -4168,10 +4197,10 @@ export function CCAgentSessionView({
]);
const handleBeforeVoiceInputStart = useCallback(async () => {
- if (cindyMakeInputLocked) return false;
+ if (cindyMakeInputLocked || voiceModeOpen) return false;
const { proceed } = await vendorAuthGate.checkAndConfirm('codex', { purpose: 'voice-input' });
return proceed;
- }, [vendorAuthGate, cindyMakeInputLocked]);
+ }, [vendorAuthGate, cindyMakeInputLocked, voiceModeOpen]);
// M32: Retry — ErrorBanner 的 retryText 现在只是兼容展示值。真正的
// recovery target 由 main coordinator 持有,避免把已发出的文本重新走普通
@@ -4844,7 +4873,7 @@ export function CCAgentSessionView({
// 伙伴对话不是用户经营的任务:它拿的是「跟谁说话 + 进 TA 的设置」,
// 不是重命名/置顶/归档/导出那一套任务菜单。
botChatIdentity ? (
- <BotSessionContentHeaderRegistration bot={botChatIdentity} />
+ <BotSessionContentHeaderRegistration bot={botChatIdentity} onVoice={voiceModeAvailable ? toggleVoiceMode : undefined} voiceActive={voiceModeOpen} />
) : (
<SessionContentHeaderRegistration
session={session}
@@ -5536,6 +5565,7 @@ export function CCAgentSessionView({
getContentWidth={getMessageWidth}
/>
) : (
+ <div className={voiceModeOpen && voiceModeAvailable ? 'hidden' : 'contents'}>
<ChatInput
topSlot={cindyMakeRecoveryId && session ? (
<CindyMakeEditingActions
@@ -5547,7 +5577,7 @@ export function CCAgentSessionView({
onSend={handleSend}
onBeforeVoiceInputStart={handleBeforeVoiceInputStart}
sessionId={sessionId}
- ownsHardwareComposerActions={ownsHardwareTaskActions}
+ ownsHardwareComposerActions={ownsHardwareTaskActions && !voiceModeOpen}
// session=null 是冷启动 / 直链 GET 尚未回流的合法首帧;显式传 null,
// 让 ChatInput 暂不显示 Agent 身份,不能跟随 displayAgentKind 的 cc 回退。
runtimeAgentKind={session ? dbToMakerAgentKind(session.agentKind) : null}
@@ -5696,7 +5726,12 @@ export function CCAgentSessionView({
botMentions={botMentions}
hideRuntimeControls={Boolean(botChatIdentity)}
/>
+ </div>
)}
+ {voiceModeOpen && voiceModeAvailable && botChatIdentity && sessionId ? (
+ <CompanionVoicePanel key={sessionId} bot={botChatIdentity} sessionId={sessionId} messages={messages}
+ blocked={voiceModeBlocked} onSend={sendVoiceMessage} onClose={closeVoiceMode} />
+ ) : null}
{/* F-FP-5: workingDir — always rendered to prevent layout shift
worktree-parallel-sessions:worktree 创建过程的反馈(creating/failed)
diff --git a/apps/desktop/src/renderer/hooks/useCCAgentChat.ts b/apps/desktop/src/renderer/hooks/useCCAgentChat.ts
index 26342e2e19..800c2dcd79 100644
--- a/apps/desktop/src/renderer/hooks/useCCAgentChat.ts
+++ b/apps/desktop/src/renderer/hooks/useCCAgentChat.ts
@@ -133,6 +133,7 @@ interface UseCCAgentChatReturn {
pastedTextRanges?: PastedTextRange[];
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
+ onInputCreated?: (clientId: string) => void;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
annotationBurnFailure?: 'abort';
},
@@ -159,6 +160,7 @@ interface UseCCAgentChatReturn {
pastedTextRanges?: PastedTextRange[];
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
+ onInputCreated?: (clientId: string) => void;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
annotationBurnFailure?: 'abort';
},
@@ -427,6 +429,7 @@ export function useCCAgentChat(
pastedTextRanges?: PastedTextRange[];
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
+ onInputCreated?: (clientId: string) => void;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
annotationBurnFailure?: 'abort';
},
@@ -484,6 +487,7 @@ export function useCCAgentChat(
pastedTextRanges?: PastedTextRange[];
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
+ onInputCreated?: (clientId: string) => void;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
annotationBurnFailure?: 'abort';
},
diff --git a/apps/desktop/src/renderer/i18n/locales/en/common.json b/apps/desktop/src/renderer/i18n/locales/en/common.json
index e8c564c07c..e4b57b252c 100644
--- a/apps/desktop/src/renderer/i18n/locales/en/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/en/common.json
@@ -8946,6 +8946,44 @@
"memory": "Remembered: {{title}}",
"skillCreated": "Created skill: {{title}}",
"skillUpdated": "Updated skill: {{title}}"
+ },
+ "voiceMode": {
+ "finishBeforeDictation": "End voice mode before starting dictation.",
+ "title": "Voice Mode",
+ "close": "Close voice mode",
+ "model": "Speech model",
+ "refresh": "Refresh models",
+ "voice": "Voice",
+ "voiceHint": "Use a voice name supported by the selected model, such as alloy.",
+ "modelUnavailable": "The saved speech model is unavailable. Choose another model.",
+ "noModels": "No speech model is available for this account yet. You can start when one becomes available.",
+ "catalogError": "Could not load or save speech settings. Please try again.",
+ "start": "Start talking",
+ "resume": "Resume",
+ "retrySend": "Reconnect and send",
+ "mute": "Mute",
+ "interrupt": "Stop reading",
+ "send": "Finish sentence",
+ "end": "End",
+ "unsent": "Not sent · select the text to copy it",
+ "hint": "Pause to send. Replies are read aloud with an AI voice. Speak at any time to interrupt.",
+ "status": {
+ "idle": "Ready when you are",
+ "connecting": "Connecting…",
+ "listening": "Listening · pause to send",
+ "thinking": "Thinking · you can keep talking",
+ "speaking": "Speaking · interrupt at any time",
+ "paused": "Paused · microphone off",
+ "error": "Paused · microphone off",
+ "confirmation": "Waiting for your confirmation · microphone off",
+ "preparing": "Preparing voice…"
+ },
+ "errors": {
+ "connection": "Voice connection interrupted. Your unsent text is kept below.",
+ "microphone": "Microphone stopped. Check its permission and connection, then resume.",
+ "send": "This message was not accepted. Your text is kept below.",
+ "speech": "Could not read the reply aloud. The text reply remains in the conversation. Check the speech model and voice."
+ }
}
},
"common": {
diff --git a/apps/desktop/src/renderer/i18n/locales/ja/common.json b/apps/desktop/src/renderer/i18n/locales/ja/common.json
index 11cbb2ae47..586688ef5d 100644
--- a/apps/desktop/src/renderer/i18n/locales/ja/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/ja/common.json
@@ -8927,6 +8927,44 @@
"memory": "記憶しました:{{title}}",
"skillCreated": "スキルを作成:{{title}}",
"skillUpdated": "スキルを更新:{{title}}"
+ },
+ "voiceMode": {
+ "finishBeforeDictation": "音声モードを終了してから音声入力を開始してください。",
+ "title": "音声モード",
+ "close": "音声モードを閉じる",
+ "model": "読み上げモデル",
+ "refresh": "モデルを更新",
+ "voice": "音声",
+ "voiceHint": "選択したモデルが対応する音声名を入力してください(例:alloy)。",
+ "modelUnavailable": "保存した読み上げモデルは利用できません。別のモデルを選んでください。",
+ "noModels": "このアカウントで利用できる読み上げモデルはまだありません。利用可能になれば開始できます。",
+ "catalogError": "音声設定の読み込み、または保存に失敗しました。もう一度お試しください。",
+ "start": "会話を始める",
+ "resume": "再開",
+ "retrySend": "再接続して送信",
+ "mute": "ミュート",
+ "interrupt": "読み上げを停止",
+ "send": "発話を確定",
+ "end": "終了",
+ "unsent": "未送信 · テキストを選択してコピーできます",
+ "hint": "話し終えると自動送信し、返答を AI 音声で読み上げます。いつでも話しかけて割り込めます。",
+ "status": {
+ "idle": "準備ができたら開始",
+ "connecting": "接続中…",
+ "listening": "聞いています · 間を置くと送信",
+ "thinking": "考え中 · 続けて話せます",
+ "speaking": "読み上げ中 · いつでも割り込めます",
+ "paused": "一時停止 · マイクはオフ",
+ "error": "一時停止 · マイクはオフ",
+ "confirmation": "確認待ち · マイクはオフ",
+ "preparing": "音声を準備中…"
+ },
+ "errors": {
+ "connection": "音声接続が途切れました。未送信のテキストは保持されています。",
+ "microphone": "マイクが停止しました。権限と接続を確認して再開してください。",
+ "send": "メッセージは受け付けられませんでした。テキストは保持されています。",
+ "speech": "読み上げに失敗しました。返答のテキストは会話に残っています。モデルと音声を確認してください。"
+ }
}
},
"common": {
diff --git a/apps/desktop/src/renderer/i18n/locales/ko/common.json b/apps/desktop/src/renderer/i18n/locales/ko/common.json
index 894d343063..cfdba655a6 100644
--- a/apps/desktop/src/renderer/i18n/locales/ko/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/ko/common.json
@@ -8927,6 +8927,44 @@
"memory": "기억했어요: {{title}}",
"skillCreated": "스킬 생성: {{title}}",
"skillUpdated": "스킬 업데이트: {{title}}"
+ },
+ "voiceMode": {
+ "finishBeforeDictation": "음성 모드를 종료한 후 받아쓰기를 시작하세요.",
+ "title": "음성 모드",
+ "close": "음성 모드 닫기",
+ "model": "읽어주기 모델",
+ "refresh": "모델 새로고침",
+ "voice": "목소리",
+ "voiceHint": "선택한 모델이 지원하는 음성 이름을 입력하세요. 예: alloy.",
+ "modelUnavailable": "저장된 읽어주기 모델을 사용할 수 없습니다. 다른 모델을 선택하세요.",
+ "noModels": "이 계정에서 사용할 수 있는 읽어주기 모델이 아직 없습니다. 모델이 제공되면 시작할 수 있습니다.",
+ "catalogError": "음성 설정을 불러오거나 저장하지 못했습니다. 다시 시도하세요.",
+ "start": "대화 시작",
+ "resume": "계속",
+ "retrySend": "다시 연결하고 보내기",
+ "mute": "음소거",
+ "interrupt": "읽기 중지",
+ "send": "발화 완료",
+ "end": "종료",
+ "unsent": "전송되지 않음 · 텍스트를 선택해 복사할 수 있습니다",
+ "hint": "말을 멈추면 자동으로 보내고 답변을 AI 음성으로 읽어줍니다. 언제든 말해서 끼어들 수 있습니다.",
+ "status": {
+ "idle": "준비되면 시작하세요",
+ "connecting": "연결 중…",
+ "listening": "듣는 중 · 잠시 멈추면 전송",
+ "thinking": "생각 중 · 계속 말씀하세요",
+ "speaking": "읽는 중 · 언제든 끼어들 수 있어요",
+ "paused": "일시 중지 · 마이크 꺼짐",
+ "error": "일시 중지 · 마이크 꺼짐",
+ "confirmation": "확인 대기 · 마이크 꺼짐",
+ "preparing": "음성 준비 중…"
+ },
+ "errors": {
+ "connection": "음성 연결이 끊겼습니다. 전송되지 않은 텍스트는 유지됩니다.",
+ "microphone": "마이크가 중지되었습니다. 권한과 연결을 확인한 후 계속하세요.",
+ "send": "메시지가 접수되지 않았습니다. 텍스트는 유지됩니다.",
+ "speech": "답변을 읽지 못했습니다. 텍스트 답변은 대화에 남아 있습니다. 모델과 목소리를 확인하세요."
+ }
}
},
"common": {
diff --git a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
index b6b9a986ff..de140da12b 100644
--- a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
@@ -8927,6 +8927,44 @@
"memory": "已记住:{{title}}",
"skillCreated": "已建立技能:{{title}}",
"skillUpdated": "已更新技能:{{title}}"
+ },
+ "voiceMode": {
+ "finishBeforeDictation": "请先结束语音模式,再开始听写。",
+ "title": "语音模式",
+ "close": "关闭语音模式",
+ "model": "朗读模型",
+ "refresh": "刷新模型",
+ "voice": "音色",
+ "voiceHint": "填写所选模型支持的音色名称,例如 alloy。",
+ "modelUnavailable": "之前选择的朗读模型已不可用,请重新选择。",
+ "noModels": "当前账号还没有可用的朗读模型,开放后即可开始语音对话。",
+ "catalogError": "语音设置读取或保存失败,请重试。",
+ "start": "开始说话",
+ "resume": "继续",
+ "retrySend": "重新连接并发送",
+ "mute": "静音",
+ "interrupt": "停止朗读",
+ "send": "结束本句",
+ "end": "结束",
+ "unsent": "尚未发送 · 可选中文字复制",
+ "hint": "停顿后自动发送,回复由 AI 语音朗读。随时开口即可插话。",
+ "status": {
+ "idle": "准备好就开始",
+ "connecting": "正在连接…",
+ "listening": "正在听 · 停顿后发送",
+ "thinking": "正在思考 · 可以继续说",
+ "speaking": "正在朗读 · 随时开口插话",
+ "paused": "已暂停 · 麦克风已关闭",
+ "error": "已暂停 · 麦克风已关闭",
+ "confirmation": "等待你的确认 · 麦克风已关闭",
+ "preparing": "正在准备语音…"
+ },
+ "errors": {
+ "connection": "语音连接中断,未发送的文字已保留。",
+ "microphone": "麦克风已停止,请检查权限和设备连接后继续。",
+ "send": "这条消息未被接收,文字已保留。",
+ "speech": "回复朗读失败,文字回复仍保留在对话中。请检查朗读模型和音色。"
+ }
}
},
"common": {
diff --git a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
index 9e2f8841f7..22c074adb1 100644
--- a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
@@ -8927,6 +8927,44 @@
"memory": "已記住:{{title}}",
"skillCreated": "已建立技能:{{title}}",
"skillUpdated": "已更新技能:{{title}}"
+ },
+ "voiceMode": {
+ "finishBeforeDictation": "請先結束語音模式,再開始聽寫。",
+ "title": "語音模式",
+ "close": "關閉語音模式",
+ "model": "朗讀模型",
+ "refresh": "重新整理模型",
+ "voice": "音色",
+ "voiceHint": "填入所選模型支援的音色名稱,例如 alloy。",
+ "modelUnavailable": "先前選擇的朗讀模型已無法使用,請重新選擇。",
+ "noModels": "目前帳號還沒有可用的朗讀模型,開放後即可開始語音對話。",
+ "catalogError": "語音設定讀取或儲存失敗,請重試。",
+ "start": "開始說話",
+ "resume": "繼續",
+ "retrySend": "重新連線並傳送",
+ "mute": "靜音",
+ "interrupt": "停止朗讀",
+ "send": "結束本句",
+ "end": "結束",
+ "unsent": "尚未傳送 · 可選取文字複製",
+ "hint": "停頓後自動傳送,回覆由 AI 語音朗讀。隨時開口即可插話。",
+ "status": {
+ "idle": "準備好就開始",
+ "connecting": "正在連線…",
+ "listening": "正在聽 · 停頓後傳送",
+ "thinking": "正在思考 · 可以繼續說",
+ "speaking": "正在朗讀 · 隨時開口插話",
+ "paused": "已暫停 · 麥克風已關閉",
+ "error": "已暫停 · 麥克風已關閉",
+ "confirmation": "等待你的確認 · 麥克風已關閉",
+ "preparing": "正在準備語音…"
+ },
+ "errors": {
+ "connection": "語音連線中斷,未傳送的文字已保留。",
+ "microphone": "麥克風已停止,請檢查權限和裝置連線後繼續。",
+ "send": "這則訊息未被接收,文字已保留。",
+ "speech": "回覆朗讀失敗,文字回覆仍保留在對話中。請檢查朗讀模型和音色。"
+ }
}
},
"common": {
diff --git a/apps/desktop/src/renderer/lib/makerChatStore.ts b/apps/desktop/src/renderer/lib/makerChatStore.ts
index 2b04bd9042..0bc338de95 100644
--- a/apps/desktop/src/renderer/lib/makerChatStore.ts
+++ b/apps/desktop/src/renderer/lib/makerChatStore.ts
@@ -14531,6 +14531,8 @@ type SendMessageOpts = {
bypassGhostHooks?: boolean;
/** 远程预检在乐观投影建立后执行;false 时按 clientId 精确回滚。 */
beforeEnqueue?: () => Promise<boolean>;
+ /** Renderer-local correlation, never serialized into a message or wire payload. */
+ onInputCreated?: (clientId: string) => void;
/** 远程乐观发送在稍后确认永久失败时恢复 composer。 */
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
/**
@@ -14806,6 +14808,7 @@ async function sendMessageCore(
opts,
identity,
);
+ opts?.onInputCreated?.(queued.clientId);
// #2194: 登记本端发送——MessageStream 的强 pin 只认这个集合,外部入口
// (IM / 手机端 / 定时任务)注入的 user 消息不抢视口。后续路径 return false
// (如 beforeEnqueue 回滚)会留下一个永不渲染的 id,无害。
@@ -15043,6 +15046,8 @@ function steerMessage(
pastedTextRanges?: PastedTextRange[];
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
+ /** Renderer-local correlation, never serialized into a message or wire payload. */
+ onInputCreated?: (clientId: string) => void;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
/** 见 SendMessageOpts.annotationBurnFailure。 */
annotationBurnFailure?: 'abort';
@@ -15200,6 +15205,8 @@ async function steerMessageCore(
pastedTextRanges?: PastedTextRange[];
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
+ /** Renderer-local correlation, never serialized into a message or wire payload. */
+ onInputCreated?: (clientId: string) => void;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
},
clearGenerationAtStart = 0,
@@ -15228,6 +15235,7 @@ async function steerMessageCore(
opts,
identity,
);
+ opts?.onInputCreated?.(queued.clientId);
// #2194: 与 sendMessageCore 相同——本端 steer 也是明确的本地发送意图。
markLocalSentUserMessage(sessionId, queued.clientId);
const deviceLinkRemote = remoteScopeAtStart !== null;
diff --git a/apps/desktop/src/renderer/vite-env.d.ts b/apps/desktop/src/renderer/vite-env.d.ts
index 498926b558..1a23b828aa 100644
--- a/apps/desktop/src/renderer/vite-env.d.ts
+++ b/apps/desktop/src/renderer/vite-env.d.ts
@@ -1714,6 +1714,7 @@ interface ElectronAPI {
onConfirm: (callback: (request: unknown) => void) => () => void;
resolveConfirm: (requestId: string, confirmed: boolean) => Promise<{ handled: boolean }>;
};
+ voiceConversation: import('../shared/voiceConversation').VoiceConversationApi;
voiceInput: {
prewarm: (payload?: {
sourceLanguage?: string;
diff --git a/apps/desktop/src/renderer/voice-input/__tests__/overlayErrors.test.ts b/apps/desktop/src/renderer/voice-input/__tests__/overlayErrors.test.ts
index 1d0499497a..da027adcf5 100644
--- a/apps/desktop/src/renderer/voice-input/__tests__/overlayErrors.test.ts
+++ b/apps/desktop/src/renderer/voice-input/__tests__/overlayErrors.test.ts
@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest';
import { TEST_XD_GATEWAY_BASE_URL as XD_GATEWAY_BASE_URL } from '../../../test/vitest/clientEndpointsFixture';
import { createInstance } from 'i18next';
-import { VOICE_INPUT_RATE_LIMITED_MESSAGE } from '../../../shared/voiceInputErrors';
+import { VOICE_CONVERSATION_ACTIVE_MESSAGE, VOICE_INPUT_RATE_LIMITED_MESSAGE } from '../../../shared/voiceInputErrors';
import { getVoiceInputErrorMessageKey, isVoiceInputServiceConnectionError } from '../overlayErrors';
describe('voice input overlay error classification', () => {
@@ -18,6 +18,7 @@ describe('voice input overlay error classification', () => {
expect(local.t(getVoiceInputErrorMessageKey(VOICE_INPUT_RATE_LIMITED_MESSAGE)!)).toBe('Japanese limit message');
expect(getVoiceInputErrorMessageKey('Upstream HTTP 429')).toBeUndefined();
expect(getVoiceInputErrorMessageKey('WebSocket connection timed out')).toBe('voiceInputOverlay.asrServiceUnavailable');
+ expect(getVoiceInputErrorMessageKey(VOICE_CONVERSATION_ACTIVE_MESSAGE)).toBe('bots.voiceMode.finishBeforeDictation');
});
it('detects transport failures as service connection errors', () => {
diff --git a/apps/desktop/src/renderer/voice-input/overlayErrors.ts b/apps/desktop/src/renderer/voice-input/overlayErrors.ts
index 9a5531628a..4dae7338b4 100644
--- a/apps/desktop/src/renderer/voice-input/overlayErrors.ts
+++ b/apps/desktop/src/renderer/voice-input/overlayErrors.ts
@@ -1,5 +1,5 @@
import type { VoiceInputErrorCode } from '@cindy/voice-input-core';
-import { VOICE_INPUT_RATE_LIMITED_MESSAGE } from '../../shared/voiceInputErrors';
+import { VOICE_CONVERSATION_ACTIVE_MESSAGE, VOICE_INPUT_RATE_LIMITED_MESSAGE } from '../../shared/voiceInputErrors';
/**
* i18n keys for the failures the voice controller classifies itself. Their
@@ -22,6 +22,7 @@ export function isVoiceInputServiceConnectionError(message: string): boolean {
/** Resolve known messages at display time using this renderer's current locale. */
export function getVoiceInputErrorMessageKey(message: string): string | undefined {
+ if (message === VOICE_CONVERSATION_ACTIVE_MESSAGE) return 'bots.voiceMode.finishBeforeDictation';
if (message === VOICE_INPUT_RATE_LIMITED_MESSAGE) return 'voiceInputOverlay.rateLimited';
if (isVoiceInputServiceConnectionError(message)) return 'voiceInputOverlay.asrServiceUnavailable';
return undefined;
diff --git a/apps/desktop/src/shared/voiceConversation.ts b/apps/desktop/src/shared/voiceConversation.ts
new file mode 100644
index 0000000000..6971ce0916
--- /dev/null
+++ b/apps/desktop/src/shared/voiceConversation.ts
@@ -0,0 +1,45 @@
+import type { AudioTrace, ConversationInputEvent } from '@cindy/voice-input-core';
+
+/** Local Desktop voice UI contract. No credentials, raw paths or remote invoke access. */
+export type VoiceConversationSelection = { modelId: string; voice: string };
+export type VoiceConversationModel = { id: string; name: string };
+export type VoiceConversationEvent = {
+ callId: string;
+ event: ConversationInputEvent | { type: 'ended' };
+};
+export type VoiceConversationStart = VoiceConversationSelection & {
+ callId: string;
+ botId: string;
+ sessionId: string;
+};
+export interface VoiceConversationApi {
+ models(): Promise<VoiceConversationModel[]>;
+ start(input: VoiceConversationStart): Promise<{ callId: string }>;
+ audio(input: { callId: string; pcm: ArrayBuffer; trace?: AudioTrace }): void;
+ finishUtterance(callId: string): Promise<void>;
+ end(callId: string): Promise<void>;
+ speak(input: {
+ callId: string;
+ requestId: string;
+ text: string;
+ }): Promise<{ sampleRate: number }>;
+ readSpeech(callId: string, requestId: string): Promise<{ pcm: ArrayBuffer; done: boolean }>;
+ interrupt(callId: string): Promise<void>;
+ onEvent(callback: (event: VoiceConversationEvent) => void): () => void;
+}
+
+export function normalizeVoiceConversationSelection(
+ value: unknown,
+): VoiceConversationSelection | undefined {
+ if (!value || typeof value !== 'object') return undefined;
+ const { modelId, voice } = value as Partial<VoiceConversationSelection>;
+ if (
+ typeof modelId !== 'string' ||
+ !modelId.trim() ||
+ modelId.length > 256 ||
+ typeof voice !== 'string' ||
+ !/^[a-zA-Z0-9_-]{1,128}$/.test(voice)
+ )
+ return undefined;
+ return { modelId: modelId.trim(), voice };
+}
diff --git a/apps/desktop/src/shared/voiceInputData.ts b/apps/desktop/src/shared/voiceInputData.ts
index 386352bd30..0e68a90a9a 100644
--- a/apps/desktop/src/shared/voiceInputData.ts
+++ b/apps/desktop/src/shared/voiceInputData.ts
@@ -1,3 +1,4 @@
+import { normalizeVoiceConversationSelection, type VoiceConversationSelection } from './voiceConversation';
import type {
DictationDictionaryAdviceInput,
DictationDictionaryLearningAction,
@@ -78,6 +79,8 @@ export interface VoiceInputDictionaryCandidate {
}
export interface VoiceInputSettings {
+ /** Explicit speech selection; absent means follow the available catalog. */
+ conversationSelection?: VoiceConversationSelection | null;
language: VoiceInputLanguage;
microphoneDeviceId: string | null;
muteSystemAudio: boolean;
@@ -524,6 +527,8 @@ export function normalizeVoiceInputSettings(
? normalizeVoiceInputShortcut(candidate.shortcut)
: defaults.shortcut;
return {
+ ...(normalizeVoiceConversationSelection(candidate.conversationSelection)
+ ? { conversationSelection: normalizeVoiceConversationSelection(candidate.conversationSelection) } : {}),
language: isVoiceInputLanguage(candidate.language) ? candidate.language : defaults.language,
microphoneDeviceId:
typeof candidate.microphoneDeviceId === 'string' && candidate.microphoneDeviceId.trim().length > 0
diff --git a/apps/desktop/src/shared/voiceInputErrors.ts b/apps/desktop/src/shared/voiceInputErrors.ts
index b49e31197a..40198bdca8 100644
--- a/apps/desktop/src/shared/voiceInputErrors.ts
+++ b/apps/desktop/src/shared/voiceInputErrors.ts
@@ -1,2 +1,3 @@
/** Safe, locale-neutral fallback text carried by the existing voice error payload. */
export const VOICE_INPUT_RATE_LIMITED_MESSAGE = 'Too many voice requests. Wait a moment, then try again.';
+export const VOICE_CONVERSATION_ACTIVE_MESSAGE = 'End voice mode before starting dictation.';
diff --git a/docs/design-rules/design-inventory.md b/docs/design-rules/design-inventory.md
index 54549800f1..c1fa4c38f5 100644
--- a/docs/design-rules/design-inventory.md
+++ b/docs/design-rules/design-inventory.md
@@ -9,7 +9,7 @@
本区块由 `scripts/design-inventory.mjs` 生成,请勿手改。
重新生成:`pnpm design:inventory`;校验:`pnpm check:design-inventory`。
-计数快照日期:2026-10-09。生成命令:`pnpm design:inventory`。裸颜色匹配与 `scripts/hardcoded-color-audit.mjs` 共用 `scripts/shared/hardcoded-color-match.mjs`(HEX / RGB / HSL / OKLCH 等字面通道),两者在共享 matcher 内统一排除纯语义包装、注释及 PR 编号;数值颜色函数还需位于样式属性、CSS 声明/函数或任意值语境,普通文案字符串里的颜色函数文本不计入;字面 fallback 仍计入——语义 token 消费与注释引用不是迁移债务;裸圆角为粗粒度(`rounded*` class、`border-radius:` 与 React style 对象的 `borderRadius:`)。Token 计数为样式源里 `var(--token)` / `hsl(var(--token)` 的去重 ID 数;Mobile 的 ThemeColors / typeScale 等 RN 属性不计入此列,不能将 0 解释为未使用 Token。
+计数快照日期:2026-10-10。生成命令:`pnpm design:inventory`。裸颜色匹配与 `scripts/hardcoded-color-audit.mjs` 共用 `scripts/shared/hardcoded-color-match.mjs`(HEX / RGB / HSL / OKLCH 等字面通道),两者在共享 matcher 内统一排除纯语义包装、注释及 PR 编号;数值颜色函数还需位于样式属性、CSS 声明/函数或任意值语境,普通文案字符串里的颜色函数文本不计入;字面 fallback 仍计入——语义 token 消费与注释引用不是迁移债务;裸圆角为粗粒度(`rounded*` class、`border-radius:` 与 React style 对象的 `borderRadius:`)。Token 计数为样式源里 `var(--token)` / `hsl(var(--token)` 的去重 ID 数;Mobile 的 ThemeColors / typeScale 等 RN 属性不计入此列,不能将 0 解释为未使用 Token。
登记 surface 数:52。平台包含 Desktop 与 Mobile;静态入口发现不表示已迁移或已实机验证。
@@ -18,7 +18,7 @@
| `desktop.auth.add-account` | desktop | 添加账号 | hash `/add-account`(ProtectedRoute → AddAccountLoginPage) | AddAccountLoginPage | apps/desktop/src/renderer/components/login/AddAccountLoginPage.tsx, apps/desktop/src/renderer/components/login/LoginBrandStage.tsx, apps/desktop/src/renderer/components/login/LoginCaptchaOverlay.tsx, apps/desktop/src/renderer/components/login/LoginControls.tsx, apps/desktop/src/renderer/components/login/LoginPage.tsx, apps/desktop/src/renderer/components/login/loginDesignTokens.ts | 59 | 0 | 20 |
| `desktop.auth.legacy-migration` | desktop | 首登数据迁移弹窗 | App 顶层挂载 LegacyMigrationDialog(仅主窗;main 检测到旧版 userData 经 `legacy-migration:state` 驱动) | LegacyMigrationDialog | apps/desktop/src/renderer/components/auth/LegacyMigrationDialog.tsx | 8 | 0 | 2 |
| `desktop.auth.login` | desktop | 登录页 | hash `/login`(GuestRoute → LoginPage) | LoginBrandStage, LoginCaptchaOverlay, LoginControls, LoginPage | apps/desktop/src/renderer/components/login/LoginBrandStage.tsx, apps/desktop/src/renderer/components/login/LoginCaptchaOverlay.tsx, apps/desktop/src/renderer/components/login/LoginControls.tsx, apps/desktop/src/renderer/components/login/LoginPage.tsx, apps/desktop/src/renderer/components/login/loginDesignTokens.ts | 59 | 0 | 20 |
-| `desktop.bots` | desktop | 伙伴(列表 / 对话 / 设置 / 历史 / 伙伴私聊 / 群聊) | hash `/bots`、`/bots/list`、`/bots/:botId`、`/bots/roster` 及伙伴当前/历史任务、伙伴私聊、伙伴群聊路由(BotsFeatureLayout) | BotBasicProfileFields, BotCollaborationCard, BotDirectMessageView, BotGroupChatView, BotGroupCreateDialog, BotGroupSettingsDrawer, BotHistorySessionView, BotLifecycleSettings, BotModelChainEditor, BotRosterView, BotSessionView, BotSettingsDrawer, BotsHomeView, BotsListView, RemoteBotSessionView | apps/desktop/src/renderer/components/chat/AgentActionRow.tsx, apps/desktop/src/renderer/components/chat/AgentActionsBlock.tsx, apps/desktop/src/renderer/components/chat/AgentOnOtherDeviceContext.tsx, apps/desktop/src/renderer/components/chat/AgentTaskCard.tsx, apps/desktop/src/renderer/components/chat/AgentTaskLiveDetails.tsx, apps/desktop/src/renderer/components/chat/AnnotationStrokesSvg.tsx, apps/desktop/src/renderer/components/chat/AskUserQuestionBubble.tsx, apps/desktop/src/renderer/components/chat/AssistantMessage.tsx, apps/desktop/src/renderer/components/chat/AutomationOriginBadge.tsx, apps/desktop/src/renderer/components/chat/ChatAudioCard.tsx, apps/desktop/src/renderer/components/chat/ChatDisplaySnapshotContext.tsx, apps/desktop/src/renderer/components/chat/ChatImageView.tsx, apps/desktop/src/renderer/components/chat/ChatSessionFileContext.tsx, apps/desktop/src/renderer/components/chat/ChatSoundEffectCard.tsx, apps/desktop/src/renderer/components/chat/ChatVideoView.tsx, apps/desktop/src/renderer/components/chat/CindyMakeDoctorCard.tsx, apps/desktop/src/renderer/components/chat/CopyAsImageBlock.tsx, apps/desktop/src/renderer/components/chat/CredentialSwitchWaitBanner.tsx, apps/desktop/src/renderer/components/chat/DiffView.tsx, apps/desktop/src/renderer/components/chat/ErrorBanner.tsx, apps/desktop/src/renderer/components/chat/ErrorMessageCard.tsx, apps/desktop/src/renderer/components/chat/GeneratedFilesCard.tsx, apps/desktop/src/renderer/components/chat/GhostCardHostPrompts.tsx, apps/desktop/src/renderer/components/chat/GhostSummonCard.tsx, apps/desktop/src/renderer/components/chat/GhostToolCard.tsx, apps/desktop/src/renderer/components/chat/HookTaskCard.tsx, apps/desktop/src/renderer/components/chat/ImageGalleryContext.tsx, apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx, apps/desktop/src/renderer/components/chat/ImageLightbox.tsx, apps/desktop/src/renderer/components/chat/ImageMissingPlaceholder.tsx, apps/desktop/src/renderer/components/chat/InlineReferenceChip.tsx, apps/desktop/src/renderer/components/chat/InterruptedTurnBanner.tsx, apps/desktop/src/renderer/components/chat/JumpToBottomChip.tsx, apps/desktop/src/renderer/components/chat/MarkdownDiffBlock.tsx, apps/desktop/src/renderer/components/chat/MarkdownMermaidBlock.tsx, apps/desktop/src/renderer/components/chat/MarkdownRenderer.tsx, apps/desktop/src/renderer/components/chat/MermaidLightbox.tsx, apps/desktop/src/renderer/components/chat/MessageActionBar.tsx, apps/desktop/src/renderer/components/chat/MessageNavRail.tsx, apps/desktop/src/renderer/components/chat/MessageSourceLabels.tsx, apps/desktop/src/renderer/components/chat/MessageStream.tsx, apps/desktop/src/renderer/components/chat/MessageViewport.tsx, apps/desktop/src/renderer/components/chat/ModelLightbox.tsx, apps/desktop/src/renderer/components/chat/NewMessageIndicator.tsx, apps/desktop/src/renderer/components/chat/PlanReviewBubble.tsx, apps/desktop/src/renderer/components/chat/PrevMessageJumpChip.tsx, apps/desktop/src/renderer/components/chat/ProjectLinkChip.tsx, apps/desktop/src/renderer/components/chat/QuoteChip.tsx, apps/desktop/src/renderer/components/chat/RewindPreviewDialog.tsx, apps/desktop/src/renderer/components/chat/SelectionQuoteButton.tsx, apps/desktop/src/renderer/components/chat/SentAgentReferenceChip.tsx, apps/desktop/src/renderer/components/chat/SentInlineAtomBody.tsx, apps/desktop/src/renderer/components/chat/SessionHandoffCard.tsx, apps/desktop/src/renderer/components/chat/SessionLinkChip.tsx, apps/desktop/src/renderer/components/chat/ShareMessageCheckbox.tsx, apps/desktop/src/renderer/components/chat/ShareSelectionBar.tsx, apps/desktop/src/renderer/components/chat/SlackIcon.tsx, apps/desktop/src/renderer/components/chat/StreamFadeSpan.tsx, apps/desktop/src/renderer/components/chat/StreamingMarkdownChunk.tsx, apps/desktop/src/renderer/components/chat/SystemCard.tsx, apps/desktop/src/renderer/components/chat/TextLightbox.tsx, apps/desktop/src/renderer/components/chat/ThinkingCard.tsx, apps/desktop/src/renderer/components/chat/ThinkingText.tsx, apps/desktop/src/renderer/components/chat/TodoListCard.tsx, apps/desktop/src/renderer/components/chat/ToolCallCard.tsx, apps/desktop/src/renderer/components/chat/ToolPayloadLightbox.tsx, apps/desktop/src/renderer/components/chat/TopRightChipStack.tsx, apps/desktop/src/renderer/components/chat/TurnChangesCard.tsx, apps/desktop/src/renderer/components/chat/UnreadFailedScheduleBanner.tsx, apps/desktop/src/renderer/components/chat/UpgradeBanner.tsx, apps/desktop/src/renderer/components/chat/UserAttachmentChip.tsx, apps/desktop/src/renderer/components/chat/UserMessage.tsx, apps/desktop/src/renderer/components/chat/UserMessageEditBox.tsx, apps/desktop/src/renderer/components/chat/UserMessageUrlLink.tsx, apps/desktop/src/renderer/components/chat/VideoLightbox.tsx, apps/desktop/src/renderer/components/chat/WorkGroupBlock.tsx, apps/desktop/src/renderer/components/chat/WorktreeRestoreBanner.tsx, apps/desktop/src/renderer/components/chat/XIcon.tsx, apps/desktop/src/renderer/components/chat/activityRowChrome.ts, apps/desktop/src/renderer/components/chat/animateFocusScroll.ts, apps/desktop/src/renderer/components/chat/autoFollowIntent.ts, apps/desktop/src/renderer/components/chat/chatChrome.ts, apps/desktop/src/renderer/components/chat/errorReasonI18n.ts, apps/desktop/src/renderer/components/chat/generatedFilesProjection.ts, apps/desktop/src/renderer/components/chat/imageDisplaySize.ts, apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts, apps/desktop/src/renderer/components/chat/lightboxGestures.ts, apps/desktop/src/renderer/components/chat/markdownImageTargets.ts, apps/desktop/src/renderer/components/chat/markdownMedia.ts, apps/desktop/src/renderer/components/chat/messageNavRailModel.ts, apps/desktop/src/renderer/components/chat/messageNavRailWheel.ts, apps/desktop/src/renderer/components/chat/messageNavigationProjection.ts, apps/desktop/src/renderer/components/chat/messageProjectionCache.ts, apps/desktop/src/renderer/components/chat/messageStreamIndicatorPosition.ts, apps/desktop/src/renderer/components/chat/messageViewportCompensation.ts, apps/desktop/src/renderer/components/chat/messageViewportGeometry.ts, apps/desktop/src/renderer/components/chat/messageWorkGroups.ts, apps/desktop/src/renderer/components/chat/observeProgrammaticScroll.ts, apps/desktop/src/renderer/components/chat/rehypeFencedCodeMarker.ts, apps/desktop/src/renderer/components/chat/rehypeHighlightShared.ts, apps/desktop/src/renderer/components/chat/rehypeMathBlockMarker.ts, apps/desktop/src/renderer/components/chat/rehypeStreamWordFade.ts, apps/desktop/src/renderer/components/chat/remarkHtmlImages.ts, apps/desktop/src/renderer/components/chat/remarkLocalPathLinks.ts, apps/desktop/src/renderer/components/chat/remarkPreserveRawLocalDestinations.ts, apps/desktop/src/renderer/components/chat/remarkSessionLinks.ts, apps/desktop/src/renderer/components/chat/remarkStrictInlineMath.ts, apps/desktop/src/renderer/components/chat/remarkTruncateCjkUrls.ts, apps/desktop/src/renderer/components/chat/repairStreamingMarkdown.ts, apps/desktop/src/renderer/components/chat/scrollAnchoringDetect.ts, apps/desktop/src/renderer/components/chat/shareSelectionStore.ts, apps/desktop/src/renderer/components/chat/streamingMarkdownBlocks.tsx, apps/desktop/src/renderer/components/chat/streamingMarkdownChunks.ts, apps/desktop/src/renderer/components/chat/streamingMessageProjection.ts, apps/desktop/src/renderer/components/chat/toolLoopI18n.ts, apps/desktop/src/renderer/components/chat/unreadCount.ts, apps/desktop/src/renderer/components/chat/useAutomaticHistoryLoadBudget.ts, apps/desktop/src/renderer/components/chat/useCopyAsImage.tsx, apps/desktop/src/renderer/components/chat/useDeleteMessage.ts, apps/desktop/src/renderer/components/chat/useFileChipContextMenu.tsx, apps/desktop/src/renderer/components/chat/useForkAtMessage.ts, apps/desktop/src/renderer/components/chat/useImageClipboard.ts, apps/desktop/src/renderer/components/chat/useImageLoadFailure.ts, apps/desktop/src/renderer/components/chat/useMessageViewport.ts, apps/desktop/src/renderer/components/chat/useNavigationKeyListener.ts, apps/desktop/src/renderer/components/chat/useOpenWithMenu.tsx, apps/desktop/src/renderer/components/chat/usePrevUserMessageInView.ts, apps/desktop/src/renderer/components/chat/useTurnChangeSets.ts, apps/desktop/src/renderer/components/chat/userMessageCollapse.ts, apps/desktop/src/renderer/components/chat/userMessageDisplayText.ts, apps/desktop/src/renderer/components/chat/userMessageLinkify.ts, apps/desktop/src/renderer/components/chat/viewportFillDetect.ts, apps/desktop/src/renderer/components/new-chat/AddRemoteProjectDialog.tsx, apps/desktop/src/renderer/components/new-chat/AgentSelect.tsx, apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx, apps/desktop/src/renderer/components/new-chat/AtMentionPanel.tsx, apps/desktop/src/renderer/components/new-chat/AttachmentTypeThumb.tsx, apps/desktop/src/renderer/components/new-chat/ChatInput.tsx, apps/desktop/src/renderer/components/new-chat/CjkPunctDecoration.ts, apps/desktop/src/renderer/components/new-chat/CjkPunctuationUtils.ts, apps/desktop/src/renderer/components/new-chat/ComposerAttachments.tsx, apps/desktop/src/renderer/components/new-chat/ComposerListIndentDecoration.ts, apps/desktop/src/renderer/components/new-chat/ComposerListNodes.ts, apps/desktop/src/renderer/components/new-chat/ComposerQuoteNode.tsx, apps/desktop/src/renderer/components/new-chat/DeviceSwitcherPill.tsx, apps/desktop/src/renderer/components/new-chat/EffortSlider.tsx, apps/desktop/src/renderer/components/new-chat/EmptyDocSelectionGuard.ts, apps/desktop/src/renderer/components/new-chat/ExtraDirsButton.tsx, apps/desktop/src/renderer/components/new-chat/FastModeToggle.tsx, apps/desktop/src/renderer/components/new-chat/FolderPickerPopover.tsx, apps/desktop/src/renderer/components/new-chat/FullAccessConfirmContent.tsx, apps/desktop/src/renderer/components/new-chat/GhostCommandDecoration.ts, apps/desktop/src/renderer/components/new-chat/GoalAdvancedLimits.tsx, apps/desktop/src/renderer/components/new-chat/GoalIndicator.tsx, apps/desktop/src/renderer/components/new-chat/HomeUsageDashboard.tsx, apps/desktop/src/renderer/components/new-chat/ListComposerTextarea.tsx, apps/desktop/src/renderer/components/new-chat/LocalModelCatalogNotice.tsx, apps/desktop/src/renderer/components/new-chat/MentionChipNode.tsx, apps/desktop/src/renderer/components/new-chat/MentionDragCaretDecoration.ts, apps/desktop/src/renderer/components/new-chat/ModelCompatibilityNotice.tsx, apps/desktop/src/renderer/components/new-chat/ModelConfigFlyout.tsx, apps/desktop/src/renderer/components/new-chat/ModelHarnessPicker.tsx, apps/desktop/src/renderer/components/new-chat/ModelSelector.tsx, apps/desktop/src/renderer/components/new-chat/ModelSourceDetails.tsx, apps/desktop/src/renderer/components/new-chat/NewGoalDialog.tsx, apps/desktop/src/renderer/components/new-chat/PastedTextChipNode.tsx, apps/desktop/src/renderer/components/new-chat/PendingQueuePanel.tsx, apps/desktop/src/renderer/components/new-chat/PermissionPrompt.tsx, apps/desktop/src/renderer/components/new-chat/PermissionSelector.tsx, apps/desktop/src/renderer/components/new-chat/PinnedPlanPanel.tsx, apps/desktop/src/renderer/components/new-chat/PlanActionCard.tsx, apps/desktop/src/renderer/components/new-chat/PlanModeIndicator.tsx, apps/desktop/src/renderer/components/new-chat/PlanViewerCard.tsx, apps/desktop/src/renderer/components/new-chat/PluginConnectionForm.tsx, apps/desktop/src/renderer/components/new-chat/PluginOauthDeviceCode.tsx, apps/desktop/src/renderer/components/new-chat/PluginSetupPrompt.tsx, apps/desktop/src/renderer/components/new-chat/PluginSetupRemoteDesktopButton.tsx, apps/desktop/src/renderer/components/new-chat/PluginWriteAccessRecovery.tsx, apps/desktop/src/renderer/components/new-chat/QuickStartPillMark.ts, apps/desktop/src/renderer/components/new-chat/RemoteOauthSetupCard.tsx, apps/desktop/src/renderer/components/new-chat/SendButton.tsx, apps/desktop/src/renderer/components/new-chat/SlashCommandDecoration.ts, apps/desktop/src/renderer/components/new-chat/SlashCommandPalette.tsx, apps/desktop/src/renderer/components/new-chat/TakeoverMask.tsx, apps/desktop/src/renderer/components/new-chat/ThinkingToggle.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedFlyoutHost.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelPanel.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelRail.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelRow.tsx, apps/desktop/src/renderer/components/new-chat/UsageDailyBars.tsx, apps/desktop/src/renderer/components/new-chat/UsageHeatmap.tsx, apps/desktop/src/renderer/components/new-chat/VendorSegmentedSwitcher.tsx, apps/desktop/src/renderer/components/new-chat/VoiceInputDraftDecoration.ts, apps/desktop/src/renderer/components/new-chat/WindowsSelectionReplacement.ts, apps/desktop/src/renderer/components/new-chat/WorktreeChipsRow.tsx, apps/desktop/src/renderer/components/new-chat/WorktreeCreatingOverlay.tsx, apps/desktop/src/renderer/components/new-chat/agentOptions.ts, apps/desktop/src/renderer/components/new-chat/agentSwitchConfirmation.ts, apps/desktop/src/renderer/components/new-chat/branchPick.ts, apps/desktop/src/renderer/components/new-chat/composerBlankPointerFocus.ts, apps/desktop/src/renderer/components/new-chat/composerContentSerialization.ts, apps/desktop/src/renderer/components/new-chat/composerDocState.ts, apps/desktop/src/renderer/components/new-chat/composerFrameScheduler.ts, apps/desktop/src/renderer/components/new-chat/composerHistoryProjection.ts, apps/desktop/src/renderer/components/new-chat/composerModelSelection.ts, apps/desktop/src/renderer/components/new-chat/composerPromptRecommendation.ts, apps/desktop/src/renderer/components/new-chat/composerRenderGate.ts, apps/desktop/src/renderer/components/new-chat/composerSendOwnership.ts, apps/desktop/src/renderer/components/new-chat/composerSendSnapshot.ts, apps/desktop/src/renderer/components/new-chat/effortChangeQueue.ts, apps/desktop/src/renderer/components/new-chat/extraDirsActions.ts, apps/desktop/src/renderer/components/new-chat/ghostComposerPlacement.ts, apps/desktop/src/renderer/components/new-chat/localAttachmentPicker.ts, apps/desktop/src/renderer/components/new-chat/mentionChipInsertion.ts, apps/desktop/src/renderer/components/new-chat/modelWindowErrorToast.ts, apps/desktop/src/renderer/components/new-chat/pastePipeline.ts, apps/desktop/src/renderer/components/new-chat/pathPaste.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueLocks.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueRowActivity.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueRowPresentation.ts, apps/desktop/src/renderer/components/new-chat/planModeComposerCommand.ts, apps/desktop/src/renderer/components/new-chat/priceTierMarks.tsx, apps/desktop/src/renderer/components/new-chat/remoteBrowseAdapters.ts, apps/desktop/src/renderer/components/new-chat/remoteExistingProjects.ts, apps/desktop/src/renderer/components/new-chat/restoreComposerDocument.ts, apps/desktop/src/renderer/components/new-chat/sessionLinkPaste.ts, apps/desktop/src/renderer/components/new-chat/sourceSwitch.ts, apps/desktop/src/renderer/components/new-chat/unifiedModelSelection.ts, apps/desktop/src/renderer/components/new-chat/usagePalette.ts, apps/desktop/src/renderer/components/new-chat/useComposerSendFocusRestore.ts, apps/desktop/src/renderer/components/new-chat/useModelDiscoveryPending.ts, apps/desktop/src/renderer/components/new-chat/useProviderWeeklyQuota.ts, apps/desktop/src/renderer/components/new-chat/useUnifiedRowActions.ts, apps/desktop/src/renderer/features/bots/BotAuthorizationCard.tsx, apps/desktop/src/renderer/features/bots/BotAvatar.tsx, apps/desktop/src/renderer/features/bots/BotBasicProfileFields.tsx, apps/desktop/src/renderer/features/bots/BotCapabilitySettings.tsx, apps/desktop/src/renderer/features/bots/BotCollaborationCard.tsx, apps/desktop/src/renderer/features/bots/BotConnectionStatus.tsx, apps/desktop/src/renderer/features/bots/BotCreateMenu.tsx, apps/desktop/src/renderer/features/bots/BotDeleteDialog.tsx, apps/desktop/src/renderer/features/bots/BotDirectMessageCard.tsx, apps/desktop/src/renderer/features/bots/BotDirectMessageView.tsx, apps/desktop/src/renderer/features/bots/BotFollowMark.tsx, apps/desktop/src/renderer/features/bots/BotGenerationLabel.tsx, apps/desktop/src/renderer/features/bots/BotGroupAvatars.tsx, apps/desktop/src/renderer/features/bots/BotGroupChatView.tsx, apps/desktop/src/renderer/features/bots/BotGroupComposer.tsx, apps/desktop/src/renderer/features/bots/BotGroupCreateDialog.tsx, apps/desktop/src/renderer/features/bots/BotGroupPendingInteraction.tsx, apps/desktop/src/renderer/features/bots/BotGroupPlan.tsx, apps/desktop/src/renderer/features/bots/BotGroupSettingsDrawer.tsx, apps/desktop/src/renderer/features/bots/BotGroupSidebarRow.tsx, apps/desktop/src/renderer/features/bots/BotHistorySessionView.tsx, apps/desktop/src/renderer/features/bots/BotImportForm.tsx, apps/desktop/src/renderer/features/bots/BotInvitationWelcome.tsx, apps/desktop/src/renderer/features/bots/BotLearningFooter.tsx, apps/desktop/src/renderer/features/bots/BotLifecycleSettings.tsx, apps/desktop/src/renderer/features/bots/BotMemorySettings.tsx, apps/desktop/src/renderer/features/bots/BotModelChainEditor.tsx, apps/desktop/src/renderer/features/bots/BotPortraitPicker.tsx, apps/desktop/src/renderer/features/bots/BotRosterView.tsx, apps/desktop/src/renderer/features/bots/BotRoutines.tsx, apps/desktop/src/renderer/features/bots/BotSessionContentHeader.tsx, apps/desktop/src/renderer/features/bots/BotSessionTaskLink.tsx, apps/desktop/src/renderer/features/bots/BotSessionTaskResultCard.tsx, apps/desktop/src/renderer/features/bots/BotSessionView.tsx, apps/desktop/src/renderer/features/bots/BotSettingsBlock.tsx, apps/desktop/src/renderer/features/bots/BotSettingsDrawer.tsx, apps/desktop/src/renderer/features/bots/BotTaskCardHeader.tsx, apps/desktop/src/renderer/features/bots/BotTaskModelEditor.tsx, apps/desktop/src/renderer/features/bots/BotWorkbench.tsx, apps/desktop/src/renderer/features/bots/BotWorkingStatus.tsx, apps/desktop/src/renderer/features/bots/BotsFeatureLayout.tsx, apps/desktop/src/renderer/features/bots/BotsGlobalSettingsSection.tsx, apps/desktop/src/renderer/features/bots/BotsHomeView.tsx, apps/desktop/src/renderer/features/bots/BotsListView.tsx, apps/desktop/src/renderer/features/bots/BotsSidebar.tsx, apps/desktop/src/renderer/features/bots/ChatBotAccessField.tsx, apps/desktop/src/renderer/features/bots/ChatGroupSettings.tsx, apps/desktop/src/renderer/features/bots/ChatInviteDialog.tsx, apps/desktop/src/renderer/features/bots/ChatInviteHost.tsx, apps/desktop/src/renderer/features/bots/ChatServerControls.tsx, apps/desktop/src/renderer/features/bots/ChatThreadPanel.tsx, apps/desktop/src/renderer/features/bots/CindyDevicePicker.tsx, apps/desktop/src/renderer/features/bots/CindyDeviceRow.tsx, apps/desktop/src/renderer/features/bots/RemoteBotSessionView.tsx, apps/desktop/src/renderer/features/bots/RemoteBotSettings.tsx, apps/desktop/src/renderer/features/bots/WorkbenchSessionList.tsx, apps/desktop/src/renderer/features/bots/botAvatarIdentity.ts, apps/desktop/src/renderer/features/bots/botCapabilityDefaults.ts, apps/desktop/src/renderer/features/bots/botChatPresentation.ts, apps/desktop/src/renderer/features/bots/botConversationPresentation.ts, apps/desktop/src/renderer/features/bots/botConversationTimeline.ts, apps/desktop/src/renderer/features/bots/botDelegationLive.ts, apps/desktop/src/renderer/features/bots/botFollowScopes.ts, apps/desktop/src/renderer/features/bots/botGeneratedArtifacts.ts, apps/desktop/src/renderer/features/bots/botGroupAttachments.ts, apps/desktop/src/renderer/features/bots/botGroupLane.ts, apps/desktop/src/renderer/features/bots/botGroupMentions.ts, apps/desktop/src/renderer/features/bots/botGroupPresentation.ts, apps/desktop/src/renderer/features/bots/botGroupStore.ts, apps/desktop/src/renderer/features/bots/botListDisplay.ts, apps/desktop/src/renderer/features/bots/botNavigation.ts, apps/desktop/src/renderer/features/bots/botPronounContext.tsx, apps/desktop/src/renderer/features/bots/botReadState.ts, apps/desktop/src/renderer/features/bots/botRosterDisplay.ts, apps/desktop/src/renderer/features/bots/botSessionOwners.ts, apps/desktop/src/renderer/features/bots/botSettingsAutosave.ts, apps/desktop/src/renderer/features/bots/botStore.ts, apps/desktop/src/renderer/features/bots/botWelcomeContext.ts, apps/desktop/src/renderer/features/bots/botWorkbenchProjection.ts, apps/desktop/src/renderer/features/bots/chatError.ts, apps/desktop/src/renderer/features/bots/cindyDeviceRoster.ts, apps/desktop/src/renderer/features/bots/remoteBotRoster.ts, apps/desktop/src/renderer/features/bots/remoteBotSettingsData.ts, apps/desktop/src/renderer/features/bots/useBotGeneratedFileDeliveries.ts, apps/desktop/src/renderer/features/bots/useBotIslandVisibleSession.ts, apps/desktop/src/renderer/features/bots/useBotSettingsAutosave.ts, apps/desktop/src/renderer/features/bots/useBotUnreadSync.ts, apps/desktop/src/renderer/features/bots/useRemoteBots.ts, apps/desktop/src/renderer/features/bots/useWorkbenchSessionPages.ts, apps/desktop/src/renderer/features/bots/useWorkingStatusCopy.ts, apps/desktop/src/renderer/features/cc-agent/CCAgentFeatureLayout.tsx, apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx | 277 | 38 | 468 |
+| `desktop.bots` | desktop | 伙伴(列表 / 对话 / 设置 / 历史 / 伙伴私聊 / 群聊) | hash `/bots`、`/bots/list`、`/bots/:botId`、`/bots/roster` 及伙伴当前/历史任务、伙伴私聊、伙伴群聊路由(BotsFeatureLayout) | BotBasicProfileFields, BotCollaborationCard, BotDirectMessageView, BotGroupChatView, BotGroupCreateDialog, BotGroupSettingsDrawer, BotHistorySessionView, BotLifecycleSettings, BotModelChainEditor, BotRosterView, BotSessionView, BotSettingsDrawer, BotsHomeView, BotsListView, RemoteBotSessionView | apps/desktop/src/renderer/components/chat/AgentActionRow.tsx, apps/desktop/src/renderer/components/chat/AgentActionsBlock.tsx, apps/desktop/src/renderer/components/chat/AgentOnOtherDeviceContext.tsx, apps/desktop/src/renderer/components/chat/AgentTaskCard.tsx, apps/desktop/src/renderer/components/chat/AgentTaskLiveDetails.tsx, apps/desktop/src/renderer/components/chat/AnnotationStrokesSvg.tsx, apps/desktop/src/renderer/components/chat/AskUserQuestionBubble.tsx, apps/desktop/src/renderer/components/chat/AssistantMessage.tsx, apps/desktop/src/renderer/components/chat/AutomationOriginBadge.tsx, apps/desktop/src/renderer/components/chat/ChatAudioCard.tsx, apps/desktop/src/renderer/components/chat/ChatDisplaySnapshotContext.tsx, apps/desktop/src/renderer/components/chat/ChatImageView.tsx, apps/desktop/src/renderer/components/chat/ChatSessionFileContext.tsx, apps/desktop/src/renderer/components/chat/ChatSoundEffectCard.tsx, apps/desktop/src/renderer/components/chat/ChatVideoView.tsx, apps/desktop/src/renderer/components/chat/CindyMakeDoctorCard.tsx, apps/desktop/src/renderer/components/chat/CopyAsImageBlock.tsx, apps/desktop/src/renderer/components/chat/CredentialSwitchWaitBanner.tsx, apps/desktop/src/renderer/components/chat/DiffView.tsx, apps/desktop/src/renderer/components/chat/ErrorBanner.tsx, apps/desktop/src/renderer/components/chat/ErrorMessageCard.tsx, apps/desktop/src/renderer/components/chat/GeneratedFilesCard.tsx, apps/desktop/src/renderer/components/chat/GhostCardHostPrompts.tsx, apps/desktop/src/renderer/components/chat/GhostSummonCard.tsx, apps/desktop/src/renderer/components/chat/GhostToolCard.tsx, apps/desktop/src/renderer/components/chat/HookTaskCard.tsx, apps/desktop/src/renderer/components/chat/ImageGalleryContext.tsx, apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx, apps/desktop/src/renderer/components/chat/ImageLightbox.tsx, apps/desktop/src/renderer/components/chat/ImageMissingPlaceholder.tsx, apps/desktop/src/renderer/components/chat/InlineReferenceChip.tsx, apps/desktop/src/renderer/components/chat/InterruptedTurnBanner.tsx, apps/desktop/src/renderer/components/chat/JumpToBottomChip.tsx, apps/desktop/src/renderer/components/chat/MarkdownDiffBlock.tsx, apps/desktop/src/renderer/components/chat/MarkdownMermaidBlock.tsx, apps/desktop/src/renderer/components/chat/MarkdownRenderer.tsx, apps/desktop/src/renderer/components/chat/MermaidLightbox.tsx, apps/desktop/src/renderer/components/chat/MessageActionBar.tsx, apps/desktop/src/renderer/components/chat/MessageNavRail.tsx, apps/desktop/src/renderer/components/chat/MessageSourceLabels.tsx, apps/desktop/src/renderer/components/chat/MessageStream.tsx, apps/desktop/src/renderer/components/chat/MessageViewport.tsx, apps/desktop/src/renderer/components/chat/ModelLightbox.tsx, apps/desktop/src/renderer/components/chat/NewMessageIndicator.tsx, apps/desktop/src/renderer/components/chat/PlanReviewBubble.tsx, apps/desktop/src/renderer/components/chat/PrevMessageJumpChip.tsx, apps/desktop/src/renderer/components/chat/ProjectLinkChip.tsx, apps/desktop/src/renderer/components/chat/QuoteChip.tsx, apps/desktop/src/renderer/components/chat/RewindPreviewDialog.tsx, apps/desktop/src/renderer/components/chat/SelectionQuoteButton.tsx, apps/desktop/src/renderer/components/chat/SentAgentReferenceChip.tsx, apps/desktop/src/renderer/components/chat/SentInlineAtomBody.tsx, apps/desktop/src/renderer/components/chat/SessionHandoffCard.tsx, apps/desktop/src/renderer/components/chat/SessionLinkChip.tsx, apps/desktop/src/renderer/components/chat/ShareMessageCheckbox.tsx, apps/desktop/src/renderer/components/chat/ShareSelectionBar.tsx, apps/desktop/src/renderer/components/chat/SlackIcon.tsx, apps/desktop/src/renderer/components/chat/StreamFadeSpan.tsx, apps/desktop/src/renderer/components/chat/StreamingMarkdownChunk.tsx, apps/desktop/src/renderer/components/chat/SystemCard.tsx, apps/desktop/src/renderer/components/chat/TextLightbox.tsx, apps/desktop/src/renderer/components/chat/ThinkingCard.tsx, apps/desktop/src/renderer/components/chat/ThinkingText.tsx, apps/desktop/src/renderer/components/chat/TodoListCard.tsx, apps/desktop/src/renderer/components/chat/ToolCallCard.tsx, apps/desktop/src/renderer/components/chat/ToolPayloadLightbox.tsx, apps/desktop/src/renderer/components/chat/TopRightChipStack.tsx, apps/desktop/src/renderer/components/chat/TurnChangesCard.tsx, apps/desktop/src/renderer/components/chat/UnreadFailedScheduleBanner.tsx, apps/desktop/src/renderer/components/chat/UpgradeBanner.tsx, apps/desktop/src/renderer/components/chat/UserAttachmentChip.tsx, apps/desktop/src/renderer/components/chat/UserMessage.tsx, apps/desktop/src/renderer/components/chat/UserMessageEditBox.tsx, apps/desktop/src/renderer/components/chat/UserMessageUrlLink.tsx, apps/desktop/src/renderer/components/chat/VideoLightbox.tsx, apps/desktop/src/renderer/components/chat/WorkGroupBlock.tsx, apps/desktop/src/renderer/components/chat/WorktreeRestoreBanner.tsx, apps/desktop/src/renderer/components/chat/XIcon.tsx, apps/desktop/src/renderer/components/chat/activityRowChrome.ts, apps/desktop/src/renderer/components/chat/animateFocusScroll.ts, apps/desktop/src/renderer/components/chat/autoFollowIntent.ts, apps/desktop/src/renderer/components/chat/chatChrome.ts, apps/desktop/src/renderer/components/chat/errorReasonI18n.ts, apps/desktop/src/renderer/components/chat/generatedFilesProjection.ts, apps/desktop/src/renderer/components/chat/imageDisplaySize.ts, apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts, apps/desktop/src/renderer/components/chat/lightboxGestures.ts, apps/desktop/src/renderer/components/chat/markdownImageTargets.ts, apps/desktop/src/renderer/components/chat/markdownMedia.ts, apps/desktop/src/renderer/components/chat/messageNavRailModel.ts, apps/desktop/src/renderer/components/chat/messageNavRailWheel.ts, apps/desktop/src/renderer/components/chat/messageNavigationProjection.ts, apps/desktop/src/renderer/components/chat/messageProjectionCache.ts, apps/desktop/src/renderer/components/chat/messageStreamIndicatorPosition.ts, apps/desktop/src/renderer/components/chat/messageViewportCompensation.ts, apps/desktop/src/renderer/components/chat/messageViewportGeometry.ts, apps/desktop/src/renderer/components/chat/messageWorkGroups.ts, apps/desktop/src/renderer/components/chat/observeProgrammaticScroll.ts, apps/desktop/src/renderer/components/chat/rehypeFencedCodeMarker.ts, apps/desktop/src/renderer/components/chat/rehypeHighlightShared.ts, apps/desktop/src/renderer/components/chat/rehypeMathBlockMarker.ts, apps/desktop/src/renderer/components/chat/rehypeStreamWordFade.ts, apps/desktop/src/renderer/components/chat/remarkHtmlImages.ts, apps/desktop/src/renderer/components/chat/remarkLocalPathLinks.ts, apps/desktop/src/renderer/components/chat/remarkPreserveRawLocalDestinations.ts, apps/desktop/src/renderer/components/chat/remarkSessionLinks.ts, apps/desktop/src/renderer/components/chat/remarkStrictInlineMath.ts, apps/desktop/src/renderer/components/chat/remarkTruncateCjkUrls.ts, apps/desktop/src/renderer/components/chat/repairStreamingMarkdown.ts, apps/desktop/src/renderer/components/chat/scrollAnchoringDetect.ts, apps/desktop/src/renderer/components/chat/shareSelectionStore.ts, apps/desktop/src/renderer/components/chat/streamingMarkdownBlocks.tsx, apps/desktop/src/renderer/components/chat/streamingMarkdownChunks.ts, apps/desktop/src/renderer/components/chat/streamingMessageProjection.ts, apps/desktop/src/renderer/components/chat/toolLoopI18n.ts, apps/desktop/src/renderer/components/chat/unreadCount.ts, apps/desktop/src/renderer/components/chat/useAutomaticHistoryLoadBudget.ts, apps/desktop/src/renderer/components/chat/useCopyAsImage.tsx, apps/desktop/src/renderer/components/chat/useDeleteMessage.ts, apps/desktop/src/renderer/components/chat/useFileChipContextMenu.tsx, apps/desktop/src/renderer/components/chat/useForkAtMessage.ts, apps/desktop/src/renderer/components/chat/useImageClipboard.ts, apps/desktop/src/renderer/components/chat/useImageLoadFailure.ts, apps/desktop/src/renderer/components/chat/useMessageViewport.ts, apps/desktop/src/renderer/components/chat/useNavigationKeyListener.ts, apps/desktop/src/renderer/components/chat/useOpenWithMenu.tsx, apps/desktop/src/renderer/components/chat/usePrevUserMessageInView.ts, apps/desktop/src/renderer/components/chat/useTurnChangeSets.ts, apps/desktop/src/renderer/components/chat/userMessageCollapse.ts, apps/desktop/src/renderer/components/chat/userMessageDisplayText.ts, apps/desktop/src/renderer/components/chat/userMessageLinkify.ts, apps/desktop/src/renderer/components/chat/viewportFillDetect.ts, apps/desktop/src/renderer/components/new-chat/AddRemoteProjectDialog.tsx, apps/desktop/src/renderer/components/new-chat/AgentSelect.tsx, apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx, apps/desktop/src/renderer/components/new-chat/AtMentionPanel.tsx, apps/desktop/src/renderer/components/new-chat/AttachmentTypeThumb.tsx, apps/desktop/src/renderer/components/new-chat/ChatInput.tsx, apps/desktop/src/renderer/components/new-chat/CjkPunctDecoration.ts, apps/desktop/src/renderer/components/new-chat/CjkPunctuationUtils.ts, apps/desktop/src/renderer/components/new-chat/ComposerAttachments.tsx, apps/desktop/src/renderer/components/new-chat/ComposerListIndentDecoration.ts, apps/desktop/src/renderer/components/new-chat/ComposerListNodes.ts, apps/desktop/src/renderer/components/new-chat/ComposerQuoteNode.tsx, apps/desktop/src/renderer/components/new-chat/DeviceSwitcherPill.tsx, apps/desktop/src/renderer/components/new-chat/EffortSlider.tsx, apps/desktop/src/renderer/components/new-chat/EmptyDocSelectionGuard.ts, apps/desktop/src/renderer/components/new-chat/ExtraDirsButton.tsx, apps/desktop/src/renderer/components/new-chat/FastModeToggle.tsx, apps/desktop/src/renderer/components/new-chat/FolderPickerPopover.tsx, apps/desktop/src/renderer/components/new-chat/FullAccessConfirmContent.tsx, apps/desktop/src/renderer/components/new-chat/GhostCommandDecoration.ts, apps/desktop/src/renderer/components/new-chat/GoalAdvancedLimits.tsx, apps/desktop/src/renderer/components/new-chat/GoalIndicator.tsx, apps/desktop/src/renderer/components/new-chat/HomeUsageDashboard.tsx, apps/desktop/src/renderer/components/new-chat/ListComposerTextarea.tsx, apps/desktop/src/renderer/components/new-chat/LocalModelCatalogNotice.tsx, apps/desktop/src/renderer/components/new-chat/MentionChipNode.tsx, apps/desktop/src/renderer/components/new-chat/MentionDragCaretDecoration.ts, apps/desktop/src/renderer/components/new-chat/ModelCompatibilityNotice.tsx, apps/desktop/src/renderer/components/new-chat/ModelConfigFlyout.tsx, apps/desktop/src/renderer/components/new-chat/ModelHarnessPicker.tsx, apps/desktop/src/renderer/components/new-chat/ModelSelector.tsx, apps/desktop/src/renderer/components/new-chat/ModelSourceDetails.tsx, apps/desktop/src/renderer/components/new-chat/NewGoalDialog.tsx, apps/desktop/src/renderer/components/new-chat/PastedTextChipNode.tsx, apps/desktop/src/renderer/components/new-chat/PendingQueuePanel.tsx, apps/desktop/src/renderer/components/new-chat/PermissionPrompt.tsx, apps/desktop/src/renderer/components/new-chat/PermissionSelector.tsx, apps/desktop/src/renderer/components/new-chat/PinnedPlanPanel.tsx, apps/desktop/src/renderer/components/new-chat/PlanActionCard.tsx, apps/desktop/src/renderer/components/new-chat/PlanModeIndicator.tsx, apps/desktop/src/renderer/components/new-chat/PlanViewerCard.tsx, apps/desktop/src/renderer/components/new-chat/PluginConnectionForm.tsx, apps/desktop/src/renderer/components/new-chat/PluginOauthDeviceCode.tsx, apps/desktop/src/renderer/components/new-chat/PluginSetupPrompt.tsx, apps/desktop/src/renderer/components/new-chat/PluginSetupRemoteDesktopButton.tsx, apps/desktop/src/renderer/components/new-chat/PluginWriteAccessRecovery.tsx, apps/desktop/src/renderer/components/new-chat/QuickStartPillMark.ts, apps/desktop/src/renderer/components/new-chat/RemoteOauthSetupCard.tsx, apps/desktop/src/renderer/components/new-chat/SendButton.tsx, apps/desktop/src/renderer/components/new-chat/SlashCommandDecoration.ts, apps/desktop/src/renderer/components/new-chat/SlashCommandPalette.tsx, apps/desktop/src/renderer/components/new-chat/TakeoverMask.tsx, apps/desktop/src/renderer/components/new-chat/ThinkingToggle.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedFlyoutHost.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelPanel.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelRail.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelRow.tsx, apps/desktop/src/renderer/components/new-chat/UsageDailyBars.tsx, apps/desktop/src/renderer/components/new-chat/UsageHeatmap.tsx, apps/desktop/src/renderer/components/new-chat/VendorSegmentedSwitcher.tsx, apps/desktop/src/renderer/components/new-chat/VoiceInputDraftDecoration.ts, apps/desktop/src/renderer/components/new-chat/WindowsSelectionReplacement.ts, apps/desktop/src/renderer/components/new-chat/WorktreeChipsRow.tsx, apps/desktop/src/renderer/components/new-chat/WorktreeCreatingOverlay.tsx, apps/desktop/src/renderer/components/new-chat/agentOptions.ts, apps/desktop/src/renderer/components/new-chat/agentSwitchConfirmation.ts, apps/desktop/src/renderer/components/new-chat/branchPick.ts, apps/desktop/src/renderer/components/new-chat/composerBlankPointerFocus.ts, apps/desktop/src/renderer/components/new-chat/composerContentSerialization.ts, apps/desktop/src/renderer/components/new-chat/composerDocState.ts, apps/desktop/src/renderer/components/new-chat/composerFrameScheduler.ts, apps/desktop/src/renderer/components/new-chat/composerHistoryProjection.ts, apps/desktop/src/renderer/components/new-chat/composerModelSelection.ts, apps/desktop/src/renderer/components/new-chat/composerPromptRecommendation.ts, apps/desktop/src/renderer/components/new-chat/composerRenderGate.ts, apps/desktop/src/renderer/components/new-chat/composerSendOwnership.ts, apps/desktop/src/renderer/components/new-chat/composerSendSnapshot.ts, apps/desktop/src/renderer/components/new-chat/effortChangeQueue.ts, apps/desktop/src/renderer/components/new-chat/extraDirsActions.ts, apps/desktop/src/renderer/components/new-chat/ghostComposerPlacement.ts, apps/desktop/src/renderer/components/new-chat/localAttachmentPicker.ts, apps/desktop/src/renderer/components/new-chat/mentionChipInsertion.ts, apps/desktop/src/renderer/components/new-chat/modelWindowErrorToast.ts, apps/desktop/src/renderer/components/new-chat/pastePipeline.ts, apps/desktop/src/renderer/components/new-chat/pathPaste.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueLocks.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueRowActivity.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueRowPresentation.ts, apps/desktop/src/renderer/components/new-chat/planModeComposerCommand.ts, apps/desktop/src/renderer/components/new-chat/priceTierMarks.tsx, apps/desktop/src/renderer/components/new-chat/remoteBrowseAdapters.ts, apps/desktop/src/renderer/components/new-chat/remoteExistingProjects.ts, apps/desktop/src/renderer/components/new-chat/restoreComposerDocument.ts, apps/desktop/src/renderer/components/new-chat/sessionLinkPaste.ts, apps/desktop/src/renderer/components/new-chat/sourceSwitch.ts, apps/desktop/src/renderer/components/new-chat/unifiedModelSelection.ts, apps/desktop/src/renderer/components/new-chat/usagePalette.ts, apps/desktop/src/renderer/components/new-chat/useComposerSendFocusRestore.ts, apps/desktop/src/renderer/components/new-chat/useModelDiscoveryPending.ts, apps/desktop/src/renderer/components/new-chat/useProviderWeeklyQuota.ts, apps/desktop/src/renderer/components/new-chat/useUnifiedRowActions.ts, apps/desktop/src/renderer/features/bots/BotAuthorizationCard.tsx, apps/desktop/src/renderer/features/bots/BotAvatar.tsx, apps/desktop/src/renderer/features/bots/BotBasicProfileFields.tsx, apps/desktop/src/renderer/features/bots/BotCapabilitySettings.tsx, apps/desktop/src/renderer/features/bots/BotCollaborationCard.tsx, apps/desktop/src/renderer/features/bots/BotConnectionStatus.tsx, apps/desktop/src/renderer/features/bots/BotCreateMenu.tsx, apps/desktop/src/renderer/features/bots/BotDeleteDialog.tsx, apps/desktop/src/renderer/features/bots/BotDirectMessageCard.tsx, apps/desktop/src/renderer/features/bots/BotDirectMessageView.tsx, apps/desktop/src/renderer/features/bots/BotFollowMark.tsx, apps/desktop/src/renderer/features/bots/BotGenerationLabel.tsx, apps/desktop/src/renderer/features/bots/BotGroupAvatars.tsx, apps/desktop/src/renderer/features/bots/BotGroupChatView.tsx, apps/desktop/src/renderer/features/bots/BotGroupComposer.tsx, apps/desktop/src/renderer/features/bots/BotGroupCreateDialog.tsx, apps/desktop/src/renderer/features/bots/BotGroupPendingInteraction.tsx, apps/desktop/src/renderer/features/bots/BotGroupPlan.tsx, apps/desktop/src/renderer/features/bots/BotGroupSettingsDrawer.tsx, apps/desktop/src/renderer/features/bots/BotGroupSidebarRow.tsx, apps/desktop/src/renderer/features/bots/BotHistorySessionView.tsx, apps/desktop/src/renderer/features/bots/BotImportForm.tsx, apps/desktop/src/renderer/features/bots/BotInvitationWelcome.tsx, apps/desktop/src/renderer/features/bots/BotLearningFooter.tsx, apps/desktop/src/renderer/features/bots/BotLifecycleSettings.tsx, apps/desktop/src/renderer/features/bots/BotMemorySettings.tsx, apps/desktop/src/renderer/features/bots/BotModelChainEditor.tsx, apps/desktop/src/renderer/features/bots/BotPortraitPicker.tsx, apps/desktop/src/renderer/features/bots/BotRosterView.tsx, apps/desktop/src/renderer/features/bots/BotRoutines.tsx, apps/desktop/src/renderer/features/bots/BotSessionContentHeader.tsx, apps/desktop/src/renderer/features/bots/BotSessionTaskLink.tsx, apps/desktop/src/renderer/features/bots/BotSessionTaskResultCard.tsx, apps/desktop/src/renderer/features/bots/BotSessionView.tsx, apps/desktop/src/renderer/features/bots/BotSettingsBlock.tsx, apps/desktop/src/renderer/features/bots/BotSettingsDrawer.tsx, apps/desktop/src/renderer/features/bots/BotTaskCardHeader.tsx, apps/desktop/src/renderer/features/bots/BotTaskModelEditor.tsx, apps/desktop/src/renderer/features/bots/BotWorkbench.tsx, apps/desktop/src/renderer/features/bots/BotWorkingStatus.tsx, apps/desktop/src/renderer/features/bots/BotsFeatureLayout.tsx, apps/desktop/src/renderer/features/bots/BotsGlobalSettingsSection.tsx, apps/desktop/src/renderer/features/bots/BotsHomeView.tsx, apps/desktop/src/renderer/features/bots/BotsListView.tsx, apps/desktop/src/renderer/features/bots/BotsSidebar.tsx, apps/desktop/src/renderer/features/bots/ChatBotAccessField.tsx, apps/desktop/src/renderer/features/bots/ChatGroupSettings.tsx, apps/desktop/src/renderer/features/bots/ChatInviteDialog.tsx, apps/desktop/src/renderer/features/bots/ChatInviteHost.tsx, apps/desktop/src/renderer/features/bots/ChatServerControls.tsx, apps/desktop/src/renderer/features/bots/ChatThreadPanel.tsx, apps/desktop/src/renderer/features/bots/CindyDevicePicker.tsx, apps/desktop/src/renderer/features/bots/CindyDeviceRow.tsx, apps/desktop/src/renderer/features/bots/RemoteBotSessionView.tsx, apps/desktop/src/renderer/features/bots/RemoteBotSettings.tsx, apps/desktop/src/renderer/features/bots/WorkbenchSessionList.tsx, apps/desktop/src/renderer/features/bots/botAvatarIdentity.ts, apps/desktop/src/renderer/features/bots/botCapabilityDefaults.ts, apps/desktop/src/renderer/features/bots/botChatPresentation.ts, apps/desktop/src/renderer/features/bots/botConversationPresentation.ts, apps/desktop/src/renderer/features/bots/botConversationTimeline.ts, apps/desktop/src/renderer/features/bots/botDelegationLive.ts, apps/desktop/src/renderer/features/bots/botFollowScopes.ts, apps/desktop/src/renderer/features/bots/botGeneratedArtifacts.ts, apps/desktop/src/renderer/features/bots/botGroupAttachments.ts, apps/desktop/src/renderer/features/bots/botGroupLane.ts, apps/desktop/src/renderer/features/bots/botGroupMentions.ts, apps/desktop/src/renderer/features/bots/botGroupPresentation.ts, apps/desktop/src/renderer/features/bots/botGroupStore.ts, apps/desktop/src/renderer/features/bots/botListDisplay.ts, apps/desktop/src/renderer/features/bots/botNavigation.ts, apps/desktop/src/renderer/features/bots/botPronounContext.tsx, apps/desktop/src/renderer/features/bots/botReadState.ts, apps/desktop/src/renderer/features/bots/botRosterDisplay.ts, apps/desktop/src/renderer/features/bots/botSessionOwners.ts, apps/desktop/src/renderer/features/bots/botSettingsAutosave.ts, apps/desktop/src/renderer/features/bots/botStore.ts, apps/desktop/src/renderer/features/bots/botWelcomeContext.ts, apps/desktop/src/renderer/features/bots/botWorkbenchProjection.ts, apps/desktop/src/renderer/features/bots/chatError.ts, apps/desktop/src/renderer/features/bots/cindyDeviceRoster.ts, apps/desktop/src/renderer/features/bots/remoteBotRoster.ts, apps/desktop/src/renderer/features/bots/remoteBotSettingsData.ts, apps/desktop/src/renderer/features/bots/useBotGeneratedFileDeliveries.ts, apps/desktop/src/renderer/features/bots/useBotIslandVisibleSession.ts, apps/desktop/src/renderer/features/bots/useBotSettingsAutosave.ts, apps/desktop/src/renderer/features/bots/useBotUnreadSync.ts, apps/desktop/src/renderer/features/bots/useRemoteBots.ts, apps/desktop/src/renderer/features/bots/useWorkbenchSessionPages.ts, apps/desktop/src/renderer/features/bots/useWorkingStatusCopy.ts, apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx, apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts, apps/desktop/src/renderer/features/bots/voice/VoiceConversationController.ts, apps/desktop/src/renderer/features/bots/voice/VoiceReplyTracker.ts, apps/desktop/src/renderer/features/bots/voice/voiceConversation.css, apps/desktop/src/renderer/features/cc-agent/CCAgentFeatureLayout.tsx, apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx | 278 | 38 | 472 |
| `desktop.chat.files` | desktop | 工作目录文件浏览 | hash `/cc-agent/files/:sessionId`(WorkdirBrowseRoute) | FileBodyView, FileTreeView, WorkdirBrowseRoute | apps/desktop/src/renderer/features/cc-agent/workdir-browse/DocSearchBar.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/DrawioPreview.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/FileBodyView.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/FileFilterInput.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/FileTabsBar.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/FileTreeView.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/FilterResultList.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/ImagePreview.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/OpenInSystemActions.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/PdfPreview.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/SessionTabsBar.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/UnrenderablePlaceholder.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/WorkdirBrowseRoute.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/WorkdirBrowseSidebar.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useChatRailCollapsed.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useChatRailResize.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useConfirmSwitchAwayIfDirty.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useDelayedFlag.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useFileContent.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useFileTree.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useOpenTabs.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useProjectFileList.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/hooks/useRevealFileInTree.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/activeFileBodyHandle.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/docModeSwitchProjects.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/drawioExt.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/expandedStore.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/fileContentCache.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/fileIcon.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/fileMeta.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/fileScrollStore.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/fileSelectionParams.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/filterFiles.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/imageExt.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/markdownAutosave.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/openTabsStore.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/pdfExt.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/lib/selectedFileStore.ts, apps/desktop/src/renderer/features/cc-agent/workdir-browse/search/SearchInput.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/search/SearchPanel.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/search/SearchResults.tsx, apps/desktop/src/renderer/features/cc-agent/workdir-browse/search/hooks/useProjectSearch.ts | 17 | 7 | 45 |
| `desktop.chat.new-draft` | desktop | 新建任务草稿页 | hash `/cc-agent/new`(NewMakerDraftRoute) | NewMakerDraftRoute | apps/desktop/src/renderer/features/cc-agent/NewMakerDraftRoute.tsx | 15 | 0 | 3 |
| `desktop.chat.orca-workflow` | desktop | Orca 协同工作台 | hash `/cc-agent/orca/:sessionId`(OrcaWorkflowRoute) | CCAgentSessionView, OrcaSplitView, OrcaWorkerPanel, OrcaWorkflowRoute | apps/desktop/src/renderer/components/chat/AgentActionRow.tsx, apps/desktop/src/renderer/components/chat/AgentActionsBlock.tsx, apps/desktop/src/renderer/components/chat/AgentOnOtherDeviceContext.tsx, apps/desktop/src/renderer/components/chat/AgentTaskCard.tsx, apps/desktop/src/renderer/components/chat/AgentTaskLiveDetails.tsx, apps/desktop/src/renderer/components/chat/AnnotationStrokesSvg.tsx, apps/desktop/src/renderer/components/chat/AskUserQuestionBubble.tsx, apps/desktop/src/renderer/components/chat/AssistantMessage.tsx, apps/desktop/src/renderer/components/chat/AutomationOriginBadge.tsx, apps/desktop/src/renderer/components/chat/ChatAudioCard.tsx, apps/desktop/src/renderer/components/chat/ChatDisplaySnapshotContext.tsx, apps/desktop/src/renderer/components/chat/ChatImageView.tsx, apps/desktop/src/renderer/components/chat/ChatSessionFileContext.tsx, apps/desktop/src/renderer/components/chat/ChatSoundEffectCard.tsx, apps/desktop/src/renderer/components/chat/ChatVideoView.tsx, apps/desktop/src/renderer/components/chat/CindyMakeDoctorCard.tsx, apps/desktop/src/renderer/components/chat/CopyAsImageBlock.tsx, apps/desktop/src/renderer/components/chat/CredentialSwitchWaitBanner.tsx, apps/desktop/src/renderer/components/chat/DiffView.tsx, apps/desktop/src/renderer/components/chat/ErrorBanner.tsx, apps/desktop/src/renderer/components/chat/ErrorMessageCard.tsx, apps/desktop/src/renderer/components/chat/GeneratedFilesCard.tsx, apps/desktop/src/renderer/components/chat/GhostCardHostPrompts.tsx, apps/desktop/src/renderer/components/chat/GhostSummonCard.tsx, apps/desktop/src/renderer/components/chat/GhostToolCard.tsx, apps/desktop/src/renderer/components/chat/HookTaskCard.tsx, apps/desktop/src/renderer/components/chat/ImageGalleryContext.tsx, apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx, apps/desktop/src/renderer/components/chat/ImageLightbox.tsx, apps/desktop/src/renderer/components/chat/ImageMissingPlaceholder.tsx, apps/desktop/src/renderer/components/chat/InlineReferenceChip.tsx, apps/desktop/src/renderer/components/chat/InterruptedTurnBanner.tsx, apps/desktop/src/renderer/components/chat/JumpToBottomChip.tsx, apps/desktop/src/renderer/components/chat/MarkdownDiffBlock.tsx, apps/desktop/src/renderer/components/chat/MarkdownMermaidBlock.tsx, apps/desktop/src/renderer/components/chat/MarkdownRenderer.tsx, apps/desktop/src/renderer/components/chat/MermaidLightbox.tsx, apps/desktop/src/renderer/components/chat/MessageActionBar.tsx, apps/desktop/src/renderer/components/chat/MessageNavRail.tsx, apps/desktop/src/renderer/components/chat/MessageSourceLabels.tsx, apps/desktop/src/renderer/components/chat/MessageStream.tsx, apps/desktop/src/renderer/components/chat/MessageViewport.tsx, apps/desktop/src/renderer/components/chat/ModelLightbox.tsx, apps/desktop/src/renderer/components/chat/NewMessageIndicator.tsx, apps/desktop/src/renderer/components/chat/PlanReviewBubble.tsx, apps/desktop/src/renderer/components/chat/PrevMessageJumpChip.tsx, apps/desktop/src/renderer/components/chat/ProjectLinkChip.tsx, apps/desktop/src/renderer/components/chat/QuoteChip.tsx, apps/desktop/src/renderer/components/chat/RewindPreviewDialog.tsx, apps/desktop/src/renderer/components/chat/SelectionQuoteButton.tsx, apps/desktop/src/renderer/components/chat/SentAgentReferenceChip.tsx, apps/desktop/src/renderer/components/chat/SentInlineAtomBody.tsx, apps/desktop/src/renderer/components/chat/SessionHandoffCard.tsx, apps/desktop/src/renderer/components/chat/SessionLinkChip.tsx, apps/desktop/src/renderer/components/chat/ShareMessageCheckbox.tsx, apps/desktop/src/renderer/components/chat/ShareSelectionBar.tsx, apps/desktop/src/renderer/components/chat/SlackIcon.tsx, apps/desktop/src/renderer/components/chat/StreamFadeSpan.tsx, apps/desktop/src/renderer/components/chat/StreamingMarkdownChunk.tsx, apps/desktop/src/renderer/components/chat/SystemCard.tsx, apps/desktop/src/renderer/components/chat/TextLightbox.tsx, apps/desktop/src/renderer/components/chat/ThinkingCard.tsx, apps/desktop/src/renderer/components/chat/ThinkingText.tsx, apps/desktop/src/renderer/components/chat/TodoListCard.tsx, apps/desktop/src/renderer/components/chat/ToolCallCard.tsx, apps/desktop/src/renderer/components/chat/ToolPayloadLightbox.tsx, apps/desktop/src/renderer/components/chat/TopRightChipStack.tsx, apps/desktop/src/renderer/components/chat/TurnChangesCard.tsx, apps/desktop/src/renderer/components/chat/UnreadFailedScheduleBanner.tsx, apps/desktop/src/renderer/components/chat/UpgradeBanner.tsx, apps/desktop/src/renderer/components/chat/UserAttachmentChip.tsx, apps/desktop/src/renderer/components/chat/UserMessage.tsx, apps/desktop/src/renderer/components/chat/UserMessageEditBox.tsx, apps/desktop/src/renderer/components/chat/UserMessageUrlLink.tsx, apps/desktop/src/renderer/components/chat/VideoLightbox.tsx, apps/desktop/src/renderer/components/chat/WorkGroupBlock.tsx, apps/desktop/src/renderer/components/chat/WorktreeRestoreBanner.tsx, apps/desktop/src/renderer/components/chat/XIcon.tsx, apps/desktop/src/renderer/components/chat/activityRowChrome.ts, apps/desktop/src/renderer/components/chat/animateFocusScroll.ts, apps/desktop/src/renderer/components/chat/autoFollowIntent.ts, apps/desktop/src/renderer/components/chat/chatChrome.ts, apps/desktop/src/renderer/components/chat/errorReasonI18n.ts, apps/desktop/src/renderer/components/chat/generatedFilesProjection.ts, apps/desktop/src/renderer/components/chat/imageDisplaySize.ts, apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts, apps/desktop/src/renderer/components/chat/lightboxGestures.ts, apps/desktop/src/renderer/components/chat/markdownImageTargets.ts, apps/desktop/src/renderer/components/chat/markdownMedia.ts, apps/desktop/src/renderer/components/chat/messageNavRailModel.ts, apps/desktop/src/renderer/components/chat/messageNavRailWheel.ts, apps/desktop/src/renderer/components/chat/messageNavigationProjection.ts, apps/desktop/src/renderer/components/chat/messageProjectionCache.ts, apps/desktop/src/renderer/components/chat/messageStreamIndicatorPosition.ts, apps/desktop/src/renderer/components/chat/messageViewportCompensation.ts, apps/desktop/src/renderer/components/chat/messageViewportGeometry.ts, apps/desktop/src/renderer/components/chat/messageWorkGroups.ts, apps/desktop/src/renderer/components/chat/observeProgrammaticScroll.ts, apps/desktop/src/renderer/components/chat/rehypeFencedCodeMarker.ts, apps/desktop/src/renderer/components/chat/rehypeHighlightShared.ts, apps/desktop/src/renderer/components/chat/rehypeMathBlockMarker.ts, apps/desktop/src/renderer/components/chat/rehypeStreamWordFade.ts, apps/desktop/src/renderer/components/chat/remarkHtmlImages.ts, apps/desktop/src/renderer/components/chat/remarkLocalPathLinks.ts, apps/desktop/src/renderer/components/chat/remarkPreserveRawLocalDestinations.ts, apps/desktop/src/renderer/components/chat/remarkSessionLinks.ts, apps/desktop/src/renderer/components/chat/remarkStrictInlineMath.ts, apps/desktop/src/renderer/components/chat/remarkTruncateCjkUrls.ts, apps/desktop/src/renderer/components/chat/repairStreamingMarkdown.ts, apps/desktop/src/renderer/components/chat/scrollAnchoringDetect.ts, apps/desktop/src/renderer/components/chat/shareSelectionStore.ts, apps/desktop/src/renderer/components/chat/streamingMarkdownBlocks.tsx, apps/desktop/src/renderer/components/chat/streamingMarkdownChunks.ts, apps/desktop/src/renderer/components/chat/streamingMessageProjection.ts, apps/desktop/src/renderer/components/chat/toolLoopI18n.ts, apps/desktop/src/renderer/components/chat/unreadCount.ts, apps/desktop/src/renderer/components/chat/useAutomaticHistoryLoadBudget.ts, apps/desktop/src/renderer/components/chat/useCopyAsImage.tsx, apps/desktop/src/renderer/components/chat/useDeleteMessage.ts, apps/desktop/src/renderer/components/chat/useFileChipContextMenu.tsx, apps/desktop/src/renderer/components/chat/useForkAtMessage.ts, apps/desktop/src/renderer/components/chat/useImageClipboard.ts, apps/desktop/src/renderer/components/chat/useImageLoadFailure.ts, apps/desktop/src/renderer/components/chat/useMessageViewport.ts, apps/desktop/src/renderer/components/chat/useNavigationKeyListener.ts, apps/desktop/src/renderer/components/chat/useOpenWithMenu.tsx, apps/desktop/src/renderer/components/chat/usePrevUserMessageInView.ts, apps/desktop/src/renderer/components/chat/useTurnChangeSets.ts, apps/desktop/src/renderer/components/chat/userMessageCollapse.ts, apps/desktop/src/renderer/components/chat/userMessageDisplayText.ts, apps/desktop/src/renderer/components/chat/userMessageLinkify.ts, apps/desktop/src/renderer/components/chat/viewportFillDetect.ts, apps/desktop/src/renderer/components/new-chat/AddRemoteProjectDialog.tsx, apps/desktop/src/renderer/components/new-chat/AgentSelect.tsx, apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx, apps/desktop/src/renderer/components/new-chat/AtMentionPanel.tsx, apps/desktop/src/renderer/components/new-chat/AttachmentTypeThumb.tsx, apps/desktop/src/renderer/components/new-chat/ChatInput.tsx, apps/desktop/src/renderer/components/new-chat/CjkPunctDecoration.ts, apps/desktop/src/renderer/components/new-chat/CjkPunctuationUtils.ts, apps/desktop/src/renderer/components/new-chat/ComposerAttachments.tsx, apps/desktop/src/renderer/components/new-chat/ComposerListIndentDecoration.ts, apps/desktop/src/renderer/components/new-chat/ComposerListNodes.ts, apps/desktop/src/renderer/components/new-chat/ComposerQuoteNode.tsx, apps/desktop/src/renderer/components/new-chat/DeviceSwitcherPill.tsx, apps/desktop/src/renderer/components/new-chat/EffortSlider.tsx, apps/desktop/src/renderer/components/new-chat/EmptyDocSelectionGuard.ts, apps/desktop/src/renderer/components/new-chat/ExtraDirsButton.tsx, apps/desktop/src/renderer/components/new-chat/FastModeToggle.tsx, apps/desktop/src/renderer/components/new-chat/FolderPickerPopover.tsx, apps/desktop/src/renderer/components/new-chat/FullAccessConfirmContent.tsx, apps/desktop/src/renderer/components/new-chat/GhostCommandDecoration.ts, apps/desktop/src/renderer/components/new-chat/GoalAdvancedLimits.tsx, apps/desktop/src/renderer/components/new-chat/GoalIndicator.tsx, apps/desktop/src/renderer/components/new-chat/HomeUsageDashboard.tsx, apps/desktop/src/renderer/components/new-chat/ListComposerTextarea.tsx, apps/desktop/src/renderer/components/new-chat/LocalModelCatalogNotice.tsx, apps/desktop/src/renderer/components/new-chat/MentionChipNode.tsx, apps/desktop/src/renderer/components/new-chat/MentionDragCaretDecoration.ts, apps/desktop/src/renderer/components/new-chat/ModelCompatibilityNotice.tsx, apps/desktop/src/renderer/components/new-chat/ModelConfigFlyout.tsx, apps/desktop/src/renderer/components/new-chat/ModelHarnessPicker.tsx, apps/desktop/src/renderer/components/new-chat/ModelSelector.tsx, apps/desktop/src/renderer/components/new-chat/ModelSourceDetails.tsx, apps/desktop/src/renderer/components/new-chat/NewGoalDialog.tsx, apps/desktop/src/renderer/components/new-chat/PastedTextChipNode.tsx, apps/desktop/src/renderer/components/new-chat/PendingQueuePanel.tsx, apps/desktop/src/renderer/components/new-chat/PermissionPrompt.tsx, apps/desktop/src/renderer/components/new-chat/PermissionSelector.tsx, apps/desktop/src/renderer/components/new-chat/PinnedPlanPanel.tsx, apps/desktop/src/renderer/components/new-chat/PlanActionCard.tsx, apps/desktop/src/renderer/components/new-chat/PlanModeIndicator.tsx, apps/desktop/src/renderer/components/new-chat/PlanViewerCard.tsx, apps/desktop/src/renderer/components/new-chat/PluginConnectionForm.tsx, apps/desktop/src/renderer/components/new-chat/PluginOauthDeviceCode.tsx, apps/desktop/src/renderer/components/new-chat/PluginSetupPrompt.tsx, apps/desktop/src/renderer/components/new-chat/PluginSetupRemoteDesktopButton.tsx, apps/desktop/src/renderer/components/new-chat/PluginWriteAccessRecovery.tsx, apps/desktop/src/renderer/components/new-chat/QuickStartPillMark.ts, apps/desktop/src/renderer/components/new-chat/RemoteOauthSetupCard.tsx, apps/desktop/src/renderer/components/new-chat/SendButton.tsx, apps/desktop/src/renderer/components/new-chat/SlashCommandDecoration.ts, apps/desktop/src/renderer/components/new-chat/SlashCommandPalette.tsx, apps/desktop/src/renderer/components/new-chat/TakeoverMask.tsx, apps/desktop/src/renderer/components/new-chat/ThinkingToggle.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedFlyoutHost.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelPanel.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelRail.tsx, apps/desktop/src/renderer/components/new-chat/UnifiedModelRow.tsx, apps/desktop/src/renderer/components/new-chat/UsageDailyBars.tsx, apps/desktop/src/renderer/components/new-chat/UsageHeatmap.tsx, apps/desktop/src/renderer/components/new-chat/VendorSegmentedSwitcher.tsx, apps/desktop/src/renderer/components/new-chat/VoiceInputDraftDecoration.ts, apps/desktop/src/renderer/components/new-chat/WindowsSelectionReplacement.ts, apps/desktop/src/renderer/components/new-chat/WorktreeChipsRow.tsx, apps/desktop/src/renderer/components/new-chat/WorktreeCreatingOverlay.tsx, apps/desktop/src/renderer/components/new-chat/agentOptions.ts, apps/desktop/src/renderer/components/new-chat/agentSwitchConfirmation.ts, apps/desktop/src/renderer/components/new-chat/branchPick.ts, apps/desktop/src/renderer/components/new-chat/composerBlankPointerFocus.ts, apps/desktop/src/renderer/components/new-chat/composerContentSerialization.ts, apps/desktop/src/renderer/components/new-chat/composerDocState.ts, apps/desktop/src/renderer/components/new-chat/composerFrameScheduler.ts, apps/desktop/src/renderer/components/new-chat/composerHistoryProjection.ts, apps/desktop/src/renderer/components/new-chat/composerModelSelection.ts, apps/desktop/src/renderer/components/new-chat/composerPromptRecommendation.ts, apps/desktop/src/renderer/components/new-chat/composerRenderGate.ts, apps/desktop/src/renderer/components/new-chat/composerSendOwnership.ts, apps/desktop/src/renderer/components/new-chat/composerSendSnapshot.ts, apps/desktop/src/renderer/components/new-chat/effortChangeQueue.ts, apps/desktop/src/renderer/components/new-chat/extraDirsActions.ts, apps/desktop/src/renderer/components/new-chat/ghostComposerPlacement.ts, apps/desktop/src/renderer/components/new-chat/localAttachmentPicker.ts, apps/desktop/src/renderer/components/new-chat/mentionChipInsertion.ts, apps/desktop/src/renderer/components/new-chat/modelWindowErrorToast.ts, apps/desktop/src/renderer/components/new-chat/pastePipeline.ts, apps/desktop/src/renderer/components/new-chat/pathPaste.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueLocks.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueRowActivity.ts, apps/desktop/src/renderer/components/new-chat/pendingQueueRowPresentation.ts, apps/desktop/src/renderer/components/new-chat/planModeComposerCommand.ts, apps/desktop/src/renderer/components/new-chat/priceTierMarks.tsx, apps/desktop/src/renderer/components/new-chat/remoteBrowseAdapters.ts, apps/desktop/src/renderer/components/new-chat/remoteExistingProjects.ts, apps/desktop/src/renderer/components/new-chat/restoreComposerDocument.ts, apps/desktop/src/renderer/components/new-chat/sessionLinkPaste.ts, apps/desktop/src/renderer/components/new-chat/sourceSwitch.ts, apps/desktop/src/renderer/components/new-chat/unifiedModelSelection.ts, apps/desktop/src/renderer/components/new-chat/usagePalette.ts, apps/desktop/src/renderer/components/new-chat/useComposerSendFocusRestore.ts, apps/desktop/src/renderer/components/new-chat/useModelDiscoveryPending.ts, apps/desktop/src/renderer/components/new-chat/useProviderWeeklyQuota.ts, apps/desktop/src/renderer/components/new-chat/useUnifiedRowActions.ts, apps/desktop/src/renderer/features/cc-agent/CCAgentFeatureLayout.tsx, apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx, apps/desktop/src/renderer/features/cc-agent/OrcaSplitView.tsx, apps/desktop/src/renderer/features/cc-agent/OrcaWorkerPanel.tsx, apps/desktop/src/renderer/features/cc-agent/OrcaWorkflowRoute.tsx | 252 | 38 | 336 |
diff --git a/docs/product-rules/cindy-bots-runtime.md b/docs/product-rules/cindy-bots-runtime.md
index 708760ff06..2e550a3997 100644
--- a/docs/product-rules/cindy-bots-runtime.md
+++ b/docs/product-rules/cindy-bots-runtime.md
@@ -3,6 +3,9 @@
> 本文是伙伴(Bot)身份、任务、模型路由、工作目录、Skill / MCP、伙伴消息与后台任务的产品契约。
> 修改伙伴运行时、伙伴设置、伙伴消息、后台任务或伙伴提示词前必须先读本文。
+桌面一对一的连续语音交互沿用本文件的 canonical 任务与权限边界,具体行为和验证范围见
+[桌面伙伴语音模式](./companion-voice-mode.md)。
+
## 1. 用户心智:伙伴拥有一条长期主任务
伙伴是长期存在的数字同事。用户面对的是同一个名字、性格、记忆、能力和连续的主时间线,
diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.md
new file mode 100644
index 0000000000..f76b588049
--- /dev/null
+++ b/docs/product-rules/companion-voice-mode.md
@@ -0,0 +1,34 @@
+# 桌面伙伴语音模式
+
+## 范围与行为
+
+- 入口位于本机伙伴的一对一对话标题栏。沿用伙伴 canonical 任务、模型、上下文、工作目录和工具权限,不创建第二条任务或切换聊天模型。
+- 用户明确点击开始后采集麦克风。使用现有语音输入的 ASR 设置和当前账号的服务模式;不启用听写润色,不修改原有听写行为。
+- 停顿约 900ms 后等待 ASR 的最终文字,再用短暂的稳定窗口合并同一段发言。只有最终、非空文字能自动发送;声音活动检测本身不能产生消息。
+- 原任务空闲时正常发送,仍在运行时沿既有插话路径投递。语音是本机用户本人输入,不盖自动化或远程设备来源标签,不增加系统提示词。识别结果按普通文字发送,不执行桌面斜杠命令。
+- 最终回复经过持久化完成标记确认后才朗读。按本次语音发送的 clientId 关联,不朗读历史、过程文字、工具、子代理、内部协调或群来源私信。文本消息始终是对话记录的正本。
+- 独立选择朗读模型和音色。只展示当前账号可用、未停用的 Cindy AI `audio_speech` 模型;账号无模型时显示空状态,不能开始。刷新复用现有 Cindy AI 目录刷新。
+- 朗读结束继续监听。用户开口时立即取消待生成及正在播放的声音;新的文字仍投递到原任务。停止朗读不会取消 Agent 工具、清空任务队列或代答权限确认。
+- 需要确认权限、问题或计划,或任务队列暂停时,关闭麦克风并暂停语音模式;用户完成确认后显式继续。静音、锁屏、休眠、切走任务、关面板、账号切换同样释放采集和播放资源,不在后台重新开始。
+- 未发送的文字在可恢复错误或暂停后留在面板内;用户可选中复制或明确重新连接并发送。TTS 失败保留任务中的文字回复,不自动重发用户消息。
+
+## 交互与动效
+
+面板保留伙伴头像和姓名,持续显示连接、监听、思考、准备语音、朗读或暂停状态。音量条反映实际输入或输出音量,没有循环伪造的声波。进入动效和音量条只变换 opacity/transform,使用现有 motion token,并支持减少动态效果。模型和音色可在开始前或暂停后更换;进行中的结束、静音及面板关闭按钮不随内容滚走,暂停后可继续或关闭。面板最多占视口高度的 65%,中间内容独立滚动。Light/Dark 使用语义颜色。
+
+设计依据为 `DESIGN.md` §4 标准控件、§5 圆角分层、§14.4 动效与 `design-governance.md` §13 点击层分类:标题栏语音入口是 32px 的普通动作按钮,沿用同栏的 pill;面板和未发送文字框属于容器,使用 12px 圆角;音量条是无交互的指示层,不是按钮。设置、开始、静音、结束等操作复用现有 Button / Select / Input。
+
+## 执行边界
+
+- 当前仅支持 Desktop、本机伙伴一对一。群聊、Mobile、远程控制的麦克风、原生 Realtime speech-to-speech 不在此实现内。
+- 新 IPC 只接受可信应用顶层 Renderer,按窗口、账号代次、callId 和 canonical 任务绑定。callId 在发起请求前生成,使结束按钮也能取消连接过程;它不是授权凭证。接口不加入 device-link 或插件白名单。模型权限在开始和每次 TTS 请求、读取时复核。
+- ASR 适配器额外提供按上游 utterance/item ID 标识的分段事件,现有累积 `AsrEvent` 合同保持不变。连接在 150 秒轮换,沿用 voice-server 的 dictation 协议;切换期间最多缓存 5 秒输入,不能无声丢弃或重复发送前文。
+- Gateway 执行合同为 OpenAI 兼容 `POST <下发的 baseUrl>/audio/speech`,body 包含 `model`、`voice`、`input`、`response_format: pcm`。PCM 固定为该协议规定的 24kHz、单声道、16bit little-endian。显式声明其他 native API 或 `null` 未验证 API 的模型不进入选择器;未声明协议的 Gateway 新模型沿用此端点合同。下发 `mode` 只代表能力分类,不证明具体音色、音频格式或端点可用。
+- 凭证只在 Main,使用当前账号配对的 Gateway baseUrl 和密钥;不自动改走 BYOK 或系统朗读。响应按 Renderer 播放速度读取,播放只预排约 300ms;中断丢弃旧请求和迟到音频。原始音频只在内存,不新增录音文件或对话音频归档。
+- 初始实现的声音活动判断是带迟滞的音量检测,配合浏览器回声消除与降噪。它不是语义 VAD,不能保证所有扬声器、蓝牙或噪声环境都能准确分辨插话。
+
+## 验证
+
+自动化覆盖分段去重、乱序完成、空转写、连接轮换、权限与账号边界、取消启动、插话抢占、迟到音频、PCM 字节边界、回复关联、无模型空状态及现有 ASR 回归。
+
+Gateway 尚未给测试账号开放真实 TTS 模型时,自动化使用明确的测试替身;产品目录不添加演示模型。真实模型开放后,还需验证其音色和 PCM 合同,并在 macOS/Windows 的耳机、扬声器、蓝牙设备上验证连续多轮、回声与插话延迟。测试替身不构成这些项目已通过的证据。
diff --git a/i18n/GLOSSARY.md b/i18n/GLOSSARY.md
index 45e0ac0559..ef7d17aa43 100644
--- a/i18n/GLOSSARY.md
+++ b/i18n/GLOSSARY.md
@@ -589,6 +589,10 @@ Cindy 里的**持久 AI 助手实体**(原名 Bot):有长期身份、自
语音输入的用户自定义术语表(人名、产品名、代号与其常见误识别写法),在同账号的电脑之间自动同步、手机只读查看。当前先采用四语直译并登记为待讨论术语,避免与「自定义词典」「术语表」「用户词库」等说法混用。
+### Voice Mode
+
+桌面伙伴一对一的连续语音交互方式;停顿发送、最终回复朗读和插话,沿用伙伴主任务。
+
### Wallpaper
Desktop appearance setting for an in-app visual background layer.
diff --git a/i18n/glossary.json b/i18n/glossary.json
index 9bdb988776..3640368165 100644
--- a/i18n/glossary.json
+++ b/i18n/glossary.json
@@ -5,6 +5,7 @@
"sourceLocale": "en",
"punctuationExempt": ["desktop:settings.help.qnaFeedbackBodyTemplate"],
"terms": [
+ {"id": "voice-mode", "status": "proposed", "en": "Voice Mode", "translations": {"zh-CN": "语音模式", "zh-TW": "語音模式", "ja": "音声モード", "ko": "음성 모드"}, "checkCase": false, "note": "桌面伙伴一对一的连续语音交互方式;停顿发送、最终回复朗读和插话,沿用伙伴主任务。"},
{"id": "wallpaper-blur", "status": "proposed", "en": "Wallpaper Blur", "translations": {"zh-CN": "壁纸模糊", "zh-TW": "桌布模糊", "ja": "壁紙のぼかし", "ko": "배경화면 흐림"}, "note": "仅使应用壁纸柔化,不模糊文字或控件;0 为关闭,按用户选择应用于静态图片和视频。"},
{"id": "group-owner", "status": "proposed", "en": "Group Owner", "translations": {"zh-CN": "群主", "zh-TW": "群主", "ja": "グループオーナー", "ko": "그룹 소유자"}, "checkCase": false, "note": "Chat Server 的群角色,可由人或伙伴担任;不要与伙伴主人或分工负责人混同。群设置成员行英文可简写 Owner。"},
{"id": "group-admin", "status": "proposed", "en": "Group Admin", "translations": {"zh-CN": "群管理员", "zh-TW": "群管理員", "ja": "グループ管理者", "ko": "그룹 관리자"}, "checkCase": false, "note": "可管理群资料和成员,但不能替伙伴主人授权工具或私人资料。群设置内可简称管理员 / Admin。"},
diff --git a/packages/voice-input-core/src/VoiceConversationInput.ts b/packages/voice-input-core/src/VoiceConversationInput.ts
new file mode 100644
index 0000000000..43db464b61
--- /dev/null
+++ b/packages/voice-input-core/src/VoiceConversationInput.ts
@@ -0,0 +1,211 @@
+import type { AsrProvider, AsrSegment, AudioTrace } from "./types.js";
+import { ConversationActivity } from "./conversationActivity.js";
+
+export type ConversationInputEvent =
+ | { type: "ready" }
+ | { type: "speech-started" }
+ | { type: "draft"; text: string }
+ | { type: "utterance"; id: string; text: string }
+ | { type: "error"; text: string };
+
+/**
+ * Continuous ASR ownership and endpointing. Provider IDs, never string-prefix
+ * subtraction, identify submitted speech. Connections rotate below the managed
+ * service's default lifetime while capture is buffered; the product call lives on.
+ */
+export class VoiceConversationInput {
+ private provider: AsrProvider | null = null;
+ private generation = 0;
+ private sequence = 0;
+ private closed = false;
+ private rotating = false;
+ private activity = new ConversationActivity();
+ private speaking = false;
+ private manual = false;
+ private segments = new Map<string, AsrSegment & { generation: number }>();
+ private consumed = new Set<string>();
+ private pending: Array<{ pcm: ArrayBuffer; trace?: AudioTrace }> = [];
+ private pendingBytes = 0;
+ private settleTimer?: ReturnType<typeof setTimeout>;
+ private rotationTimer?: ReturnType<typeof setTimeout>;
+ private finalTimer?: ReturnType<typeof setTimeout>;
+
+ constructor(
+ private readonly options: {
+ createProvider: () => Promise<AsrProvider>;
+ emit: (event: ConversationInputEvent) => void;
+ rotationMs?: number;
+ },
+ ) {}
+
+ async start(): Promise<void> {
+ if (this.closed) return;
+ await this.open();
+ }
+
+ appendAudio(pcm: ArrayBuffer, trace?: AudioTrace): void {
+ if (this.closed) return;
+ const activity = this.activity.push(pcm);
+ this.speaking = activity.active;
+ if (activity.started) {
+ clearTimeout(this.finalTimer);
+ if (!this.manual) clearTimeout(this.settleTimer);
+ this.options.emit({ type: "speech-started" });
+ }
+ if (!this.provider || this.rotating) {
+ this.pending.push({ pcm, trace });
+ this.pendingBytes += pcm.byteLength;
+ // Five seconds of 16 kHz PCM. Never silently discard a user's speech.
+ if (this.pendingBytes > 160_000) this.fail();
+ return;
+ }
+ this.provider.appendAudio(pcm, trace);
+ if (activity.ended) this.finishUtterance();
+ }
+
+ finishUtterance(force = false): void {
+ if (this.closed) return;
+ this.manual ||= force;
+ this.provider?.commitUtterance?.();
+ this.scheduleSettlement();
+ clearTimeout(this.finalTimer);
+ this.finalTimer = setTimeout(() => {
+ if (
+ this.segments.size &&
+ [...this.segments.values()].some((part) => !part.final)
+ )
+ this.fail();
+ }, 8_000);
+ }
+
+ async stop(): Promise<void> {
+ if (this.closed) return;
+ this.closed = true;
+ ++this.generation;
+ clearTimeout(this.settleTimer);
+ clearTimeout(this.rotationTimer);
+ clearTimeout(this.finalTimer);
+ this.pending = [];
+ this.pendingBytes = 0;
+ const provider = this.provider;
+ this.provider = null;
+ if (provider) {
+ try {
+ await provider.stop();
+ } finally {
+ await provider.dispose?.();
+ }
+ }
+ }
+
+ private async open(): Promise<void> {
+ const generation = ++this.generation;
+ const provider = await this.options.createProvider();
+ if (this.closed || generation !== this.generation) {
+ await provider.stop();
+ await provider.dispose?.();
+ return;
+ }
+ this.provider = provider;
+ if (!provider.onSegment)
+ throw new Error("Continuous transcription is unavailable.");
+ provider.onSegment((segment) => {
+ if (this.closed || generation !== this.generation) return;
+ const key = `${generation}:${segment.id}`;
+ if (this.consumed.has(key)) return;
+ const prior = this.segments.get(key);
+ if (prior?.final) return;
+ if (!prior?.text && segment.text.trim() && !this.speaking)
+ this.options.emit({ type: "speech-started" });
+ this.segments.set(key, { ...segment, generation });
+ this.options.emit({ type: "draft", text: this.draft() });
+ if (this.draft().length > 16_000) {
+ this.fail();
+ return;
+ }
+ this.scheduleSettlement();
+ });
+ provider.onEvent((event) => {
+ if (this.closed || generation !== this.generation || this.rotating)
+ return;
+ if (event.type === "error" || event.type === "disconnected") this.fail();
+ });
+ await provider.start();
+ if (this.closed || generation !== this.generation) return;
+ this.rotating = false;
+ const hadBufferedAudio = this.pending.length > 0;
+ for (const chunk of this.pending.splice(0))
+ provider.appendAudio(chunk.pcm, chunk.trace);
+ this.pendingBytes = 0;
+ if (hadBufferedAudio && !this.speaking) provider.commitUtterance?.();
+ this.rotationTimer = setTimeout(() => {
+ void this.rotate().catch(() => this.fail());
+ }, this.options.rotationMs ?? 150_000);
+ this.options.emit({ type: "ready" });
+ }
+
+ private draft(): string {
+ return [...this.segments.values()]
+ .sort(
+ (a, b) =>
+ a.generation - b.generation || (a.order ?? 0) - (b.order ?? 0),
+ )
+ .map((segment) => segment.text.trim())
+ .join(" ")
+ .trim();
+ }
+
+ private scheduleSettlement(): void {
+ clearTimeout(this.settleTimer);
+ if ((this.speaking && !this.manual) || this.rotating || !this.segments.size)
+ return;
+ if ([...this.segments.values()].some((segment) => !segment.final)) return;
+ this.settleTimer = setTimeout(
+ () => {
+ if (this.closed || (this.speaking && !this.manual) || this.rotating)
+ return;
+ const text = this.draft();
+ for (const key of this.segments.keys()) this.consumed.add(key);
+ this.segments.clear();
+ this.manual = false;
+ clearTimeout(this.finalTimer);
+ if (text)
+ this.options.emit({
+ type: "utterance",
+ id: String(++this.sequence),
+ text,
+ });
+ this.options.emit({ type: "draft", text: "" });
+ },
+ this.manual ? 0 : 350,
+ );
+ }
+
+ private async rotate(): Promise<void> {
+ if (this.closed || !this.provider) return;
+ this.rotating = true;
+ clearTimeout(this.settleTimer);
+ const previous = this.provider;
+ // Finalize the previous transport before accepting its last segments. New
+ // audio is already buffered for the replacement, so boundaries cannot overlap.
+ await previous.flushAudio();
+ if (this.closed) return;
+ if ([...this.segments.values()].some((segment) => !segment.final)) {
+ this.fail();
+ return;
+ }
+ ++this.generation;
+ await previous.stop();
+ await previous.dispose?.();
+ this.provider = null;
+ this.consumed.clear();
+ await this.open();
+ this.scheduleSettlement();
+ }
+
+ private fail(): void {
+ if (this.closed) return;
+ this.options.emit({ type: "error", text: this.draft() });
+ void this.stop().catch(() => undefined);
+ }
+}
diff --git a/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
new file mode 100644
index 0000000000..e1ce118106
--- /dev/null
+++ b/packages/voice-input-core/src/__tests__/VoiceConversationInput.test.ts
@@ -0,0 +1,219 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { AsrEvent, AsrProvider, AsrSegment } from "../types.js";
+import {
+ VoiceConversationInput,
+ type ConversationInputEvent,
+} from "../VoiceConversationInput.js";
+import { ConversationActivity } from "../conversationActivity.js";
+
+class Provider implements AsrProvider {
+ segment: (value: AsrSegment) => void = () => {};
+ event: (value: AsrEvent) => void = () => {};
+ start = vi.fn(async () => {});
+ stop = vi.fn(async () => {});
+ dispose = vi.fn(async () => {});
+ appendAudio = vi.fn();
+ flushAudio = vi.fn(async () => {});
+ commitUtterance = vi.fn();
+ onSegment(callback: (value: AsrSegment) => void) {
+ this.segment = callback;
+ }
+ onEvent(callback: (value: AsrEvent) => void) {
+ this.event = callback;
+ }
+}
+const pcm = (amplitude: number, ms = 100) =>
+ new Int16Array(16 * ms).fill(amplitude).buffer;
+
+describe("continuous speech input", () => {
+ beforeEach(() => vi.useFakeTimers());
+ afterEach(() => vi.useRealTimers());
+
+ it("rejects a short click, commits on a pause, waits for final text and sends each segment once", async () => {
+ const provider = new Provider();
+ const events: ConversationInputEvent[] = [];
+ const input = new VoiceConversationInput({
+ createProvider: async () => provider,
+ emit: (event) => events.push(event),
+ });
+ await input.start();
+ input.appendAudio(pcm(2000, 40));
+ input.appendAudio(pcm(0, 1000));
+ expect(events.filter((e) => e.type === "speech-started")).toHaveLength(0);
+ input.appendAudio(pcm(2000));
+ input.appendAudio(pcm(2000));
+ provider.segment({ id: "one", text: "你好", final: false });
+ input.appendAudio(pcm(0, 800));
+ expect(provider.commitUtterance).not.toHaveBeenCalled();
+ input.appendAudio(pcm(0, 100));
+ expect(provider.commitUtterance).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(400);
+ expect(events.filter((e) => e.type === "utterance")).toHaveLength(0);
+ provider.segment({ id: "one", text: "你好。", final: true });
+ await vi.advanceTimersByTimeAsync(350);
+ provider.segment({ id: "one", text: "你好。", final: true });
+ provider.segment({ id: "two", text: "你好。", final: true });
+ await vi.advanceTimersByTimeAsync(350);
+ expect(events.filter((e) => e.type === "utterance")).toEqual([
+ { type: "utterance", id: "1", text: "你好。" },
+ { type: "utterance", id: "2", text: "你好。" },
+ ]);
+ await input.stop();
+ });
+
+ it("does not mistake an out-of-order final for the complete utterance", async () => {
+ const provider = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => provider,
+ emit,
+ });
+ await input.start();
+ provider.segment({ id: "a", text: "", final: false, order: 0 });
+ provider.segment({ id: "b", text: "second", final: true, order: 1 });
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(emit).not.toHaveBeenCalledWith(
+ expect.objectContaining({ type: "utterance" }),
+ );
+ provider.segment({ id: "a", text: "first", final: true, order: 0 });
+ await vi.advanceTimersByTimeAsync(350);
+ expect(emit).toHaveBeenCalledWith({
+ type: "utterance",
+ id: "1",
+ text: "first second",
+ });
+ await input.stop();
+ });
+
+ it("lets Send finish a finalized sentence despite background activity, but never sends a partial", async () => {
+ const provider = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => provider,
+ emit,
+ });
+ await input.start();
+ input.appendAudio(pcm(2000, 200));
+ provider.segment({ id: "one", text: "partial", final: false });
+ input.finishUtterance(true);
+ await vi.advanceTimersByTimeAsync(400);
+ expect(emit).not.toHaveBeenCalledWith(
+ expect.objectContaining({ type: "utterance" }),
+ );
+ provider.segment({ id: "one", text: "final", final: true });
+ await vi.advanceTimersByTimeAsync(1);
+ expect(emit).toHaveBeenCalledWith({
+ type: "utterance",
+ id: "1",
+ text: "final",
+ });
+ await input.stop();
+ });
+
+ it("buffers capture during rotation and fences repeated/late segments from the previous socket", async () => {
+ const a = new Provider();
+ const b = new Provider();
+ const emit = vi.fn();
+ let finish!: () => void;
+ a.flushAudio.mockImplementationOnce(
+ () =>
+ new Promise<void>((resolve) => {
+ finish = resolve;
+ }),
+ );
+ const createProvider = vi
+ .fn()
+ .mockResolvedValueOnce(a)
+ .mockResolvedValueOnce(b);
+ const input = new VoiceConversationInput({
+ createProvider,
+ emit,
+ rotationMs: 1000,
+ });
+ await input.start();
+ a.segment({ id: "same", text: "before", final: true });
+ await vi.advanceTimersByTimeAsync(350);
+ await vi.advanceTimersByTimeAsync(650);
+ const buffered = pcm(1000, 200);
+ input.appendAudio(buffered);
+ finish();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(b.appendAudio).toHaveBeenCalledWith(buffered, undefined);
+ a.segment({ id: "late", text: "ignore", final: true });
+ b.segment({ id: "same", text: "after", final: true });
+ input.appendAudio(pcm(0, 900));
+ await vi.advanceTimersByTimeAsync(350);
+ expect(
+ emit.mock.calls
+ .filter(([e]) => e.type === "utterance")
+ .map(([e]) => e.text),
+ ).toEqual(["before", "after"]);
+ expect(a.dispose).toHaveBeenCalledOnce();
+ await input.stop();
+ });
+
+ it("retains a partial transcript on disconnect and ignores late finals after stopping", async () => {
+ const p = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => p,
+ emit,
+ });
+ await input.start();
+ p.segment({ id: "a", text: "keep me", final: false });
+ p.event({ type: "disconnected", at: 0 });
+ p.segment({ id: "a", text: "late", final: true });
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(emit).toHaveBeenCalledWith({ type: "error", text: "keep me" });
+ expect(emit).not.toHaveBeenCalledWith(
+ expect.objectContaining({ type: "utterance" }),
+ );
+ expect(p.stop).toHaveBeenCalledOnce();
+ });
+
+ it("disposes a provider whose creation resolves after cancellation", async () => {
+ const p = new Provider();
+ let resolve!: (provider: AsrProvider) => void;
+ const input = new VoiceConversationInput({
+ createProvider: () =>
+ new Promise((done) => {
+ resolve = done;
+ }),
+ emit: vi.fn(),
+ });
+ const starting = input.start();
+ await input.stop();
+ resolve(p);
+ await starting;
+ expect(p.start).not.toHaveBeenCalled();
+ expect(p.dispose).toHaveBeenCalledOnce();
+ });
+
+ it("settles an empty recognition without sending noise or leaving a stale pending item", async () => {
+ const p = new Provider();
+ const emit = vi.fn();
+ const input = new VoiceConversationInput({
+ createProvider: async () => p,
+ emit,
+ });
+ await input.start();
+ p.segment({ id: "noise", text: "", final: false });
+ p.segment({ id: "noise", text: "", final: true });
+ await vi.advanceTimersByTimeAsync(350);
+ p.segment({ id: "speech", text: "hello", final: true });
+ await vi.advanceTimersByTimeAsync(350);
+ expect(emit.mock.calls.filter(([e]) => e.type === "utterance")).toEqual([
+ [{ type: "utterance", id: "1", text: "hello" }],
+ ]);
+ await input.stop();
+ });
+});
+
+it("keeps soft speech active and does not end it on a short pause", () => {
+ const gate = new ConversationActivity();
+ expect(gate.push(pcm(800, 200)).started).toBe(true);
+ expect(gate.push(pcm(240, 500)).active).toBe(true);
+ expect(gate.push(pcm(0, 500)).ended).toBe(false);
+ expect(gate.push(pcm(400, 200)).active).toBe(true);
+ expect(gate.push(pcm(0, 900)).ended).toBe(true);
+});
diff --git a/packages/voice-input-core/src/conversationActivity.ts b/packages/voice-input-core/src/conversationActivity.ts
new file mode 100644
index 0000000000..c0bdc65008
--- /dev/null
+++ b/packages/voice-input-core/src/conversationActivity.ts
@@ -0,0 +1,53 @@
+/**
+ * Acoustic activity gate, not a semantic speech classifier. It commits manual
+ * ASR buffers after silence; only provider-finalized nonempty text may be sent.
+ * Hysteresis avoids clipping quiet syllables and transient clicks. Capture must
+ * enable echo cancellation when this gate runs alongside speaker playback.
+ */
+export class ConversationActivity {
+ private noise = 80;
+ private attackMs = 0;
+ private silenceMs = 0;
+ private active = false;
+
+ push(
+ pcm: ArrayBuffer,
+ sampleRate = 16_000,
+ ): { level: number; started: boolean; ended: boolean; active: boolean } {
+ const samples = new Int16Array(pcm);
+ let energy = 0;
+ for (const sample of samples) energy += sample * sample;
+ const rms = Math.sqrt(energy / Math.max(1, samples.length));
+ const duration = (samples.length / sampleRate) * 1_000;
+ const threshold = Math.max(
+ this.active ? 180 : 320,
+ this.noise * (this.active ? 2 : 3.5),
+ );
+ const sound = rms > threshold;
+ let started = false;
+ let ended = false;
+ if (!this.active && !sound)
+ this.noise = this.noise * 0.98 + Math.min(rms, 250) * 0.02;
+ if (sound) {
+ this.attackMs += duration;
+ this.silenceMs = 0;
+ if (!this.active && this.attackMs >= 180) {
+ this.active = true;
+ started = true;
+ }
+ } else {
+ this.attackMs = 0;
+ this.silenceMs += duration;
+ if (this.active && this.silenceMs >= 900) {
+ this.active = false;
+ ended = true;
+ }
+ }
+ return {
+ level: Math.min(1, rms / 5_000),
+ started,
+ ended,
+ active: this.active,
+ };
+ }
+}
diff --git a/packages/voice-input-core/src/index.ts b/packages/voice-input-core/src/index.ts
index 28b7e9c0e6..888a0b7003 100644
--- a/packages/voice-input-core/src/index.ts
+++ b/packages/voice-input-core/src/index.ts
@@ -10,3 +10,6 @@ export * from './dictionary-sync';
export * from './streamingJson';
export * from './refinementContext';
export * from './pcmActivity';
+
+export * from './VoiceConversationInput';
+export * from './conversationActivity';
diff --git a/packages/voice-input-core/src/types.ts b/packages/voice-input-core/src/types.ts
index 5f01f669e3..57f17ec07f 100644
--- a/packages/voice-input-core/src/types.ts
+++ b/packages/voice-input-core/src/types.ts
@@ -43,6 +43,9 @@ export type AsrEvent =
| { type: 'stable'; text: string; at: number }
| { type: 'error'; message: string; at: number };
+/** One provider-owned utterance, independent of the cumulative dictation transcript. */
+export type AsrSegment = { id: string; text: string; final: boolean; order?: number };
+
export type VoiceInputDraftSource = 'partial' | 'stable' | 'refinement';
export type VoiceInputDraftReason = 'asr_partial' | 'asr_stable' | 'refinement_preview';
@@ -56,6 +59,10 @@ export type AudioTrace = {
};
export interface AsrProvider {
+ /** Optional continuous-input contract. Existing dictation consumers are unchanged. */
+ onSegment?(callback: (segment: AsrSegment) => void): void;
+ /** Commit an utterance without closing the connection; server-VAD providers need no action. */
+ commitUtterance?(): void;
start(): Promise<void>;
stop(): Promise<void>;
appendAudio(chunk: ArrayBuffer, trace?: AudioTrace): void;