Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
feat(desktop): support BYOK companion speech and per-companion voices
JiangJiahao01a984a61149
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.tsindex 882a339f4d..b539c2a3c3 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/ipc.test.ts@@ -32,6 +32,7 @@ vi.mock('../../maker-host/active-catalog.js', () => ({}),}));vi.mock('../../secrets/providerSecretStore.js', () => ({+ readCustomProviderKey: () => null,getProviderSecretStore: () => ({ get: () => 'test-key' }),}));vi.mock('../../model-access/effectiveEndpoint.js', () => ({@@ -52,6 +53,7 @@ vi.mock('../../utility-model/oneShotCandidates.js', () => ({isProviderModelRouteDisabled: () => false,}));+import { normalizeVoiceInputSettings } from '../../../shared/voiceInputData.js';import { registerVoiceConversationIpc } from '../index.js';const sender = (id = 1) => ({ sender: { id, once: vi.fn() } });const call = (name: string, ...args: unknown[]) =>@@ -62,7 +64,13 @@ beforeEach(() => {h.trusted.mockReset();h.append.mockReset();h.boundary = false;+ let settings = normalizeVoiceInputSettings({});registerVoiceConversationIpc({+ getSettings: () => settings,+ updateSettings: (patch) => {+ settings = normalizeVoiceInputSettings({ ...settings, ...patch });+ return settings;+ },hasDictation: () => false,createProvider: async () => ({start: async () => {},@@ -125,3 +133,28 @@ it('refuses a start during an account transition, malformed voice parameters andcall('speak', { callId: 'call', requestId: 'x', text: 'x'.repeat(4097) }),).rejects.toThrow();});++it('saves and resets one companion without overwriting another, and guards settings IPC', async () => {+ const choice = {+ providerId: 'byok',+ agent: 'codex',+ modelId: 'tts',+ voice: 'nova',+ protocol: 'openai',+ };+ await call('set-preference', 'bot-a', choice);+ await call('set-preference', 'bot-b', { ...choice, voice: 'alloy' });+ expect(await call('preference', 'bot-a')).toEqual({ selection: choice, customized: true });+ await call('set-preference', 'bot-a', null);+ expect(await call('preference', 'bot-a')).toEqual({ selection: null, customized: false });+ expect(await call('preference', 'bot-b')).toMatchObject({ selection: { voice: 'alloy' } });+ h.boundary = true;+ await expect(call('preference', 'bot-b')).rejects.toThrow();+ await expect(call('set-preference', 'bot-b', null)).rejects.toThrow();+ h.boundary = false;+ h.trusted.mockImplementation(() => {+ throw new Error('untrusted');+ });+ await expect(call('preference', 'bot-b')).rejects.toThrow('untrusted');+ await expect(call('set-preference', 'bot-b', null)).rejects.toThrow('untrusted');+});diff --git a/apps/desktop/src/main/voice-conversation/__tests__/models.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/models.test.tsindex c217ae3777..04790ed3a5 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/models.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/models.test.ts@@ -1,7 +1,14 @@-import { expect, it } from 'vitest';-import type { Catalog, ProviderMediaModel } from '@cindy/model-providers';-import { speechModels } from '../models.js';+import { expect, it, vi } from 'vitest';+import {+ BUNDLED_CATALOG,+ buildUserProvider,+ type Catalog,+ type ProviderMediaModel,+ type ProviderRuntimeModelConfig,+} from '@cindy/model-providers';+import { resolveSpeechConnection, speechModels, type SpeechCatalogDeps } from '../models.js';import { normalizeVoiceInputSettings } from '../../../shared/voiceInputData.js';+import { voiceModelKey } from '../../../shared/voiceConversation.js';const model = (id: string, extra: Partial<ProviderMediaModel> = {}): ProviderMediaModel => ({id,@@ -10,32 +17,160 @@ const model = (id: string, extra: Partial<ProviderMediaModel> = {}): ProviderMedavailability: 'available',...extra,});-it('offers only available, enabled Gateway speech models with a supported protocol', () => {- const catalog = {- providers: [- {- id: 'xd',- audioModels: [- model('new-speech'),- model('paid', { availability: 'requires_payment' }),- model('registry-only', { availability: undefined }),- model('disabled', { disabled: true }),- model('override'),- model('asr', { mode: 'audio_transcription' }),- model('wrong-protocol', { nativeApi: 'openai-responses' }),- model('unverified', { nativeApi: null }),- ],- },- { id: 'other', audioModels: [model('other-speech')] },- ],- } as unknown as Catalog;- expect(speechModels(catalog, true, (id) => id === 'override')).toEqual([- { id: 'new-speech', name: 'new-speech' },+function deps(providers: Catalog['providers']): SpeechCatalogDeps {+ return {+ catalog: { ...BUNDLED_CATALOG, providers },+ gatewayAllowed: true,+ disabled: () => false,+ gatewayConnection: () => ({ baseUrl: 'https://gateway.test/v1', apiKey: 'gateway-key' }),+ readKey: vi.fn((id, agent) => `${id}-${agent}-key`),+ };+}+function custom(+ id: string,+ baseUrl: string,+ models: ProviderRuntimeModelConfig[] = [+ { id: 'tts', name: 'Speech', mode: 'audio_speech', nativeApi: 'openai-audio-speech' },+ ],+) {+ return buildUserProvider(+ { id, name: id, runtimes: { codex: { baseUrl, wireProtocol: 'openai-chat', models } } },+ { modelRegistry: BUNDLED_CATALOG.modelRegistry },+ );+}++it('keeps Gateway entitlement and supported-protocol gates', () => {+ const d = deps([+ {+ id: 'xd',+ audioModels: [+ model('new-speech'),+ model('paid', { availability: 'requires_payment' }),+ model('registry-only', { availability: undefined }),+ model('disabled', { disabled: true }),+ model('override'),+ model('asr', { mode: 'audio_transcription' }),+ model('wrong', { nativeApi: 'openai-responses' }),+ model('unverified', { nativeApi: null }),+ ],+ },+ ] as Catalog['providers']);+ d.disabled = (provider, id) => provider === 'xd' && id === 'override';+ expect(speechModels(d).map((m) => m.id)).toEqual(['new-speech']);+ expect(speechModels({ ...d, gatewayAllowed: false })).toEqual([]);+});++it('distinguishes two user connections with the same upstream ID and uses only the selected runtime key', () => {+ const d = deps([custom('a', 'https://a.test/prefix/v1'), custom('b', 'https://b.test/v1')]);+ const models = speechModels({ ...d, gatewayAllowed: false });+ expect(models).toHaveLength(2);+ expect(voiceModelKey(models[0])).not.toBe(voiceModelKey(models[1]));+ expect(JSON.stringify(models)).not.toContain('-key');+ vi.mocked(d.readKey).mockClear();+ const connection = resolveSpeechConnection(d, {+ providerId: 'b',+ agent: 'codex',+ modelId: 'tts',+ voice: 'alloy',+ });+ expect(d.readKey).toHaveBeenCalledExactlyOnceWith('b', 'codex');+ expect(connection).toEqual({+ baseUrl: 'https://b.test/v1',+ protocol: 'openai',+ headers: { authorization: 'Bearer b-codex-key' },+ });+ expect(() => resolveSpeechConnection(d, { modelId: 'tts', voice: 'alloy' })).toThrow(+ 'model_unavailable',+ );+ expect(() =>+ resolveSpeechConnection(d, { providerId: 'b', agent: 'pi', modelId: 'tts', voice: 'alloy' }),+ ).toThrow('model_unavailable');+});++it('supports documented native endpoints without guessing protocols from model IDs or chat wire', () => {+ const tts: ProviderRuntimeModelConfig[] = [+ { id: 'private-voice', name: 'Voice', discoveredMetadata: { mode: 'audio_speech' } },+ ];+ const d = deps([+ custom('eleven', 'https://api.elevenlabs.io/v1', tts),+ custom('mini', 'https://api.minimax.io/anthropic', tts),+ custom('openai', 'https://api.openai.com/v1', tts),+ custom('unknown', 'https://proxy.test/v1', tts),+ custom('chat', 'https://api.elevenlabs.io/v1', [{ id: 'chat', name: 'Chat' }]),]);- expect(speechModels(catalog, false, () => false)).toEqual([]);+ expect(speechModels(d).map((m) => m.protocol)).toEqual([+ 'elevenlabs',+ 'minimax',+ 'openai',+ undefined,+ ]);+ expect(+ resolveSpeechConnection(d, {+ providerId: 'eleven',+ agent: 'codex',+ modelId: 'private-voice',+ voice: 'voice',+ }).headers,+ ).toEqual({ 'xi-api-key': 'eleven-codex-key' });+ expect(+ resolveSpeechConnection(d, {+ providerId: 'mini',+ agent: 'codex',+ modelId: 'private-voice',+ voice: 'voice',+ }).baseUrl,+ ).toBe('https://api.minimax.io/v1');});-it('does not pin a default speech model and lets null reset an explicit preference', () => {+it('requires an explicit choice for null protocols and excludes unusable credentials/routes', () => {+ const provider = custom('p', 'https://api.elevenlabs.io/v1', [+ { id: 'voice', name: 'Voice', mode: 'audio_speech', nativeApi: null },+ ]);+ const d = deps([provider]);+ expect(speechModels(d)[0].protocol).toBeUndefined();+ const selected = { providerId: 'p', agent: 'codex' as const, modelId: 'voice', voice: 'v' };+ expect(() => resolveSpeechConnection(d, selected)).toThrow('model_unavailable');+ expect(resolveSpeechConnection(d, { ...selected, protocol: 'elevenlabs' }).protocol).toBe(+ 'elevenlabs',+ );+ const p = custom('p', 'https://proxy.test/v1');+ d.catalog.providers = [p];+ expect(speechModels({ ...d, readKey: () => null })).toEqual([]);+ expect(speechModels({ ...d, disabled: () => true })).toEqual([]);+ p.routing.codex!.disabled = true;+ expect(speechModels(d)).toEqual([]);+ p.routing.codex!.disabled = false;+ p.auth.native = 'codex';+ expect(speechModels(d)).toEqual([]);+});++it('rejects cross-origin model overrides and stale credentials in headers; permits explicit local no-auth', () => {+ const p = custom('p', 'https://api.elevenlabs.io/v1', [+ { id: 'v', name: 'V', mode: 'audio_speech' },+ ]);+ const d = deps([p]);+ p.routing.codex!.headerOverride = {+ Authorization: 'old',+ 'xi-api-key': 'stale',+ Cookie: 'private',+ 'Content-Type': 'bad',+ 'X-Project': 'project',+ };+ expect(+ resolveSpeechConnection(d, { providerId: 'p', agent: 'codex', modelId: 'v', voice: 'v' })+ .headers,+ ).toEqual({ 'xi-api-key': 'p-codex-key', 'x-project': 'project' });+ p.models.codex![0].route = { baseUrl: 'https://foreign.test/v1', wireProtocol: 'openai-chat' };+ expect(speechModels(d)).toEqual([]);+ const local = custom('local', 'http://127.0.0.1:8000/v1');+ local.auth.method = 'none';+ local.routing.codex!.authStrategy = 'none';+ expect(speechModels(deps([local]))).toHaveLength(1);+ local.routing.codex!.upstream = 'https://remote.test/v1';+ expect(speechModels(deps([local]))).toEqual([]);+});++it('normalizes legacy settings while validating connection/runtime identity', () => {expect(normalizeVoiceInputSettings({}).conversationSelection).toBeUndefined();expect(normalizeVoiceInputSettings({ conversationSelection: { modelId: 'tts', voice: 'alloy' } })@@ -44,8 +179,38 @@ it('does not pin a default speech model and lets null reset an explicit preferenexpect(normalizeVoiceInputSettings({ conversationSelection: null }).conversationSelection,).toBeUndefined();- expect(- normalizeVoiceInputSettings({ conversationSelection: { modelId: 'tts', voice: 'bad\nvoice' } })- .conversationSelection,- ).toBeUndefined();+ for (const invalid of [+ { modelId: 'tts', voice: 'bad\nvoice' },+ { modelId: 'tts', voice: 'alloy', providerId: 'byok' },+ ])+ expect(+ normalizeVoiceInputSettings({ conversationSelection: invalid }).conversationSelection,+ ).toBeUndefined();});++it.each(['openai', 'elevenlabs', 'minimax'] as const)(+ 'requires a user choice to execute an unknown proxy with %s and keeps its prefix',+ (protocol) => {+ const d = deps([+ custom('proxy', 'https://proxy.test/prefix/v1', [+ { id: 'private-tts', name: 'TTS', mode: 'audio_speech' },+ ]),+ ]);+ const selection = {+ providerId: 'proxy',+ agent: 'codex' as const,+ modelId: 'private-tts',+ voice: 'voice-id',+ };+ expect(speechModels(d)[0].protocol).toBeUndefined();+ expect(() => resolveSpeechConnection(d, selection)).toThrow('model_unavailable');+ expect(resolveSpeechConnection(d, { ...selection, protocol })).toEqual({+ baseUrl: 'https://proxy.test/prefix/v1',+ protocol,+ headers:+ protocol === 'elevenlabs'+ ? { 'xi-api-key': 'proxy-codex-key' }+ : { authorization: 'Bearer proxy-codex-key' },+ });+ },+);diff --git a/apps/desktop/src/main/voice-conversation/__tests__/preferences.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/preferences.test.tsnew file mode 100644index 0000000000..8f895c350f--- /dev/null+++ b/apps/desktop/src/main/voice-conversation/__tests__/preferences.test.ts@@ -0,0 +1,39 @@+import { expect, it } from 'vitest';+import { normalizeVoiceInputSettings } from '../../../shared/voiceInputData.js';+import { companionVoicePreference, updateCompanionVoicePreference } from '../preferences.js';+const a = { providerId: 'a', agent: 'codex' as const, modelId: 'tts', voice: 'alloy' };+const b = { ...a, providerId: 'b', voice: 'nova' };+it('persists each companion independently, preserves legacy inheritance, and deletes only a reset override', () => {+ let settings = normalizeVoiceInputSettings({+ conversationSelection: { modelId: 'old', voice: 'alloy' },+ });+ const update = (id: string, value: typeof a | null) => {+ settings = normalizeVoiceInputSettings({+ ...settings,+ conversationSelections: updateCompanionVoicePreference(settings, id, value),+ });+ };+ update('bot-a', a);+ update('bot-b', b);+ settings = normalizeVoiceInputSettings(JSON.parse(JSON.stringify(settings)));+ expect(companionVoicePreference(settings, 'bot-a')).toEqual({ selection: a, customized: true });+ expect(companionVoicePreference(settings, 'bot-b')).toEqual({ selection: b, customized: true });+ update('bot-a', null);+ expect(settings.conversationSelections!['bot-a']).toBeNull();+ expect(companionVoicePreference(settings, 'bot-a')).toEqual({+ selection: null,+ customized: false,+ });+ expect(companionVoicePreference(settings, 'bot-b').selection).toEqual(b);+ expect(companionVoicePreference(normalizeVoiceInputSettings({}), 'bot-b')).toEqual({+ selection: null,+ customized: false,+ });+});+it('does not read prototype keys or retain malformed saved choices', () => {+ const settings = normalizeVoiceInputSettings({+ conversationSelections: { invalid: { modelId: 'tts', voice: '/' }, valid: a },+ });+ expect(companionVoicePreference(settings, 'toString').selection).toBe(null);+ expect(Object.keys(settings.conversationSelections!)).toEqual(['valid']);+});diff --git a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/service.test.tsindex 2902b075ef..116d3a2a64 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts@@ -29,7 +29,11 @@ function fixture() {assertCompanion: vi.fn(async () => {}),models: vi.fn(() => [{ id: 'tts', name: 'Speech' }]),createProvider: vi.fn(async () => provider),- connection: () => ({ baseUrl: 'https://gateway.test/v1', apiKey: 'test-key' }),+ connection: vi.fn(() => ({+ baseUrl: 'https://gateway.test/v1',+ protocol: 'openai' as const,+ headers: { authorization: 'Bearer test-key' },+ })),fetch: vi.fn(async () =>new Response(new Uint8Array([1, 2]), { headers: { 'Content-Type': 'audio/pcm' } }),@@ -78,7 +82,9 @@ describe('local companion call ownership', () => {it('rechecks catalog eligibility before speech dispatch', async () => {const { service, deps } = fixture();const { callId } = await service.start(1, selection);- deps.models.mockReturnValue([]);+ deps.connection.mockImplementation(() => {+ throw new Error('model_unavailable');+ });await expect(service.speak(1, callId, 'request', 'hello')).rejects.toThrow('model_unavailable');expect(deps.fetch).not.toHaveBeenCalled();});@@ -101,3 +107,35 @@ describe('local companion call ownership', () => {expect(provider.dispose).toHaveBeenCalledOnce();});});++it('pins the selected connection and stops before sending to changed credentials or endpoints', async () => {+ const { service, deps } = fixture();+ await service.start(1, selection);+ deps.connection.mockReturnValue({+ baseUrl: 'https://changed.test/v1',+ protocol: 'openai',+ headers: { authorization: 'Bearer different-key' },+ });+ await expect(service.speak(1, selection.callId, 'req', 'private reply')).rejects.toThrow(+ 'connection_changed',+ );+ expect(deps.fetch).not.toHaveBeenCalled();+});++it('cancels an existing body when credentials change before a PCM read', async () => {+ const { service, deps } = fixture();+ const cancelled = vi.fn();+ deps.fetch.mockImplementation(+ async () =>+ new Response(new ReadableStream<Uint8Array>({ cancel: cancelled }), {+ headers: { 'Content-Type': 'audio/pcm' },+ }),+ );+ await service.start(1, selection);+ await service.speak(1, selection.callId, 'req', 'hello');+ deps.connection.mockImplementation(() => {+ throw new Error('model_unavailable');+ });+ await expect(service.readSpeech(1, selection.callId, 'req')).rejects.toThrow('model_unavailable');+ expect(cancelled).toHaveBeenCalledOnce();+});diff --git a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.tsindex 20b42a672b..a8664c2064 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts@@ -8,7 +8,11 @@ const make = () => {const assertCurrent = vi.fn();const speech = new SpeechSynthesisSession({- connection: () => ({ baseUrl: 'https://gateway.test/prefix/v1/', apiKey: 'test-key' }),+ connection: () => ({+ baseUrl: 'https://gateway.test/prefix/v1/',+ protocol: 'openai' as const,+ headers: { authorization: 'Bearer test-key' },+ }),fetch: fetcher,assertCurrent,});@@ -105,3 +109,132 @@ describe('speech transport', () => {speech.cancel();});});++function nativeSession(protocol: 'elevenlabs' | 'minimax', response: Response) {+ const fetcher = vi.fn(async () => response);+ const speech = new SpeechSynthesisSession({+ connection: () => ({+ baseUrl: 'https://provider.test/proxy/v1',+ protocol,+ headers:+ protocol === 'elevenlabs'+ ? ({ 'xi-api-key': 'test-eleven-key' } as Record<string, string>)+ : ({ authorization: 'Bearer test-mini-key' } as Record<string, string>),+ }),+ assertCurrent: () => {},+ fetch: fetcher,+ });+ return { speech, fetcher };+}+function sse(events: unknown[], split = 7) {+ const bytes = new TextEncoder().encode(+ events.map((event) => `data: ${JSON.stringify(event)}\r\n\r\n`).join(''),+ );+ return new Response(+ new ReadableStream<Uint8Array>({+ start(controller) {+ for (let i = 0; i < bytes.length; i += split) controller.enqueue(bytes.slice(i, i + split));+ controller.close();+ },+ }),+ { headers: { 'Content-Type': 'text/event-stream' } },+ );+}++it('uses the ElevenLabs stream endpoint, voice ID, model_id and xi-api-key with raw 24 kHz PCM', async () => {+ const { speech, fetcher } = nativeSession(+ 'elevenlabs',+ new Response(new Uint8Array([1, 2]), {+ headers: { 'Content-Type': 'application/octet-stream' },+ }),+ );+ expect(await speech.start({ ...input, voice: 'a_voice-id' })).toEqual({ sampleRate: 24000 });+ expect(fetcher).toHaveBeenCalledWith(+ new URL(+ 'https://provider.test/proxy/v1/text-to-speech/a_voice-id/stream?output_format=pcm_24000',+ ),+ expect.objectContaining({+ redirect: 'error',+ headers: { 'xi-api-key': 'test-eleven-key', 'Content-Type': 'application/json' },+ body: JSON.stringify({ model_id: input.modelId, text: input.text }),+ }),+ );+ expect([...new Uint8Array((await speech.read('one')).pcm)]).toEqual([1, 2]);+ expect((await speech.read('one')).done).toBe(true);+});++it('decodes fragmented MiniMax SSE/hex without replaying the terminal aggregate', async () => {+ const { speech, fetcher } = nativeSession(+ 'minimax',+ sse([+ { data: { status: 1, audio: '0102' }, base_resp: { status_code: 0 } },+ { data: { status: 1, audio: '0304' } },+ {+ data: { status: 2, audio: '01020304' },+ extra_info: { audio_sample_rate: 24000, audio_channel: 1, audio_format: 'pcm' },+ base_resp: { status_code: 0 },+ },+ ]),+ );+ await speech.start(input);+ expect(fetcher).toHaveBeenCalledWith(+ new URL('https://provider.test/proxy/v1/t2a_v2'),+ expect.objectContaining({+ headers: { authorization: 'Bearer test-mini-key', 'Content-Type': 'application/json' },+ body: JSON.stringify({+ model: input.modelId,+ text: input.text,+ stream: true,+ stream_options: { exclude_aggregated_audio: true },+ output_format: 'hex',+ voice_setting: { voice_id: input.voice },+ audio_setting: { format: 'pcm', sample_rate: 24000, channel: 1 },+ }),+ }),+ );+ expect([...new Uint8Array((await speech.read('one')).pcm)]).toEqual([1, 2]);+ expect([...new Uint8Array((await speech.read('one')).pcm)]).toEqual([3, 4]);+ expect(await speech.read('one')).toEqual({ done: true, pcm: new ArrayBuffer(0) });+});++it.each([+ [{ data: { status: 1, audio: 'zz' } }, 'format'],+ [{ data: { status: 1, audio: '0' } }, 'format'],+ [+ { data: { status: 2 }, base_resp: { status_code: 1004, status_msg: 'private token detail' } },+ 'request',+ ],+ [{ data: { status: 2 }, extra_info: { audio_format: 'mp3' } }, 'format'],+ [{ data: { status: 2 }, extra_info: { audio_sample_rate: 32000 } }, 'format'],+ [{ data: { status: 2 }, extra_info: { audio_channel: 2 } }, 'format'],+])('rejects invalid MiniMax output with a safe error', async (event, message) => {+ const { speech } = nativeSession('minimax', sse([event]));+ await speech.start(input);+ await expect(speech.read('one')).rejects.toThrow(new RegExp(`^${message}$`));+});++it('rejects a truncated MiniMax stream instead of claiming completion', async () => {+ const { speech } = nativeSession('minimax', sse([{ data: { status: 1, audio: '0102' } }]));+ await speech.start(input);+ await speech.read('one');+ await expect(speech.read('one')).rejects.toThrow('format');+});++it('cancels a MiniMax read waiting for the remainder of an SSE event', async () => {+ const cancelled = vi.fn();+ const body = new ReadableStream<Uint8Array>({+ start(c) {+ c.enqueue(new TextEncoder().encode('data: {'));+ },+ cancel: cancelled,+ });+ const { speech } = nativeSession(+ 'minimax',+ new Response(body, { headers: { 'Content-Type': 'text/event-stream' } }),+ );+ await speech.start(input);+ const reading = speech.read('one');+ speech.cancel();+ await expect(reading).rejects.toThrow('cancelled');+ expect(cancelled).toHaveBeenCalledOnce();+});diff --git a/apps/desktop/src/main/voice-conversation/index.ts b/apps/desktop/src/main/voice-conversation/index.tsindex 838d46bb07..cd60230069 100644--- a/apps/desktop/src/main/voice-conversation/index.ts+++ b/apps/desktop/src/main/voice-conversation/index.ts@@ -1,8 +1,11 @@import { ipcMain, webContents, type IpcMainInvokeEvent } from 'electron';import type { AsrProvider } from '@cindy/voice-input-core';-import { normalizeVoiceConversationSelection } from '../../shared/voiceConversation.js';+import {+ normalizeVoiceConversationSelection,+ type VoiceConversationPreference,+} from '../../shared/voiceConversation.js';import { getActiveCatalog } from '../maker-host/active-catalog.js';-import { getProviderSecretStore } from '../secrets/providerSecretStore.js';+import { getProviderSecretStore, readCustomProviderKey } from '../secrets/providerSecretStore.js';import { effectiveXdGatewayBaseUrl } from '../model-access/effectiveEndpoint.js';import { outboundFetch } from '../maker-host/outbound-fetch.js';import { getAppCapabilities } from '../appCapabilities.js';@@ -12,7 +15,9 @@ import { isProviderModelRouteDisabled } from '../utility-model/oneShotCandidatesimport { assertTrustedAppRendererEvent } from '../security/trustedAppRenderer.js';import { requireObject, requireString, throwIpcError } from '../utils/ipcValidate.js';import { VoiceConversationService } from './service.js';-import { speechModels } from './models.js';+import { resolveSpeechConnection, speechModels, type SpeechCatalogDeps } from './models.js';+import { companionVoicePreference, updateCompanionVoicePreference } from './preferences.js';+import type { VoiceInputSettings } from '../../shared/voiceInputData.js';function boundedString(value: unknown, name: string, limit: number): string {const text = requireString(value, name);@@ -24,7 +29,22 @@ function boundedString(value: unknown, name: string, limit: number): string {export function registerVoiceConversationIpc(deps: {createProvider(): Promise<AsrProvider>;hasDictation(): boolean;+ getSettings(): VoiceInputSettings;+ updateSettings(patch: Partial<VoiceInputSettings>): VoiceInputSettings;}): void {+ const catalogDeps = (): SpeechCatalogDeps => {+ if (isAppSessionBoundaryPending()) throw new Error('changing_account');+ return {+ catalog: getActiveCatalog(),+ gatewayAllowed: getAppCapabilities().canUseCindyGateway,+ disabled: isProviderModelRouteDisabled,+ readKey: readCustomProviderKey,+ gatewayConnection: () => ({+ baseUrl: effectiveXdGatewayBaseUrl(),+ apiKey: getProviderSecretStore().get('xd') ?? '',+ }),+ };+ };const service = new VoiceConversationService({...deps,owner: () => (isAppSessionBoundaryPending() ? 'changing-account' : activeOwnerScopeKey()),@@ -33,17 +53,8 @@ export function registerVoiceConversationIpc(deps: {if (bot.hiddenAt !== null || bot.canonicalSessionId !== sessionId)throw new Error('invalid_companion');},- models: () =>- speechModels(getActiveCatalog(), getAppCapabilities().canUseCindyGateway, (id) =>- isProviderModelRouteDisabled('xd', id),- ),- connection: () => {- if (!getAppCapabilities().canUseCindyGateway) throw new Error('unavailable');- return {- baseUrl: effectiveXdGatewayBaseUrl(),- apiKey: getProviderSecretStore().get('xd') ?? '',- };- },+ models: () => speechModels(catalogDeps()),+ connection: (selection) => resolveSpeechConnection(catalogDeps(), selection),fetch: outboundFetch,emit: (sender, event) => {const target = webContents.fromId(sender);@@ -65,6 +76,29 @@ export function registerVoiceConversationIpc(deps: {});};handle('models', () => service.models());+ const preferenceBot = async (+ raw: unknown,+ action: (botId: string) => VoiceConversationPreference,+ ): Promise<VoiceConversationPreference> => {+ if (isAppSessionBoundaryPending()) throw new Error('changing_account');+ const owner = activeOwnerScopeKey();+ const botId = boundedString(raw, 'botId', 256);+ const bot = await getBotRemoteResourceSource(botId);+ if (isAppSessionBoundaryPending() || activeOwnerScopeKey() !== owner || bot.hiddenAt !== null)+ throw new Error('invalid_companion');+ return action(botId);+ };+ handle('preference', (_event, raw) =>+ preferenceBot(raw, (botId) => companionVoicePreference(deps.getSettings(), botId)),+ );+ handle('set-preference', (_event, raw, value) =>+ preferenceBot(raw, (botId) => {+ const selection = value === null ? null : normalizeVoiceConversationSelection(value);+ if (selection === undefined) throwIpcError('INVALID_PARAMS', 'Invalid speech selection');+ const map = updateCompanionVoicePreference(deps.getSettings(), botId, selection);+ return companionVoicePreference(deps.updateSettings({ conversationSelections: map }), botId);+ }),+ );handle('start', async (event, raw) => {if (isAppSessionBoundaryPending()) throw new Error('changing_account');const value = requireObject(raw);diff --git a/apps/desktop/src/main/voice-conversation/models.ts b/apps/desktop/src/main/voice-conversation/models.tsindex 65c9380b6f..4fc5b5292d 100644--- a/apps/desktop/src/main/voice-conversation/models.ts+++ b/apps/desktop/src/main/voice-conversation/models.ts@@ -1,33 +1,207 @@-import { BUNDLED_CATALOG, resolveModelNativeApi, type Catalog } from '@cindy/model-providers';-import type { VoiceConversationModel } from '../../shared/voiceConversation.js';+import {+ BUNDLED_CATALOG,+ isLoopbackProviderUrl,+ resolveModelNativeApi,+ type AgentKind,+ type Catalog,+ type ProviderMediaModel,+} from '@cindy/model-providers';+import {+ voiceModelKey,+ type SpeechProtocol,+ type VoiceConversationModel,+ type VoiceConversationSelection,+} from '../../shared/voiceConversation.js';+import type { SpeechConnection } from './speech.js';-/** Registry metadata explains the protocol; only the account catalog grants access. */-export function speechModels(- catalog: Catalog,- gatewayAllowed: boolean,- disabled: (modelId: string) => boolean,-): VoiceConversationModel[] {- if (!gatewayAllowed) return [];- return (catalog.providers.find((provider) => provider.id === 'xd')?.audioModels ?? [])- .filter((model) => {+export type SpeechCatalogDeps = {+ catalog: Catalog;+ gatewayAllowed: boolean;+ disabled(providerId: string, modelId: string): boolean;+ gatewayConnection(): { baseUrl: string; apiKey: string };+ readKey(providerId: string, agent: AgentKind): string | null;+};+type Binding = { model: VoiceConversationModel; connection: SpeechConnection | null };++function apiBase(raw: string): URL | null {+ try {+ const url = new URL(raw);+ if (+ !['http:', 'https:'].includes(url.protocol) ||+ url.username ||+ url.password ||+ url.search ||+ url.hash+ )+ return null;+ url.pathname = url.pathname.replace(/\/+$/, '');+ return url;+ } catch {+ return null;+ }+}++function nativeProtocol(catalog: Catalog, providerId: string, model: ProviderMediaModel) {+ const declared =+ model.nativeApi !== undefined+ ? model.nativeApi+ : resolveModelNativeApi(catalog.modelRegistry, providerId, model.id);+ return declared !== undefined+ ? declared+ : resolveModelNativeApi(BUNDLED_CATALOG.modelRegistry, providerId, model.id);+}++/** Endpoint adapters are explicit contracts, not guesses based on a model name or chat wire. */+function customProtocol(+ base: URL,+ declared: ReturnType<typeof nativeProtocol>,+): SpeechProtocol | null {+ if (declared === 'openai-audio-speech') return 'openai';+ if (declared !== undefined) return null; // Includes an explicit unverified/null declaration.+ return endpointProtocol(base);+}++function endpointProtocol(base: URL): SpeechProtocol | null {+ if (base.protocol !== 'https:') return null;+ if (base.hostname === 'api.elevenlabs.io') return 'elevenlabs';+ if (+ [+ 'api.minimax.io',+ 'api.minimax.cn',+ 'api.minimaxi.com',+ 'api.minimax.chat',+ 'api-uw.minimax.io',+ ].includes(base.hostname)+ )+ return 'minimax';+ if (base.hostname === 'api.openai.com') return 'openai';+ return null;+}++function bindings(deps: SpeechCatalogDeps, selection?: VoiceConversationSelection): Binding[] {+ const result: Binding[] = [];+ for (const provider of deps.catalog.providers) {+ if (selection && provider.id !== (selection.providerId ?? 'xd')) continue;+ for (const media of provider.audioModels ?? []) {+ if (selection && media.id !== selection.modelId) continue;+ if (+ media.mode !== 'audio_speech' ||+ media.disabled ||+ deps.disabled(provider.id, media.id) ||+ (media.availability !== undefined && media.availability !== 'available')+ )+ continue;+ const declared = nativeProtocol(deps.catalog, provider.id, media);+ if (provider.id === 'xd') {+ if (+ !deps.gatewayAllowed ||+ media.availability !== 'available' ||+ (declared !== undefined && declared !== 'openai-audio-speech')+ )+ continue;+ const connection = deps.gatewayConnection();+ if (!connection.apiKey || !apiBase(connection.baseUrl)) continue;+ result.push({+ model: {+ id: media.id,+ name: media.name,+ providerId: 'xd',+ providerName: provider.name,+ protocol: 'openai',+ defaultVoice: 'alloy',+ },+ connection: {+ baseUrl: connection.baseUrl,+ protocol: 'openai',+ headers: { authorization: `Bearer ${connection.apiKey}` },+ },+ });+ continue;+ }if (- model.mode !== 'audio_speech' ||- model.availability !== 'available' ||- model.disabled ||- disabled(model.id)+ provider.source !== 'user' ||+ provider.auth.native ||+ !['apiKey', 'none'].includes(provider.auth.method))- return false;- const declared =- model.nativeApi !== undefined- ? model.nativeApi- : resolveModelNativeApi(catalog.modelRegistry, 'xd', model.id);- const protocol =- declared !== undefined- ? declared- : resolveModelNativeApi(BUNDLED_CATALOG.modelRegistry, 'xd', model.id);- // Unknown Gateway entries use its OpenAI speech contract. Explicitly unverified- // (null) or another native API must not be silently treated as that protocol.- return protocol === undefined || protocol === 'openai-audio-speech';- })- .map((model) => ({ id: model.id, name: model.name }));+ continue;+ const agent = media.sourceAgent;+ if (!agent) continue;+ const route = provider.routing[agent];+ const model = provider.models[agent]?.find((entry) => entry.id === media.id);+ if (!route || route.disabled || route.headerOverrideState === 'unknown' || !model) continue;+ const base = apiBase(model.route?.baseUrl ?? route.upstream);+ const ownerBase = apiBase(route.upstream);+ if (!base || !ownerBase || base.origin !== ownerBase.origin) continue;+ const detectedProtocol = customProtocol(base, declared);+ const protocol = selection?.protocol ?? detectedProtocol;+ // Official native adapters use their documented origin endpoint. Compatible+ // OpenAI proxies keep the configured prefix (including a non-/v1 API base).+ if (protocol !== 'openai' && endpointProtocol(base) === protocol) base.pathname = '/v1';+ else if (!base.pathname || base.pathname === '/') base.pathname = '/v1';+ const headers: Record<string, string> = {};+ for (const [name, value] of Object.entries(route.headerOverride ?? {})) {+ const lower = name.toLowerCase();+ if (+ [+ 'authorization',+ 'x-api-key',+ 'xi-api-key',+ 'api-key',+ 'cookie',+ 'host',+ 'content-type',+ 'content-length',+ 'connection',+ 'chatgpt-account-id',+ 'openai-account-id',+ 'openai-beta',+ 'originator',+ 'session_id',+ ].includes(lower)+ )+ continue;+ headers[lower] = value;+ }+ for (const name of route.headerDelete ?? []) delete headers[name.toLowerCase()];+ if (provider.auth.method === 'none') {+ if (route.authStrategy !== 'none' || !isLoopbackProviderUrl(base.toString())) continue;+ } else {+ if (route.authStrategy !== 'api-key-header') continue;+ const key = deps.readKey(provider.id, agent)?.trim();+ if (!key) continue;+ headers[protocol === 'elevenlabs' ? 'xi-api-key' : 'authorization'] =+ protocol === 'elevenlabs' ? key : `Bearer ${key}`;+ }+ result.push({+ model: {+ id: media.id,+ name: media.name,+ providerId: provider.id,+ providerName: provider.name,+ agent,+ protocol: detectedProtocol ?? undefined,+ defaultVoice: protocol === 'openai' ? 'alloy' : '',+ },+ connection: protocol+ ? { baseUrl: base.toString().replace(/\/+$/, ''), headers, protocol }+ : null,+ });+ }+ }+ return result;+}++/** Registry entries alone never add catalog membership or grant a credential. */+export function speechModels(deps: SpeechCatalogDeps): VoiceConversationModel[] {+ return bindings(deps).map((binding) => binding.model);+}+export function resolveSpeechConnection(+ deps: SpeechCatalogDeps,+ selection: VoiceConversationSelection,+): SpeechConnection {+ const binding = bindings(deps, selection).find(+ (entry) => voiceModelKey(entry.model) === voiceModelKey(selection),+ );+ if (!binding?.connection) throw new Error('model_unavailable');+ return binding.connection;}diff --git a/apps/desktop/src/main/voice-conversation/preferences.ts b/apps/desktop/src/main/voice-conversation/preferences.tsnew file mode 100644index 0000000000..7da98d3223--- /dev/null+++ b/apps/desktop/src/main/voice-conversation/preferences.ts@@ -0,0 +1,37 @@+import {+ normalizeVoiceConversationPreferences,+ type VoiceConversationPreference,+ type VoiceConversationSelection,+} from '../../shared/voiceConversation.js';+import type { VoiceInputSettings } from '../../shared/voiceInputData.js';++export function companionVoicePreference(+ settings: VoiceInputSettings,+ botId: string,+): VoiceConversationPreference {+ const map = normalizeVoiceConversationPreferences(settings.conversationSelections);+ const selection = Object.hasOwn(map, botId)+ ? map[botId]+ : (settings.conversationSelection ?? null);+ return { selection, customized: !!selection };+}++/** Read the latest map in Main before changing one key; windows never overwrite each other. */+export function updateCompanionVoicePreference(+ settings: VoiceInputSettings,+ botId: string,+ selection: VoiceConversationSelection | null,+): Record<string, VoiceConversationSelection | null> {+ const map = normalizeVoiceConversationPreferences(settings.conversationSelections);+ // A null entry clears only this companion's legacy global selection. It stores+ // no derived model/voice default; other companions retain their old preference.+ if (selection || settings.conversationSelection)+ Object.defineProperty(map, botId, {+ value: selection,+ enumerable: true,+ configurable: true,+ writable: true,+ });+ else delete map[botId];+ return map;+}diff --git a/apps/desktop/src/main/voice-conversation/service.ts b/apps/desktop/src/main/voice-conversation/service.tsindex 08d0c5ba9c..c6a9203f83 100644--- a/apps/desktop/src/main/voice-conversation/service.ts+++ b/apps/desktop/src/main/voice-conversation/service.ts@@ -1,4 +1,5 @@import { VoiceConversationInput, type AsrProvider, type AudioTrace } from '@cindy/voice-input-core';+import { voiceModelKey, type VoiceConversationSelection } from '../../shared/voiceConversation.js';import type {VoiceConversationEvent,VoiceConversationModel,@@ -27,7 +28,7 @@ export class VoiceConversationService {assertCompanion(botId: string, sessionId: string): Promise<void>;models(): VoiceConversationModel[];createProvider(): Promise<AsrProvider>;- connection(): SpeechConnection;+ connection(selection: VoiceConversationSelection): SpeechConnection;fetch: typeof fetch;emit(sender: number, event: VoiceConversationEvent): void;},@@ -39,8 +40,10 @@ export class VoiceConversationService {async start(sender: number, selection: VoiceConversationStart): Promise<{ callId: string }> {if (this.call || this.deps.hasDictation()) throw new Error('busy');- if (!this.models().some((model) => model.id === selection.modelId))+ if (!this.models().some((model) => voiceModelKey(model) === voiceModelKey(selection)))throw new Error('model_unavailable');+ const binding = this.deps.connection(selection);+ const bindingIdentity = JSON.stringify(binding);const id = selection.callId;const owner = this.deps.owner();if (!acquireVoiceConversationLease(id)) throw new Error('busy');@@ -63,12 +66,13 @@ export class VoiceConversationService {},});const speech = new SpeechSynthesisSession({- connection: this.deps.connection,+ connection: () => binding,fetch: this.deps.fetch,assertCurrent: () => {assertCurrent();- if (!this.models().some((model) => model.id === selection.modelId))- throw new Error('model_unavailable');+ // Resolving the selected binding rechecks its current catalog, route and key.+ if (JSON.stringify(this.deps.connection(selection)) !== bindingIdentity)+ throw new Error('connection_changed');},});this.call = { id, owner, sender, selection, input, speech };diff --git a/apps/desktop/src/main/voice-conversation/speech.ts b/apps/desktop/src/main/voice-conversation/speech.tsindex 787f4cfea6..dc81beefa4 100644--- a/apps/desktop/src/main/voice-conversation/speech.ts+++ b/apps/desktop/src/main/voice-conversation/speech.ts@@ -1,13 +1,7 @@-export type SpeechConnection = { baseUrl: string; apiKey: string };--/** Safe, fixed-protocol errors: upstream bodies and credentials never cross IPC. */-export class SpeechError extends Error {- constructor(readonly code: 'unavailable' | 'request' | 'format' | 'cancelled') {- super(code);- }-}--/** OpenAI-compatible speech transport. Reads are pulled by playback for bounded buffering. */+import { miniMaxPcmStream, speechRequest, type SpeechConnection } from './speechProtocols.js';+import { SpeechError } from './speechError.js';+export type { SpeechConnection } from './speechProtocols.js';+/** PCM speech transport with provider adapters. Reads are pulled by playback for bounded buffering. */export class SpeechSynthesisSession {private epoch = 0;private controller: AbortController | null = null;@@ -33,26 +27,20 @@ export class SpeechSynthesisSession {this.cancel();const epoch = this.epoch;this.deps.assertCurrent();- const { baseUrl, apiKey } = this.deps.connection();- const url = new URL(baseUrl.replace(/\/+$/, '') + '/audio/speech');- if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || !apiKey)- throw new SpeechError('unavailable');+ const connection = this.deps.connection();+ const { url, body } = speechRequest(connection, input);const controller = new AbortController();this.controller = controller;this.requestId = input.requestId;const timer = setTimeout(() => controller.abort(), 30_000);+ let response: Response | undefined;try {- const response = await this.deps.fetch(url, {+ response = await this.deps.fetch(url, {method: 'POST',redirect: 'error',signal: controller.signal,- headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },- body: JSON.stringify({- model: input.modelId,- voice: input.voice,- input: input.text,- response_format: 'pcm',- }),+ headers: { ...connection.headers, 'Content-Type': 'application/json' },+ body,});this.deps.assertCurrent();if (epoch !== this.epoch || controller.signal.aborted) {@@ -66,15 +54,23 @@ export class SpeechSynthesisSession {const contentType = response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase();if (!contentType ||- !['audio/pcm', 'audio/x-pcm', 'application/octet-stream'].includes(contentType)+ !(+ connection.protocol === 'minimax'+ ? ['text/event-stream', 'application/json']+ : ['audio/pcm', 'audio/x-pcm', 'application/octet-stream']+ ).includes(contentType)) {await response.body.cancel();throw new SpeechError('format');}- this.reader = response.body.getReader();- // This sample rate is the OpenAI PCM protocol contract, not guessed from model mode.+ this.reader = (+ connection.protocol === 'minimax' ? miniMaxPcmStream(response.body) : response.body+ ).getReader();+ // All three adapters request 24 kHz mono PCM; this is not inferred from model mode.return { sampleRate: 24_000 };} catch (error) {+ if (response?.body && !response.body.locked)+ void response.body.cancel().catch(() => undefined);if (epoch === this.epoch) this.cancel();throw error instanceof SpeechError ? error : new SpeechError('request');} finally {@@ -83,7 +79,12 @@ export class SpeechSynthesisSession {}async read(requestId: string): Promise<{ pcm: ArrayBuffer; done: boolean }> {- this.deps.assertCurrent();+ try {+ this.deps.assertCurrent();+ } catch (error) {+ this.cancel();+ throw error;+ }if (requestId !== this.requestId || !this.reader || this.reading)throw new SpeechError('cancelled');const reader = this.reader;diff --git a/apps/desktop/src/main/voice-conversation/speechError.ts b/apps/desktop/src/main/voice-conversation/speechError.tsnew file mode 100644index 0000000000..6b4dcc918a--- /dev/null+++ b/apps/desktop/src/main/voice-conversation/speechError.ts@@ -0,0 +1,6 @@+/** Safe errors only: never send upstream bodies, paths or credentials across IPC. */+export class SpeechError extends Error {+ constructor(readonly code: 'unavailable' | 'request' | 'format' | 'cancelled') {+ super(code);+ }+}diff --git a/apps/desktop/src/main/voice-conversation/speechProtocols.ts b/apps/desktop/src/main/voice-conversation/speechProtocols.tsnew file mode 100644index 0000000000..593e5e3466--- /dev/null+++ b/apps/desktop/src/main/voice-conversation/speechProtocols.ts@@ -0,0 +1,145 @@+import type { SpeechProtocol } from '../../shared/voiceConversation.js';+import { SpeechError } from './speechError.js';++export type SpeechConnection = {+ baseUrl: string;+ headers: Record<string, string>;+ protocol: SpeechProtocol;+};+export type SpeechInput = { modelId: string; voice: string; text: string };++export function speechRequest(+ connection: SpeechConnection,+ input: SpeechInput,+): { url: URL; body: string } {+ const base = new URL(connection.baseUrl);+ if (+ !['https:', 'http:'].includes(base.protocol) ||+ base.username ||+ base.password ||+ base.search ||+ base.hash+ )+ throw new SpeechError('unavailable');+ const prefix = connection.baseUrl.replace(/\/+$/, '');+ switch (connection.protocol) {+ case 'openai':+ return {+ url: new URL(`${prefix}/audio/speech`),+ body: JSON.stringify({+ model: input.modelId,+ voice: input.voice,+ input: input.text,+ response_format: 'pcm',+ }),+ };+ case 'elevenlabs':+ return {+ url: new URL(+ `${prefix}/text-to-speech/${encodeURIComponent(input.voice)}/stream?output_format=pcm_24000`,+ ),+ body: JSON.stringify({ model_id: input.modelId, text: input.text }),+ };+ case 'minimax':+ return {+ url: new URL(`${prefix}/t2a_v2`),+ body: JSON.stringify({+ model: input.modelId,+ text: input.text,+ stream: true,+ stream_options: { exclude_aggregated_audio: true },+ output_format: 'hex',+ voice_setting: { voice_id: input.voice },+ audio_setting: { format: 'pcm', sample_rate: 24000, channel: 1 },+ }),+ };+ }+}++/** MiniMax SSE is JSON + hex. Terminal audio is an aggregate, never another chunk.+ * https://platform.minimax.io/docs/api-reference/speech-t2a-http+ * Read only on demand; bound both encoded frames and decoded PCM before returning bytes.+ */+export function miniMaxPcmStream(body: ReadableStream<Uint8Array>): ReadableStream<Uint8Array> {+ const reader = body.getReader();+ const decoder = new TextDecoder('utf-8', { fatal: true });+ let buffer = '';+ let finished = false;+ let eof = false;+ let encodedBytes = 0;+ return new ReadableStream<Uint8Array>(+ {+ async pull(controller) {+ try {+ while (true) {+ const delimiter = /\r?\n\r?\n/.exec(buffer);+ if (delimiter || (eof && buffer)) {+ const frame = delimiter ? buffer.slice(0, delimiter.index) : buffer;+ buffer = delimiter ? buffer.slice(delimiter.index + delimiter[0].length) : '';+ const data = frame+ .split(/\r?\n/)+ .filter((line) => line.startsWith('data:'))+ .map((line) => line.slice(5).trimStart())+ .join('\n');+ if (!data) continue; // SSE comments/keep-alives.+ if (data === '[DONE]') {+ if (!finished) throw new SpeechError('format');+ continue;+ }+ const event = JSON.parse(data);+ if (!event || typeof event !== 'object') throw new SpeechError('format');+ if (event.base_resp?.status_code !== undefined && event.base_resp.status_code !== 0)+ throw new SpeechError('request');+ const info = event.extra_info;+ if (+ info &&+ ((info.audio_sample_rate !== undefined && info.audio_sample_rate !== 24000) ||+ (info.audio_channel !== undefined && info.audio_channel !== 1) ||+ (info.audio_format !== undefined && info.audio_format !== 'pcm'))+ )+ throw new SpeechError('format');+ const part = event.data;+ if (!part) continue;+ if (finished || ![1, 2].includes(part.status)) throw new SpeechError('format');+ if (part.status === 2) {+ finished = true;+ // The provider may still return the full aggregate despite exclude_aggregated_audio.+ // Do not replay it after chunks already delivered to the speaker.+ continue;+ }+ const hex = part.audio;+ if (hex === undefined || hex === '') continue;+ if (+ typeof hex !== 'string' ||+ hex.length > 2_097_152 ||+ hex.length % 2 !== 0 ||+ /[^\da-f]/i.test(hex)+ )+ throw new SpeechError('format');+ controller.enqueue(Uint8Array.from(Buffer.from(hex, 'hex')));+ return;+ }+ if (eof) {+ if (!finished) throw new SpeechError('format');+ reader.releaseLock();+ controller.close();+ return;+ }+ const chunk = await reader.read();+ eof = chunk.done;+ encodedBytes += chunk.value?.byteLength ?? 0;+ if ((chunk.value?.byteLength ?? 0) > 2_097_152 || encodedBytes > 64 * 1024 * 1024)+ throw new SpeechError('format');+ buffer += decoder.decode(chunk.value, { stream: !eof });+ if (buffer.length > 2_097_152) throw new SpeechError('format');+ }+ } catch (error) {+ void reader.cancel().catch(() => undefined);+ controller.error(error instanceof SpeechError ? error : new SpeechError('format'));+ }+ },+ cancel: () => reader.cancel(),+ },+ { highWaterMark: 0 },+ );+}diff --git a/apps/desktop/src/main/voice-input/__tests__/VoiceInputDataStore.test.ts b/apps/desktop/src/main/voice-input/__tests__/VoiceInputDataStore.test.tsindex 8ede4225f6..aed5feff1b 100644--- a/apps/desktop/src/main/voice-input/__tests__/VoiceInputDataStore.test.ts+++ b/apps/desktop/src/main/voice-input/__tests__/VoiceInputDataStore.test.ts@@ -38,6 +38,7 @@ import {voiceInputDataStore,VoiceInputDataStore,} from '../VoiceInputDataStore.js';+import { companionVoicePreference, updateCompanionVoicePreference } from '../../voice-conversation/preferences.js';import { createVoiceInputHistoryEntry, getDefaultVoiceInputSettings } from '../../../shared/voiceInputData.js';describe('VoiceInputDataStore persistence', () => {@@ -54,6 +55,16 @@ describe('VoiceInputDataStore persistence', () => {vi.restoreAllMocks();});+ it('reopens per-companion speech preferences and persists reset without freezing a default', () => {+ const store = new VoiceInputDataStore();+ const choice = { providerId: 'tts-connection', agent: 'codex' as const, modelId: 'same-model', voice: 'voice-id', protocol: 'elevenlabs' as const };+ store.updateSettings({ conversationSelections: updateCompanionVoicePreference(store.getSettings(), 'companion', choice) });+ const reopened = new VoiceInputDataStore();+ expect(companionVoicePreference(reopened.getSettings(), 'companion')).toEqual({ selection: choice, customized: true });+ reopened.updateSettings({ conversationSelections: updateCompanionVoicePreference(reopened.getSettings(), 'companion', null) });+ expect(companionVoicePreference(new VoiceInputDataStore().getSettings(), 'companion')).toEqual({ selection: null, customized: false });+ });+it('关闭同步开关也会立刻广播,让在线手机清掉旧词典', () => {const store = new VoiceInputDataStore();const listener = vi.fn();diff --git a/apps/desktop/src/main/voice-input/index.ts b/apps/desktop/src/main/voice-input/index.tsindex baa1e28ad5..6c0db89297 100644--- a/apps/desktop/src/main/voice-input/index.ts+++ b/apps/desktop/src/main/voice-input/index.ts@@ -1881,6 +1881,8 @@ export function registerVoiceInputIpc(): void {registerVoiceInputDataStoreIpc();let pendingDictationStarts = 0;registerVoiceConversationIpc({+ getSettings: () => voiceInputDataStore.getSettings(),+ updateSettings: patch => voiceInputDataStore.updateSettings(patch),hasDictation: () => activeByWebContentsId.size > 0 || pendingDictationStarts > 0 || systemAudioMuteGuard.hasPendingOrActiveMute,createProvider: async () => {const owner = activeOwnerScopeKey();diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.tsindex 421c71aa17..f230326cdb 100644--- a/apps/desktop/src/preload/preload.ts+++ b/apps/desktop/src/preload/preload.ts@@ -1595,6 +1595,8 @@ contextBridge.exposeInMainWorld('electronAPI', {},voiceConversation: {models: () => ipcRenderer.invoke('voice-conversation:models'),+ preference: (botId) => ipcRenderer.invoke('voice-conversation:preference', botId),+ setPreference: (botId, selection) => ipcRenderer.invoke('voice-conversation:set-preference', botId, selection),start: (input) => ipcRenderer.invoke('voice-conversation:start', input),audio: (input) => ipcRenderer.send('voice-conversation:audio', input),finishUtterance: (callId) => ipcRenderer.invoke('voice-conversation:finish-utterance', callId),diff --git a/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx b/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsxindex 4bb9df956a..53352c868d 100644--- a/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx+++ b/apps/desktop/src/renderer/features/bots/voice/CompanionVoicePanel.tsx@@ -11,6 +11,7 @@ import { AudioLines, Mic, MicOff, PhoneOff, RefreshCw, Send, VolumeX, X } from 'import { useTranslation } from 'react-i18next';import { Button } from '@/components/ui/button';import { Input } from '@/components/ui/input';+import { FormField } from '@/components/ui/form-field';import { Select } from '@/components/ui/select';import { getDataOwnerGeneration } from '@/contexts/dataOwnerGeneration';import { getVoiceInputSettings } from '@/hooks/useVoiceInputSettings';@@ -18,10 +19,11 @@ import type { ChatMessage } from '@/lib/makerChatStore';import { WebMicAudioEngine } from '@/voice-input/WebMicAudioEngine';import { getVoiceInputWorkletUrl } from '@/voice-input/workletUrl';import type {+ SpeechProtocol,VoiceConversationModel,VoiceConversationSelection,} from '@/../shared/voiceConversation';-import { normalizeVoiceConversationSelection } from '@/../shared/voiceConversation';+import { normalizeVoiceConversationSelection, voiceModelKey } from '@/../shared/voiceConversation';import { BotAvatar } from '../BotAvatar';import type { BotChatIdentity } from '../BotSessionContentHeader';import { VoiceConversationController } from './VoiceConversationController';@@ -47,13 +49,31 @@ export function CompanionVoicePanel(props: Props) {const [models, setModels] = useState<VoiceConversationModel[]>([]);const [loading, setLoading] = useState(true);const [catalogError, setCatalogError] = useState(false);- const [preference, setPreference] = useState<VoiceConversationSelection | null>(- () => getVoiceInputSettings().conversationSelection ?? null,- );- const [voice, setVoice] = useState(preference?.voice ?? 'alloy');- const modelId = preference?.modelId ?? models[0]?.id ?? '';- const validModel = models.some((model) => model.id === modelId);- const selection = normalizeVoiceConversationSelection({ modelId, voice });+ const [preference, setPreference] = useState<VoiceConversationSelection | null>(null);+ const [voice, setVoice] = useState<string>();+ const [customized, setCustomized] = useState(false);+ const [preferenceOwner, setPreferenceOwner] = useState<string>();+ const [preferenceError, setPreferenceError] = useState(false);+ const [saving, setSaving] = useState(false);+ const dirty = useRef(false);+ const pendingSave = useRef<Promise<boolean> | null>(null);+ const ready = preferenceOwner === props.bot.id;+ const selectedModel = preference+ ? models.find((model) => voiceModelKey(model) === voiceModelKey(preference))+ : models[0];+ const validModel = !!selectedModel;+ const selectedVoice = voice ?? preference?.voice ?? selectedModel?.defaultVoice ?? 'alloy';+ const selection = normalizeVoiceConversationSelection({+ providerId: selectedModel?.providerId,+ agent: selectedModel?.agent,+ modelId: selectedModel?.id,+ protocol: preference?.protocol,+ voice: selectedVoice,+ });+ const protocol =+ preference?.protocol ??+ selectedModel?.protocol ??+ ((selectedModel?.providerId ?? 'xd') === 'xd' ? 'openai' : undefined);const controller = useMemo(() =>new VoiceConversationController({@@ -84,12 +104,15 @@ export function CompanionVoicePanel(props: Props) {state.phase,);const refreshVersion = useRef(0);+ const preferenceVersion = useRef(0);const refresh = useCallback(async (fetchCatalog = false) => {const version = ++refreshVersion.current;setLoading(true);setCatalogError(false);try {- if (fetchCatalog) await window.electronAPI.maker.refreshBuiltinProviderModels('xd');+ // A Gateway refresh failure must not hide usable local BYOK connections.+ if (fetchCatalog)+ await window.electronAPI.maker.refreshBuiltinProviderModels('xd').catch(() => undefined);const next = await window.electronAPI.voiceConversation.models();if (version === refreshVersion.current) setModels(next);} catch {@@ -99,14 +122,39 @@ export function CompanionVoicePanel(props: Props) {}}, []);+ const loadPreference = useCallback(() => {+ const version = ++preferenceVersion.current;+ setPreferenceOwner(undefined);+ setPreferenceError(false);+ setPreference(null);+ setVoice(undefined);+ setCustomized(false);+ setSaving(false);+ dirty.current = false;+ pendingSave.current = null;+ void window.electronAPI.voiceConversation+ .preference(props.bot.id)+ .then((result) => {+ if (version !== preferenceVersion.current) return;+ setPreference(result.selection);+ setCustomized(result.customized);+ setPreferenceOwner(props.bot.id);+ })+ .catch(() => {+ if (version === preferenceVersion.current) setPreferenceError(true);+ });+ }, [props.bot.id]);+useEffect(() => {void refresh();+ loadPreference();titleRef.current?.focus();return () => {++refreshVersion.current;+ ++preferenceVersion.current;controller.end();};- }, [controller, refresh]);+ }, [controller, loadPreference, refresh]);useEffect(() => controller.setBlocked(props.blocked), [controller, props.blocked]);useEffect(() => controller.updateMessages(props.messages), [controller, props.messages]);useEffect(() => {@@ -129,16 +177,60 @@ export function CompanionVoicePanel(props: Props) {};}, [controller]);- const persist = (next: VoiceConversationSelection) => {- setPreference(next);- void window.electronAPI.voiceInput- .updateSettings({ conversationSelection: next })- .catch(() => setCatalogError(true));+ const persist = (next: VoiceConversationSelection | null): Promise<boolean> => {+ if (pendingSave.current) return pendingSave.current;+ const version = preferenceVersion.current;+ setSaving(true);+ setPreferenceError(false);+ const pending = window.electronAPI.voiceConversation+ .setPreference(props.bot.id, next)+ .then((result) => {+ if (version !== preferenceVersion.current) return false;+ setPreference(result.selection);+ setCustomized(result.customized);+ setVoice(undefined);+ dirty.current = false;+ return true;+ })+ .catch(() => {+ if (version === preferenceVersion.current) setPreferenceError(true);+ return false;+ })+ .finally(() => {+ if (version === preferenceVersion.current) {+ setSaving(false);+ pendingSave.current = null;+ }+ });+ pendingSave.current = pending;+ return pending;};- const start = () => {- if (!selection || !validModel || props.blocked) return;+ const start = async () => {+ if (!selection || !validModel || !protocol || props.blocked || !ready) return;+ const version = preferenceVersion.current;+ if (dirty.current && !(await persist(selection))) return;+ if (version !== preferenceVersion.current || latest.current.blocked) return;controller.setBlocked(false);- void controller.start(selection).then(() => controller.retryUnsent());+ await controller.start(selection);+ controller.retryUnsent();+ };+ const chooseModel = (key: string) => {+ const model = models.find((item) => voiceModelKey(item) === key);+ if (!model) return;+ const next = {+ providerId: model.providerId,+ agent: model.agent,+ modelId: model.id,+ voice:+ model.providerId === selectedModel?.providerId && model.protocol === selectedModel?.protocol+ ? selectedVoice+ : (model.defaultVoice ?? 'alloy'),+ };+ dirty.current = true;+ setCustomized(true);+ setPreference(next);+ setVoice(next.voice);+ if (normalizeVoiceConversationSelection(next)) void persist(next);};const status = props.blocked ? 'confirmation' : state.phase;@@ -164,7 +256,13 @@ export function CompanionVoicePanel(props: Props) {</div><div className="min-h-0 flex-1 overflow-y-auto overscroll-contain">- <div className="flex flex-col items-center gap-3 py-5">+ <div+ className={+ active+ ? 'flex flex-col items-center gap-3 py-5'+ : 'flex items-center justify-center gap-4 py-3'+ }+ ><div className="flex items-center gap-5"><BotAvatar bot={props.bot} size="xl" /><div@@ -221,52 +319,127 @@ export function CompanionVoicePanel(props: Props) {{t(catalogError ? 'bots.voiceMode.catalogError' : 'bots.voiceMode.noModels')}</p>) : null}- <div className="flex flex-wrap items-center gap-2">- <Select- label={t('bots.voiceMode.model')}- value={validModel ? modelId : ''}- options={models.map((model) => ({ value: model.id, label: model.name }))}- onValueChange={(id) => persist({ modelId: id, voice: selection?.voice ?? 'alloy' })}- disabled={loading || models.length === 0}- className="min-w-40 flex-1"- />- <Button- variant="secondary"- size="md"- disabled={loading}- aria-label={t('bots.voiceMode.refresh')}- title={t('bots.voiceMode.refresh')}- onClick={() => void refresh(true)}- >- <RefreshCw size={15} />- </Button>- </div>+ <FormField label={t('bots.voiceMode.model')}>+ {(control) => (+ <div className="flex items-center gap-2">+ <Select+ {...control}+ label={t('bots.voiceMode.model')}+ value={selectedModel ? voiceModelKey(selectedModel) : ''}+ options={models.map((model) => ({+ value: voiceModelKey(model),+ label: [model.name, model.providerName].filter(Boolean).join(' · '),+ }))}+ onValueChange={chooseModel}+ disabled={loading || !ready || saving || models.length === 0}+ className="flex-1"+ />+ <Button+ variant="secondary"+ size="lg"+ disabled={loading}+ aria-label={t('bots.voiceMode.refresh')}+ title={t('bots.voiceMode.refresh')}+ onClick={() => {+ void refresh(true);+ if (!ready) loadPreference();+ }}+ >+ <RefreshCw size={15} />+ </Button>+ </div>+ )}+ </FormField>{!validModel && preference && models.length > 0 ? (<p className="text-12 text-[var(--text-secondary)]">{t('bots.voiceMode.modelUnavailable')}</p>) : null}- {models.length > 0 ? (- <div className="flex flex-wrap items-center gap-2">- <label htmlFor={voiceId} className="text-13 text-[var(--text-secondary)]">- {t('bots.voiceMode.voice')}- </label>- <Input+ <div className="flex flex-wrap items-start gap-3">+ {selectedModel && selectedModel.providerId !== 'xd' && selectedModel.agent ? (+ <FormField+ label={t('bots.voiceMode.protocol')}+ hint={t('bots.voiceMode.protocolHint')}+ className="min-w-52 flex-1"+ >+ {(control) => (+ <Select+ {...control}+ label={t('bots.voiceMode.protocol')}+ value={protocol ?? ''}+ options={[+ { value: 'openai', label: t('bots.voiceMode.protocolOpenai') },+ { value: 'elevenlabs', label: 'ElevenLabs' },+ { value: 'minimax', label: 'MiniMax' },+ ]}+ disabled={!ready || saving}+ onValueChange={(value) => {+ const next = {+ providerId: selectedModel.providerId,+ agent: selectedModel.agent,+ modelId: selectedModel.id,+ voice: selectedVoice || (value === 'openai' ? 'alloy' : ''),+ protocol: value as SpeechProtocol,+ };+ dirty.current = true;+ setCustomized(true);+ setPreference(next);+ setVoice(next.voice);+ if (normalizeVoiceConversationSelection(next)) void persist(next);+ }}+ className="w-full"+ />+ )}+ </FormField>+ ) : null}+ {models.length > 0 ? (+ <FormFieldid={voiceId}- value={voice}- maxLength={128}- onChange={setVoice}- onBlur={() => {- if (selection && validModel) persist(selection);- }}- className="min-w-24 flex-1"- aria-describedby={`${voiceId}-hint`}- />- <p id={`${voiceId}-hint`} className="w-full text-12 text-[var(--text-tertiary)]">- {t('bots.voiceMode.voiceHint')}- </p>- </div>+ label={t('bots.voiceMode.voice')}+ hint={t(+ protocol === 'elevenlabs' || protocol === 'minimax'+ ? 'bots.voiceMode.voiceIdHint'+ : 'bots.voiceMode.voiceHint',+ )}+ className="min-w-52 flex-1"+ >+ {(control) => (+ <Input+ {...control}+ value={selectedVoice}+ disabled={!ready || saving}+ maxLength={128}+ onChange={(value) => {+ dirty.current = true;+ setVoice(value);+ }}+ onBlur={() => {+ if (dirty.current && selection && validModel) void persist(selection);+ }}+ className="w-full"+ />+ )}+ </FormField>+ ) : null}+ </div>+ {preferenceError ? (+ <p role="alert" className="text-13 text-[var(--error-fg)]">+ {t('bots.voiceMode.preferenceError')}+ </p>) : null}+ <div className="flex flex-wrap items-center gap-2">+ <p className="flex-1 text-12 text-[var(--text-tertiary)]">+ {t('bots.voiceMode.preferenceHint')}+ </p>+ <Button+ variant="secondary"+ size="sm"+ disabled={!ready || saving || (!customized && !dirty.current)}+ onClick={() => void persist(null)}+ >+ {t('bots.voiceMode.resetPreference')}+ </Button>+ </div></div>) : null}</div>@@ -275,8 +448,8 @@ export function CompanionVoicePanel(props: Props) {<Buttonvariant="primary"size="lg"- onClick={start}- disabled={!validModel || !selection || props.blocked || loading}+ onClick={() => void start()}+ disabled={!validModel || !selection || !protocol || props.blocked || loading || !ready}><Mic size={16} />{t(diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsxindex 92540bcfba..b311d7305a 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoicePanel.test.tsx@@ -1,7 +1,37 @@// @vitest-environment jsdomimport { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';+import { voiceModelKey, type VoiceConversationModel } from '@/../shared/voiceConversation';import { afterEach, beforeEach, expect, it, vi } from 'vitest';+vi.mock('@/components/ui/select', () => ({+ Select: ({+ label,+ value,+ options,+ onValueChange,+ disabled,+ }: {+ label: string;+ value: string;+ options: { value: string; label: string }[];+ onValueChange(value: string): void;+ disabled: boolean;+ }) => (+ <select+ aria-label={label}+ value={value}+ disabled={disabled}+ onChange={(e) => onValueChange(e.target.value)}+ >+ <option value="" />+ {options.map((option) => (+ <option key={option.value} value={option.value}>+ {option.label}+ </option>+ ))}+ </select>+ ),+}));vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }));vi.mock('@/hooks/useVoiceInputSettings', () => ({getVoiceInputSettings: () => ({ microphoneDeviceId: null }),@@ -30,7 +60,7 @@ vi.mock('../PcmSpeechPlayer', () => ({}));import { CompanionVoicePanel } from '../CompanionVoicePanel';-let models: Array<{ id: string; name: string }>;+let models: VoiceConversationModel[];const start = vi.fn(async () => ({ callId: 'call' }));const refresh = vi.fn(async () => undefined);beforeEach(() => {@@ -41,6 +71,8 @@ beforeEach(() => {value: {voiceConversation: {models: async () => models,+ preference: vi.fn(async () => ({ selection: null, customized: false })),+ setPreference: vi.fn(async (_botId, selection) => ({ selection, customized: !!selection })),start,end: vi.fn(async () => {}),onEvent: () => () => {},@@ -52,10 +84,10 @@ beforeEach(() => {});});afterEach(cleanup);-const renderPanel = (blocked = false) =>+const renderPanel = (blocked = false, botId = 'bot') =>render(<CompanionVoicePanel- bot={{ id: 'bot', name: 'Partner' }}+ bot={{ id: botId, name: 'Partner' }}sessionId="canonical"messages={[]}blocked={blocked}@@ -110,3 +142,138 @@ it('keeps permission cards authoritative and never starts through a pending confscreen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),).toBe(true);});++it('selects the exact BYOK connection for duplicate IDs and persists it for this companion', async () => {+ models = ['a', 'b'].map((providerId) => ({+ id: 'same',+ name: 'Speech',+ providerId,+ providerName: providerId,+ agent: 'codex',+ protocol: 'openai',+ defaultVoice: 'alloy',+ }));+ renderPanel();+ await waitFor(() =>+ expect(+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),+ ).toBe(false),+ );+ expect(screen.getByRole('option', { name: 'Speech · b' })).toBeTruthy();+ fireEvent.change(screen.getByLabelText('bots.voiceMode.model'), {+ target: { value: voiceModelKey(models[1]) },+ });+ await waitFor(() =>+ expect(window.electronAPI.voiceConversation.setPreference).toHaveBeenCalledWith('bot', {+ providerId: 'b',+ agent: 'codex',+ modelId: 'same',+ voice: 'alloy',+ }),+ );+ fireEvent.click(screen.getByRole('button', { name: 'bots.voiceMode.start' }));+ await waitFor(() =>+ expect(start).toHaveBeenCalledWith(+ expect.objectContaining({ providerId: 'b', agent: 'codex', modelId: 'same' }),+ ),+ );+});++it('requires an explicit API choice for an unknown endpoint before requesting a microphone', async () => {+ models = [+ { id: 'private', name: 'Voice', providerId: 'custom', agent: 'codex', defaultVoice: '' },+ ];+ renderPanel();+ await screen.findByText('bots.voiceMode.protocolHint');+ fireEvent.change(screen.getByLabelText('bots.voiceMode.voice'), {+ target: { value: 'voice-id' },+ });+ expect(+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),+ ).toBe(true);+ fireEvent.change(screen.getByLabelText('bots.voiceMode.protocol'), {+ target: { value: 'elevenlabs' },+ });+ await waitFor(() =>+ expect(window.electronAPI.voiceConversation.setPreference).toHaveBeenCalledWith(+ 'bot',+ expect.objectContaining({ protocol: 'elevenlabs', voice: 'voice-id' }),+ ),+ );+ fireEvent.click(screen.getByRole('button', { name: 'bots.voiceMode.start' }));+ await waitFor(() =>+ expect(start).toHaveBeenCalledWith(+ expect.objectContaining({ providerId: 'custom', protocol: 'elevenlabs', voice: 'voice-id' }),+ ),+ );+});++it('preserves an unavailable saved connection until the user restores defaults', async () => {+ models = [+ {+ id: 'same',+ name: 'Available',+ providerId: 'other',+ agent: 'codex',+ protocol: 'openai',+ defaultVoice: 'alloy',+ },+ ];+ vi.mocked(window.electronAPI.voiceConversation.preference).mockResolvedValue({+ selection: { providerId: 'missing', agent: 'codex', modelId: 'same', voice: 'nova' },+ customized: true,+ });+ renderPanel();+ await screen.findByText('bots.voiceMode.modelUnavailable');+ expect(+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),+ ).toBe(true);+ fireEvent.click(screen.getByRole('button', { name: 'bots.voiceMode.resetPreference' }));+ await waitFor(() =>+ expect(window.electronAPI.voiceConversation.setPreference).toHaveBeenCalledWith('bot', null),+ );+ await waitFor(() =>+ expect(+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),+ ).toBe(false),+ );+ expect(start).not.toHaveBeenCalled();+});++it('reloads the right companion preference after reopening and does not save a derived default', async () => {+ models = [{ id: 'tts', name: 'Speech' }];+ vi.mocked(window.electronAPI.voiceConversation.preference).mockImplementation(async (id) => ({+ selection: { modelId: 'tts', voice: id === 'a' ? 'nova' : 'alloy' },+ customized: true,+ }));+ const first = renderPanel(false, 'a');+ await waitFor(() =>+ expect((screen.getByLabelText('bots.voiceMode.voice') as HTMLInputElement).value).toBe('nova'),+ );+ first.unmount();+ renderPanel(false, 'b');+ await waitFor(() =>+ expect((screen.getByLabelText('bots.voiceMode.voice') as HTMLInputElement).value).toBe('alloy'),+ );+ expect(window.electronAPI.voiceConversation.setPreference).not.toHaveBeenCalled();+});++it('retries a failed preference load and still lists BYOK when Gateway refresh fails', async () => {+ models = [{ id: 'tts', name: 'Speech' }];+ vi.mocked(window.electronAPI.voiceConversation.preference).mockRejectedValueOnce(+ new Error('load failed'),+ );+ refresh.mockRejectedValueOnce(new Error('Gateway unavailable'));+ renderPanel();+ await screen.findByText('bots.voiceMode.preferenceError');+ expect(+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),+ ).toBe(true);+ fireEvent.click(screen.getByRole('button', { name: 'bots.voiceMode.refresh' }));+ await waitFor(() =>+ expect(+ screen.getByRole('button', { name: 'bots.voiceMode.start' }).hasAttribute('disabled'),+ ).toBe(false),+ );+ expect(window.electronAPI.voiceConversation.preference).toHaveBeenCalledTimes(2);+});diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.tsindex bfa22ff4fd..b49289b26e 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts@@ -14,6 +14,8 @@ function fixture() {let calls = 0;const api = {models: vi.fn(async () => []),+ preference: vi.fn(async () => ({ selection: null, customized: false })),+ setPreference: vi.fn(async () => ({ selection: null, customized: false })),start: vi.fn(async () => ({ callId: `call-${++calls}` })),audio: vi.fn(),finishUtterance: vi.fn(async () => {}),@@ -239,3 +241,27 @@ describe('companion voice loop', () => {f.controller.end();});});++it('keeps the ordinary final text and stops voice on a BYOK synthesis failure without fallback or resend', async () => {+ const f = fixture();+ f.api.speak.mockRejectedValueOnce(new Error('upstream error'));+ await f.controller.start({+ ...selection,+ providerId: 'byok',+ agent: 'codex',+ protocol: 'openai',+ });+ f.utterance();+ await vi.waitFor(() => expect(f.send).toHaveBeenCalledOnce());+ await Promise.resolve();+ const messages = [user(), reply()];+ f.controller.updateMessages(messages);+ await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('error'));+ expect(f.controller.getSnapshot().error).toBe('speech');+ expect(f.microphone.stop).toHaveBeenCalledOnce();+ expect(messages[1]).toEqual(reply());+ expect(f.api.start).toHaveBeenCalledOnce();+ expect(f.api.speak).toHaveBeenCalledOnce();+ expect(f.send).toHaveBeenCalledOnce();+ f.controller.end();+});diff --git a/apps/desktop/src/renderer/i18n/locales/en/common.json b/apps/desktop/src/renderer/i18n/locales/en/common.jsonindex 1588d6d142..07e49a80a3 100644--- a/apps/desktop/src/renderer/i18n/locales/en/common.json+++ b/apps/desktop/src/renderer/i18n/locales/en/common.json@@ -9355,7 +9355,7 @@"voice": "Voice","voiceHint": "Use a voice name supported by the selected model, such as alloy.","modelUnavailable": "The saved speech model is unavailable. Choose another model.",- "noModels": "No speech model is available for this account yet. You can start when one becomes available.",+ "noModels": "No usable speech model. Connect a speech provider in Settings, enable a model, then refresh here.","catalogError": "Could not load or save speech settings. Please try again.","start": "Start talking","resume": "Resume",@@ -9382,7 +9382,14 @@"microphone": "Microphone stopped. Check its permission and connection, then resume.","send": "This message was not accepted. Your text is kept below.","speech": "Could not read the reply aloud. The text reply remains in the conversation. Check the speech model and voice."- }+ },+ "voiceIdHint": "Enter a voice ID available to this provider account.",+ "preferenceHint": "The model and voice are saved for this companion only.",+ "resetPreference": "Use defaults",+ "preferenceError": "Could not load or save voice settings. Please try again.",+ "protocol": "Speech API",+ "protocolOpenai": "OpenAI compatible",+ "protocolHint": "Choose the speech API used by this connection. The chat API is unchanged."}},"common": {diff --git a/apps/desktop/src/renderer/i18n/locales/ja/common.json b/apps/desktop/src/renderer/i18n/locales/ja/common.jsonindex 572c46d55e..643b482f76 100644--- a/apps/desktop/src/renderer/i18n/locales/ja/common.json+++ b/apps/desktop/src/renderer/i18n/locales/ja/common.json@@ -9317,7 +9317,7 @@"voice": "音声","voiceHint": "選択したモデルが対応する音声名を入力してください(例:alloy)。","modelUnavailable": "保存した読み上げモデルは利用できません。別のモデルを選んでください。",- "noModels": "このアカウントで利用できる読み上げモデルはまだありません。利用可能になれば開始できます。",+ "noModels": "利用できる音声合成モデルがありません。設定でプロバイダーに接続し、モデルを有効にしてから更新してください。","catalogError": "音声設定の読み込み、または保存に失敗しました。もう一度お試しください。","start": "会話を始める","resume": "再開",@@ -9344,7 +9344,14 @@"microphone": "マイクが停止しました。権限と接続を確認して再開してください。","send": "メッセージは受け付けられませんでした。テキストは保持されています。","speech": "読み上げに失敗しました。返答のテキストは会話に残っています。モデルと音声を確認してください。"- }+ },+ "voiceIdHint": "このプロバイダーのアカウントで利用できる音声 ID を入力してください。",+ "preferenceHint": "モデルと音声は、このパートナー専用に保存されます。",+ "resetPreference": "既定値に戻す",+ "preferenceError": "音声設定を読み込むか保存できませんでした。もう一度お試しください。",+ "protocol": "音声 API",+ "protocolOpenai": "OpenAI 互換",+ "protocolHint": "この接続が使用する音声 API を選択します。チャット API は変更されません。"}},"common": {diff --git a/apps/desktop/src/renderer/i18n/locales/ko/common.json b/apps/desktop/src/renderer/i18n/locales/ko/common.jsonindex 8212f17ba8..cc9b22f987 100644--- a/apps/desktop/src/renderer/i18n/locales/ko/common.json+++ b/apps/desktop/src/renderer/i18n/locales/ko/common.json@@ -9317,7 +9317,7 @@"voice": "목소리","voiceHint": "선택한 모델이 지원하는 음성 이름을 입력하세요. 예: alloy.","modelUnavailable": "저장된 읽어주기 모델을 사용할 수 없습니다. 다른 모델을 선택하세요.",- "noModels": "이 계정에서 사용할 수 있는 읽어주기 모델이 아직 없습니다. 모델이 제공되면 시작할 수 있습니다.",+ "noModels": "사용 가능한 음성 합성 모델이 없습니다. 설정에서 제공자를 연결하고 모델을 활성화한 다음 새로고침하세요.","catalogError": "음성 설정을 불러오거나 저장하지 못했습니다. 다시 시도하세요.","start": "대화 시작","resume": "계속",@@ -9344,7 +9344,14 @@"microphone": "마이크가 중지되었습니다. 권한과 연결을 확인한 후 계속하세요.","send": "메시지가 접수되지 않았습니다. 텍스트는 유지됩니다.","speech": "답변을 읽지 못했습니다. 텍스트 답변은 대화에 남아 있습니다. 모델과 목소리를 확인하세요."- }+ },+ "voiceIdHint": "이 제공자 계정에서 사용할 수 있는 음성 ID를 입력하세요.",+ "preferenceHint": "모델과 음성은 이 파트너에만 저장됩니다.",+ "resetPreference": "기본값 복원",+ "preferenceError": "음성 설정을 불러오거나 저장하지 못했습니다. 다시 시도해 주세요.",+ "protocol": "음성 API",+ "protocolOpenai": "OpenAI 호환",+ "protocolHint": "이 연결에서 사용하는 음성 API를 선택하세요. 채팅 API는 변경되지 않습니다."}},"common": {diff --git a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.jsonindex c3b654fee6..9d75aff44d 100644--- a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json+++ b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json@@ -9317,7 +9317,7 @@"voice": "音色","voiceHint": "填写所选模型支持的音色名称,例如 alloy。","modelUnavailable": "之前选择的朗读模型已不可用,请重新选择。",- "noModels": "当前账号还没有可用的朗读模型,开放后即可开始语音对话。",+ "noModels": "暂无可用的语音合成模型。请在设置中连接供应商并启用模型,然后刷新。","catalogError": "语音设置读取或保存失败,请重试。","start": "开始说话","resume": "继续",@@ -9344,7 +9344,14 @@"microphone": "麦克风已停止,请检查权限和设备连接后继续。","send": "这条消息未被接收,文字已保留。","speech": "回复朗读失败,文字回复仍保留在对话中。请检查朗读模型和音色。"- }+ },+ "voiceIdHint": "填写该供应商账号可用的音色 ID。",+ "preferenceHint": "模型和音色仅为当前伙伴保存。",+ "resetPreference": "恢复默认",+ "preferenceError": "未能读取或保存语音设置,请重试。",+ "protocol": "语音接口",+ "protocolOpenai": "OpenAI 兼容",+ "protocolHint": "选择该连接使用的语音接口,不影响对话接口。"}},"common": {diff --git a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.jsonindex 613fa9f357..83774aeec3 100644--- a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json+++ b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json@@ -9317,7 +9317,7 @@"voice": "音色","voiceHint": "填入所選模型支援的音色名稱,例如 alloy。","modelUnavailable": "先前選擇的朗讀模型已無法使用,請重新選擇。",- "noModels": "目前帳號還沒有可用的朗讀模型,開放後即可開始語音對話。",+ "noModels": "暫無可用的語音合成模型。請在設定中連接供應商並啟用模型,然後重新整理。","catalogError": "語音設定讀取或儲存失敗,請重試。","start": "開始說話","resume": "繼續",@@ -9344,7 +9344,14 @@"microphone": "麥克風已停止,請檢查權限和裝置連線後繼續。","send": "這則訊息未被接收,文字已保留。","speech": "回覆朗讀失敗,文字回覆仍保留在對話中。請檢查朗讀模型和音色。"- }+ },+ "voiceIdHint": "填寫該供應商帳號可用的音色 ID。",+ "preferenceHint": "模型和音色僅為目前夥伴儲存。",+ "resetPreference": "恢復預設",+ "preferenceError": "無法讀取或儲存語音設定,請重試。",+ "protocol": "語音介面",+ "protocolOpenai": "OpenAI 相容",+ "protocolHint": "選擇此連接使用的語音介面,不影響對話介面。"}},"common": {diff --git a/apps/desktop/src/shared/voiceConversation.ts b/apps/desktop/src/shared/voiceConversation.tsindex 6971ce0916..4346ea5383 100644--- a/apps/desktop/src/shared/voiceConversation.ts+++ b/apps/desktop/src/shared/voiceConversation.ts@@ -1,8 +1,38 @@import type { AudioTrace, ConversationInputEvent } from '@cindy/voice-input-core';+import type { AgentKind } from '@cindy/model-providers';/** Local Desktop voice UI contract. No credentials, raw paths or remote invoke access. */-export type VoiceConversationSelection = { modelId: string; voice: string };-export type VoiceConversationModel = { id: string; name: string };+export type SpeechProtocol = 'openai' | 'elevenlabs' | 'minimax';+// Missing provider/runtime is the original Cindy AI selection, never another connection.+export type VoiceConversationSelection = {+ providerId?: string;+ agent?: AgentKind;+ modelId: string;+ voice: string;+ /** Explicit user choice for a custom speech endpoint; absent follows a known adapter. */+ protocol?: SpeechProtocol;+};+export type VoiceConversationModel = {+ id: string;+ name: string;+ providerId?: string;+ providerName?: string;+ agent?: AgentKind;+ protocol?: SpeechProtocol;+ defaultVoice?: string;+};+export type VoiceConversationPreference = {+ selection: VoiceConversationSelection | null;+ customized: boolean;+};+export function voiceModelKey(model: {+ providerId?: string;+ agent?: AgentKind;+ modelId?: string;+ id?: string;+}): string {+ return JSON.stringify([model.providerId ?? 'xd', model.agent ?? '', model.modelId ?? model.id]);+}export type VoiceConversationEvent = {callId: string;event: ConversationInputEvent | { type: 'ended' };@@ -14,6 +44,11 @@ export type VoiceConversationStart = VoiceConversationSelection & {};export interface VoiceConversationApi {models(): Promise<VoiceConversationModel[]>;+ preference(botId: string): Promise<VoiceConversationPreference>;+ setPreference(+ botId: string,+ selection: VoiceConversationSelection | null,+ ): Promise<VoiceConversationPreference>;start(input: VoiceConversationStart): Promise<{ callId: string }>;audio(input: { callId: string; pcm: ArrayBuffer; trace?: AudioTrace }): void;finishUtterance(callId: string): Promise<void>;@@ -32,14 +67,42 @@ export function normalizeVoiceConversationSelection(value: unknown,): VoiceConversationSelection | undefined {if (!value || typeof value !== 'object') return undefined;- const { modelId, voice } = value as Partial<VoiceConversationSelection>;+ const { modelId, voice, providerId, agent, protocol } =+ value as Partial<VoiceConversationSelection>;if (typeof modelId !== 'string' ||!modelId.trim() ||modelId.length > 256 ||typeof voice !== 'string' ||- !/^[a-zA-Z0-9_-]{1,128}$/.test(voice)+ !/^[a-zA-Z0-9_-]{1,128}$/.test(voice) ||+ (providerId !== undefined &&+ (typeof providerId !== 'string' || !providerId.trim() || providerId.length > 256)) ||+ (protocol !== undefined && !['openai', 'elevenlabs', 'minimax'].includes(protocol)) ||+ ((providerId === undefined || providerId === 'xd') &&+ protocol !== undefined &&+ protocol !== 'openai') ||+ (agent !== undefined && !['claude-code', 'codex', 'pi'].includes(agent)) ||+ (providerId !== undefined && providerId !== 'xd' && !agent) ||+ ((providerId === undefined || providerId === 'xd') && agent !== undefined))return undefined;- return { modelId: modelId.trim(), voice };+ return {+ modelId: modelId.trim(),+ voice,+ ...(providerId ? { providerId } : {}),+ ...(agent ? { agent } : {}),+ ...(protocol ? { protocol } : {}),+ };+}++export function normalizeVoiceConversationPreferences(+ value: unknown,+): Record<string, VoiceConversationSelection | null> {+ if (!value || typeof value !== 'object' || Array.isArray(value)) return {};+ return Object.fromEntries(+ Object.entries(value).flatMap(([id, raw]) => {+ const selection = raw === null ? null : normalizeVoiceConversationSelection(raw);+ return id.trim() && id.length <= 256 && selection !== undefined ? [[id, selection]] : [];+ }),+ );}diff --git a/apps/desktop/src/shared/voiceInputData.ts b/apps/desktop/src/shared/voiceInputData.tsindex 0e68a90a9a..9ed9fe5bdc 100644--- a/apps/desktop/src/shared/voiceInputData.ts+++ b/apps/desktop/src/shared/voiceInputData.ts@@ -1,4 +1,4 @@-import { normalizeVoiceConversationSelection, type VoiceConversationSelection } from './voiceConversation';+import { normalizeVoiceConversationPreferences, normalizeVoiceConversationSelection, type VoiceConversationSelection } from './voiceConversation';import type {DictationDictionaryAdviceInput,DictationDictionaryLearningAction,@@ -81,6 +81,8 @@ export interface VoiceInputDictionaryCandidate {export interface VoiceInputSettings {/** Explicit speech selection; absent means follow the available catalog. */conversationSelection?: VoiceConversationSelection | null;+ /** Per-companion overrides; absence inherits the default without copying it. */+ conversationSelections?: Record<string, VoiceConversationSelection | null>;language: VoiceInputLanguage;microphoneDeviceId: string | null;muteSystemAudio: boolean;@@ -521,6 +523,7 @@ export function normalizeVoiceInputSettings(const defaults = getDefaultVoiceInputSettings(platform, region);if (!raw || typeof raw !== 'object') return defaults;const candidate = raw as Partial<VoiceInputSettings>;+ const conversationSelections = normalizeVoiceConversationPreferences(candidate.conversationSelections);const legacyPlayStartSound = (candidate as { playStartSound?: unknown }).playStartSound;const shortcut ='shortcut' in candidate@@ -529,6 +532,7 @@ export function normalizeVoiceInputSettings(return {...(normalizeVoiceConversationSelection(candidate.conversationSelection)? { conversationSelection: normalizeVoiceConversationSelection(candidate.conversationSelection) } : {}),+ ...(Object.keys(conversationSelections).length ? { conversationSelections } : {}),language: isVoiceInputLanguage(candidate.language) ? candidate.language : defaults.language,microphoneDeviceId:typeof candidate.microphoneDeviceId === 'string' && candidate.microphoneDeviceId.trim().length > 0diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.mdindex f76b588049..79c4dcb50c 100644--- a/docs/product-rules/companion-voice-mode.md+++ b/docs/product-rules/companion-voice-mode.md@@ -7,28 +7,43 @@- 停顿约 900ms 后等待 ASR 的最终文字,再用短暂的稳定窗口合并同一段发言。只有最终、非空文字能自动发送;声音活动检测本身不能产生消息。- 原任务空闲时正常发送,仍在运行时沿既有插话路径投递。语音是本机用户本人输入,不盖自动化或远程设备来源标签,不增加系统提示词。识别结果按普通文字发送,不执行桌面斜杠命令。- 最终回复经过持久化完成标记确认后才朗读。按本次语音发送的 clientId 关联,不朗读历史、过程文字、工具、子代理、内部协调或群来源私信。文本消息始终是对话记录的正本。-- 独立选择朗读模型和音色。只展示当前账号可用、未停用的 Cindy AI `audio_speech` 模型;账号无模型时显示空状态,不能开始。刷新复用现有 Cindy AI 目录刷新。+- 独立选择朗读模型和音色,按当前账号下的伙伴保存,不改变聊天模型。来源包括有权限的 Cindy AI,以及已连接的本地 BYOK 供应商;候选必须是 `audio_speech`、未停用且具备可用的 API key 或明确的本机免鉴权连接。同名模型按连接 ID、runtime 和上游模型 ID 区分,不能借另一连接的密钥。无候选时显示空状态,不采集麦克风。+- BYOK 支持 OpenAI 兼容、ElevenLabs、MiniMax 三种语音接口。已声明的 OpenAI speech 协议或已知官方端点可确定默认接口;其余候选(包括显式未验证协议)必须由用户在语音面板明确选择接口,不能只凭模型名、`mode` 或聊天协议推断。这个覆盖仅作用于当前伙伴的朗读,不重写公共 Registry 或供应商的聊天协议。ElevenLabs / MiniMax 需填写账号可用的音色 ID;本次不做音色目录、试听或克隆。+- 模型、语音接口和音色为伙伴级显式偏好,打开面板、重启后保留。恢复默认删除该伙伴的选择,继续跟随可用目录;已有旧版全局偏好仍可继承,单独重置一个伙伴时以 `null` 清除该伙伴的旧偏好继承,不复制默认模型、不影响其他伙伴。保存失败明确提示。保存的连接消失或停用时保留选择并禁止开始,不静默改用同名模型。+- 刷新重新读取当前有效目录,并尝试刷新 Cindy AI;Gateway 刷新失败不阻止本机 BYOK 目录显示。新增第三方型号仍沿设置页已有的获取列表、手动添加和启用流程。- 朗读结束继续监听。用户开口时立即取消待生成及正在播放的声音;新的文字仍投递到原任务。停止朗读不会取消 Agent 工具、清空任务队列或代答权限确认。- 需要确认权限、问题或计划,或任务队列暂停时,关闭麦克风并暂停语音模式;用户完成确认后显式继续。静音、锁屏、休眠、切走任务、关面板、账号切换同样释放采集和播放资源,不在后台重新开始。- 未发送的文字在可恢复错误或暂停后留在面板内;用户可选中复制或明确重新连接并发送。TTS 失败保留任务中的文字回复,不自动重发用户消息。## 交互与动效-面板保留伙伴头像和姓名,持续显示连接、监听、思考、准备语音、朗读或暂停状态。音量条反映实际输入或输出音量,没有循环伪造的声波。进入动效和音量条只变换 opacity/transform,使用现有 motion token,并支持减少动态效果。模型和音色可在开始前或暂停后更换;进行中的结束、静音及面板关闭按钮不随内容滚走,暂停后可继续或关闭。面板最多占视口高度的 65%,中间内容独立滚动。Light/Dark 使用语义颜色。+面板保留伙伴头像和姓名,持续显示连接、监听、思考、准备语音、朗读或暂停状态。音量条反映实际输入或输出音量,没有循环伪造的声波。进入动效和音量条只变换 opacity/transform,使用现有 motion token,并支持减少动态效果。模型、接口和音色可在开始前或暂停后更换,静止配置区采用紧凑的头像与状态行;语音运行时恢复集中呈现的伙伴头像和音量条。进行中的结束、静音及面板关闭按钮不随内容滚走,暂停后可继续或关闭。面板最多占视口高度的 65%,中间内容独立滚动。Light/Dark 使用语义颜色。-设计依据为 `DESIGN.md` §4 标准控件、§5 圆角分层、§14.4 动效与 `design-governance.md` §13 点击层分类:标题栏语音入口是 32px 的普通动作按钮,沿用同栏的 pill;面板和未发送文字框属于容器,使用 12px 圆角;音量条是无交互的指示层,不是按钮。设置、开始、静音、结束等操作复用现有 Button / Select / Input。+设计依据为 `DESIGN.md` §4 标准控件、§5 圆角分层、§14.4 动效与 `design-governance.md` §13 点击层分类:标题栏语音入口是 32px 的普通动作按钮,沿用同栏的 pill;面板和未发送文字框属于容器,使用 12px 圆角;音量条是无交互的指示层,不是按钮。设置、开始、静音、结束等操作复用现有 Button / Select / Input / FormField,模型、接口和音色均有可见标签。较窄的窗口将接口和音色折行,操作区保持可见。## 执行边界- 当前仅支持 Desktop、本机伙伴一对一。群聊、Mobile、远程控制的麦克风、原生 Realtime speech-to-speech 不在此实现内。- 新 IPC 只接受可信应用顶层 Renderer,按窗口、账号代次、callId 和 canonical 任务绑定。callId 在发起请求前生成,使结束按钮也能取消连接过程;它不是授权凭证。接口不加入 device-link 或插件白名单。模型权限在开始和每次 TTS 请求、读取时复核。- ASR 适配器额外提供按上游 utterance/item ID 标识的分段事件,现有累积 `AsrEvent` 合同保持不变。连接在 150 秒轮换,沿用 voice-server 的 dictation 协议;切换期间最多缓存 5 秒输入,不能无声丢弃或重复发送前文。-- Gateway 执行合同为 OpenAI 兼容 `POST <下发的 baseUrl>/audio/speech`,body 包含 `model`、`voice`、`input`、`response_format: pcm`。PCM 固定为该协议规定的 24kHz、单声道、16bit little-endian。显式声明其他 native API 或 `null` 未验证 API 的模型不进入选择器;未声明协议的 Gateway 新模型沿用此端点合同。下发 `mode` 只代表能力分类,不证明具体音色、音频格式或端点可用。-- 凭证只在 Main,使用当前账号配对的 Gateway baseUrl 和密钥;不自动改走 BYOK 或系统朗读。响应按 Renderer 播放速度读取,播放只预排约 300ms;中断丢弃旧请求和迟到音频。原始音频只在内存,不新增录音文件或对话音频归档。+- Gateway 执行合同为 OpenAI 兼容 `POST <下发的 baseUrl>/audio/speech`,body 包含 `model`、`voice`、`input`、`response_format: pcm`。PCM 固定为该协议规定的 24kHz、单声道、16bit little-endian。显式声明其他 native API 或 `null` 未验证 API 的 Cindy AI 模型不进入选择器;未声明协议的 Gateway 新模型沿用此端点合同。下发 `mode` 只代表能力分类,不证明具体音色、音频格式或端点可用。+- 凭证只在 Main:Gateway 使用当前账号配对的 baseUrl / 密钥,BYOK 复用对应连接、对应 runtime 的 safeStorage 密钥和固定请求头。排除原生订阅 OAuth、远程或共享供应商;per-model 地址不得跨越凭证所属连接的 origin。开始时固定连接,每次合成与音频读取复核资格及路由/密钥是否变化;变化则停止该语音请求。禁止自动切换到其他供应商、系统朗读或另一条凭证。响应按 Renderer 播放速度读取,播放只预排约 300ms;中断丢弃旧请求和迟到音频。原始音频只在内存,不新增录音文件或对话音频归档。- 初始实现的声音活动判断是带迟滞的音量检测,配合浏览器回声消除与降噪。它不是语义 VAD,不能保证所有扬声器、蓝牙或噪声环境都能准确分辨插话。+## BYOK 音频合同++| 接口 | 请求与鉴权 | 响应 |+| --- | --- | --- |+| OpenAI 兼容 | `<base>/audio/speech`,Bearer,`model / voice / input / response_format: pcm` | 原始 24kHz、单声道、16bit little-endian PCM |+| ElevenLabs | `<base>/text-to-speech/<voiceId>/stream?output_format=pcm_24000`,`xi-api-key`,`model_id / text` | 原始 PCM |+| MiniMax | `<base>/t2a_v2`,Bearer,`stream: true`、`exclude_aggregated_audio: true`、`output_format: hex`、`voice_setting.voice_id`、24kHz 单声道 PCM | SSE JSON 的 hex 音频;检查业务错误及格式,结束包的聚合音频不重复播放 |++官方域名使用文档规定的 `/v1` 语音端点,MiniMax 的 `/anthropic` 聊天路径不会被沿用。用户指定的兼容代理保留配置的 API base 路径前缀;如果代理提供不同的语音根路径,应在设置中新增独立连接。接口选择不证明供应商的型号、付费方案或音色可用,执行失败仍回到文字。++原生接口依据:[ElevenLabs stream](https://elevenlabs.io/docs/api-reference/text-to-speech/stream)、[MiniMax HTTP TTS](https://platform.minimax.io/docs/api-reference/speech-t2a-http)。响应有类型、大小、超时和完整性检查,重定向被拒绝,上游原始错误不穿过 IPC。流式传输与插话取消共用原有代次检查和播放背压,不增加音频落盘。+## 验证-自动化覆盖分段去重、乱序完成、空转写、连接轮换、权限与账号边界、取消启动、插话抢占、迟到音频、PCM 字节边界、回复关联、无模型空状态及现有 ASR 回归。+自动化覆盖分段去重、乱序完成、空转写、连接轮换、权限与账号边界、取消启动、插话抢占、迟到音频、PCM 字节边界、回复关联、无模型空状态及现有 ASR 回归。BYOK 回归覆盖接口请求、SSE 断帧与结束聚合包、连接/密钥隔离、停用与账号切换、伙伴偏好重开与重置、未知接口须用户选择,以及合成失败不换供应商、不重发消息。-Gateway 尚未给测试账号开放真实 TTS 模型时,自动化使用明确的测试替身;产品目录不添加演示模型。真实模型开放后,还需验证其音色和 PCM 合同,并在 macOS/Windows 的耳机、扬声器、蓝牙设备上验证连续多轮、回声与插话延迟。测试替身不构成这些项目已通过的证据。+没有真实 Gateway / BYOK TTS 账号可用于验收时,自动化和组件预览使用明确的测试替身;产品目录不添加演示模型。真实模型开放后,还需验证其音色和 PCM 合同,并在 macOS/Windows 的耳机、扬声器、蓝牙设备上验证连续多轮、回声与插话延迟。测试替身不构成这些项目已通过的证据。