Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(model-providers): normalize ElevenLabs TTS capabilities
JiangJiahao0c3b65ef351b
diff --git a/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts b/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.tsindex 0842049b8b..b33ed727e4 100644--- a/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts+++ b/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts@@ -58,6 +58,23 @@ describe('import discovery limits', () => {});describe('buildModelsFetchRequest', () => {+ it('discovers ElevenLabs speech capabilities through the existing custom-header connection', async () => {+ const result = await fetchProviderModels(spec({ agent: 'codex', apiKey: undefined,+ baseUrl: 'https://api.elevenlabs.io/v1', headers: { 'xi-api-key': 'fake-elevenlabs-key' },+ }), async (url, init) => {+ expect(url).toBe('https://api.elevenlabs.io/v1/models');+ expect(init?.headers).toEqual({ 'xi-api-key': 'fake-elevenlabs-key' });+ return fakeResponse(200, JSON.stringify([+ { model_id: 'eleven_multilingual_v2', name: 'Eleven Multilingual v2', can_do_text_to_speech: true },+ { model_id: 'voice-conversion', can_do_text_to_speech: false, can_do_voice_conversion: true },+ ]));+ });+ expect(result).toMatchObject({ ok: true, models: [+ { id: 'eleven_multilingual_v2', discoveredMetadata: { mode: 'audio_speech' } },+ { id: 'voice-conversion', discoveredMetadata: { mode: 'other' } },+ ] });+ });+it('discovers Sub2API Codex capabilities at the backend path', async () => {const result = await fetchProviderModels(spec({ agent: 'codex',baseUrl: 'https://sub2api.example/backend-api/codex', wireProtocol: 'openai-responses',diff --git a/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts b/apps/desktop/src/renderer/__tests__/unifiedModelList.test.tsindex cc44bfa0d7..0e276d0259 100644--- a/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts+++ b/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts@@ -69,6 +69,19 @@ afterEach(() => {});describe('buildUnionRows', () => {+ it('classifies ElevenLabs capability declarations as TTS or other in model management', () => {+ const projected = buildUserProvider({ id: 'elevenlabs-byok', name: 'Speech', runtimes: { codex: {+ baseUrl: 'https://api.elevenlabs.io/v1', models: mergeDiscoveredRuntimeModels([], parseModelsListResponse([+ { model_id: 'eleven_multilingual_v2', can_do_text_to_speech: true },+ { model_id: 'voice-conversion', can_do_text_to_speech: false },+ ])!),+ } } }, { modelRegistry: BUNDLED_CATALOG.modelRegistry });+ const rows = buildUnionRows({ ...projected, connected: true } as ProviderView);+ expect(rows).toHaveLength(2);+ expect(managementKindsOfRow(rows.find(row => row.id === 'eleven_multilingual_v2')!, true)).toEqual(['tts']);+ expect(managementKindsOfRow(rows.find(row => row.id === 'voice-conversion')!, true)).toEqual(['other']);+ });+it('groups discovered BYOK speech models as TTS and keeps the normal disabled management state', () => {const projected = buildUserProvider({ id: 'speech-byok', name: 'Speech', runtimes: { codex: {baseUrl: 'https://speech.example.test/v1', models: mergeDiscoveredRuntimeModels([], parseModelsListResponse({ data: [diff --git a/docs/model-registry-v4-media.md b/docs/model-registry-v4-media.mdindex 1d9e68e2a5..0ab0325e96 100644--- a/docs/model-registry-v4-media.md+++ b/docs/model-registry-v4-media.md@@ -88,6 +88,13 @@型号改成语音模型。不提供类型的服务可在连接设置中手动指定「语音合成」;同一 ID可更新类型,「自动识别」删除类型覆盖,重新使用发现快照/目录资料。+ElevenLabs 及兼容转发的[模型列表响应](https://elevenlabs.io/docs/api-reference/models/list)+以匹配的 `model_id` 和布尔 `can_do_text_to_speech` 声明能力:`true` 归一为+`audio_speech`;明确 `false` 归为 `other`,避免刷新继续保留旧 TTS 分类,也不推断+聊天或语音识别能力。缺失/非法值不补类型,沿用已有缺项继承;接口显式 `mode` /+`model_info.mode` 和用户类型覆盖仍优先。只解析能力,不推断 `nativeApi`。+直连目录鉴权可复用自定义请求头 `xi-api-key`;没有新增凭证存储或 TTS 执行器。+显式媒体类型不套用聊天协议的名称前缀规则(例如 `gpt-*`);语音协议缺失仍保持未知,不凭 `audio_speech` 推断具体厂商协议。模型的原生协议资料与连接的聊天 `wireProtocol`是两件事,保留资料不会新增语音合成执行器。diff --git a/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts b/packages/model-providers/src/__tests__/speechModelDiscovery.test.tsindex 78b1d9eeee..86f7efc5a2 100644--- a/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts+++ b/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts@@ -81,6 +81,77 @@ describe('BYOK speech discovery and projection', () => {});});+describe('ElevenLabs speech capability discovery', () => {+ const modelId = 'eleven_multilingual_v2';+ const response = (fields: Record<string, unknown> = {}) => [{ model_id: modelId, name: 'Eleven Multilingual v2', ...fields }];++ it('projects the explicit TTS capability into audioModels, including through compatible relays', () => {+ const discovered = parseModelsListResponse(response({ can_do_text_to_speech: true }),+ 'https://speech-relay.example.test/v1/models')!;+ expect(discovered[0]).toMatchObject({ id: modelId,+ discoveredMetadata: { name: 'Eleven Multilingual v2', mode: 'audio_speech' } });+ const p = provider(mergeDiscoveredRuntimeModels([], discovered));+ expect(p.audioModels).toEqual([expect.objectContaining({ id: modelId, mode: 'audio_speech' })]);+ expect(p.audioModels?.[0].nativeApi).toBeUndefined();+ expect(isAgentSelectableModel(p.models.codex![0], { userProvider: true })).toBe(false);+ });++ it.each([+ { value: false, mode: 'other' },+ { value: undefined, mode: undefined },+ { value: 'true', mode: undefined },+ ])('does not advertise TTS for capability $value', ({ value, mode }) => {+ const discovered = parseModelsListResponse(response({ can_do_text_to_speech: value }))!;+ expect(discovered[0].discoveredMetadata?.mode).toBe(mode);+ const p = provider(mergeDiscoveredRuntimeModels([], discovered));+ expect(p.audioModels).toBeUndefined();+ if (value === false) expect(isAgentSelectableModel(p.models.codex![0], { userProvider: true })).toBe(false);+ });++ it.each([+ { fields: { mode: 'chat' }, mode: 'chat' },+ { fields: { model_info: { mode: 'audio_transcription' } }, mode: 'audio_transcription' },+ ])('keeps explicit $mode ahead of the vendor capability', ({ fields, mode }) => {+ const discovered = parseModelsListResponse(response({ can_do_text_to_speech: true, ...fields }))!;+ expect(discovered[0].discoveredMetadata?.mode).toBe(mode);+ });++ it('does not apply the vendor capability without a matching model_id', () => {+ const discovered = parseModelsListResponse([+ { id: 'chat-one', can_do_text_to_speech: true },+ { id: 'chat-two', model_id: modelId, can_do_text_to_speech: false },+ ])!;+ const p = provider(mergeDiscoveredRuntimeModels([], discovered));+ expect(p.audioModels).toBeUndefined();+ for (const model of p.models.codex!) {+ expect(model.mode).toBeUndefined();+ expect(isAgentSelectableModel(model, { userProvider: true })).toBe(true);+ }+ });++ it('refreshes true to false, keeps sparse snapshots, and preserves user overrides', () => {+ const tts = parseModelsListResponse(response({ can_do_text_to_speech: true }))!;+ const noTts = parseModelsListResponse(response({ can_do_text_to_speech: false }))!;+ const sparse = parseModelsListResponse(response())!;+ const models = mergeDiscoveredRuntimeModels([], tts);+ expect(provider(mergeDiscoveredRuntimeModels(models, sparse)).audioModels?.[0].mode).toBe('audio_speech');+ const refreshed = mergeDiscoveredRuntimeModels(models, noTts);+ const p = provider(refreshed);+ expect(p.audioModels).toBeUndefined();+ expect(p.models.codex![0].mode).toBe('other');+ expect(isAgentSelectableModel(p.models.codex![0], { userProvider: true })).toBe(false);+ expect(provider(mergeDiscoveredRuntimeModels(refreshed, sparse)).models.codex![0].mode).toBe('other');+ expect(provider(mergeDiscoveredRuntimeModels(refreshed, tts)).audioModels?.[0].mode).toBe('audio_speech');++ const manual = mergeDiscoveredRuntimeModels([{ ...models[0], name: 'My voice',+ mode: 'audio_speech', defaultEnabled: false }], noTts);+ expect(manual[0]).toMatchObject({ name: 'My voice', mode: 'audio_speech', defaultEnabled: false,+ discoveredMetadata: { mode: 'other' } });+ expect(provider(manual).audioModels?.[0].mode).toBe('audio_speech');+ expect(provider([{ ...manual[0], mode: undefined }]).audioModels).toBeUndefined();+ });+});+describe('media native API metadata', () => {it('preserves known media protocols and explicit unknown, and rejects arbitrary protocols', () => {expect(validModelMetadata({ nativeApi: 'openai-audio-speech' })).toBe(true);diff --git a/packages/model-providers/src/modelDiscovery.ts b/packages/model-providers/src/modelDiscovery.tsindex f1671ecf29..4af7615cb4 100644--- a/packages/model-providers/src/modelDiscovery.ts+++ b/packages/model-providers/src/modelDiscovery.ts@@ -158,6 +158,12 @@ export function parseModelsListResponse(// Vercel marks image/video/etc. as type, but Cindy chat import only executes language models.// Keep them out of the picker instead of saving a mode that later disappears from every list.if (isVercel && record.type !== undefined && record.type !== 'language') continue;+ // ElevenLabs /v1/models (and compatible relays) declares TTS with a boolean,+ // not mode. Match the response contract, never a model-name prefix or a transport API.+ // Explicit false must replace an earlier TTS snapshot without inventing a chat/STT capability.+ const elevenLabsMode = record.model_id === id && typeof record.can_do_text_to_speech === 'boolean'+ ? record.can_do_text_to_speech ? 'audio_speech' : 'other'+ : undefined;const discoveredMetadata = { ...pickModelMetadata(record), ...pickModelMetadata({nativeApi: record.nativeApi !== undefined ? record.nativeApi : record.native_api,...([rec?.display_name, rec?.name, google?.displayName].some(@@ -165,7 +171,8 @@ export function parseModelsListResponse()? { name }: {}),- mode: record.mode ?? info.mode ?? (isVercel && record.type === 'language' ? 'chat' : isVercel ? record.type : undefined),+ mode: record.mode ?? info.mode ?? elevenLabsMode ??+ (isVercel && record.type === 'language' ? 'chat' : isVercel ? record.type : undefined),modalities,officialDocs: record.officialDocs,description: record.description,