Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(model-providers): normalize ElevenLabs TTS capabilities

JiangJiahao0c3b65ef351b
diff --git a/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts b/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts
index 0842049b8b..b33ed727e4 100644
--- a/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts
+++ b/apps/desktop/src/main/maker-host/__tests__/providerModelFetch.test.ts
@@ -58,6 +58,23 @@ describe('import discovery limits', () => {
});
describe('buildModelsFetchRequest', () => {
+ it('discovers ElevenLabs speech capabilities through the existing custom-header connection', async () => {
+ const result = await fetchProviderModels(spec({ agent: 'codex', apiKey: undefined,
+ baseUrl: 'https://api.elevenlabs.io/v1', headers: { 'xi-api-key': 'fake-elevenlabs-key' },
+ }), async (url, init) => {
+ expect(url).toBe('https://api.elevenlabs.io/v1/models');
+ expect(init?.headers).toEqual({ 'xi-api-key': 'fake-elevenlabs-key' });
+ return fakeResponse(200, JSON.stringify([
+ { model_id: 'eleven_multilingual_v2', name: 'Eleven Multilingual v2', can_do_text_to_speech: true },
+ { model_id: 'voice-conversion', can_do_text_to_speech: false, can_do_voice_conversion: true },
+ ]));
+ });
+ expect(result).toMatchObject({ ok: true, models: [
+ { id: 'eleven_multilingual_v2', discoveredMetadata: { mode: 'audio_speech' } },
+ { id: 'voice-conversion', discoveredMetadata: { mode: 'other' } },
+ ] });
+ });
+
it('discovers Sub2API Codex capabilities at the backend path', async () => {
const result = await fetchProviderModels(spec({ agent: 'codex',
baseUrl: 'https://sub2api.example/backend-api/codex', wireProtocol: 'openai-responses',
diff --git a/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts b/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts
index cc44bfa0d7..0e276d0259 100644
--- a/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts
+++ b/apps/desktop/src/renderer/__tests__/unifiedModelList.test.ts
@@ -69,6 +69,19 @@ afterEach(() => {
});
describe('buildUnionRows', () => {
+ it('classifies ElevenLabs capability declarations as TTS or other in model management', () => {
+ const projected = buildUserProvider({ id: 'elevenlabs-byok', name: 'Speech', runtimes: { codex: {
+ baseUrl: 'https://api.elevenlabs.io/v1', models: mergeDiscoveredRuntimeModels([], parseModelsListResponse([
+ { model_id: 'eleven_multilingual_v2', can_do_text_to_speech: true },
+ { model_id: 'voice-conversion', can_do_text_to_speech: false },
+ ])!),
+ } } }, { modelRegistry: BUNDLED_CATALOG.modelRegistry });
+ const rows = buildUnionRows({ ...projected, connected: true } as ProviderView);
+ expect(rows).toHaveLength(2);
+ expect(managementKindsOfRow(rows.find(row => row.id === 'eleven_multilingual_v2')!, true)).toEqual(['tts']);
+ expect(managementKindsOfRow(rows.find(row => row.id === 'voice-conversion')!, true)).toEqual(['other']);
+ });
+
it('groups discovered BYOK speech models as TTS and keeps the normal disabled management state', () => {
const projected = buildUserProvider({ id: 'speech-byok', name: 'Speech', runtimes: { codex: {
baseUrl: 'https://speech.example.test/v1', models: mergeDiscoveredRuntimeModels([], parseModelsListResponse({ data: [
diff --git a/docs/model-registry-v4-media.md b/docs/model-registry-v4-media.md
index 1d9e68e2a5..0ab0325e96 100644
--- a/docs/model-registry-v4-media.md
+++ b/docs/model-registry-v4-media.md
@@ -88,6 +88,13 @@
型号改成语音模型。不提供类型的服务可在连接设置中手动指定「语音合成」;同一 ID
可更新类型,「自动识别」删除类型覆盖,重新使用发现快照/目录资料。
+ElevenLabs 及兼容转发的[模型列表响应](https://elevenlabs.io/docs/api-reference/models/list)
+以匹配的 `model_id` 和布尔 `can_do_text_to_speech` 声明能力:`true` 归一为
+`audio_speech`;明确 `false` 归为 `other`,避免刷新继续保留旧 TTS 分类,也不推断
+聊天或语音识别能力。缺失/非法值不补类型,沿用已有缺项继承;接口显式 `mode` /
+`model_info.mode` 和用户类型覆盖仍优先。只解析能力,不推断 `nativeApi`。
+直连目录鉴权可复用自定义请求头 `xi-api-key`;没有新增凭证存储或 TTS 执行器。
+
显式媒体类型不套用聊天协议的名称前缀规则(例如 `gpt-*`);语音协议缺失仍保持未知,
不凭 `audio_speech` 推断具体厂商协议。模型的原生协议资料与连接的聊天 `wireProtocol`
是两件事,保留资料不会新增语音合成执行器。
diff --git a/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts b/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts
index 78b1d9eeee..86f7efc5a2 100644
--- a/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts
+++ b/packages/model-providers/src/__tests__/speechModelDiscovery.test.ts
@@ -81,6 +81,77 @@ describe('BYOK speech discovery and projection', () => {
});
});
+describe('ElevenLabs speech capability discovery', () => {
+ const modelId = 'eleven_multilingual_v2';
+ const response = (fields: Record<string, unknown> = {}) => [{ model_id: modelId, name: 'Eleven Multilingual v2', ...fields }];
+
+ it('projects the explicit TTS capability into audioModels, including through compatible relays', () => {
+ const discovered = parseModelsListResponse(response({ can_do_text_to_speech: true }),
+ 'https://speech-relay.example.test/v1/models')!;
+ expect(discovered[0]).toMatchObject({ id: modelId,
+ discoveredMetadata: { name: 'Eleven Multilingual v2', mode: 'audio_speech' } });
+ const p = provider(mergeDiscoveredRuntimeModels([], discovered));
+ expect(p.audioModels).toEqual([expect.objectContaining({ id: modelId, mode: 'audio_speech' })]);
+ expect(p.audioModels?.[0].nativeApi).toBeUndefined();
+ expect(isAgentSelectableModel(p.models.codex![0], { userProvider: true })).toBe(false);
+ });
+
+ it.each([
+ { value: false, mode: 'other' },
+ { value: undefined, mode: undefined },
+ { value: 'true', mode: undefined },
+ ])('does not advertise TTS for capability $value', ({ value, mode }) => {
+ const discovered = parseModelsListResponse(response({ can_do_text_to_speech: value }))!;
+ expect(discovered[0].discoveredMetadata?.mode).toBe(mode);
+ const p = provider(mergeDiscoveredRuntimeModels([], discovered));
+ expect(p.audioModels).toBeUndefined();
+ if (value === false) expect(isAgentSelectableModel(p.models.codex![0], { userProvider: true })).toBe(false);
+ });
+
+ it.each([
+ { fields: { mode: 'chat' }, mode: 'chat' },
+ { fields: { model_info: { mode: 'audio_transcription' } }, mode: 'audio_transcription' },
+ ])('keeps explicit $mode ahead of the vendor capability', ({ fields, mode }) => {
+ const discovered = parseModelsListResponse(response({ can_do_text_to_speech: true, ...fields }))!;
+ expect(discovered[0].discoveredMetadata?.mode).toBe(mode);
+ });
+
+ it('does not apply the vendor capability without a matching model_id', () => {
+ const discovered = parseModelsListResponse([
+ { id: 'chat-one', can_do_text_to_speech: true },
+ { id: 'chat-two', model_id: modelId, can_do_text_to_speech: false },
+ ])!;
+ const p = provider(mergeDiscoveredRuntimeModels([], discovered));
+ expect(p.audioModels).toBeUndefined();
+ for (const model of p.models.codex!) {
+ expect(model.mode).toBeUndefined();
+ expect(isAgentSelectableModel(model, { userProvider: true })).toBe(true);
+ }
+ });
+
+ it('refreshes true to false, keeps sparse snapshots, and preserves user overrides', () => {
+ const tts = parseModelsListResponse(response({ can_do_text_to_speech: true }))!;
+ const noTts = parseModelsListResponse(response({ can_do_text_to_speech: false }))!;
+ const sparse = parseModelsListResponse(response())!;
+ const models = mergeDiscoveredRuntimeModels([], tts);
+ expect(provider(mergeDiscoveredRuntimeModels(models, sparse)).audioModels?.[0].mode).toBe('audio_speech');
+ const refreshed = mergeDiscoveredRuntimeModels(models, noTts);
+ const p = provider(refreshed);
+ expect(p.audioModels).toBeUndefined();
+ expect(p.models.codex![0].mode).toBe('other');
+ expect(isAgentSelectableModel(p.models.codex![0], { userProvider: true })).toBe(false);
+ expect(provider(mergeDiscoveredRuntimeModels(refreshed, sparse)).models.codex![0].mode).toBe('other');
+ expect(provider(mergeDiscoveredRuntimeModels(refreshed, tts)).audioModels?.[0].mode).toBe('audio_speech');
+
+ const manual = mergeDiscoveredRuntimeModels([{ ...models[0], name: 'My voice',
+ mode: 'audio_speech', defaultEnabled: false }], noTts);
+ expect(manual[0]).toMatchObject({ name: 'My voice', mode: 'audio_speech', defaultEnabled: false,
+ discoveredMetadata: { mode: 'other' } });
+ expect(provider(manual).audioModels?.[0].mode).toBe('audio_speech');
+ expect(provider([{ ...manual[0], mode: undefined }]).audioModels).toBeUndefined();
+ });
+});
+
describe('media native API metadata', () => {
it('preserves known media protocols and explicit unknown, and rejects arbitrary protocols', () => {
expect(validModelMetadata({ nativeApi: 'openai-audio-speech' })).toBe(true);
diff --git a/packages/model-providers/src/modelDiscovery.ts b/packages/model-providers/src/modelDiscovery.ts
index f1671ecf29..4af7615cb4 100644
--- a/packages/model-providers/src/modelDiscovery.ts
+++ b/packages/model-providers/src/modelDiscovery.ts
@@ -158,6 +158,12 @@ export function parseModelsListResponse(
// Vercel marks image/video/etc. as type, but Cindy chat import only executes language models.
// Keep them out of the picker instead of saving a mode that later disappears from every list.
if (isVercel && record.type !== undefined && record.type !== 'language') continue;
+ // ElevenLabs /v1/models (and compatible relays) declares TTS with a boolean,
+ // not mode. Match the response contract, never a model-name prefix or a transport API.
+ // Explicit false must replace an earlier TTS snapshot without inventing a chat/STT capability.
+ const elevenLabsMode = record.model_id === id && typeof record.can_do_text_to_speech === 'boolean'
+ ? record.can_do_text_to_speech ? 'audio_speech' : 'other'
+ : undefined;
const discoveredMetadata = { ...pickModelMetadata(record), ...pickModelMetadata({
nativeApi: record.nativeApi !== undefined ? record.nativeApi : record.native_api,
...([rec?.display_name, rec?.name, google?.displayName].some(
@@ -165,7 +171,8 @@ export function parseModelsListResponse(
)
? { name }
: {}),
- mode: record.mode ?? info.mode ?? (isVercel && record.type === 'language' ? 'chat' : isVercel ? record.type : undefined),
+ mode: record.mode ?? info.mode ?? elevenLabsMode ??
+ (isVercel && record.type === 'language' ? 'chat' : isVercel ? record.type : undefined),
modalities,
officialDocs: record.officialDocs,
description: record.description,