Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat(voice): support OpenRouter speech models and voices

JiangJiahao3fd0cd6a9525
diff --git a/apps/desktop/scripts/fixtures/voice-settings/ui-page.mjs b/apps/desktop/scripts/fixtures/voice-settings/ui-page.mjs
index 855425d239..0386cd57d7 100644
--- a/apps/desktop/scripts/fixtures/voice-settings/ui-page.mjs
+++ b/apps/desktop/scripts/fixtures/voice-settings/ui-page.mjs
@@ -12,6 +12,15 @@ export default {
'const empty = {}; export const EMPTY_LIGHT_STATE = empty; export const makerChatStore = { subscribeLight: () => () => {}, getLightSnapshot: () => empty };',
},
cases: {
+ openrouter: {
+ about: 'OpenRouter provider voice IDs and no unsupported speed slider',
+ data: { openRouter: true, protocol: 'openai', modelId: 'elevenlabs/eleven-v3', voiceId: 'sarah', speed: 1.5 },
+ },
+ 'openrouter-manual': {
+ about: 'A saved OpenRouter manual voice keeps audition without carrying an old speed',
+ viewport: { width: 320, height: 900 },
+ data: { openRouter: true, protocol: 'openai', modelId: 'elevenlabs/eleven-v3', voiceId: 'sarah', manual: true, speed: 1.5 },
+ },
'gateway-elevenlabs': {
about: 'Gateway replacement reference voices with separate selection and audition',
data: {
@@ -173,6 +182,31 @@ export default {
}
},
async check(page, ui) {
+ if (ui.data.openRouter) {
+ assert.equal(await page.getByRole('slider').count(), 0);
+ await page.getByText(await ui.t('bots.voiceMode.speedUnavailable'), { exact: true }).waitFor();
+ assert.ok((await page.getByRole('combobox', { name: await ui.t('bots.voiceMode.model'), exact: true }).textContent()).includes('OpenRouter'));
+ if (ui.data.manual) {
+ await page.getByRole('button', { name: await ui.t('bots.voiceMode.preview'), exact: true }).click();
+ } else {
+ await page.getByRole('button', { name: await ui.t('bots.voiceMode.refreshVoices'), exact: true }).click();
+ await page.getByRole('button', { name: await ui.t('bots.voiceMode.voice'), exact: true }).click();
+ const menu = page.getByRole('dialog', { name: await ui.t('bots.voiceMode.voice') });
+ await menu.getByRole('button', { name: 'george', exact: true }).waitFor();
+ await ui.capture('provider-voices');
+ await menu.getByRole('button', { name: 'george', exact: true }).hover();
+ await menu.getByRole('button', { name: await ui.t('bots.voiceMode.previewVoice', { voice: 'george' }), exact: true }).click();
+ }
+ await page.waitForFunction(() => window.__uiPage.ctx.data.previewRequests.length === 1);
+ const request = await page.evaluate(() => window.__uiPage.ctx.data.previewRequests[0]);
+ assert.equal(request.voice, ui.data.manual ? 'custom-voice' : 'george');
+ assert.equal(request.speed, undefined);
+ assert.equal(await page.evaluate(() => window.__uiPage.ctx.data.savedVoice), ui.data.manual ? 'custom-voice' : 'sarah');
+ await ui.capture('preview');
+ if (ui.data.manual) await page.getByRole('button', { name: await ui.t('bots.voiceMode.stopPreview'), exact: true }).click();
+ else await page.keyboard.press('Escape');
+ return;
+ }
if (ui.data.gatewayReference) {
// Use raw catalog names in the host fixture; localization belongs to the UI.
if (ui.data.menu) await page.keyboard.press('Escape');
diff --git a/apps/desktop/scripts/fixtures/voice-settings/voice-settings-fixture.tsx b/apps/desktop/scripts/fixtures/voice-settings/voice-settings-fixture.tsx
index 171812b69f..a78db154af 100644
--- a/apps/desktop/scripts/fixtures/voice-settings/voice-settings-fixture.tsx
+++ b/apps/desktop/scripts/fixtures/voice-settings/voice-settings-fixture.tsx
@@ -65,7 +65,7 @@ mountPage({
...(ctx.data.hidden ? { pickerVisible: false } : {}),
name: modelId,
providerId,
- providerName: ctx.data.gatewayReference
+ providerName: ctx.data.openRouter ? 'OpenRouter' : ctx.data.gatewayReference
? 'XD Gateway'
: protocol === 'elevenlabs'
? 'ElevenLabs'
@@ -77,13 +77,16 @@ mountPage({
agent,
protocol: ctx.data.custom ? undefined : protocol,
protocolLocked: !ctx.data.custom,
- defaultVoice: referenceVoice,
+ defaultVoice: ctx.data.openRouter ? '' : referenceVoice,
+ ...(ctx.data.openRouter ? { speedRange: null } : {}),
},
...(ctx.data.hidden ? [{ id: 'unchecked', name: 'Unchecked speech', providerId: 'fixture', agent: 'codex' as const, pickerVisible: false }] : []),
...(ctx.data.hidden && !ctx.data.noPreference ? [{ id: 'visible', name: 'Visible speech', providerId: 'fixture', agent: 'codex' as const, protocol, protocolLocked: true, defaultVoice: 'river' }] : []),
],
voices: async () =>
- ctx.data.gatewayReference
+ ctx.data.openRouter
+ ? { source: 'provider', voices: ['sarah', 'george'].map(id => ({ id, name: id })) }
+ : ctx.data.gatewayReference
? { source: 'reference', voices: gatewayVoices.map((voice) => ({ ...voice })) }
: ctx.data.systemCatalog
? {
diff --git a/apps/desktop/src/main/maker-host/model-discovery-pages.ts b/apps/desktop/src/main/maker-host/model-discovery-pages.ts
index 446b0d7ef3..34312bd2b3 100644
--- a/apps/desktop/src/main/maker-host/model-discovery-pages.ts
+++ b/apps/desktop/src/main/maker-host/model-discovery-pages.ts
@@ -1,4 +1,4 @@
-import { mergeModelMetadata, parseModelsListResponse, type DiscoveredModel } from '@cindy/model-providers';
+import { isOpenRouterModelsUrl, mergeModelMetadata, parseModelsListResponse, type DiscoveredModel } from '@cindy/model-providers';
/** Follow catalog cursors only on the original endpoint. Never forward credentials to a next-link host. */
export async function collectModelDiscoveryPages(
@@ -9,12 +9,9 @@ export async function collectModelDiscoveryPages(
const origin = new URL(endpoint);
const visited = new Set([origin.toString()]);
const models = new Map<string, DiscoveredModel>();
- let page = first;
- let current = origin;
- for (let count = 0; count < 100; count++) {
- const parsed = parseModelsListResponse(page, endpoint);
- if (!parsed) return models.size ? [...models.values()] : null;
+ const append = (parsed: DiscoveredModel[]) => {
for (const model of parsed) {
+ if (models.size >= 10_000) break;
const previous = models.get(model.id);
models.set(model.id, previous ? {
...previous,
@@ -25,8 +22,15 @@ export async function collectModelDiscoveryPages(
...(previous.discoveredCost || model.discoveredCost
? { discoveredCost: { ...previous.discoveredCost, ...model.discoveredCost } } : {}),
} : model);
- if (models.size >= 10_000) return [...models.values()];
}
+ };
+ let page = first;
+ let current = origin;
+ for (let count = 0; count < 100; count++) {
+ const parsed = parseModelsListResponse(page, endpoint);
+ if (!parsed) return models.size ? [...models.values()] : null;
+ append(parsed);
+ if (models.size >= 10_000) return [...models.values()];
if (!page || typeof page !== 'object' || Array.isArray(page)) break;
const data = page as Record<string, unknown>;
const next = new URL(current);
@@ -48,5 +52,16 @@ export async function collectModelDiscoveryPages(
try { page = await fetchPage(next.toString()); } catch { break; }
current = next;
}
+ // The ordinary OpenRouter list excludes speech. Fetch its separate catalog on
+ // the same credential-owned endpoint, using the caller's timeout/byte limits.
+ // Explicit filtered catalogs stay user-owned. A speech outage must not discard chat.
+ if (isOpenRouterModelsUrl(endpoint) && !origin.search && !origin.username && !origin.password) {
+ const speech = new URL(origin);
+ speech.searchParams.set('output_modalities', 'speech');
+ try {
+ const parsed = parseModelsListResponse(await fetchPage(speech.toString()), speech.toString());
+ append((parsed ?? []).filter(model => model.discoveredMetadata?.mode === 'audio_speech'));
+ } catch { /* Keep the successfully fetched catalog, as for a failed pagination request. */ }
+ }
return [...models.values()];
}
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/openrouter.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/openrouter.test.ts
new file mode 100644
index 0000000000..4e2b342e5a
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/__tests__/openrouter.test.ts
@@ -0,0 +1,321 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+ BUNDLED_CATALOG,
+ buildUserProvider,
+ isAgentSelectableModel,
+ mergeDiscoveredRuntimeModels,
+ parseModelsListResponse,
+} from '@cindy/model-providers';
+import { fetchProviderModels } from '../../maker-host/provider-model-fetch.js';
+import { resolveSpeechConnection, speechModels, type SpeechCatalogDeps } from '../models.js';
+import { SpeechSynthesisSession, type SpeechConnection } from '../speech.js';
+import { speechRequest } from '../speechProtocols.js';
+import { loadSpeechVoices } from '../voices.js';
+
+// Unit tier: exercise the actual discovery -> saved runtime -> picker -> transport path.
+// The ordinary OpenRouter catalog omits speech; its dedicated list has no Cindy mode.
+const catalogUrl = 'https://openrouter.ai/api/v1/models';
+const speechUrl = `${catalogUrl}?output_modalities=speech`;
+const speechId = 'elevenlabs/eleven-v3';
+const speechEntry = {
+ id: speechId,
+ name: 'Eleven v3',
+ architecture: { input_modalities: ['text'], output_modalities: ['speech'] },
+ supported_voices: ['george', 'sarah'],
+ pricing: { prompt: '0.00018' }, // per character, not per token
+};
+const chatEntry = {
+ id: 'openai/audio-chat',
+ architecture: { input_modalities: ['text', 'audio'], output_modalities: ['text', 'audio'] },
+ pricing: { prompt: '0.000001' },
+};
+const json = (data: unknown) => new Response(JSON.stringify(data));
+const connection: SpeechConnection = {
+ protocol: 'openai',
+ baseUrl: 'https://openrouter.ai/api/v1',
+ headers: { authorization: 'Bearer test-openrouter-key' },
+};
+const current = () => {};
+
+describe('OpenRouter speech integration', () => {
+ it.each(['claude-code', 'codex', 'pi'] as const)(
+ 'discovers speech alongside chat and executes the owning %s connection',
+ async (agent) => {
+ const baseUrl = agent === 'claude-code' ? 'https://openrouter.ai/api' : connection.baseUrl;
+ const fetcher = vi.fn<typeof fetch>(async (url, init) => {
+ expect(init?.headers).toEqual({ authorization: 'Bearer test-openrouter-key' });
+ return json({ data: String(url) === speechUrl ? [speechEntry] : [chatEntry] });
+ });
+ const result = await fetchProviderModels(
+ {
+ agent,
+ baseUrl,
+ apiKey: 'test-openrouter-key',
+ redirect: 'error',
+ },
+ fetcher,
+ );
+ expect(fetcher.mock.calls.map(([url]) => String(url))).toEqual([catalogUrl, speechUrl]);
+ expect(result.ok).toBe(true);
+ const discovered = result.models!;
+ expect(discovered.find((m) => m.id === speechId)?.discoveredCost).toBeUndefined();
+ expect(discovered.find((m) => m.id === chatEntry.id)?.discoveredCost).toEqual({ input: 1 });
+ const saved = mergeDiscoveredRuntimeModels([], discovered);
+ const provider = buildUserProvider({
+ id: 'router',
+ name: 'My OpenRouter',
+ runtimes: {
+ [agent]: { baseUrl, models: saved },
+ },
+ });
+ provider.routing[agent]!.headerOverride = {
+ 'anthropic-version': '2023-06-01',
+ 'x-api-key': 'stale-key',
+ };
+ expect(provider.audioModels).toEqual([
+ expect.objectContaining({ id: speechId, mode: 'audio_speech' }),
+ ]);
+ expect(
+ provider.models[agent]!.filter((m) =>
+ isAgentSelectableModel(m, { userProvider: true }),
+ ).map((m) => m.id),
+ ).toEqual([chatEntry.id]);
+ const deps: SpeechCatalogDeps = {
+ catalog: { ...BUNDLED_CATALOG, providers: [provider] },
+ gatewayAllowed: false,
+ disabled: () => false,
+ readKey: vi.fn(() => 'test-openrouter-key'),
+ gatewayConnection: () => {
+ throw new Error('not Gateway');
+ },
+ };
+ expect(speechModels(deps)).toEqual([
+ expect.objectContaining({
+ id: speechId,
+ providerId: 'router',
+ providerName: 'My OpenRouter',
+ agent,
+ protocol: 'openai',
+ protocolLocked: true,
+ defaultVoice: '',
+ speedRange: null,
+ }),
+ ]);
+ const selection = { providerId: 'router', agent, modelId: speechId, voice: 'sarah' };
+ const bound = resolveSpeechConnection(deps, { ...selection, protocol: 'elevenlabs' });
+ expect(bound.baseUrl).toBe(connection.baseUrl);
+ expect(bound.protocol).toBe('openai');
+ expect(bound.headers).toEqual(connection.headers);
+ expect(deps.readKey).toHaveBeenCalledWith('router', agent);
+ expect(provider.routing[agent]?.upstream).toBe(baseUrl);
+ expect(() => resolveSpeechConnection(deps, { ...selection, speed: 1.5 })).toThrow(
+ 'invalid_speech_speed',
+ );
+ const request = speechRequest(bound, { modelId: speechId, voice: 'sarah', text: 'Hello' });
+ expect(request.url.href).toBe('https://openrouter.ai/api/v1/audio/speech');
+ expect(JSON.parse(request.body)).toEqual({
+ model: speechId,
+ voice: 'sarah',
+ input: 'Hello',
+ response_format: 'pcm',
+ });
+ expect(() =>
+ speechRequest(bound, { modelId: speechId, voice: 'sarah', text: 'Hello', speed: 1.5 }),
+ ).toThrow();
+ },
+ );
+
+ it('retains chat on a failed optional speech query, and honors an explicit catalog filter', async () => {
+ const spec = { agent: 'codex' as const, baseUrl: connection.baseUrl, apiKey: 'test-key' };
+ const fetcher = vi.fn<typeof fetch>(async (url) =>
+ String(url) === speechUrl ? new Response(null, { status: 503 }) : json({ data: [chatEntry] }),
+ );
+ expect((await fetchProviderModels(spec, fetcher)).models?.map((m) => m.id)).toEqual([
+ chatEntry.id,
+ ]);
+ expect(fetcher).toHaveBeenCalledTimes(2);
+ fetcher.mockClear();
+ await fetchProviderModels(
+ { ...spec, modelsUrl: `${catalogUrl}?output_modalities=text` },
+ fetcher,
+ );
+ expect(fetcher).toHaveBeenCalledOnce();
+ });
+
+ it('keeps user type/name overrides and does not infer TTS from audio or a different origin', () => {
+ const discovered = parseModelsListResponse({ data: [speechEntry, chatEntry] }, speechUrl)!;
+ expect(discovered[0].discoveredMetadata?.mode).toBe('audio_speech');
+ expect(discovered[0].discoveredMetadata?.nativeApi).toBeUndefined();
+ expect(discovered[1].discoveredMetadata?.mode).toBeUndefined();
+ const saved = mergeDiscoveredRuntimeModels(
+ [{ id: speechId, name: 'My name', mode: 'chat' }],
+ discovered,
+ );
+ expect(saved.find((m) => m.id === speechId)).toMatchObject({
+ name: 'My name',
+ mode: 'chat',
+ discoveredMetadata: { mode: 'audio_speech' },
+ });
+ expect(
+ parseModelsListResponse({ data: [speechEntry] }, 'https://proxy.test/api/v1/models')![0]
+ .discoveredMetadata?.mode,
+ ).toBeUndefined();
+ });
+
+ it.each([
+ 'https://openrouter.ai/proxy/v1',
+ 'https://openrouter.ai:8443/api/v1',
+ 'https://openrouter.ai.untrusted.test/api/v1',
+ 'http://openrouter.ai/api/v1',
+ ])('does not apply official routing or speed limits to %s', (baseUrl) => {
+ const provider = buildUserProvider({
+ id: 'proxy',
+ name: 'Proxy',
+ runtimes: {
+ codex: {
+ baseUrl,
+ models: [{ id: speechId, name: speechId, mode: 'audio_speech', nativeApi: null }],
+ },
+ },
+ });
+ const deps: SpeechCatalogDeps = {
+ catalog: { ...BUNDLED_CATALOG, providers: [provider] },
+ gatewayAllowed: false,
+ disabled: () => false,
+ readKey: () => 'test-key',
+ gatewayConnection: () => {
+ throw new Error('not Gateway');
+ },
+ };
+ expect(speechModels(deps)[0]).toMatchObject({ protocolLocked: false });
+ expect(() =>
+ resolveSpeechConnection(deps, { providerId: 'proxy', agent: 'codex', modelId: speechId }),
+ ).toThrow('model_unavailable');
+ const explicit = resolveSpeechConnection(deps, {
+ providerId: 'proxy',
+ agent: 'codex',
+ modelId: speechId,
+ protocol: 'openai',
+ });
+ expect(explicit.baseUrl).toBe(baseUrl);
+ expect(explicit.speedRange).toBeUndefined();
+ });
+
+ it('loads and refreshes only the selected model voices, keeping manual IDs valid', async () => {
+ const fetcher = vi
+ .fn<typeof fetch>()
+ .mockResolvedValueOnce(
+ json({
+ data: [
+ { ...speechEntry, supported_voices: ['george', 'sarah', 'george', null, '../bad'] },
+ { id: 'another-model', supported_voices: ['wrong-model'] },
+ ],
+ }),
+ )
+ .mockResolvedValueOnce(json({ data: [{ ...speechEntry, supported_voices: ['alice'] }] }));
+ expect(await loadSpeechVoices(connection, speechId, fetcher, current, 'router')).toEqual({
+ source: 'provider',
+ voices: [
+ { id: 'george', name: 'george' },
+ { id: 'sarah', name: 'sarah' },
+ ],
+ });
+ expect(await loadSpeechVoices(connection, speechId, fetcher, current, 'router')).toEqual({
+ source: 'provider',
+ voices: [{ id: 'alice', name: 'alice' }],
+ });
+ expect(fetcher).toHaveBeenCalledWith(
+ speechUrl,
+ expect.objectContaining({ method: 'GET', headers: connection.headers, redirect: 'error' }),
+ );
+ expect(
+ JSON.parse(
+ speechRequest(connection, { modelId: speechId, voice: 'custom-voice-id', text: 'Hello' })
+ .body,
+ ).voice,
+ ).toBe('custom-voice-id');
+ });
+
+ it.each([null, undefined])(
+ 'keeps manual entry when supported_voices is %s',
+ async (supported_voices) => {
+ expect(
+ await loadSpeechVoices(
+ connection,
+ speechId,
+ vi.fn(async () => json({ data: [{ ...speechEntry, supported_voices }] })),
+ current,
+ ),
+ ).toEqual({ source: 'manual', voices: [] });
+ },
+ );
+
+ it('rejects missing models, malformed catalogs and stale account results', async () => {
+ for (const data of [
+ { data: [] },
+ { data: [{ ...speechEntry, supported_voices: 'sarah' }] },
+ {},
+ ]) {
+ await expect(
+ loadSpeechVoices(
+ connection,
+ speechId,
+ vi.fn(async () => json(data)),
+ current,
+ ),
+ ).rejects.toThrow();
+ }
+ let active = true;
+ await expect(
+ loadSpeechVoices(
+ connection,
+ speechId,
+ vi.fn(async () => {
+ active = false;
+ return json({ data: [speechEntry] });
+ }),
+ () => {
+ if (!active) throw new Error('stale account');
+ },
+ ),
+ ).rejects.toThrow('stale account');
+ });
+
+ it('never queries OpenRouter for Gateway or a third-party proxy', async () => {
+ const fetcher = vi.fn();
+ for (const baseUrl of ['https://proxy.test/api/v1', 'https://openrouter.ai/proxy/v1']) {
+ expect(
+ await loadSpeechVoices({ ...connection, baseUrl }, speechId, fetcher, current),
+ ).toEqual({ source: 'manual', voices: [] });
+ }
+ await loadSpeechVoices(connection, 'elevenlabs/eleven_v3', fetcher, current, 'xd');
+ expect(fetcher).not.toHaveBeenCalled();
+ });
+
+ it('streams binary PCM through the existing cancellable playback contract', async () => {
+ const fetcher = vi.fn<typeof fetch>(
+ async () =>
+ new Response(new Uint8Array([1, 0, 255, 127]), {
+ headers: { 'Content-Type': 'audio/pcm' },
+ }),
+ );
+ const speech = new SpeechSynthesisSession({
+ connection: () => connection,
+ fetch: fetcher,
+ assertCurrent: current,
+ });
+ expect(
+ await speech.start({
+ requestId: 'or-test',
+ modelId: speechId,
+ voice: 'sarah',
+ text: 'Hello',
+ }),
+ ).toEqual({ sampleRate: 24000 });
+ expect(new Uint8Array((await speech.read('or-test')).pcm)).toEqual(
+ new Uint8Array([1, 0, 255, 127]),
+ );
+ expect((await speech.read('or-test')).done).toBe(true);
+ speech.cancel();
+ });
+});
diff --git a/apps/desktop/src/main/voice-conversation/adapters/types.ts b/apps/desktop/src/main/voice-conversation/adapters/types.ts
index 60324543d3..9698a3105e 100644
--- a/apps/desktop/src/main/voice-conversation/adapters/types.ts
+++ b/apps/desktop/src/main/voice-conversation/adapters/types.ts
@@ -1,9 +1,10 @@
-import type { SpeechProtocol } from '../../../shared/speechProtocols.js';
+import type { SpeechProtocol, SpeechSpeedRange } from '../../../shared/speechProtocols.js';
export type SpeechConnection = {
baseUrl: string;
headers: Record<string, string>;
protocol: SpeechProtocol;
+ speedRange?: SpeechSpeedRange;
};
export type SpeechInput = { modelId: string; voice: string; text: string; speed?: number };
export type SpeechRequest = { url: URL; body: string; headers?: Record<string, string> };
diff --git a/apps/desktop/src/main/voice-conversation/models.ts b/apps/desktop/src/main/voice-conversation/models.ts
index 65b1254251..11e3ded2fd 100644
--- a/apps/desktop/src/main/voice-conversation/models.ts
+++ b/apps/desktop/src/main/voice-conversation/models.ts
@@ -22,6 +22,7 @@ import {
} from '../../shared/voiceConversation.js';
import { gatewayReferenceSpeechVoices, referenceSpeechVoices } from '../../shared/speechVoices.js';
import type { SpeechConnection } from './speech.js';
+import { openRouterSpeechBaseUrl } from './openrouter.js';
export type SpeechCatalogDeps = {
catalog: Catalog;
@@ -170,7 +171,8 @@ function bindings(deps: SpeechCatalogDeps, selection?: SpeechModelSelection): Bi
const base = apiBase(model.route?.baseUrl ?? route.upstream);
const ownerBase = apiBase(route.upstream);
if (!base || !ownerBase || base.origin !== ownerBase.origin) continue;
- const fixedProtocol = endpointProtocol(base, media.id);
+ const openRouterBase = openRouterSpeechBaseUrl(base.toString());
+ const fixedProtocol = openRouterBase ? 'openai' : endpointProtocol(base, media.id);
const detectedProtocol =
fixedProtocol ?? (declared === 'openai-audio-speech' ? 'openai' : null);
// A saved override cannot turn an official endpoint into another provider's API.
@@ -180,11 +182,13 @@ function bindings(deps: SpeechCatalogDeps, selection?: SpeechModelSelection): Bi
// OpenAI proxies keep the configured prefix (including a non-/v1 API base).
const nativeBase =
protocol === 'qwen-tts' || protocol === 'dashscope-tts' ? '/api/v1' : '/v1';
- if (fixedProtocol && fixedProtocol !== 'openai') base.pathname = nativeBase;
+ if (openRouterBase) base.href = openRouterBase;
+ else if (fixedProtocol && fixedProtocol !== 'openai') base.pathname = nativeBase;
else if (!base.pathname || base.pathname === '/') base.pathname = nativeBase;
const headers: Record<string, string> = {};
for (const [name, value] of Object.entries(route.headerOverride ?? {})) {
const lower = name.toLowerCase();
+ if (openRouterBase && ['anthropic-version', 'anthropic-beta'].includes(lower)) continue;
if (
[
'authorization',
@@ -226,10 +230,18 @@ function bindings(deps: SpeechCatalogDeps, selection?: SpeechModelSelection): Bi
agent,
protocol: detectedProtocol ?? undefined,
protocolLocked: !!fixedProtocol,
- defaultVoice: referenceSpeechVoices(media.id, protocol ?? undefined)[0] ?? '',
+ // OpenRouter's voices come from supported_voices, not vendor references.
+ defaultVoice: openRouterBase
+ ? '' : referenceSpeechVoices(media.id, protocol ?? undefined)[0] ?? '',
+ ...(openRouterBase ? { speedRange: null } : {}),
},
connection: protocol
- ? { baseUrl: base.toString().replace(/\/+$/, ''), headers, protocol }
+ ? {
+ baseUrl: base.toString().replace(/\/+$/, ''), headers, protocol,
+ // OpenRouter speed is model/provider-dependent. Keep the provider
+ // default until its route advertises a verified range; many reject overrides.
+ ...(openRouterBase ? { speedRange: null } : {}),
+ }
: null,
});
}
@@ -252,7 +264,9 @@ export function resolveSpeechConnection(
if (!binding?.connection) throw new Error('model_unavailable');
if (
'speed' in selection &&
- !isSpeechSpeedSupported(selection.modelId, binding.connection.protocol, selection.speed)
+ !isSpeechSpeedSupported(
+ selection.modelId, binding.connection.protocol, selection.speed, binding.connection.speedRange,
+ )
)
throw new Error('invalid_speech_speed');
return binding.connection;
diff --git a/apps/desktop/src/main/voice-conversation/openrouter.ts b/apps/desktop/src/main/voice-conversation/openrouter.ts
new file mode 100644
index 0000000000..a8657fa66f
--- /dev/null
+++ b/apps/desktop/src/main/voice-conversation/openrouter.ts
@@ -0,0 +1,21 @@
+/** Only the documented OpenRouter API bases own this TTS contract. Chat keeps
+ * its original /api (Messages) or /api/v1 (OpenAI) route; custom prefixes stay untouched.
+ * https://openrouter.ai/docs/guides/overview/multimodal/tts
+ */
+export function openRouterSpeechBaseUrl(value: string): string | undefined {
+ try {
+ const url = new URL(value);
+ if (
+ url.origin === 'https://openrouter.ai' &&
+ !url.username &&
+ !url.password &&
+ !url.search &&
+ !url.hash &&
+ ['/api', '/api/v1'].includes(url.pathname.replace(/\/+$/, ''))
+ )
+ return `${url.origin}/api/v1`;
+ } catch {
+ /* Not an official API base. */
+ }
+ return undefined;
+}
diff --git a/apps/desktop/src/main/voice-conversation/speechProtocols.ts b/apps/desktop/src/main/voice-conversation/speechProtocols.ts
index f788a2e0cf..eed5aa512e 100644
--- a/apps/desktop/src/main/voice-conversation/speechProtocols.ts
+++ b/apps/desktop/src/main/voice-conversation/speechProtocols.ts
@@ -39,7 +39,7 @@ export function speechRequest(connection: SpeechConnection, input: SpeechInput):
base.hash
)
throw new SpeechError('unavailable');
- if (!isSpeechSpeedSupported(input.modelId, connection.protocol, input.speed))
+ if (!isSpeechSpeedSupported(input.modelId, connection.protocol, input.speed, connection.speedRange))
throw new SpeechError('unavailable');
return speechAdapter(connection.protocol).request(connection.baseUrl.replace(/\/+$/, ''), input);
}
diff --git a/apps/desktop/src/main/voice-conversation/voices.ts b/apps/desktop/src/main/voice-conversation/voices.ts
index d017dc1292..9299122dfb 100644
--- a/apps/desktop/src/main/voice-conversation/voices.ts
+++ b/apps/desktop/src/main/voice-conversation/voices.ts
@@ -7,6 +7,7 @@ import { gatewayReferenceSpeechVoices, referenceSpeechVoices } from '../../share
import { alibabaSystemVoices } from '../../shared/alibabaSpeechVoices.js';
import { loadDashscopeVoices } from './dashscopeVoices.js';
import type { SpeechConnection } from './speech.js';
+import { openRouterSpeechBaseUrl } from './openrouter.js';
type RecordValue = Record<string, unknown>;
const record = (value: unknown): RecordValue =>
@@ -59,7 +60,9 @@ export async function loadSpeechVoices(
if (voices.length)
return { source: 'reference', voices: voices.map((voice) => ({ ...voice })) };
}
- if (connection.protocol === 'openai' || connection.protocol === 'qwen-tts') {
+ const openRouterBase = providerId !== 'xd' && connection.protocol === 'openai'
+ ? openRouterSpeechBaseUrl(connection.baseUrl) : undefined;
+ if ((connection.protocol === 'openai' && !openRouterBase) || connection.protocol === 'qwen-tts') {
const voices =
connection.protocol === 'qwen-tts'
? alibabaSystemVoices(modelId, connection.protocol, connection.baseUrl)
@@ -95,6 +98,20 @@ export async function loadSpeechVoices(
};
try {
const base = connection.baseUrl.replace(/\/+$/, '');
+ if (openRouterBase) {
+ const json = await request(new URL(`${openRouterBase}/models?output_modalities=speech`));
+ if (!Array.isArray(json.data)) throw new Error('invalid_voice_catalog');
+ const model = json.data.map(record).find(entry => entry.id === modelId);
+ if (!model) throw new Error('voice_catalog_unavailable');
+ if (model.supported_voices == null) return { source: 'manual', voices: [] };
+ if (!Array.isArray(model.supported_voices)) throw new Error('invalid_voice_catalog');
+ const ids = [...new Set(model.supported_voices
+ .map(normalizeSpeechVoiceId).filter((id): id is string => !!id))];
+ return {
+ source: 'provider', voices: ids.slice(0, 1000).map(id => ({ id, name: id })),
+ ...(ids.length > 1000 ? { truncated: true } : {}),
+ };
+ }
if (connection.protocol === 'dashscope-tts')
return await loadDashscopeVoices(modelId, base, request, assertCurrent);
if (connection.protocol === 'minimax') {
diff --git a/apps/desktop/src/renderer/features/bots/voice/CompanionVoiceSettings.tsx b/apps/desktop/src/renderer/features/bots/voice/CompanionVoiceSettings.tsx
index aaa736e504..0823f2f47d 100644
--- a/apps/desktop/src/renderer/features/bots/voice/CompanionVoiceSettings.tsx
+++ b/apps/desktop/src/renderer/features/bots/voice/CompanionVoiceSettings.tsx
@@ -24,7 +24,6 @@ import type {
} from '@/../shared/voiceConversation';
import {
normalizeVoiceConversationSelection,
- resolveSpeechSpeed,
voiceModelKey,
} from '@/../shared/voiceConversation';
import { referenceSpeechVoices } from '@/../shared/speechVoices';
@@ -251,8 +250,7 @@ export function CompanionVoiceSettings(props: Props) {
}, [props.beforeLeaveRef, persist, ready, selection, protocol, preview]);
const carrySpeed = (next: VoiceConversationSelection): VoiceConversationSelection => {
const previousSpeed = speed ?? preference?.speed;
- const nextProtocol = resolveVoiceConfiguration(models, next).protocol;
- const nextSpeed = resolveSpeechSpeed(next.modelId, nextProtocol, previousSpeed);
+ const nextSpeed = resolveVoiceConfiguration(models, next, undefined, previousSpeed).speed;
setSpeed(undefined);
setSpeedNotice(previousSpeed !== nextSpeed ? (nextSpeed ?? 1) : undefined);
return { ...next, ...(nextSpeed !== undefined ? { speed: nextSpeed } : {}) };
@@ -450,7 +448,7 @@ export function CompanionVoiceSettings(props: Props) {
modelId: selectedModel.id,
protocol: protocolOverride,
voice: value,
- speed: resolveSpeechSpeed(selectedModel.id, protocol, speed ?? preference?.speed),
+ speed: effectiveSpeed,
});
if (save && next && validModel && protocol) void persist(next);
}}
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx
index 1a9b755428..f1354917ec 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/CompanionVoiceSettings.test.tsx
@@ -435,6 +435,35 @@ it('reconciles the saved speed on model changes and keeps it when changing the v
expect(saved).not.toHaveProperty('speed');
});
+// OpenRouter uses OpenAI transport, but does not promise OpenAI's speed range.
+// Exercise real settings save/preview and call-start resolution with an old saved speed.
+it('honors route speed limits when reopening, editing and previewing an OpenRouter voice', async () => {
+ const router: VoiceConversationModel = {
+ id: 'elevenlabs/eleven-v3', name: 'Eleven v3', providerId: 'router', agent: 'codex',
+ protocol: 'openai', protocolLocked: true, speedRange: null,
+ };
+ models = [router];
+ const saved = { providerId: 'router', agent: 'codex' as const, modelId: router.id, voice: 'sarah', speed: 1.5 };
+ vi.mocked(window.electronAPI.voiceConversation.preference).mockResolvedValue({ selection: saved, customized: true });
+ renderSettings();
+ await screen.findByText('bots.voiceMode.speedUnavailable');
+ expect(screen.queryByRole('slider')).toBeNull();
+ expect(resolveVoiceConfiguration(models, saved).selection).not.toHaveProperty('speed');
+ const voice = await screen.findByLabelText('bots.voiceMode.voiceId');
+ await waitFor(() => expect((voice as HTMLInputElement).disabled).toBe(false));
+ fireEvent.change(voice, { target: { value: 'custom-voice-id' } });
+ fireEvent.blur(voice);
+ await waitFor(() => expect(window.electronAPI.voiceConversation.setPreference).toHaveBeenCalledWith(
+ 'bot', { providerId: 'router', agent: 'codex', modelId: router.id, voice: 'custom-voice-id' },
+ ));
+ const previewButton = await screen.findByRole('button', { name: 'bots.voiceMode.preview' });
+ await waitFor(() => expect((previewButton as HTMLButtonElement).disabled).toBe(false));
+ fireEvent.click(previewButton);
+ await waitFor(() => expect(window.electronAPI.voiceConversation.preview).toHaveBeenCalled());
+ expect(vi.mocked(window.electronAPI.voiceConversation.preview).mock.calls[0][0]).toMatchObject({ voice: 'custom-voice-id', modelId: router.id });
+ expect(vi.mocked(window.electronAPI.voiceConversation.preview).mock.calls[0][0]).not.toHaveProperty('speed');
+});
+
it('defaults to sentence playback, persists a playback-only change, and restores the default', async () => {
models = [{ id: 'tts', name: 'Speech', defaultVoice: 'alloy' }];
const view = renderSettings();
diff --git a/apps/desktop/src/renderer/features/bots/voice/voiceConfiguration.ts b/apps/desktop/src/renderer/features/bots/voice/voiceConfiguration.ts
index f61156bfd4..0d764d0a0a 100644
--- a/apps/desktop/src/renderer/features/bots/voice/voiceConfiguration.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/voiceConfiguration.ts
@@ -33,7 +33,7 @@ export function resolveVoiceConfiguration(
selectedModel?.protocol ??
((selectedModel?.providerId ?? 'xd') === 'xd' ? 'openai' : undefined);
const resolvedSpeed = selectedModel
- ? resolveSpeechSpeed(selectedModel.id, protocol, speed)
+ ? resolveSpeechSpeed(selectedModel.id, protocol, speed, selectedModel.speedRange)
: undefined;
const selection = selectedModel
? normalizeVoiceConversationSelection({
@@ -52,7 +52,7 @@ export function resolveVoiceConfiguration(
selectedVoice,
protocol,
selection,
- speedRange: selectedModel ? speechSpeedRange(selectedModel.id, protocol) : null,
+ speedRange: selectedModel ? speechSpeedRange(selectedModel.id, protocol, selectedModel.speedRange) : null,
speed: resolvedSpeed,
speedAdjusted: speed !== undefined && speed !== resolvedSpeed,
};
diff --git a/apps/desktop/src/shared/speechProtocols.ts b/apps/desktop/src/shared/speechProtocols.ts
index 4085ad0592..2d5511379b 100644
--- a/apps/desktop/src/shared/speechProtocols.ts
+++ b/apps/desktop/src/shared/speechProtocols.ts
@@ -7,12 +7,18 @@ export const SPEECH_PROTOCOLS = [
'dashscope-tts',
] as const;
export type SpeechProtocol = (typeof SPEECH_PROTOCOLS)[number];
+export type SpeechSpeedRange = { min: number; max: number } | null;
/** Native synthesis limits, shared by settings and Main. Compatible proxies must
* implement their selected protocol; this is not a claim about gateway forwarding.
* Eleven v3 currently excludes speed: https://elevenlabs.io/docs/eleven-creative/playground/text-to-speech#speed
*/
-export function speechSpeedRange(modelId: string, protocol?: SpeechProtocol) {
+export function speechSpeedRange(
+ modelId: string,
+ protocol?: SpeechProtocol,
+ routeRange?: SpeechSpeedRange,
+): SpeechSpeedRange {
+ if (routeRange !== undefined) return routeRange;
switch (protocol) {
case 'elevenlabs':
return /(?:^|\/)eleven_v3(?:_|$)/i.test(modelId) ? null : { min: 0.7, max: 1.2 };
@@ -30,9 +36,10 @@ export function isSpeechSpeedSupported(
modelId: string,
protocol: SpeechProtocol,
speed: number | undefined,
+ routeRange?: SpeechSpeedRange,
): boolean {
if (speed === undefined) return true;
- const range = speechSpeedRange(modelId, protocol);
+ const range = speechSpeedRange(modelId, protocol, routeRange);
return !!range && Number.isFinite(speed) && speed >= range.min && speed <= range.max;
}
@@ -42,8 +49,9 @@ export function resolveSpeechSpeed(
modelId: string,
protocol: SpeechProtocol | undefined,
speed?: number,
+ routeRange?: SpeechSpeedRange,
) {
- const range = speechSpeedRange(modelId, protocol);
+ const range = speechSpeedRange(modelId, protocol, routeRange);
return range && speed !== undefined && Number.isFinite(speed)
? Math.min(range.max, Math.max(range.min, speed))
: undefined;
diff --git a/apps/desktop/src/shared/voiceConversation.ts b/apps/desktop/src/shared/voiceConversation.ts
index 6ed8183992..3f3dae09b8 100644
--- a/apps/desktop/src/shared/voiceConversation.ts
+++ b/apps/desktop/src/shared/voiceConversation.ts
@@ -50,7 +50,7 @@ export type VoiceExitDecision = boolean | 'submit_then_exit';
export { speechSpeedRange, isSpeechSpeedSupported, resolveSpeechSpeed } from './speechProtocols.js';
export type { SpeechProtocol } from './speechProtocols.js';
-import { SPEECH_PROTOCOLS, type SpeechProtocol } from './speechProtocols.js';
+import { SPEECH_PROTOCOLS, type SpeechProtocol, type SpeechSpeedRange } from './speechProtocols.js';
// Missing provider/runtime is the original Cindy AI selection, never another connection.
export type SpeechModelSelection = {
@@ -91,6 +91,9 @@ export type VoiceConversationModel = {
protocol?: SpeechProtocol;
/** The official endpoint fixes protocol; saved overrides cannot select another adapter. */
protocolLocked?: boolean;
+ /** Host-owned route capability. Absent uses protocol defaults; null omits speed.
+ * Display only, never persisted or accepted as a caller-provided capability. */
+ speedRange?: SpeechSpeedRange;
defaultVoice?: string;
};
export type VoiceConversationPreference = {
diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.md
index 21d14dae12..4f4d4b991c 100644
--- a/docs/product-rules/companion-voice-mode.md
+++ b/docs/product-rules/companion-voice-mode.md
@@ -57,6 +57,26 @@
## BYOK 音频合同
+OpenRouter 的普通 `/api/v1/models` 目录不包含完整语音列表。对未指定查询条件的官方目录,
+模型发现额外查询 `?output_modalities=speech`,按 ID 合并,并仅将明确输出 `speech` 的条目
+识别为 `audio_speech`;`audio` 输入/输出的聊天模型保持聊天分类,已有名称、类型及开关覆盖
+继续生效。额外查询失败时保留已获取的聊天目录;用户指定的过滤参数或代理目录不被改写。
+语音的按字符/时长报价不写入聊天的 token 单价字段。
+
+OpenRouter BYOK 的官方 `/api` 和 `/api/v1` 连接只在语音请求中统一到
+`https://openrouter.ai/api/v1/audio/speech`,使用对应连接的 Bearer key、OpenAI 兼容 PCM
+适配器及原模型 ID;聊天地址不变。每次进入型号、切换型号或点击音色刷新时,从语音模型
+目录读取该型号的 `supported_voices`,不套用 Gateway 或厂商直连的参考音色;缺少该字段时
+保留手动输入,查询失败沿用现有错误/重试行为,不覆盖已选音色。仍可手动输入并试听。
+音色目录仅在内存,不进入公共 Registry 或持久配置。
+
+OpenRouter 没有统一的型号语速范围,部分上游会拒绝非默认值,因此本阶段使用供应商默认
+语速,不发送 `speed`;设置页复用“不支持调整语速”的现有状态。Main 投影的可选
+`speedRange` 为本机能力信息(`null` 表示不支持、缺省沿用原协议范围),不接受 Renderer
+自报能力、不写入偏好、不加入远程协议。旧偏好中的语速在运行时忽略,明确修改设置时才
+通过原保存流程移除。可发现和可选择不等于账户具有执行权限;PCM 听感仍需按实际上游验收。
+接口依据:[OpenRouter TTS](https://openrouter.ai/docs/guides/overview/multimodal/tts)。
+
| 接口 | 请求与鉴权 | 响应 |
| --- | --- | --- |
| OpenAI 兼容 | `<base>/audio/speech`,Bearer,`model / voice / input / response_format: pcm` | 原始 24kHz、单声道、16bit little-endian PCM |
diff --git a/packages/model-providers/src/modelDiscovery.ts b/packages/model-providers/src/modelDiscovery.ts
index 711f37dc3c..997fc87520 100644
--- a/packages/model-providers/src/modelDiscovery.ts
+++ b/packages/model-providers/src/modelDiscovery.ts
@@ -169,6 +169,11 @@ export function parseModelsListResponse(
const elevenLabsMode = record.model_id === id && typeof record.can_do_text_to_speech === 'boolean'
? record.can_do_text_to_speech ? 'audio_speech' : 'other'
: undefined;
+ // OpenRouter separates TTS (`speech`) from audio-capable chat (`audio`).
+ // This is its catalog contract, not a generic inference from model names.
+ const openRouterSpeech = sourceUrl !== undefined && isOpenRouterModelsUrl(sourceUrl)
+ && Array.isArray(architecture?.output_modalities)
+ && architecture.output_modalities.includes('speech');
const discoveredMetadata = { ...pickModelMetadata(record), ...pickModelMetadata({
nativeApi: record.nativeApi !== undefined ? record.nativeApi : record.native_api,
...([rec?.display_name, rec?.name, google?.displayName].some(
@@ -176,7 +181,7 @@ export function parseModelsListResponse(
)
? { name }
: {}),
- mode: record.mode ?? info.mode ?? elevenLabsMode ??
+ mode: record.mode ?? info.mode ?? elevenLabsMode ?? (openRouterSpeech ? 'audio_speech' : undefined) ??
(isVercel && record.type === 'language' ? 'chat' : isVercel ? record.type : undefined),
modalities,
officialDocs: record.officialDocs,
@@ -232,9 +237,11 @@ export function parseModelsListResponse(
discoveredMetadata.contextWindowMax < discoveredMetadata.contextWindow) {
delete discoveredMetadata.contextWindowMax;
}
- // OpenRouter and Vercel document USD per token. Never apply these units to arbitrary proxies.
+ // OpenRouter speech pricing is per character or generated second, not per token.
+ // Do not project it into the chat cost fields. Other OpenRouter/Vercel entries
+ // document USD per token; never apply those units to arbitrary proxies.
const prices =
- sourceUrl && (isOpenRouterModelsUrl(sourceUrl) || (isVercel && record.type === 'language'))
+ sourceUrl && !openRouterSpeech && (isOpenRouterModelsUrl(sourceUrl) || (isVercel && record.type === 'language'))
? (record.pricing as Record<string, unknown> | undefined)
: undefined;
const discoveredCost = Object.fromEntries(