Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(voice): support OpenRouter MiniMax MP3 playback
JiangJiahaoda503e736986
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/service.test.tsindex b99b89c3f5..13db3c07e2 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts@@ -392,6 +392,24 @@ describe('local companion call ownership', () => {await expect(service.readSpeech(1, selection.callId, 'three')).rejects.toThrow();});+ // Unit regression: MP3 completes during start/prefetch rather than readSpeech.+ // Retaining those slots would make the third sentence fail as a full queue.+ it('releases completed MP3 prefetch slots without requiring a PCM read', async () => {+ const { service, deps } = fixture();+ const modelId = 'minimax/speech-2.8-turbo';+ deps.models.mockReturnValue([{ id: modelId, name: 'MiniMax speech' }]);+ deps.connection.mockReturnValue({ baseUrl: 'https://openrouter.ai/api/v1', protocol: 'openai', headers: {} });+ deps.fetch.mockImplementation(async () => new Response(new Uint8Array([0xff, 0xfb, 1]), {+ headers: { 'Content-Type': 'audio/mpeg' },+ }));+ await service.start(1, { ...selection, modelId });+ const ready = await Promise.all(['one', 'two'].map(id => service.speak(1, selection.callId, id, id, true)));+ expect(ready).toEqual(Array.from({ length: 2 }, () => ({ format: 'mp3', data: new Uint8Array([0xff, 0xfb, 1]).buffer })));+ await expect(service.speak(1, selection.callId, 'three', 'three', true)).resolves.toHaveProperty('format', 'mp3');+ expect(deps.fetch).toHaveBeenCalledTimes(3);+ await expect(service.readSpeech(1, selection.callId, 'three')).rejects.toThrow('cancelled');+ });+it('cancels pending lookahead headers and prevents their late audio from joining a new reply', async () => {const { service, deps } = fixture();let headers!: (response: Response) => void;diff --git a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.tsindex c72252baf1..7b58e7e4e8 100644--- a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts+++ b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts@@ -1,5 +1,6 @@import { describe, expect, it, vi } from 'vitest';import { SpeechSynthesisSession } from '../speech.js';+import { MAX_ENCODED_SPEECH_BYTES } from '../../../shared/voiceConversation.js';const make = (fetcher: typeof fetch = vi.fn(@@ -22,6 +23,102 @@ const make = (};const input = { requestId: 'one', modelId: 'speech-model', voice: 'alloy', text: 'hello' };+// Unit regression: OpenRouter MiniMax rejects PCM (HTTP 400). Existing native+// MiniMax/PCM cases cannot catch this route-specific format or prefetch contract.+it('prefetches a complete OpenRouter MiniMax MP3 without treating compressed bytes as PCM', async () => {+ let body!: ReadableStreamDefaultController<Uint8Array>;+ const fetcher = vi.fn<typeof fetch>(async () => new Response(new ReadableStream<Uint8Array>({+ start(controller) { body = controller; },+ }), { headers: { 'Content-Type': 'audio/mpeg' } }));+ const speech = new SpeechSynthesisSession({+ connection: () => ({ baseUrl: 'https://openrouter.ai/api/v1', protocol: 'openai', headers: {} }),+ assertCurrent: () => {},+ fetch: fetcher,+ });+ let ready = false;+ const starting = speech.start({ ...input, modelId: 'minimax/speech-2.8-turbo' })+ .then(result => { ready = true; return result; });+ void starting.catch(() => {});+ expect(JSON.parse(String(fetcher.mock.calls[0][1]?.body))).toMatchObject({+ model: 'minimax/speech-2.8-turbo', response_format: 'mp3',+ });+ body.enqueue(new Uint8Array([0xff, 0xfb]));+ await Promise.resolve();+ expect(ready).toBe(false);+ body.enqueue(new Uint8Array([1]));+ body.close();+ expect(await starting).toEqual({ format: 'mp3', data: new Uint8Array([0xff, 0xfb, 1]).buffer });+ await expect(speech.read(input.requestId)).rejects.toThrow('cancelled');+});++function openRouterSession(fetcher: typeof fetch, baseUrl = 'https://openrouter.ai/api/v1') {+ const assertCurrent = vi.fn();+ const speech = new SpeechSynthesisSession({+ connection: () => ({ baseUrl, protocol: 'openai', headers: {} }),+ assertCurrent, fetch: fetcher,+ });+ return { speech, assertCurrent };+}+const miniInput = { ...input, modelId: 'minimax/speech-2.8-turbo' };++it.each([+ ['https://gateway.test/v1', 'minimax/speech-2.8-turbo'],+ ['https://openrouter.ai/custom', 'minimax/speech-2.8-turbo'],+ ['https://openrouter.ai.evil.test/api/v1', 'minimax/speech-2.8-turbo'],+ ['https://openrouter.ai/api/v1', 'elevenlabs/eleven-v3'],+])('retains streaming PCM on %s for %s', async (baseUrl, modelId) => {+ const fetcher = vi.fn<typeof fetch>(async () => new Response(new Uint8Array([1, 2]), {+ headers: { 'Content-Type': 'audio/pcm' },+ }));+ const { speech } = openRouterSession(fetcher, baseUrl);+ expect(await speech.start({ ...input, modelId })).toEqual({ sampleRate: 24000 });+ expect(JSON.parse(String(fetcher.mock.calls[0][1]?.body)).response_format).toBe('pcm');+ expect((await speech.read(input.requestId)).pcm.byteLength).toBe(2);+ speech.cancel();+});++it.each(['empty', 'oversize', 'wrong-format'] as const)(+ 'rejects %s MP3 responses before returning audio to the player', async kind => {+ const cancel = vi.fn();+ const response = new Response(new ReadableStream<Uint8Array>({+ start(body) {+ if (kind === 'empty') body.close();+ else if (kind === 'oversize') body.enqueue(new Uint8Array(MAX_ENCODED_SPEECH_BYTES + 1));+ else body.enqueue(new TextEncoder().encode('private upstream error'));+ }, cancel,+ }), { headers: { 'Content-Type': kind === 'wrong-format' ? 'application/json' : 'audio/mpeg' } });+ const { speech } = openRouterSession(vi.fn(async () => response));+ await expect(speech.start(miniInput)).rejects.toThrow('format');+ if (kind !== 'empty') expect(cancel).toHaveBeenCalled();+ },+);++it.each(['interrupt', 'owner-change', 'timeout'] as const)(+ 'revokes an MP3 prefetch on %s without returning late encoded audio', async reason => {+ vi.useFakeTimers();+ try {+ let body!: ReadableStreamDefaultController<Uint8Array>;+ const cancel = vi.fn();+ const { speech, assertCurrent } = openRouterSession(vi.fn(async () => new Response(+ new ReadableStream<Uint8Array>({ start(controller) { body = controller; }, cancel }),+ { headers: { 'Content-Type': 'audio/mpeg' } },+ )));+ const pending = speech.start(miniInput);+ const rejected = expect(pending).rejects.toThrow(reason === 'timeout' ? 'timeout' : reason === 'interrupt' ? 'cancelled' : 'request');+ body.enqueue(new Uint8Array([0xff, 0xfb]));+ await vi.advanceTimersByTimeAsync(0);+ if (reason === 'interrupt') speech.cancel();+ else if (reason === 'owner-change') {+ assertCurrent.mockImplementation(() => { throw new Error('owner changed'); });+ body.enqueue(new Uint8Array([1]));+ } else await vi.advanceTimersByTimeAsync(15_001);+ await rejected;+ expect(cancel).toHaveBeenCalled();+ await expect(speech.read(input.requestId)).rejects.toThrow();+ } finally { vi.useRealTimers(); }+ },+);+describe('speech transport', () => {it('preserves the server-provided endpoint prefix, sends the selected model, and pulls binary PCM', async () => {const { speech, fetcher, diagnostic } = make();diff --git a/apps/desktop/src/main/voice-conversation/adapters/openai.ts b/apps/desktop/src/main/voice-conversation/adapters/openai.tsindex bd1dc6533c..d463a46e39 100644--- a/apps/desktop/src/main/voice-conversation/adapters/openai.ts+++ b/apps/desktop/src/main/voice-conversation/adapters/openai.ts@@ -1,15 +1,20 @@import type { SpeechAdapter } from './types.js';+import { openRouterSpeechBaseUrl } from '../openrouter.js';export const openaiSpeechAdapter: SpeechAdapter = {contentTypes: ['audio/pcm', 'audio/x-pcm', 'application/octet-stream'],request(prefix, input) {+ // OpenRouter's MiniMax adapter rejects PCM; direct MiniMax and compatible+ // gateways keep their existing contracts. Do not retry a paid PCM request.+ const mp3 = Boolean(openRouterSpeechBaseUrl(prefix)) && input.modelId.startsWith('minimax/speech-');return {url: new URL(`${prefix}/audio/speech`),+ ...(mp3 ? { format: 'mp3' as const } : {}),body: JSON.stringify({model: input.modelId,voice: input.voice,input: input.text,- response_format: 'pcm',+ response_format: mp3 ? 'mp3' : 'pcm',...(input.speed !== undefined ? { speed: input.speed } : {}),}),};diff --git a/apps/desktop/src/main/voice-conversation/adapters/types.ts b/apps/desktop/src/main/voice-conversation/adapters/types.tsindex 9698a3105e..4dc2b4da89 100644--- a/apps/desktop/src/main/voice-conversation/adapters/types.ts+++ b/apps/desktop/src/main/voice-conversation/adapters/types.ts@@ -7,10 +7,17 @@ export type SpeechConnection = {speedRange?: SpeechSpeedRange;};export type SpeechInput = { modelId: string; voice: string; text: string; speed?: number };-export type SpeechRequest = { url: URL; body: string; headers?: Record<string, string> };+export type SpeechRequest = {+ url: URL;+ body: string;+ headers?: Record<string, string>;+ /** Explicit route exception; absent keeps the adapter's streaming PCM contract. */+ format?: 'mp3';+};-/** Every adapter produces pull-driven 24 kHz, 16-bit LE mono PCM. A successful- * EOF means synthesis completed; cancellation must cancel the provider reader. */+/** Adapters default to pull-driven 24 kHz, 16-bit LE mono PCM; a request may+ * explicitly select bounded MP3. EOF means synthesis completed, and cancellation+ * must cancel the provider reader for either format. */export type SpeechAdapter = {contentTypes: readonly string[];request(prefix: string, input: SpeechInput): SpeechRequest;diff --git a/apps/desktop/src/main/voice-conversation/service.ts b/apps/desktop/src/main/voice-conversation/service.tsindex bb3861ee9d..514a46c5ef 100644--- a/apps/desktop/src/main/voice-conversation/service.ts+++ b/apps/desktop/src/main/voice-conversation/service.ts@@ -11,6 +11,7 @@ import {type SpeechPreviewStart,type SpeechVoiceCatalog,type SpeechStartResult,+ type SpeechAudioResult,type VoiceExitDecision,} from '../../shared/voiceConversation.js';import type {@@ -99,7 +100,7 @@ export class VoiceConversationService {async startPreview(sender: number,selection: SpeechPreviewStart,- ): Promise<{ sampleRate: number }> {+ ): Promise<SpeechAudioResult> {if (this.call || this.preview || this.deps.hasDictation()) throw new Error('busy');if (!this.models().some((model) => voiceModelKey(model) === voiceModelKey(selection)))throw new Error('model_unavailable');diff --git a/apps/desktop/src/main/voice-conversation/speech.ts b/apps/desktop/src/main/voice-conversation/speech.tsindex 3ad4ae35eb..88afbce66e 100644--- a/apps/desktop/src/main/voice-conversation/speech.ts+++ b/apps/desktop/src/main/voice-conversation/speech.ts@@ -5,9 +5,10 @@ import {type SpeechConnection,} from './speechProtocols.js';import { SpeechError } from './speechError.js';+import { MAX_ENCODED_SPEECH_BYTES, type SpeechAudioResult } from '../../shared/voiceConversation.js';export type { SpeechConnection } from './speechProtocols.js';export type SpeechDiagnostic = {- stage: 'request' | 'headers' | 'first_pcm' | 'complete' | 'cancelled' | 'error';+ stage: 'request' | 'headers' | 'first_pcm' | 'first_encoded' | 'complete' | 'cancelled' | 'error';requestId: string;protocol: SpeechConnection['protocol'];elapsedMs: number;@@ -15,7 +16,7 @@ export type SpeechDiagnostic = {httpStatus?: number;code?: SpeechError['code'] | 'headers_timeout' | 'read_timeout';};-/** PCM speech transport with provider adapters. Reads are pulled by playback for bounded buffering. */+/** Provider speech transport: pull-driven PCM or a bounded, complete MP3 segment. */export class SpeechSynthesisSession {private epoch = 0;private controller: AbortController | null = null;@@ -43,13 +44,14 @@ export class SpeechSynthesisSession {voice: string;text: string;speed?: number;- }): Promise<{ sampleRate: number }> {+ }): Promise<SpeechAudioResult> {this.cancel();const epoch = this.epoch;this.deps.assertCurrent();const connection = this.deps.connection();const texts = speechTextSegments(input.text, connection.protocol);const requests = texts.map((text) => speechRequest(connection, { ...input, text }));+ const encoded = requests[0].format === 'mp3';const adapter = speechAdapter(connection.protocol);const controller = new AbortController();this.controller = controller;@@ -93,9 +95,10 @@ export class SpeechSynthesisSession {?.split(';')[0]?.trim().toLowerCase();- if (!contentType || !adapter.contentTypes.includes(contentType))+ const contentTypes = encoded ? ['audio/mpeg'] : adapter.contentTypes;+ if (!contentType || !contentTypes.includes(contentType))throw new SpeechError('format');- return adapter.pcm(response.body);+ return encoded ? response.body : adapter.pcm(response.body);} catch (error) {if (response?.body && !response.body.locked)void response.body.cancel().catch(() => undefined);@@ -118,9 +121,36 @@ export class SpeechSynthesisSession {void first.cancel().catch(() => undefined);throw new SpeechError('cancelled');}- this.reader = sequentialPcm(first, requests.length, open, () => {+ this.reader = sequentialAudio(first, requests.length, open, () => {if (epoch === this.epoch) this.diagnostic?.('error', { code: 'read_timeout' });- }).getReader();+ }, !encoded).getReader();+ if (encoded) {+ // Finish this sentence during the existing one-segment lookahead. MP3+ // decoding needs the complete segment, never an unbounded full reply.+ const reader = this.reader;+ const chunks: Uint8Array[] = [];+ for (;;) {+ const { value, done } = await reader.read();+ this.deps.assertCurrent();+ if (epoch !== this.epoch || controller.signal.aborted) throw new SpeechError('cancelled');+ if (value?.byteLength) {+ this.totalBytes += value.byteLength;+ if (this.totalBytes > MAX_ENCODED_SPEECH_BYTES) throw new SpeechError('format');+ if (!chunks.length) this.diagnostic?.('first_encoded');+ chunks.push(value);+ }+ if (done) break;+ }+ const data = new Uint8Array(this.totalBytes);+ let offset = 0;+ for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.byteLength; }+ reader.releaseLock();+ this.reader = null;+ this.requestId = null;+ this.diagnostic?.('complete');+ this.diagnostic = undefined;+ return { format: 'mp3', data: data.buffer };+ }return { sampleRate: 24_000 };} catch (error) {if (epoch === this.epoch) {@@ -195,20 +225,21 @@ export class SpeechSynthesisSession {}}-/** Open the next segment only after the current PCM is consumed. Cancelling a+/** Open the next segment only after the current audio is consumed. Cancelling a* preview/call aborts its fetch and prevents any remaining billable requests. */-function sequentialPcm(+function sequentialAudio(first: ReadableStream<Uint8Array>,count: number,open: (index: number) => Promise<ReadableStream<Uint8Array>>,onReadTimeout: () => void,+ pcm: boolean,): ReadableStream<Uint8Array> {let reader = first.getReader();let index = 0;let bytes = 0;let cancelled = false;- // Only PCM reads consume this deadline. Opening a later segment has its own- // headers deadline, and its first PCM read starts a fresh 15-second budget.+ // Only body reads consume this deadline. Opening a later segment has its own+ // headers deadline, and its first body read starts a fresh 15-second budget.const read = async () => {let timer: ReturnType<typeof setTimeout> | undefined;try {@@ -237,7 +268,7 @@ function sequentialPcm(controller.enqueue(result.value);return;}- if (bytes === 0 || bytes % 2 !== 0) throw new SpeechError('format');+ if (bytes === 0 || (pcm && bytes % 2 !== 0)) throw new SpeechError('format');reader.releaseLock();if (++index === count) {controller.close();diff --git a/apps/desktop/src/main/voice-conversation/speechQueue.ts b/apps/desktop/src/main/voice-conversation/speechQueue.tsindex 84ba691dc4..dbe65fa3fc 100644--- a/apps/desktop/src/main/voice-conversation/speechQueue.ts+++ b/apps/desktop/src/main/voice-conversation/speechQueue.ts@@ -1,5 +1,6 @@import { SpeechSynthesisSession } from './speech.js';import { SpeechError } from './speechError.js';+import type { SpeechAudioResult } from '../../shared/voiceConversation.js';/** Two independent transports: the playing segment and at most one lookahead.* Readers still apply backpressure; cancellation revokes both, including pending headers.@@ -12,13 +13,20 @@ export class SpeechSynthesisQueue {async start(input: Parameters<SpeechSynthesisSession['start']>[0],- ): Promise<{ sampleRate: number }> {+ ): Promise<SpeechAudioResult> {if (this.sessions.has(input.requestId) || this.sessions.size >= 2)throw new SpeechError('request');const session = this.create();this.sessions.set(input.requestId, session);try {- return await session.start(input);+ const result = await session.start(input);+ // Encoded audio has reached EOF during prefetch; there is no PCM reader+ // to drain later. Only release this session, never a replacement request.+ if ('format' in result && this.sessions.get(input.requestId) === session) {+ this.sessions.delete(input.requestId);+ session.cancel();+ }+ return result;} catch (error) {if (this.sessions.get(input.requestId) === session) this.sessions.delete(input.requestId);session.cancel();diff --git a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.tsindex 4b9e2cfac5..14f32e82dd 100644--- a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts+++ b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts@@ -1,3 +1,5 @@+import { MAX_ENCODED_SPEECH_BYTES, type EncodedSpeechAudio } from '@/../shared/voiceConversation';+/** PCM may cross arbitrary HTTP chunk boundaries, including the middle of a 16-bit sample. */export class Pcm16Decoder {private carry: number | undefined;@@ -20,7 +22,7 @@ export class Pcm16Decoder {export interface SpeechPlayer {play(read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,- sampleRate: number,+ audio: number | EncodedSpeechAudio,onStarted?: () => void,): Promise<void>;pause(): void;@@ -48,26 +50,51 @@ export class PcmSpeechPlayer implements SpeechPlayer {async play(read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,- sampleRate: number,+ audio: number | EncodedSpeechAudio,onStarted?: () => void,): Promise<void> {this.cancel();const generation = this.generation;const decoder = new Pcm16Decoder();+ let sampleRate: number;+ let decoded: Float32Array<ArrayBuffer> | undefined;+ if (typeof audio === 'number') {+ sampleRate = audio;+ } else {+ if (audio.format !== 'mp3' || !audio.data.byteLength || audio.data.byteLength > MAX_ENCODED_SPEECH_BYTES)+ throw new Error('Invalid speech audio');+ await this.waitUntilRunning(generation);+ // decodeAudioData resamples to this AudioContext. Use its actual rate,+ // rather than interpreting compressed bytes (or 44.1 kHz audio) as 24 kHz PCM.+ const buffer = await this.context.decodeAudioData(audio.data.slice(0));+ await this.waitUntilRunning(generation);+ if (!Number.isFinite(buffer.duration) || buffer.duration <= 0 || buffer.duration > 600+ || buffer.numberOfChannels < 1 || buffer.numberOfChannels > 2)+ throw new Error('Invalid speech audio');+ sampleRate = buffer.sampleRate;+ decoded = buffer.getChannelData(0);+ if (buffer.numberOfChannels === 2) {+ const right = buffer.getChannelData(1);+ const mono = new Float32Array(buffer.length);+ for (let i = 0; i < mono.length; i++) mono[i] = (decoded[i] + right[i]) / 2;+ decoded = mono;+ }+ }let nextAt = this.context.currentTime;let started = false;this.updateContext();for (;;) {await this.waitUntilRunning(generation);- const chunk = await read();+ const chunk = decoded ? null : await read();await this.waitUntilRunning(generation);- const samples = decoder.decode(chunk.pcm);- for (let offset = 0; offset < samples.length; offset += 2_400) {+ const samples = decoded ?? decoder.decode(chunk!.pcm);+ const partLength = Math.round(sampleRate / 10);+ for (let offset = 0; offset < samples.length; offset += partLength) {while (nextAt - this.context.currentTime > 0.3) {await new Promise((resolve) => setTimeout(resolve, 25));await this.waitUntilRunning(generation);}- const part = samples.subarray(offset, offset + 2_400);+ const part = samples.subarray(offset, offset + partLength);const buffer = this.context.createBuffer(1, part.length, sampleRate);buffer.copyToChannel(part, 0);const source = this.context.createBufferSource();@@ -98,7 +125,7 @@ export class PcmSpeechPlayer implements SpeechPlayer {}nextAt += part.length / sampleRate;}- if (chunk.done) break;+ if (decoded || chunk?.done) break;}decoder.finish();while (nextAt > this.context.currentTime) {diff --git a/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts b/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.tsindex b26b3241c5..6273341786 100644--- a/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts+++ b/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts@@ -113,7 +113,7 @@ export class SpeechPlaybackQueue {if ('error' in result) throw result.error;await this.deps.player.play(() => this.deps.api.readSpeech(this.deps.callId, segment.requestId),- result.sampleRate,+ 'format' in result ? result : result.sampleRate,() => {if (this.cancelled) return;this.deps.diagnostic('playback scheduled', {diff --git a/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts b/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.tsindex e05074da84..1a155b8763 100644--- a/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts+++ b/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts@@ -64,7 +64,7 @@ export class VoicePreviewController {try {await stopped;if (generation !== this.generation) return;- const { sampleRate } = await this.deps.api.preview({+ const audio = await this.deps.api.preview({...selection,previewId: id,botId: this.deps.botId,@@ -73,7 +73,7 @@ export class VoicePreviewController {});if (generation !== this.generation) return;this.update('playing');- await player.play(() => this.deps.api.readPreview(id), sampleRate);+ await player.play(() => this.deps.api.readPreview(id), 'format' in audio ? audio : audio.sampleRate);if (generation === this.generation) this.update('complete');} catch (error) {if (generation === this.generation)diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.tsindex 8e9684fe18..b399408624 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts@@ -38,6 +38,7 @@ class TestAudioContext {this.state = 'closed';});createBuffer = vi.fn(() => ({ copyToChannel: vi.fn() }));+ decodeAudioData = vi.fn<(data: ArrayBuffer) => Promise<AudioBuffer>>();createBufferSource() {const source = createTestSource();this.sources.push(source);@@ -135,3 +136,56 @@ it('holds a synthesis response that arrives during noise and cancels a suspendedawait vi.advanceTimersByTimeAsync(0);expect(TestAudioContext.latest.state).toBe('closed');});++// Unit regression: transport can now deliver a complete MP3. Old coverage only+// supplied PCM; compressed bytes must decode at their actual rate and never play+// after cancellation, including a late browser decoder result.+function decodedAudio() {+ return {+ duration: 0.1, length: 4800, sampleRate: 48000, numberOfChannels: 2,+ getChannelData: (channel: number) => new Float32Array(4800).fill(channel ? 0.6 : 0.2),+ } as AudioBuffer;+}+it('decodes MP3 with the browser rate and downmixes stereo without reading PCM', async () => {+ const player = new PcmSpeechPlayer(vi.fn());+ const context = TestAudioContext.latest;+ context.decodeAudioData.mockResolvedValue(decodedAudio());+ const read = vi.fn();+ const started = vi.fn();+ const data = new Uint8Array([0xff, 0xfb, 1]).buffer;+ const playing = player.play(read, { format: 'mp3', data }, started);+ await vi.advanceTimersByTimeAsync(200);+ await playing;+ expect(context.decodeAudioData).toHaveBeenCalledWith(data);+ expect(context.createBuffer).toHaveBeenCalledWith(1, 4800, 48000);+ const samples = context.createBuffer.mock.results[0].value.copyToChannel.mock.calls[0][0];+ expect(samples[0]).toBeCloseTo(0.4);+ expect(read).not.toHaveBeenCalled();+ expect(started).toHaveBeenCalledOnce();+ player.close();+});++it.each(['cancel', 'close'] as const)('discards a decoder result arriving after %s', async action => {+ const player = new PcmSpeechPlayer(vi.fn());+ const context = TestAudioContext.latest;+ let finish!: (audio: AudioBuffer) => void;+ context.decodeAudioData.mockReturnValue(new Promise(resolve => { finish = resolve; }));+ const playing = player.play(vi.fn(), { format: 'mp3', data: new Uint8Array([1]).buffer });+ const rejected = expect(playing).rejects.toThrow('Playback interrupted');+ await vi.advanceTimersByTimeAsync(0);+ player[action]();+ finish(decodedAudio());+ await rejected;+ expect(context.sources).toHaveLength(0);+ player.close();+});++it('fails corrupt MP3 without scheduling noise or falling back to another request', async () => {+ const player = new PcmSpeechPlayer(vi.fn());+ TestAudioContext.latest.decodeAudioData.mockRejectedValue(new Error('corrupt audio'));+ const read = vi.fn();+ await expect(player.play(read, { format: 'mp3', data: new Uint8Array([1]).buffer })).rejects.toThrow('corrupt audio');+ expect(TestAudioContext.latest.sources).toHaveLength(0);+ expect(read).not.toHaveBeenCalled();+ player.close();+});diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.tsindex de84aadd95..2696466d54 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts@@ -1289,8 +1289,11 @@ describe('sentence playback', () => {f.controller.end();});- it('prefetches only one segment and drops pending audio on a committed interruption', async () => {+ // Unit: encoded results must retain the same prefetch/interruption contract;+ // the old PCM-only fixtures could not detect a dropped MP3 format at playback.+ it.each(['pcm', 'mp3'] as const)('prefetches only one %s segment and drops pending audio on a committed interruption', async format => {const f = fixture();+ if (format === 'mp3') f.api.speak.mockResolvedValue({ format: 'mp3', data: new Uint8Array([1]).buffer });let finish!: () => void;f.player.play.mockImplementationOnce(async (_read, _rate, started) => {started?.();@@ -1304,6 +1307,8 @@ describe('sentence playback', () => {await Promise.resolve();f.controller.updateMessages([user(), streamingReply('第一句话。')]);await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());+ expect(f.player.play.mock.calls[0][1]).toEqual(format === 'mp3'+ ? { format: 'mp3', data: new Uint8Array([1]).buffer } : 24000);f.controller.updateMessages([user(), streamingReply('第一句话。第二句话。')]);f.controller.updateMessages([user(), streamingReply('第一句话。第二句话。第三句话。')]);expect(f.api.speak.mock.calls.map(([value]) => value.text)).toEqual([@@ -1321,11 +1326,12 @@ describe('sentence playback', () => {f.controller.end();});- it('plays prepared sentences in order, pauses lookahead, and listens only after the final tail drains', async () => {+ it.each(['pcm', 'mp3'] as const)('plays prepared %s sentences in order, pauses lookahead, and listens only after the final tail drains', async format => {const f = fixture();+ if (format === 'mp3') f.api.speak.mockResolvedValue({ format: 'mp3', data: new Uint8Array([1]).buffer });const finishes: Array<() => void> = [];- f.player.play.mockImplementation(async (read, _rate, started) => {- await read();+ f.player.play.mockImplementation(async (read, audio, started) => {+ if (typeof audio === 'number') await read();started?.();await new Promise<void>((resolve) => finishes.push(resolve));});@@ -1351,7 +1357,7 @@ describe('sentence playback', () => {finishes[1]();await vi.waitFor(() => expect(finishes).toHaveLength(3));expect(f.api.readSpeech.mock.calls.map((call) => call[1])).toEqual(- f.api.speak.mock.calls.map(([request]) => request.requestId),+ format === 'mp3' ? [] : f.api.speak.mock.calls.map(([request]) => request.requestId),);finishes[2]();await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('listening'));diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.tsindex f8141a9a11..2944fe295c 100644--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts@@ -1,15 +1,17 @@import { expect, it, vi } from 'vitest';import { VoicePreviewController } from '../VoicePreviewController';+import type { VoiceConversationApi } from '@/../shared/voiceConversation';+import type { SpeechPlayer } from '../PcmSpeechPlayer';function fixture() {const api = {- preview: vi.fn(async () => ({ sampleRate: 24000 })),+ preview: vi.fn<VoiceConversationApi['preview']>(async () => ({ sampleRate: 24000 })),readPreview: vi.fn(async () => ({ pcm: new ArrayBuffer(2), done: true })),stopPreview: vi.fn<(id: string) => Promise<void>>(async () => {}),};const player = {- play: vi.fn(async (read: () => Promise<unknown>) => {- await read();+ play: vi.fn<SpeechPlayer['play']>(async (read, audio) => {+ if (typeof audio === 'number') await read();}),pause: vi.fn(),resume: vi.fn(),@@ -31,8 +33,12 @@ const choice = {voice: 'voice-id',};-it('previews through the selected model and closes playback after draining', async () => {+// Unit: preview must preserve the host-selected encoded format as well as PCM;+// otherwise a successful MiniMax request still fails at the preview player.+it.each(['pcm', 'mp3'] as const)('previews %s through the selected model and closes playback after draining', async format => {const { controller, api, player } = fixture();+ const encoded = { format: 'mp3' as const, data: new Uint8Array([1]).buffer };+ if (format === 'mp3') api.preview.mockResolvedValue(encoded);await controller.start(choice, 'sample');expect(api.preview).toHaveBeenCalledWith({...choice,@@ -41,8 +47,8 @@ it('previews through the selected model and closes playback after draining', asypreviewId: expect.any(String),text: 'sample',});- expect(api.readPreview).toHaveBeenCalledOnce();- expect(player.play).toHaveBeenCalledWith(expect.any(Function), 24000);+ expect(api.readPreview).toHaveBeenCalledTimes(format === 'mp3' ? 0 : 1);+ expect(player.play).toHaveBeenCalledWith(expect.any(Function), format === 'mp3' ? encoded : 24000);expect(player.close).toHaveBeenCalled();expect(api.stopPreview).toHaveBeenCalled();expect(controller.getSnapshot()).toBe('complete');diff --git a/apps/desktop/src/shared/voiceConversation.ts b/apps/desktop/src/shared/voiceConversation.tsindex 3f3dae09b8..0bfdea3050 100644--- a/apps/desktop/src/shared/voiceConversation.ts+++ b/apps/desktop/src/shared/voiceConversation.ts@@ -102,7 +102,12 @@ export type VoiceConversationPreference = {/** Absent on older local bridges; the current default is enabled. */sentencePlayback?: boolean;};-export type SpeechStartResult = { sampleRate: number } | { deferred: true };+/** Local-only synthesis result. PCM remains pull-driven; encoded segments are+ * received during bounded prefetch and decoded by the sandboxed audio player. */+export type EncodedSpeechAudio = { format: 'mp3'; data: ArrayBuffer };+export type SpeechAudioResult = { sampleRate: number } | EncodedSpeechAudio;+export const MAX_ENCODED_SPEECH_BYTES = 8 * 1024 * 1024;+export type SpeechStartResult = SpeechAudioResult | { deferred: true };export function voiceModelKey(model: {providerId?: string;agent?: AgentKind;@@ -125,7 +130,7 @@ export type VoiceConversationStart = VoiceConversationSelection & {export interface VoiceConversationApi {models(): Promise<VoiceConversationModel[]>;voices(selection: SpeechModelSelection): Promise<SpeechVoiceCatalog>;- preview(input: SpeechPreviewStart): Promise<{ sampleRate: number }>;+ preview(input: SpeechPreviewStart): Promise<SpeechAudioResult>;readPreview(previewId: string): Promise<{ pcm: ArrayBuffer; done: boolean }>;stopPreview(previewId: string): Promise<void>;preference(botId: string): Promise<VoiceConversationPreference>;diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.mdindex 4f4d4b991c..0ead62deea 100644--- a/docs/product-rules/companion-voice-mode.md+++ b/docs/product-rules/companion-voice-mode.md@@ -64,17 +64,25 @@ OpenRouter 的普通 `/api/v1/models` 目录不包含完整语音列表。对未语音的按字符/时长报价不写入聊天的 token 单价字段。OpenRouter BYOK 的官方 `/api` 和 `/api/v1` 连接只在语音请求中统一到-`https://openrouter.ai/api/v1/audio/speech`,使用对应连接的 Bearer key、OpenAI 兼容 PCM+`https://openrouter.ai/api/v1/audio/speech`,使用对应连接的 Bearer key、OpenAI 兼容音频适配器及原模型 ID;聊天地址不变。每次进入型号、切换型号或点击音色刷新时,从语音模型目录读取该型号的 `supported_voices`,不套用 Gateway 或厂商直连的参考音色;缺少该字段时保留手动输入,查询失败沿用现有错误/重试行为,不覆盖已选音色。仍可手动输入并试听。音色目录仅在内存,不进入公共 Registry 或持久配置。+OpenRouter 的 `minimax/speech-*` 型号实测拒绝 PCM,因此直接请求 MP3,不先发起会失败的+付费 PCM 请求或自动换模型。仅匹配官方 API 地址;Gateway、自定义代理与 MiniMax 原生+SSE 仍保持原合同。MP3 每个句段在现有预取阶段完整接收,最多 8 MiB,保留响应头 30 秒、+读取停滞 15 秒及账号/连接/取消校验。通过本机 IPC 返回明确标记的内存音频,由浏览器+原生解码器按实际采样率解码,再复用有界播放、暂停与插话取消;不落盘、不新增解码依赖。+解码后最多 10 分钟,迟到解码结果不可在取消后重新播放。按句切分及一个预取段保持不变,+但 MP3 需收齐当前句段才能解码,不承诺逐音频帧开播;其他模型仍边收 PCM 边播放。+OpenRouter 没有统一的型号语速范围,部分上游会拒绝非默认值,因此本阶段使用供应商默认语速,不发送 `speed`;设置页复用“不支持调整语速”的现有状态。Main 投影的可选`speedRange` 为本机能力信息(`null` 表示不支持、缺省沿用原协议范围),不接受 Renderer自报能力、不写入偏好、不加入远程协议。旧偏好中的语速在运行时忽略,明确修改设置时才-通过原保存流程移除。可发现和可选择不等于账户具有执行权限;PCM 听感仍需按实际上游验收。+通过原保存流程移除。可发现和可选择不等于账户具有执行权限;音频听感仍需按实际上游验收。接口依据:[OpenRouter TTS](https://openrouter.ai/docs/guides/overview/multimodal/tts)。| 接口 | 请求与鉴权 | 响应 |