Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(voice): support OpenRouter MiniMax MP3 playback

JiangJiahaoda503e736986
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
index b99b89c3f5..13db3c07e2 100644
--- a/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
+++ b/apps/desktop/src/main/voice-conversation/__tests__/service.test.ts
@@ -392,6 +392,24 @@ describe('local companion call ownership', () => {
await expect(service.readSpeech(1, selection.callId, 'three')).rejects.toThrow();
});
+ // Unit regression: MP3 completes during start/prefetch rather than readSpeech.
+ // Retaining those slots would make the third sentence fail as a full queue.
+ it('releases completed MP3 prefetch slots without requiring a PCM read', async () => {
+ const { service, deps } = fixture();
+ const modelId = 'minimax/speech-2.8-turbo';
+ deps.models.mockReturnValue([{ id: modelId, name: 'MiniMax speech' }]);
+ deps.connection.mockReturnValue({ baseUrl: 'https://openrouter.ai/api/v1', protocol: 'openai', headers: {} });
+ deps.fetch.mockImplementation(async () => new Response(new Uint8Array([0xff, 0xfb, 1]), {
+ headers: { 'Content-Type': 'audio/mpeg' },
+ }));
+ await service.start(1, { ...selection, modelId });
+ const ready = await Promise.all(['one', 'two'].map(id => service.speak(1, selection.callId, id, id, true)));
+ expect(ready).toEqual(Array.from({ length: 2 }, () => ({ format: 'mp3', data: new Uint8Array([0xff, 0xfb, 1]).buffer })));
+ await expect(service.speak(1, selection.callId, 'three', 'three', true)).resolves.toHaveProperty('format', 'mp3');
+ expect(deps.fetch).toHaveBeenCalledTimes(3);
+ await expect(service.readSpeech(1, selection.callId, 'three')).rejects.toThrow('cancelled');
+ });
+
it('cancels pending lookahead headers and prevents their late audio from joining a new reply', async () => {
const { service, deps } = fixture();
let headers!: (response: Response) => void;
diff --git a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
index c72252baf1..7b58e7e4e8 100644
--- a/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
+++ b/apps/desktop/src/main/voice-conversation/__tests__/speech.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import { SpeechSynthesisSession } from '../speech.js';
+import { MAX_ENCODED_SPEECH_BYTES } from '../../../shared/voiceConversation.js';
const make = (
fetcher: typeof fetch = vi.fn(
@@ -22,6 +23,102 @@ const make = (
};
const input = { requestId: 'one', modelId: 'speech-model', voice: 'alloy', text: 'hello' };
+// Unit regression: OpenRouter MiniMax rejects PCM (HTTP 400). Existing native
+// MiniMax/PCM cases cannot catch this route-specific format or prefetch contract.
+it('prefetches a complete OpenRouter MiniMax MP3 without treating compressed bytes as PCM', async () => {
+ let body!: ReadableStreamDefaultController<Uint8Array>;
+ const fetcher = vi.fn<typeof fetch>(async () => new Response(new ReadableStream<Uint8Array>({
+ start(controller) { body = controller; },
+ }), { headers: { 'Content-Type': 'audio/mpeg' } }));
+ const speech = new SpeechSynthesisSession({
+ connection: () => ({ baseUrl: 'https://openrouter.ai/api/v1', protocol: 'openai', headers: {} }),
+ assertCurrent: () => {},
+ fetch: fetcher,
+ });
+ let ready = false;
+ const starting = speech.start({ ...input, modelId: 'minimax/speech-2.8-turbo' })
+ .then(result => { ready = true; return result; });
+ void starting.catch(() => {});
+ expect(JSON.parse(String(fetcher.mock.calls[0][1]?.body))).toMatchObject({
+ model: 'minimax/speech-2.8-turbo', response_format: 'mp3',
+ });
+ body.enqueue(new Uint8Array([0xff, 0xfb]));
+ await Promise.resolve();
+ expect(ready).toBe(false);
+ body.enqueue(new Uint8Array([1]));
+ body.close();
+ expect(await starting).toEqual({ format: 'mp3', data: new Uint8Array([0xff, 0xfb, 1]).buffer });
+ await expect(speech.read(input.requestId)).rejects.toThrow('cancelled');
+});
+
+function openRouterSession(fetcher: typeof fetch, baseUrl = 'https://openrouter.ai/api/v1') {
+ const assertCurrent = vi.fn();
+ const speech = new SpeechSynthesisSession({
+ connection: () => ({ baseUrl, protocol: 'openai', headers: {} }),
+ assertCurrent, fetch: fetcher,
+ });
+ return { speech, assertCurrent };
+}
+const miniInput = { ...input, modelId: 'minimax/speech-2.8-turbo' };
+
+it.each([
+ ['https://gateway.test/v1', 'minimax/speech-2.8-turbo'],
+ ['https://openrouter.ai/custom', 'minimax/speech-2.8-turbo'],
+ ['https://openrouter.ai.evil.test/api/v1', 'minimax/speech-2.8-turbo'],
+ ['https://openrouter.ai/api/v1', 'elevenlabs/eleven-v3'],
+])('retains streaming PCM on %s for %s', async (baseUrl, modelId) => {
+ const fetcher = vi.fn<typeof fetch>(async () => new Response(new Uint8Array([1, 2]), {
+ headers: { 'Content-Type': 'audio/pcm' },
+ }));
+ const { speech } = openRouterSession(fetcher, baseUrl);
+ expect(await speech.start({ ...input, modelId })).toEqual({ sampleRate: 24000 });
+ expect(JSON.parse(String(fetcher.mock.calls[0][1]?.body)).response_format).toBe('pcm');
+ expect((await speech.read(input.requestId)).pcm.byteLength).toBe(2);
+ speech.cancel();
+});
+
+it.each(['empty', 'oversize', 'wrong-format'] as const)(
+ 'rejects %s MP3 responses before returning audio to the player', async kind => {
+ const cancel = vi.fn();
+ const response = new Response(new ReadableStream<Uint8Array>({
+ start(body) {
+ if (kind === 'empty') body.close();
+ else if (kind === 'oversize') body.enqueue(new Uint8Array(MAX_ENCODED_SPEECH_BYTES + 1));
+ else body.enqueue(new TextEncoder().encode('private upstream error'));
+ }, cancel,
+ }), { headers: { 'Content-Type': kind === 'wrong-format' ? 'application/json' : 'audio/mpeg' } });
+ const { speech } = openRouterSession(vi.fn(async () => response));
+ await expect(speech.start(miniInput)).rejects.toThrow('format');
+ if (kind !== 'empty') expect(cancel).toHaveBeenCalled();
+ },
+);
+
+it.each(['interrupt', 'owner-change', 'timeout'] as const)(
+ 'revokes an MP3 prefetch on %s without returning late encoded audio', async reason => {
+ vi.useFakeTimers();
+ try {
+ let body!: ReadableStreamDefaultController<Uint8Array>;
+ const cancel = vi.fn();
+ const { speech, assertCurrent } = openRouterSession(vi.fn(async () => new Response(
+ new ReadableStream<Uint8Array>({ start(controller) { body = controller; }, cancel }),
+ { headers: { 'Content-Type': 'audio/mpeg' } },
+ )));
+ const pending = speech.start(miniInput);
+ const rejected = expect(pending).rejects.toThrow(reason === 'timeout' ? 'timeout' : reason === 'interrupt' ? 'cancelled' : 'request');
+ body.enqueue(new Uint8Array([0xff, 0xfb]));
+ await vi.advanceTimersByTimeAsync(0);
+ if (reason === 'interrupt') speech.cancel();
+ else if (reason === 'owner-change') {
+ assertCurrent.mockImplementation(() => { throw new Error('owner changed'); });
+ body.enqueue(new Uint8Array([1]));
+ } else await vi.advanceTimersByTimeAsync(15_001);
+ await rejected;
+ expect(cancel).toHaveBeenCalled();
+ await expect(speech.read(input.requestId)).rejects.toThrow();
+ } finally { vi.useRealTimers(); }
+ },
+);
+
describe('speech transport', () => {
it('preserves the server-provided endpoint prefix, sends the selected model, and pulls binary PCM', async () => {
const { speech, fetcher, diagnostic } = make();
diff --git a/apps/desktop/src/main/voice-conversation/adapters/openai.ts b/apps/desktop/src/main/voice-conversation/adapters/openai.ts
index bd1dc6533c..d463a46e39 100644
--- a/apps/desktop/src/main/voice-conversation/adapters/openai.ts
+++ b/apps/desktop/src/main/voice-conversation/adapters/openai.ts
@@ -1,15 +1,20 @@
import type { SpeechAdapter } from './types.js';
+import { openRouterSpeechBaseUrl } from '../openrouter.js';
export const openaiSpeechAdapter: SpeechAdapter = {
contentTypes: ['audio/pcm', 'audio/x-pcm', 'application/octet-stream'],
request(prefix, input) {
+ // OpenRouter's MiniMax adapter rejects PCM; direct MiniMax and compatible
+ // gateways keep their existing contracts. Do not retry a paid PCM request.
+ const mp3 = Boolean(openRouterSpeechBaseUrl(prefix)) && input.modelId.startsWith('minimax/speech-');
return {
url: new URL(`${prefix}/audio/speech`),
+ ...(mp3 ? { format: 'mp3' as const } : {}),
body: JSON.stringify({
model: input.modelId,
voice: input.voice,
input: input.text,
- response_format: 'pcm',
+ response_format: mp3 ? 'mp3' : 'pcm',
...(input.speed !== undefined ? { speed: input.speed } : {}),
}),
};
diff --git a/apps/desktop/src/main/voice-conversation/adapters/types.ts b/apps/desktop/src/main/voice-conversation/adapters/types.ts
index 9698a3105e..4dc2b4da89 100644
--- a/apps/desktop/src/main/voice-conversation/adapters/types.ts
+++ b/apps/desktop/src/main/voice-conversation/adapters/types.ts
@@ -7,10 +7,17 @@ export type SpeechConnection = {
speedRange?: SpeechSpeedRange;
};
export type SpeechInput = { modelId: string; voice: string; text: string; speed?: number };
-export type SpeechRequest = { url: URL; body: string; headers?: Record<string, string> };
+export type SpeechRequest = {
+ url: URL;
+ body: string;
+ headers?: Record<string, string>;
+ /** Explicit route exception; absent keeps the adapter's streaming PCM contract. */
+ format?: 'mp3';
+};
-/** Every adapter produces pull-driven 24 kHz, 16-bit LE mono PCM. A successful
- * EOF means synthesis completed; cancellation must cancel the provider reader. */
+/** Adapters default to pull-driven 24 kHz, 16-bit LE mono PCM; a request may
+ * explicitly select bounded MP3. EOF means synthesis completed, and cancellation
+ * must cancel the provider reader for either format. */
export type SpeechAdapter = {
contentTypes: readonly string[];
request(prefix: string, input: SpeechInput): SpeechRequest;
diff --git a/apps/desktop/src/main/voice-conversation/service.ts b/apps/desktop/src/main/voice-conversation/service.ts
index bb3861ee9d..514a46c5ef 100644
--- a/apps/desktop/src/main/voice-conversation/service.ts
+++ b/apps/desktop/src/main/voice-conversation/service.ts
@@ -11,6 +11,7 @@ import {
type SpeechPreviewStart,
type SpeechVoiceCatalog,
type SpeechStartResult,
+ type SpeechAudioResult,
type VoiceExitDecision,
} from '../../shared/voiceConversation.js';
import type {
@@ -99,7 +100,7 @@ export class VoiceConversationService {
async startPreview(
sender: number,
selection: SpeechPreviewStart,
- ): Promise<{ sampleRate: number }> {
+ ): Promise<SpeechAudioResult> {
if (this.call || this.preview || this.deps.hasDictation()) throw new Error('busy');
if (!this.models().some((model) => voiceModelKey(model) === voiceModelKey(selection)))
throw new Error('model_unavailable');
diff --git a/apps/desktop/src/main/voice-conversation/speech.ts b/apps/desktop/src/main/voice-conversation/speech.ts
index 3ad4ae35eb..88afbce66e 100644
--- a/apps/desktop/src/main/voice-conversation/speech.ts
+++ b/apps/desktop/src/main/voice-conversation/speech.ts
@@ -5,9 +5,10 @@ import {
type SpeechConnection,
} from './speechProtocols.js';
import { SpeechError } from './speechError.js';
+import { MAX_ENCODED_SPEECH_BYTES, type SpeechAudioResult } from '../../shared/voiceConversation.js';
export type { SpeechConnection } from './speechProtocols.js';
export type SpeechDiagnostic = {
- stage: 'request' | 'headers' | 'first_pcm' | 'complete' | 'cancelled' | 'error';
+ stage: 'request' | 'headers' | 'first_pcm' | 'first_encoded' | 'complete' | 'cancelled' | 'error';
requestId: string;
protocol: SpeechConnection['protocol'];
elapsedMs: number;
@@ -15,7 +16,7 @@ export type SpeechDiagnostic = {
httpStatus?: number;
code?: SpeechError['code'] | 'headers_timeout' | 'read_timeout';
};
-/** PCM speech transport with provider adapters. Reads are pulled by playback for bounded buffering. */
+/** Provider speech transport: pull-driven PCM or a bounded, complete MP3 segment. */
export class SpeechSynthesisSession {
private epoch = 0;
private controller: AbortController | null = null;
@@ -43,13 +44,14 @@ export class SpeechSynthesisSession {
voice: string;
text: string;
speed?: number;
- }): Promise<{ sampleRate: number }> {
+ }): Promise<SpeechAudioResult> {
this.cancel();
const epoch = this.epoch;
this.deps.assertCurrent();
const connection = this.deps.connection();
const texts = speechTextSegments(input.text, connection.protocol);
const requests = texts.map((text) => speechRequest(connection, { ...input, text }));
+ const encoded = requests[0].format === 'mp3';
const adapter = speechAdapter(connection.protocol);
const controller = new AbortController();
this.controller = controller;
@@ -93,9 +95,10 @@ export class SpeechSynthesisSession {
?.split(';')[0]
?.trim()
.toLowerCase();
- if (!contentType || !adapter.contentTypes.includes(contentType))
+ const contentTypes = encoded ? ['audio/mpeg'] : adapter.contentTypes;
+ if (!contentType || !contentTypes.includes(contentType))
throw new SpeechError('format');
- return adapter.pcm(response.body);
+ return encoded ? response.body : adapter.pcm(response.body);
} catch (error) {
if (response?.body && !response.body.locked)
void response.body.cancel().catch(() => undefined);
@@ -118,9 +121,36 @@ export class SpeechSynthesisSession {
void first.cancel().catch(() => undefined);
throw new SpeechError('cancelled');
}
- this.reader = sequentialPcm(first, requests.length, open, () => {
+ this.reader = sequentialAudio(first, requests.length, open, () => {
if (epoch === this.epoch) this.diagnostic?.('error', { code: 'read_timeout' });
- }).getReader();
+ }, !encoded).getReader();
+ if (encoded) {
+ // Finish this sentence during the existing one-segment lookahead. MP3
+ // decoding needs the complete segment, never an unbounded full reply.
+ const reader = this.reader;
+ const chunks: Uint8Array[] = [];
+ for (;;) {
+ const { value, done } = await reader.read();
+ this.deps.assertCurrent();
+ if (epoch !== this.epoch || controller.signal.aborted) throw new SpeechError('cancelled');
+ if (value?.byteLength) {
+ this.totalBytes += value.byteLength;
+ if (this.totalBytes > MAX_ENCODED_SPEECH_BYTES) throw new SpeechError('format');
+ if (!chunks.length) this.diagnostic?.('first_encoded');
+ chunks.push(value);
+ }
+ if (done) break;
+ }
+ const data = new Uint8Array(this.totalBytes);
+ let offset = 0;
+ for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.byteLength; }
+ reader.releaseLock();
+ this.reader = null;
+ this.requestId = null;
+ this.diagnostic?.('complete');
+ this.diagnostic = undefined;
+ return { format: 'mp3', data: data.buffer };
+ }
return { sampleRate: 24_000 };
} catch (error) {
if (epoch === this.epoch) {
@@ -195,20 +225,21 @@ export class SpeechSynthesisSession {
}
}
-/** Open the next segment only after the current PCM is consumed. Cancelling a
+/** Open the next segment only after the current audio is consumed. Cancelling a
* preview/call aborts its fetch and prevents any remaining billable requests. */
-function sequentialPcm(
+function sequentialAudio(
first: ReadableStream<Uint8Array>,
count: number,
open: (index: number) => Promise<ReadableStream<Uint8Array>>,
onReadTimeout: () => void,
+ pcm: boolean,
): ReadableStream<Uint8Array> {
let reader = first.getReader();
let index = 0;
let bytes = 0;
let cancelled = false;
- // Only PCM reads consume this deadline. Opening a later segment has its own
- // headers deadline, and its first PCM read starts a fresh 15-second budget.
+ // Only body reads consume this deadline. Opening a later segment has its own
+ // headers deadline, and its first body read starts a fresh 15-second budget.
const read = async () => {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
@@ -237,7 +268,7 @@ function sequentialPcm(
controller.enqueue(result.value);
return;
}
- if (bytes === 0 || bytes % 2 !== 0) throw new SpeechError('format');
+ if (bytes === 0 || (pcm && bytes % 2 !== 0)) throw new SpeechError('format');
reader.releaseLock();
if (++index === count) {
controller.close();
diff --git a/apps/desktop/src/main/voice-conversation/speechQueue.ts b/apps/desktop/src/main/voice-conversation/speechQueue.ts
index 84ba691dc4..dbe65fa3fc 100644
--- a/apps/desktop/src/main/voice-conversation/speechQueue.ts
+++ b/apps/desktop/src/main/voice-conversation/speechQueue.ts
@@ -1,5 +1,6 @@
import { SpeechSynthesisSession } from './speech.js';
import { SpeechError } from './speechError.js';
+import type { SpeechAudioResult } from '../../shared/voiceConversation.js';
/** Two independent transports: the playing segment and at most one lookahead.
* Readers still apply backpressure; cancellation revokes both, including pending headers.
@@ -12,13 +13,20 @@ export class SpeechSynthesisQueue {
async start(
input: Parameters<SpeechSynthesisSession['start']>[0],
- ): Promise<{ sampleRate: number }> {
+ ): Promise<SpeechAudioResult> {
if (this.sessions.has(input.requestId) || this.sessions.size >= 2)
throw new SpeechError('request');
const session = this.create();
this.sessions.set(input.requestId, session);
try {
- return await session.start(input);
+ const result = await session.start(input);
+ // Encoded audio has reached EOF during prefetch; there is no PCM reader
+ // to drain later. Only release this session, never a replacement request.
+ if ('format' in result && this.sessions.get(input.requestId) === session) {
+ this.sessions.delete(input.requestId);
+ session.cancel();
+ }
+ return result;
} catch (error) {
if (this.sessions.get(input.requestId) === session) this.sessions.delete(input.requestId);
session.cancel();
diff --git a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
index 4b9e2cfac5..14f32e82dd 100644
--- a/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/PcmSpeechPlayer.ts
@@ -1,3 +1,5 @@
+import { MAX_ENCODED_SPEECH_BYTES, type EncodedSpeechAudio } from '@/../shared/voiceConversation';
+
/** PCM may cross arbitrary HTTP chunk boundaries, including the middle of a 16-bit sample. */
export class Pcm16Decoder {
private carry: number | undefined;
@@ -20,7 +22,7 @@ export class Pcm16Decoder {
export interface SpeechPlayer {
play(
read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,
- sampleRate: number,
+ audio: number | EncodedSpeechAudio,
onStarted?: () => void,
): Promise<void>;
pause(): void;
@@ -48,26 +50,51 @@ export class PcmSpeechPlayer implements SpeechPlayer {
async play(
read: () => Promise<{ pcm: ArrayBuffer; done: boolean }>,
- sampleRate: number,
+ audio: number | EncodedSpeechAudio,
onStarted?: () => void,
): Promise<void> {
this.cancel();
const generation = this.generation;
const decoder = new Pcm16Decoder();
+ let sampleRate: number;
+ let decoded: Float32Array<ArrayBuffer> | undefined;
+ if (typeof audio === 'number') {
+ sampleRate = audio;
+ } else {
+ if (audio.format !== 'mp3' || !audio.data.byteLength || audio.data.byteLength > MAX_ENCODED_SPEECH_BYTES)
+ throw new Error('Invalid speech audio');
+ await this.waitUntilRunning(generation);
+ // decodeAudioData resamples to this AudioContext. Use its actual rate,
+ // rather than interpreting compressed bytes (or 44.1 kHz audio) as 24 kHz PCM.
+ const buffer = await this.context.decodeAudioData(audio.data.slice(0));
+ await this.waitUntilRunning(generation);
+ if (!Number.isFinite(buffer.duration) || buffer.duration <= 0 || buffer.duration > 600
+ || buffer.numberOfChannels < 1 || buffer.numberOfChannels > 2)
+ throw new Error('Invalid speech audio');
+ sampleRate = buffer.sampleRate;
+ decoded = buffer.getChannelData(0);
+ if (buffer.numberOfChannels === 2) {
+ const right = buffer.getChannelData(1);
+ const mono = new Float32Array(buffer.length);
+ for (let i = 0; i < mono.length; i++) mono[i] = (decoded[i] + right[i]) / 2;
+ decoded = mono;
+ }
+ }
let nextAt = this.context.currentTime;
let started = false;
this.updateContext();
for (;;) {
await this.waitUntilRunning(generation);
- const chunk = await read();
+ const chunk = decoded ? null : await read();
await this.waitUntilRunning(generation);
- const samples = decoder.decode(chunk.pcm);
- for (let offset = 0; offset < samples.length; offset += 2_400) {
+ const samples = decoded ?? decoder.decode(chunk!.pcm);
+ const partLength = Math.round(sampleRate / 10);
+ for (let offset = 0; offset < samples.length; offset += partLength) {
while (nextAt - this.context.currentTime > 0.3) {
await new Promise((resolve) => setTimeout(resolve, 25));
await this.waitUntilRunning(generation);
}
- const part = samples.subarray(offset, offset + 2_400);
+ const part = samples.subarray(offset, offset + partLength);
const buffer = this.context.createBuffer(1, part.length, sampleRate);
buffer.copyToChannel(part, 0);
const source = this.context.createBufferSource();
@@ -98,7 +125,7 @@ export class PcmSpeechPlayer implements SpeechPlayer {
}
nextAt += part.length / sampleRate;
}
- if (chunk.done) break;
+ if (decoded || chunk?.done) break;
}
decoder.finish();
while (nextAt > this.context.currentTime) {
diff --git a/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts b/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts
index b26b3241c5..6273341786 100644
--- a/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/SpeechPlaybackQueue.ts
@@ -113,7 +113,7 @@ export class SpeechPlaybackQueue {
if ('error' in result) throw result.error;
await this.deps.player.play(
() => this.deps.api.readSpeech(this.deps.callId, segment.requestId),
- result.sampleRate,
+ 'format' in result ? result : result.sampleRate,
() => {
if (this.cancelled) return;
this.deps.diagnostic('playback scheduled', {
diff --git a/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts b/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts
index e05074da84..1a155b8763 100644
--- a/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/VoicePreviewController.ts
@@ -64,7 +64,7 @@ export class VoicePreviewController {
try {
await stopped;
if (generation !== this.generation) return;
- const { sampleRate } = await this.deps.api.preview({
+ const audio = await this.deps.api.preview({
...selection,
previewId: id,
botId: this.deps.botId,
@@ -73,7 +73,7 @@ export class VoicePreviewController {
});
if (generation !== this.generation) return;
this.update('playing');
- await player.play(() => this.deps.api.readPreview(id), sampleRate);
+ await player.play(() => this.deps.api.readPreview(id), 'format' in audio ? audio : audio.sampleRate);
if (generation === this.generation) this.update('complete');
} catch (error) {
if (generation === this.generation)
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
index 8e9684fe18..b399408624 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/PcmSpeechPlayer.test.ts
@@ -38,6 +38,7 @@ class TestAudioContext {
this.state = 'closed';
});
createBuffer = vi.fn(() => ({ copyToChannel: vi.fn() }));
+ decodeAudioData = vi.fn<(data: ArrayBuffer) => Promise<AudioBuffer>>();
createBufferSource() {
const source = createTestSource();
this.sources.push(source);
@@ -135,3 +136,56 @@ it('holds a synthesis response that arrives during noise and cancels a suspended
await vi.advanceTimersByTimeAsync(0);
expect(TestAudioContext.latest.state).toBe('closed');
});
+
+// Unit regression: transport can now deliver a complete MP3. Old coverage only
+// supplied PCM; compressed bytes must decode at their actual rate and never play
+// after cancellation, including a late browser decoder result.
+function decodedAudio() {
+ return {
+ duration: 0.1, length: 4800, sampleRate: 48000, numberOfChannels: 2,
+ getChannelData: (channel: number) => new Float32Array(4800).fill(channel ? 0.6 : 0.2),
+ } as AudioBuffer;
+}
+it('decodes MP3 with the browser rate and downmixes stereo without reading PCM', async () => {
+ const player = new PcmSpeechPlayer(vi.fn());
+ const context = TestAudioContext.latest;
+ context.decodeAudioData.mockResolvedValue(decodedAudio());
+ const read = vi.fn();
+ const started = vi.fn();
+ const data = new Uint8Array([0xff, 0xfb, 1]).buffer;
+ const playing = player.play(read, { format: 'mp3', data }, started);
+ await vi.advanceTimersByTimeAsync(200);
+ await playing;
+ expect(context.decodeAudioData).toHaveBeenCalledWith(data);
+ expect(context.createBuffer).toHaveBeenCalledWith(1, 4800, 48000);
+ const samples = context.createBuffer.mock.results[0].value.copyToChannel.mock.calls[0][0];
+ expect(samples[0]).toBeCloseTo(0.4);
+ expect(read).not.toHaveBeenCalled();
+ expect(started).toHaveBeenCalledOnce();
+ player.close();
+});
+
+it.each(['cancel', 'close'] as const)('discards a decoder result arriving after %s', async action => {
+ const player = new PcmSpeechPlayer(vi.fn());
+ const context = TestAudioContext.latest;
+ let finish!: (audio: AudioBuffer) => void;
+ context.decodeAudioData.mockReturnValue(new Promise(resolve => { finish = resolve; }));
+ const playing = player.play(vi.fn(), { format: 'mp3', data: new Uint8Array([1]).buffer });
+ const rejected = expect(playing).rejects.toThrow('Playback interrupted');
+ await vi.advanceTimersByTimeAsync(0);
+ player[action]();
+ finish(decodedAudio());
+ await rejected;
+ expect(context.sources).toHaveLength(0);
+ player.close();
+});
+
+it('fails corrupt MP3 without scheduling noise or falling back to another request', async () => {
+ const player = new PcmSpeechPlayer(vi.fn());
+ TestAudioContext.latest.decodeAudioData.mockRejectedValue(new Error('corrupt audio'));
+ const read = vi.fn();
+ await expect(player.play(read, { format: 'mp3', data: new Uint8Array([1]).buffer })).rejects.toThrow('corrupt audio');
+ expect(TestAudioContext.latest.sources).toHaveLength(0);
+ expect(read).not.toHaveBeenCalled();
+ player.close();
+});
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
index de84aadd95..2696466d54 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoiceConversationController.test.ts
@@ -1289,8 +1289,11 @@ describe('sentence playback', () => {
f.controller.end();
});
- it('prefetches only one segment and drops pending audio on a committed interruption', async () => {
+ // Unit: encoded results must retain the same prefetch/interruption contract;
+ // the old PCM-only fixtures could not detect a dropped MP3 format at playback.
+ it.each(['pcm', 'mp3'] as const)('prefetches only one %s segment and drops pending audio on a committed interruption', async format => {
const f = fixture();
+ if (format === 'mp3') f.api.speak.mockResolvedValue({ format: 'mp3', data: new Uint8Array([1]).buffer });
let finish!: () => void;
f.player.play.mockImplementationOnce(async (_read, _rate, started) => {
started?.();
@@ -1304,6 +1307,8 @@ describe('sentence playback', () => {
await Promise.resolve();
f.controller.updateMessages([user(), streamingReply('第一句话。')]);
await vi.waitFor(() => expect(f.player.play).toHaveBeenCalledOnce());
+ expect(f.player.play.mock.calls[0][1]).toEqual(format === 'mp3'
+ ? { format: 'mp3', data: new Uint8Array([1]).buffer } : 24000);
f.controller.updateMessages([user(), streamingReply('第一句话。第二句话。')]);
f.controller.updateMessages([user(), streamingReply('第一句话。第二句话。第三句话。')]);
expect(f.api.speak.mock.calls.map(([value]) => value.text)).toEqual([
@@ -1321,11 +1326,12 @@ describe('sentence playback', () => {
f.controller.end();
});
- it('plays prepared sentences in order, pauses lookahead, and listens only after the final tail drains', async () => {
+ it.each(['pcm', 'mp3'] as const)('plays prepared %s sentences in order, pauses lookahead, and listens only after the final tail drains', async format => {
const f = fixture();
+ if (format === 'mp3') f.api.speak.mockResolvedValue({ format: 'mp3', data: new Uint8Array([1]).buffer });
const finishes: Array<() => void> = [];
- f.player.play.mockImplementation(async (read, _rate, started) => {
- await read();
+ f.player.play.mockImplementation(async (read, audio, started) => {
+ if (typeof audio === 'number') await read();
started?.();
await new Promise<void>((resolve) => finishes.push(resolve));
});
@@ -1351,7 +1357,7 @@ describe('sentence playback', () => {
finishes[1]();
await vi.waitFor(() => expect(finishes).toHaveLength(3));
expect(f.api.readSpeech.mock.calls.map((call) => call[1])).toEqual(
- f.api.speak.mock.calls.map(([request]) => request.requestId),
+ format === 'mp3' ? [] : f.api.speak.mock.calls.map(([request]) => request.requestId),
);
finishes[2]();
await vi.waitFor(() => expect(f.controller.getSnapshot().phase).toBe('listening'));
diff --git a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts
index f8141a9a11..2944fe295c 100644
--- a/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts
+++ b/apps/desktop/src/renderer/features/bots/voice/__tests__/VoicePreviewController.test.ts
@@ -1,15 +1,17 @@
import { expect, it, vi } from 'vitest';
import { VoicePreviewController } from '../VoicePreviewController';
+import type { VoiceConversationApi } from '@/../shared/voiceConversation';
+import type { SpeechPlayer } from '../PcmSpeechPlayer';
function fixture() {
const api = {
- preview: vi.fn(async () => ({ sampleRate: 24000 })),
+ preview: vi.fn<VoiceConversationApi['preview']>(async () => ({ sampleRate: 24000 })),
readPreview: vi.fn(async () => ({ pcm: new ArrayBuffer(2), done: true })),
stopPreview: vi.fn<(id: string) => Promise<void>>(async () => {}),
};
const player = {
- play: vi.fn(async (read: () => Promise<unknown>) => {
- await read();
+ play: vi.fn<SpeechPlayer['play']>(async (read, audio) => {
+ if (typeof audio === 'number') await read();
}),
pause: vi.fn(),
resume: vi.fn(),
@@ -31,8 +33,12 @@ const choice = {
voice: 'voice-id',
};
-it('previews through the selected model and closes playback after draining', async () => {
+// Unit: preview must preserve the host-selected encoded format as well as PCM;
+// otherwise a successful MiniMax request still fails at the preview player.
+it.each(['pcm', 'mp3'] as const)('previews %s through the selected model and closes playback after draining', async format => {
const { controller, api, player } = fixture();
+ const encoded = { format: 'mp3' as const, data: new Uint8Array([1]).buffer };
+ if (format === 'mp3') api.preview.mockResolvedValue(encoded);
await controller.start(choice, 'sample');
expect(api.preview).toHaveBeenCalledWith({
...choice,
@@ -41,8 +47,8 @@ it('previews through the selected model and closes playback after draining', asy
previewId: expect.any(String),
text: 'sample',
});
- expect(api.readPreview).toHaveBeenCalledOnce();
- expect(player.play).toHaveBeenCalledWith(expect.any(Function), 24000);
+ expect(api.readPreview).toHaveBeenCalledTimes(format === 'mp3' ? 0 : 1);
+ expect(player.play).toHaveBeenCalledWith(expect.any(Function), format === 'mp3' ? encoded : 24000);
expect(player.close).toHaveBeenCalled();
expect(api.stopPreview).toHaveBeenCalled();
expect(controller.getSnapshot()).toBe('complete');
diff --git a/apps/desktop/src/shared/voiceConversation.ts b/apps/desktop/src/shared/voiceConversation.ts
index 3f3dae09b8..0bfdea3050 100644
--- a/apps/desktop/src/shared/voiceConversation.ts
+++ b/apps/desktop/src/shared/voiceConversation.ts
@@ -102,7 +102,12 @@ export type VoiceConversationPreference = {
/** Absent on older local bridges; the current default is enabled. */
sentencePlayback?: boolean;
};
-export type SpeechStartResult = { sampleRate: number } | { deferred: true };
+/** Local-only synthesis result. PCM remains pull-driven; encoded segments are
+ * received during bounded prefetch and decoded by the sandboxed audio player. */
+export type EncodedSpeechAudio = { format: 'mp3'; data: ArrayBuffer };
+export type SpeechAudioResult = { sampleRate: number } | EncodedSpeechAudio;
+export const MAX_ENCODED_SPEECH_BYTES = 8 * 1024 * 1024;
+export type SpeechStartResult = SpeechAudioResult | { deferred: true };
export function voiceModelKey(model: {
providerId?: string;
agent?: AgentKind;
@@ -125,7 +130,7 @@ export type VoiceConversationStart = VoiceConversationSelection & {
export interface VoiceConversationApi {
models(): Promise<VoiceConversationModel[]>;
voices(selection: SpeechModelSelection): Promise<SpeechVoiceCatalog>;
- preview(input: SpeechPreviewStart): Promise<{ sampleRate: number }>;
+ preview(input: SpeechPreviewStart): Promise<SpeechAudioResult>;
readPreview(previewId: string): Promise<{ pcm: ArrayBuffer; done: boolean }>;
stopPreview(previewId: string): Promise<void>;
preference(botId: string): Promise<VoiceConversationPreference>;
diff --git a/docs/product-rules/companion-voice-mode.md b/docs/product-rules/companion-voice-mode.md
index 4f4d4b991c..0ead62deea 100644
--- a/docs/product-rules/companion-voice-mode.md
+++ b/docs/product-rules/companion-voice-mode.md
@@ -64,17 +64,25 @@ OpenRouter 的普通 `/api/v1/models` 目录不包含完整语音列表。对未
语音的按字符/时长报价不写入聊天的 token 单价字段。
OpenRouter BYOK 的官方 `/api` 和 `/api/v1` 连接只在语音请求中统一到
-`https://openrouter.ai/api/v1/audio/speech`,使用对应连接的 Bearer key、OpenAI 兼容 PCM
+`https://openrouter.ai/api/v1/audio/speech`,使用对应连接的 Bearer key、OpenAI 兼容音频
适配器及原模型 ID;聊天地址不变。每次进入型号、切换型号或点击音色刷新时,从语音模型
目录读取该型号的 `supported_voices`,不套用 Gateway 或厂商直连的参考音色;缺少该字段时
保留手动输入,查询失败沿用现有错误/重试行为,不覆盖已选音色。仍可手动输入并试听。
音色目录仅在内存,不进入公共 Registry 或持久配置。
+OpenRouter 的 `minimax/speech-*` 型号实测拒绝 PCM,因此直接请求 MP3,不先发起会失败的
+付费 PCM 请求或自动换模型。仅匹配官方 API 地址;Gateway、自定义代理与 MiniMax 原生
+SSE 仍保持原合同。MP3 每个句段在现有预取阶段完整接收,最多 8 MiB,保留响应头 30 秒、
+读取停滞 15 秒及账号/连接/取消校验。通过本机 IPC 返回明确标记的内存音频,由浏览器
+原生解码器按实际采样率解码,再复用有界播放、暂停与插话取消;不落盘、不新增解码依赖。
+解码后最多 10 分钟,迟到解码结果不可在取消后重新播放。按句切分及一个预取段保持不变,
+但 MP3 需收齐当前句段才能解码,不承诺逐音频帧开播;其他模型仍边收 PCM 边播放。
+
OpenRouter 没有统一的型号语速范围,部分上游会拒绝非默认值,因此本阶段使用供应商默认
语速,不发送 `speed`;设置页复用“不支持调整语速”的现有状态。Main 投影的可选
`speedRange` 为本机能力信息(`null` 表示不支持、缺省沿用原协议范围),不接受 Renderer
自报能力、不写入偏好、不加入远程协议。旧偏好中的语速在运行时忽略,明确修改设置时才
-通过原保存流程移除。可发现和可选择不等于账户具有执行权限;PCM 听感仍需按实际上游验收。
+通过原保存流程移除。可发现和可选择不等于账户具有执行权限;音频听感仍需按实际上游验收。
接口依据:[OpenRouter TTS](https://openrouter.ai/docs/guides/overview/multimodal/tts)。
| 接口 | 请求与鉴权 | 响应 |