Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: add Volcengine ASR and TTS for video translation
liuPig83ef03486a19
diff --git a/.env.example b/.env.examplenew file mode 100644index 00000000..83285b90--- /dev/null+++ b/.env.example@@ -0,0 +1,5 @@+# Local secrets only. Copy to .env.local (gitignored) and fill in values.+# Do not commit real keys.+VOLCENGINE_APP_ID=+VOLCENGINE_ACCESS_TOKEN=+VOLCENGINE_SECRET_KEY=diff --git a/.gitignore b/.gitignoreindex 7d296ebc..60d598d8 100644--- a/.gitignore+++ b/.gitignore@@ -1,5 +1,8 @@node_modules/dist/+.env+.env.*+!.env.exampleapps/desktop/.pack/apps/desktop/.cache/apps/desktop/release/diff --git a/apps/daemon/src/api/routes.creator-services.ts b/apps/daemon/src/api/routes.creator-services.tsindex 275e3d5e..7e97daac 100644--- a/apps/daemon/src/api/routes.creator-services.ts+++ b/apps/daemon/src/api/routes.creator-services.ts@@ -146,7 +146,7 @@ function sendTtsError(reply: FastifyReply, error: unknown) {}function parseTtsProvider(value: unknown): CreatorTtsProvider | undefined {- if (value === 'openai' || value === 'aliyun' || value === 'minimax' || value === 'edge-tts') {+ if (value === 'openai' || value === 'aliyun' || value === 'minimax' || value === 'edge-tts' || value === 'volcengine') {return value;}return undefined;diff --git a/apps/daemon/src/creator-services/config-store.ts b/apps/daemon/src/creator-services/config-store.tsindex bf8097e8..0d789acb 100644--- a/apps/daemon/src/creator-services/config-store.ts+++ b/apps/daemon/src/creator-services/config-store.ts@@ -14,6 +14,7 @@ import {readOpenCreatorCredentials,updateOpenCreatorCredentials} from '../config/credentials-file.js';+import { applyVolcengineEnvironmentOverrides } from './volcengine-env.js';const boundedString = (maximum: number) => z.string().max(maximum);@@ -45,6 +46,12 @@ const aliyunSchema = z.object({oss: aliyunOssSchema,speech: aliyunSpeechSchema}).strict();+const volcengineAsrSchema = z.object({+ appId: boundedString(256),+ accessToken: boundedString(4096),+ resourceId: boundedString(128),+ baseUrl: boundedString(2048)+}).strict();const creatorServicesDefaults = createDefaultCreatorServicesConfig();const imageConfigDefault = creatorServicesDefaults.image;const videoConfigDefault = creatorServicesDefaults.video;@@ -86,27 +93,32 @@ const legacyVideoConfigSchema = z.object({const legacyAliyunTtsSchema = aliyunSchema.transform(() => (structuredClone(creatorServicesDefaults.tts.aliyun)));+const volcengineTtsSchema = ttsProviderSchema(creatorServicesDefaults.tts.volcengine.defaultVoiceId).extend({+ appId: boundedString(256).default('')+}).strict();const ttsConfigSchema = z.object({- provider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax']),+ provider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax', 'volcengine']),openai: ttsProviderSchema(creatorServicesDefaults.tts.openai.defaultVoiceId),minimax: ttsProviderSchema(creatorServicesDefaults.tts.minimax.defaultVoiceId),aliyun: z.union([ttsProviderSchema(creatorServicesDefaults.tts.aliyun.defaultVoiceId),legacyAliyunTtsSchema- ])+ ]),+ volcengine: volcengineTtsSchema.default(creatorServicesDefaults.tts.volcengine)}).strict();export const creatorServicesConfigSchema = z.object({proxy: boundedString(2048),llm: llmConfigSchema,transcription: z.object({- provider: z.enum(['openai', 'faster-whisper', 'whisperkit', 'whisper.cpp', 'aliyun']),+ provider: z.enum(['openai', 'faster-whisper', 'whisperkit', 'whisper.cpp', 'aliyun', 'volcengine']),enableGpuAcceleration: z.boolean(),openai: openAiCompatibleSchema,fasterWhisper: z.object({ model: z.enum(['tiny', 'medium', 'large-v2']) }).strict(),whisperKit: z.object({ model: z.literal('large-v2') }).strict(),whisperCpp: z.object({ model: z.enum(['tiny', 'medium', 'large-v2']) }).strict(),- aliyun: aliyunSchema+ aliyun: aliyunSchema,+ volcengine: volcengineAsrSchema.default(creatorServicesDefaults.transcription.volcengine)}).strict(),tts: ttsConfigSchema,image: z.union([imageConfigSchema, legacyImageConfigSchema]),@@ -198,7 +210,9 @@ export function createOpenCreatorCreatorServicesConfigStore(input: {async read() {const snapshot = readOpenCreatorConfig(input.configFile);if (!snapshot.configured.creatorServices) {- return createDefaultCreatorServicesConfig();+ return applyVolcengineEnvironmentOverrides(+ createDefaultCreatorServicesConfig()+ );}const publicConfig = parseCreatorServicesConfig(snapshot.document.creatorServices@@ -206,7 +220,9 @@ export function createOpenCreatorCreatorServicesConfigStore(input: {const credentials = (await readOpenCreatorCredentials(input.credentialsFile)).creatorServices ?? {};- return applyCreatorServiceCredentials(publicConfig, credentials);+ return applyVolcengineEnvironmentOverrides(+ applyCreatorServiceCredentials(publicConfig, credentials)+ );},async write(config) {const normalized = parseCreatorServicesConfig(config);@@ -232,7 +248,9 @@ export function createOpenCreatorCreatorServicesConfigStore(input: {delete next.creatorServices;return next;});- return createDefaultCreatorServicesConfig();+ return applyVolcengineEnvironmentOverrides(+ createDefaultCreatorServicesConfig()+ );}};return store;@@ -270,9 +288,13 @@ const creatorCredentialPaths = ['transcription.aliyun.speech.accessKeyId','transcription.aliyun.speech.accessKeySecret','transcription.aliyun.speech.appKey',+ 'transcription.volcengine.appId',+ 'transcription.volcengine.accessToken','tts.openai.apiKey','tts.minimax.apiKey','tts.aliyun.apiKey',+ 'tts.volcengine.appId',+ 'tts.volcengine.apiKey','image.openai.apiKey','image.jimeng.apiKey','image.kling.accessKey',@@ -347,9 +369,17 @@ export function presentCreatorServicesConfig(redacted.transcription.openai.apiKey = '';});redactAliyunCredentials('transcription.aliyun', redacted.transcription.aliyun, configuredCredentials);+ redact('transcription.volcengine.appId', redacted.transcription.volcengine.appId, () => {+ redacted.transcription.volcengine.appId = '';+ });+ redact('transcription.volcengine.accessToken', redacted.transcription.volcengine.accessToken, () => {+ redacted.transcription.volcengine.accessToken = '';+ });redact('tts.openai.apiKey', redacted.tts.openai.apiKey, () => { redacted.tts.openai.apiKey = ''; });redact('tts.minimax.apiKey', redacted.tts.minimax.apiKey, () => { redacted.tts.minimax.apiKey = ''; });redact('tts.aliyun.apiKey', redacted.tts.aliyun.apiKey, () => { redacted.tts.aliyun.apiKey = ''; });+ redact('tts.volcengine.appId', redacted.tts.volcengine.appId, () => { redacted.tts.volcengine.appId = ''; });+ redact('tts.volcengine.apiKey', redacted.tts.volcengine.apiKey, () => { redacted.tts.volcengine.apiKey = ''; });redact('image.openai.apiKey', redacted.image.openai.apiKey, () => { redacted.image.openai.apiKey = ''; });redact('image.jimeng.apiKey', redacted.image.jimeng.apiKey, () => { redacted.image.jimeng.apiKey = ''; });redact('image.kling.accessKey', redacted.image.kling.accessKey, () => { redacted.image.kling.accessKey = ''; });@@ -375,9 +405,21 @@ export function retainCreatorServicesCredentials(current.transcription.openai.apiKey);retainAliyunCredentials(merged.transcription.aliyun, current.transcription.aliyun);+ retainBlank(+ () => merged.transcription.volcengine.appId,+ value => { merged.transcription.volcengine.appId = value; },+ current.transcription.volcengine.appId+ );+ retainBlank(+ () => merged.transcription.volcengine.accessToken,+ value => { merged.transcription.volcengine.accessToken = value; },+ current.transcription.volcengine.accessToken+ );retainBlank(() => merged.tts.openai.apiKey, value => { merged.tts.openai.apiKey = value; }, current.tts.openai.apiKey);retainBlank(() => merged.tts.minimax.apiKey, value => { merged.tts.minimax.apiKey = value; }, current.tts.minimax.apiKey);retainBlank(() => merged.tts.aliyun.apiKey, value => { merged.tts.aliyun.apiKey = value; }, current.tts.aliyun.apiKey);+ retainBlank(() => merged.tts.volcengine.appId, value => { merged.tts.volcengine.appId = value; }, current.tts.volcengine.appId);+ retainBlank(() => merged.tts.volcengine.apiKey, value => { merged.tts.volcengine.apiKey = value; }, current.tts.volcengine.apiKey);retainBlank(() => merged.image.openai.apiKey, value => { merged.image.openai.apiKey = value; }, current.image.openai.apiKey);retainBlank(() => merged.image.jimeng.apiKey, value => { merged.image.jimeng.apiKey = value; }, current.image.jimeng.apiKey);retainBlank(() => merged.image.kling.accessKey, value => { merged.image.kling.accessKey = value; }, current.image.kling.accessKey);diff --git a/apps/daemon/src/creator-services/volcengine-env.ts b/apps/daemon/src/creator-services/volcengine-env.tsnew file mode 100644index 00000000..f2e4702c--- /dev/null+++ b/apps/daemon/src/creator-services/volcengine-env.ts@@ -0,0 +1,53 @@+import type { CreatorServicesConfig } from '@opencreator/protocol';++export type VolcengineEnvironment = {+ VOLCENGINE_APP_ID?: string;+ VOLCENGINE_ACCESS_TOKEN?: string;+};++export function applyVolcengineEnvironmentOverrides(+ config: CreatorServicesConfig,+ env: VolcengineEnvironment = process.env+): CreatorServicesConfig {+ const appId = env.VOLCENGINE_APP_ID?.trim() ?? '';+ const accessToken = env.VOLCENGINE_ACCESS_TOKEN?.trim() ?? '';+ if (!appId && !accessToken) return config;++ const next = structuredClone(config);+ if (appId) {+ if (!next.transcription.volcengine.appId.trim()) {+ next.transcription.volcengine.appId = appId;+ }+ if (!next.tts.volcengine.appId.trim()) {+ next.tts.volcengine.appId = appId;+ }+ }+ if (accessToken) {+ if (!next.transcription.volcengine.accessToken.trim()) {+ next.transcription.volcengine.accessToken = accessToken;+ }+ if (!next.tts.volcengine.apiKey.trim()) {+ next.tts.volcengine.apiKey = accessToken;+ }+ }++ const configured = next.transcription.volcengine.appId.trim().length > 0+ && next.transcription.volcengine.accessToken.trim().length > 0;+ if (configured && shouldPreferVolcengineTranscription(next)) {+ next.transcription.provider = 'volcengine';+ }+ if (configured && shouldPreferVolcengineTts(next)) {+ next.tts.provider = 'volcengine';+ }+ return next;+}++function shouldPreferVolcengineTranscription(config: CreatorServicesConfig): boolean {+ return config.transcription.provider === 'openai'+ && config.transcription.openai.apiKey.trim().length === 0;+}++function shouldPreferVolcengineTts(config: CreatorServicesConfig): boolean {+ return config.tts.provider === 'openai'+ && config.tts.openai.apiKey.trim().length === 0;+}diff --git a/apps/daemon/src/creator/krillin/capabilities.ts b/apps/daemon/src/creator/krillin/capabilities.tsindex c56ac65d..832263a1 100644--- a/apps/daemon/src/creator/krillin/capabilities.ts+++ b/apps/daemon/src/creator/krillin/capabilities.ts@@ -38,14 +38,15 @@ export function createKrillinCreatorServicesCapabilities(false,whisperCppAvailable ? undefined : 'unsupported_platform'),- cloudProvider('aliyun', [])+ cloudProvider('aliyun', []),+ cloudProvider('volcengine', [])]}};}function cloudProvider(- provider: 'openai' | 'aliyun',+ provider: 'openai' | 'aliyun' | 'volcengine',models: string[]): CreatorTranscriptionProviderCapability {return {diff --git a/apps/daemon/src/creator/krillin/cli-runner.ts b/apps/daemon/src/creator/krillin/cli-runner.tsindex 8831bda5..c8a04bdc 100644--- a/apps/daemon/src/creator/krillin/cli-runner.ts+++ b/apps/daemon/src/creator/krillin/cli-runner.ts@@ -181,7 +181,7 @@ export async function runKrillinCli(input: RunKrillinCliInput): Promise<KrillinR});throw error;} finally {- await rm(launcherRoot, { recursive: true, force: true });+ await removeLauncherRootBestEffort(launcherRoot);}}@@ -197,6 +197,7 @@ function stageConfig(&& provider !== 'aliyun'&& provider !== 'minimax'&& provider !== 'edge-tts'+ && provider !== 'volcengine') return source;const config = structuredClone(source);config.tts.provider = provider;@@ -785,6 +786,23 @@ function quoteShellArgument(value: string): string {return `'${value.replaceAll("'", "'\\''")}'`;}+async function removeLauncherRootBestEffort(launcherRoot: string): Promise<void> {+ for (let attempt = 0; attempt < 8; attempt += 1) {+ try {+ await rm(launcherRoot, { recursive: true, force: true });+ return;+ } catch (error) {+ if (!isBusyCleanupError(error) || attempt === 7) return;+ await new Promise(resolve => setTimeout(resolve, 150 * (attempt + 1)));+ }+ }+}++function isBusyCleanupError(error: unknown): boolean {+ if (error === null || typeof error !== 'object' || !('code' in error)) return false;+ return ['EBUSY', 'EPERM', 'EACCES'].includes(String(error.code));+}+function isWindowsLinkFallbackError(error: unknown): boolean {if (error === null || typeof error !== 'object' || !('code' in error)) return false;return ['EPERM', 'EACCES', 'EXDEV', 'ENOTSUP'].includes(String(error.code));diff --git a/apps/daemon/src/creator/krillin/config-bridge.ts b/apps/daemon/src/creator/krillin/config-bridge.tsindex a3f555ca..d0394808 100644--- a/apps/daemon/src/creator/krillin/config-bridge.ts+++ b/apps/daemon/src/creator/krillin/config-bridge.ts@@ -27,13 +27,26 @@ export function createKrillinConfigToml(config: CreatorServicesConfig): string {aliyun: {oss: snakeAliyunOss(config.transcription.aliyun.oss),speech: snakeAliyunSpeech(config.transcription.aliyun.speech)+ },+ volcengine: {+ app_id: config.transcription.volcengine.appId,+ access_token: config.transcription.volcengine.accessToken,+ resource_id: config.transcription.volcengine.resourceId,+ base_url: config.transcription.volcengine.baseUrl}},tts: {provider: config.tts.provider,openai: ttsProvider(config.tts.openai),minimax: ttsProvider(config.tts.minimax),- aliyun: ttsProvider(config.tts.aliyun)+ aliyun: ttsProvider(config.tts.aliyun),+ volcengine: {+ app_id: config.tts.volcengine.appId,+ access_token: config.tts.volcengine.apiKey,+ cluster: config.tts.volcengine.model,+ default_voice_id: config.tts.volcengine.defaultVoiceId,+ base_url: config.tts.volcengine.baseUrl+ }},image: {provider: config.image.provider,diff --git a/apps/daemon/src/creator/krillin/tts-service.ts b/apps/daemon/src/creator/krillin/tts-service.tsindex 5cae07fc..2e6c8e71 100644--- a/apps/daemon/src/creator/krillin/tts-service.ts+++ b/apps/daemon/src/creator/krillin/tts-service.ts@@ -263,6 +263,14 @@ function ensureCredentials(config: CreatorServicesConfig,provider: Exclude<CreatorTtsProvider, 'edge-tts'>): void {+ if (provider === 'volcengine') {+ if (config.tts.volcengine.appId.trim() && config.tts.volcengine.apiKey.trim()) return;+ throw new KrillinTtsServiceError(+ 'creator_tts_config_missing',+ 'Configure the Volcengine TTS App ID and Access Token before generating speech',+ 409+ );+ }if (config.tts[provider].apiKey.trim()) return;throw new KrillinTtsServiceError('creator_tts_config_missing',@@ -320,9 +328,56 @@ async function executeProviderSynthesis(): Promise<ExecuteSynthesisResult> {if (input.provider === 'aliyun') return synthesizeAliyun(input);if (input.provider === 'minimax') return synthesizeMinimax(input);+ if (input.provider === 'volcengine') return synthesizeVolcengine(input);return synthesizeOpenAi(input);}+async function synthesizeVolcengine(+ input: ExecuteSynthesisInput & { timeoutMs: number }+): Promise<ExecuteSynthesisResult> {+ const provider = input.config.tts.volcengine;+ const endpoint = appendPath(provider.baseUrl || 'https://openspeech.bytedance.com', '/api/v1/tts');+ const response = await timedFetch(endpoint, {+ method: 'POST',+ headers: {+ authorization: `Bearer;${provider.apiKey}`,+ 'content-type': 'application/json'+ },+ body: JSON.stringify({+ app: {+ appid: provider.appId,+ token: provider.apiKey,+ cluster: input.model || provider.model || 'volcano_tts'+ },+ user: { uid: 'opencreator' },+ audio: {+ voice_type: input.voiceId,+ encoding: input.format === 'wav' ? 'wav' : 'mp3',+ speed_ratio: input.speed+ },+ request: {+ reqid: crypto.randomUUID(),+ text: input.text,+ text_type: 'plain',+ operation: 'query'+ }+ })+ }, input);+ if (!response.ok) await throwProviderHttpError(response);+ const payload = await readJsonResponse(response) as {+ code?: number;+ message?: string;+ data?: string;+ };+ if (payload.code !== 3000) {+ throw new Error(`Volcengine TTS failed: ${payload.message || payload.code}`);+ }+ if (!payload.data) throw new Error('Volcengine TTS response did not contain audio');+ const content = Buffer.from(payload.data, 'base64');+ if (content.length === 0) throw new Error('Volcengine TTS returned invalid audio');+ return { content, format: detectAudioFormat(content, input.format) };+}+async function synthesizeOpenAi(input: ExecuteSynthesisInput & { timeoutMs: number }): Promise<ExecuteSynthesisResult> {diff --git a/apps/daemon/src/creator/krillin/tts-voice-catalog.ts b/apps/daemon/src/creator/krillin/tts-voice-catalog.tsindex 5f2cdd17..6e126bc5 100644--- a/apps/daemon/src/creator/krillin/tts-voice-catalog.ts+++ b/apps/daemon/src/creator/krillin/tts-voice-catalog.ts@@ -83,6 +83,21 @@ const currentQwen3Voices: AliyunVoiceDefinition[] = [{ id: 'Kiki', name: '粤语-阿清', language: 'zh-yue', gender: 'female', scenario: '粤语' }];+const volcengineVoices: CreatorTtsVoice[] = [+ { id: 'BV001_streaming', name: '通用女声', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '解说配音', kind: 'builtin', recommended: true },+ { id: 'BV002_streaming', name: '通用男声', language: 'zh-CN', gender: 'male', provider: 'volcengine', scenario: '解说配音', kind: 'builtin' },+ { id: 'BV007_streaming', name: '亲切女声', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '亲切自然', kind: 'builtin' },+ { id: 'BV009_streaming', name: '知性女声', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '知识讲解', kind: 'builtin' },+ { id: 'BV019_streaming', name: '霸气御姐', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '有声书', kind: 'builtin' },+ { id: 'BV033_streaming', name: '温柔女声', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '温柔叙事', kind: 'builtin' },+ { id: 'BV056_streaming', name: '阳光男声', language: 'zh-CN', gender: 'male', provider: 'volcengine', scenario: '阳光解说', kind: 'builtin' },+ { id: 'BV102_streaming', name: '儒雅青年', language: 'zh-CN', gender: 'male', provider: 'volcengine', scenario: '沉稳叙事', kind: 'builtin' },+ { id: 'BV113_streaming', name: '甜宠少御', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '甜宠', kind: 'builtin' },+ { id: 'BV119_streaming', name: '通用萌娃', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '童声', kind: 'builtin' },+ { id: 'BV700_streaming', name: '灿灿', language: 'zh-CN', gender: 'female', provider: 'volcengine', scenario: '豆包女声', kind: 'builtin' },+ { id: 'BV701_streaming', name: '擎苍', language: 'zh-CN', gender: 'male', provider: 'volcengine', scenario: '磁性男声', kind: 'builtin' }+];+export function listBundledTtsVoices(provider: Exclude<CreatorTtsProvider, 'edge-tts'>,model: string@@ -90,6 +105,12 @@ export function listBundledTtsVoices(if (provider === 'openai') {return openAiVoices.map(voice => ({ ...voice }));}+ if (provider === 'volcengine') {+ return volcengineVoices.map(voice => ({+ ...voice,+ supportedModels: [model.trim() || 'volcano_tts']+ }));+ }if (provider !== 'aliyun') return undefined;const normalizedModel = model.trim().toLowerCase();const definitions = normalizedModel === QWEN3_TTS_FLASH_2025_09_18diff --git a/apps/daemon/src/creator/stickman/audio-executor.ts b/apps/daemon/src/creator/stickman/audio-executor.tsindex d0a2430f..2b7b9ccc 100644--- a/apps/daemon/src/creator/stickman/audio-executor.ts+++ b/apps/daemon/src/creator/stickman/audio-executor.ts@@ -250,6 +250,7 @@ function resolveTtsSelection(stage: CreatorExecutorInput, config: CreatorServiceconst provider = stage.job.state.ttsProvider === 'openai'|| stage.job.state.ttsProvider === 'aliyun'|| stage.job.state.ttsProvider === 'minimax'+ || stage.job.state.ttsProvider === 'volcengine'? stage.job.state.ttsProvider: config.tts.provider === 'edge-tts'? undefineddiff --git a/apps/daemon/src/creator/templates/smart-dubbing.ts b/apps/daemon/src/creator/templates/smart-dubbing.tsindex c78767dd..21b719b0 100644--- a/apps/daemon/src/creator/templates/smart-dubbing.ts+++ b/apps/daemon/src/creator/templates/smart-dubbing.ts@@ -11,7 +11,7 @@ export function createSmartDubbingTemplate(): CreatorTemplateDefinition {renderer: 'smart-dubbing',inputSchema: z.object({text: z.string().max(5_000).default(''),- ttsProvider: z.enum(['openai', 'aliyun', 'minimax']).optional(),+ ttsProvider: z.enum(['openai', 'aliyun', 'minimax', 'volcengine']).optional(),ttsModel: z.string().max(256).optional(),voiceCode: z.string().max(256).optional(),voiceName: z.string().max(256).optional(),diff --git a/apps/daemon/src/creator/templates/stickman-video-actions.ts b/apps/daemon/src/creator/templates/stickman-video-actions.tsindex 6c3e81db..7d280fdc 100644--- a/apps/daemon/src/creator/templates/stickman-video-actions.ts+++ b/apps/daemon/src/creator/templates/stickman-video-actions.ts@@ -444,6 +444,7 @@ export function createStickmanVideoWorkflow(input: {const provider = job.state.ttsProvider === 'openai'|| job.state.ttsProvider === 'aliyun'|| job.state.ttsProvider === 'minimax'+ || job.state.ttsProvider === 'volcengine'? job.state.ttsProvider: config.tts.provider;if (provider === 'edge-tts') {diff --git a/apps/daemon/src/creator/templates/stickman-video.ts b/apps/daemon/src/creator/templates/stickman-video.tsindex e75c4df3..ad843a9b 100644--- a/apps/daemon/src/creator/templates/stickman-video.ts+++ b/apps/daemon/src/creator/templates/stickman-video.ts@@ -80,7 +80,7 @@ export function createStickmanVideoTemplate(): CreatorTemplateDefinition {targetDurationSeconds: z.number().positive().max(600).default(30),sourceLanguage: z.string().default('auto'),targetLanguage: z.string().default('zh-CN'),- ttsProvider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax']).optional(),+ ttsProvider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax', 'volcengine']).optional(),ttsModel: z.string().optional(),voiceCode: z.string().optional(),voiceName: z.string().optional(),diff --git a/apps/daemon/src/creator/templates/video-translation-actions.ts b/apps/daemon/src/creator/templates/video-translation-actions.tsindex 22044d75..94389ed7 100644--- a/apps/daemon/src/creator/templates/video-translation-actions.ts+++ b/apps/daemon/src/creator/templates/video-translation-actions.ts@@ -144,6 +144,9 @@ function hasTtsCredentials(if (provider === 'edge-tts') return true;if (provider === 'openai') return config.tts.openai.apiKey.length > 0;if (provider === 'minimax') return config.tts.minimax.apiKey.length > 0;+ if (provider === 'volcengine') {+ return config.tts.volcengine.appId.length > 0 && config.tts.volcengine.apiKey.length > 0;+ }return config.tts.aliyun.apiKey.length > 0;}@@ -151,7 +154,8 @@ function isTtsProvider(value: unknown): value is CreatorTtsProvider {return value === 'openai'|| value === 'aliyun'|| value === 'edge-tts'- || value === 'minimax';+ || value === 'minimax'+ || value === 'volcengine';}function queueNextStage(diff --git a/apps/daemon/src/creator/templates/video-translation.ts b/apps/daemon/src/creator/templates/video-translation.tsindex 1a64ea35..aa7c768c 100644--- a/apps/daemon/src/creator/templates/video-translation.ts+++ b/apps/daemon/src/creator/templates/video-translation.ts@@ -18,7 +18,7 @@ export function createVideoTranslationTemplate(): CreatorTemplateDefinition {bilingual: z.boolean().default(true),subtitlePosition: z.enum(['top', 'bottom']).default('top'),dubbing: z.boolean().default(false),- ttsProvider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax']).optional(),+ ttsProvider: z.enum(['openai', 'aliyun', 'edge-tts', 'minimax', 'volcengine']).optional(),ttsModel: z.string().optional(),voiceCode: z.string().optional(),voiceName: z.string().optional(),diff --git a/apps/daemon/src/env-local.ts b/apps/daemon/src/env-local.tsnew file mode 100644index 00000000..e79cb4f9--- /dev/null+++ b/apps/daemon/src/env-local.ts@@ -0,0 +1,81 @@+import { existsSync, readFileSync } from 'node:fs';+import { homedir } from 'node:os';+import { dirname, join, resolve } from 'node:path';++const MAX_WALK_DEPTH = 8;++export type LocalEnvironmentLoadOptions = {+ env?: NodeJS.ProcessEnv;+ cwd?: string;+ homeDir?: string | null;+};++export function parseEnvFile(source: string): Record<string, string> {+ const result: Record<string, string> = {};+ for (const rawLine of source.split(/\r?\n/)) {+ const line = rawLine.trim();+ if (!line || line.startsWith('#')) continue;+ const trimmed = line.startsWith('export ') ? line.slice(7).trim() : line;+ const separator = trimmed.indexOf('=');+ if (separator <= 0) continue;+ const key = trimmed.slice(0, separator).trim();+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) continue;+ let value = trimmed.slice(separator + 1).trim();+ if (+ (value.startsWith('"') && value.endsWith('"'))+ || (value.startsWith("'") && value.endsWith("'"))+ ) {+ value = value.slice(1, -1);+ }+ result[key] = value;+ }+ return result;+}++export function loadLocalEnvironment(+ options: LocalEnvironmentLoadOptions = {}+): string[] {+ const env = options.env ?? process.env;+ const loaded: string[] = [];+ for (const file of resolveLocalEnvFiles(options)) {+ applyEnvFile(file, env);+ loaded.push(file);+ }+ return loaded;+}++function resolveLocalEnvFiles(options: LocalEnvironmentLoadOptions): string[] {+ const env = options.env ?? process.env;+ const explicit = env.OPENCREATOR_ENV_FILE?.trim();+ if (explicit) {+ return existsSync(explicit) ? [resolve(explicit)] : [];+ }++ const files: string[] = [];+ if (options.homeDir !== null) {+ const homeFile = join(options.homeDir ?? homedir(), '.opencreator', '.env.local');+ if (existsSync(homeFile)) files.push(homeFile);+ }++ let dir = resolve(options.cwd ?? process.cwd());+ for (let depth = 0; depth < MAX_WALK_DEPTH; depth += 1) {+ const candidate = join(dir, '.env.local');+ if (existsSync(candidate) && !files.includes(candidate)) {+ files.push(candidate);+ break;+ }+ const parent = dirname(dir);+ if (parent === dir) break;+ dir = parent;+ }+ return files;+}++function applyEnvFile(file: string, env: NodeJS.ProcessEnv): void {+ const parsed = parseEnvFile(readFileSync(file, 'utf8'));+ for (const [key, value] of Object.entries(parsed)) {+ if (env[key] === undefined || env[key] === '') {+ env[key] = value;+ }+ }+}diff --git a/apps/daemon/src/main.ts b/apps/daemon/src/main.tsindex 44c6a7e9..2be8d187 100644--- a/apps/daemon/src/main.ts+++ b/apps/daemon/src/main.ts@@ -38,6 +38,7 @@ import {installManagedParentWatch,parseManagedParentPid} from './managed-parent.js';+import { loadLocalEnvironment } from './env-local.js';type BootstrapPhase = 'starting_runtime';@@ -62,6 +63,7 @@ await main().catch(error => {});async function main(): Promise<void> {+ loadLocalEnvironment();const token = createRuntimeToken();const environment = resolveProductionServerEnvironment();const paths = resolveProductionRuntimePaths(environment, {diff --git a/apps/daemon/src/smart-dubbing/service.ts b/apps/daemon/src/smart-dubbing/service.tsindex 8f8c401d..be053ba3 100644--- a/apps/daemon/src/smart-dubbing/service.ts+++ b/apps/daemon/src/smart-dubbing/service.ts@@ -214,7 +214,7 @@ function isSmartDubbingResult(value: unknown): value is SmartDubbingResult {&& typeof value.fileName === 'string'&& (value.mime === 'audio/mpeg' || value.mime === 'audio/wav')&& typeof value.size === 'number'- && (value.provider === 'openai' || value.provider === 'aliyun' || value.provider === 'minimax')+ && (value.provider === 'openai' || value.provider === 'aliyun' || value.provider === 'minimax' || value.provider === 'volcengine')&& typeof value.model === 'string'&& typeof value.voice === 'string'&& value.voice.length > 0diff --git a/apps/daemon/test/integration/creator-krillin-cli-runner.test.ts b/apps/daemon/test/integration/creator-krillin-cli-runner.test.tsindex 330935a4..9171c5f9 100644--- a/apps/daemon/test/integration/creator-krillin-cli-runner.test.ts+++ b/apps/daemon/test/integration/creator-krillin-cli-runner.test.ts@@ -105,6 +105,39 @@ describe('KrillinAI CLI runner', () => {});});+ it('bridges Volcengine ASR and TTS credentials into KrillinAI config', () => {+ const config = createDefaultCreatorServicesConfig();+ config.transcription.provider = 'volcengine';+ config.transcription.volcengine.appId = 'asr-app';+ config.transcription.volcengine.accessToken = 'asr-token';+ config.tts.provider = 'volcengine';+ config.tts.volcengine.appId = 'tts-app';+ config.tts.volcengine.apiKey = 'tts-token';++ const value = parse(createKrillinConfigToml(config)) as {+ transcribe: { provider: string; volcengine: Record<string, string> };+ tts: { provider: string; volcengine: Record<string, string> };+ };++ expect(value.transcribe).toMatchObject({+ provider: 'volcengine',+ volcengine: {+ app_id: 'asr-app',+ access_token: 'asr-token',+ resource_id: 'volc.seedasr.auc'+ }+ });+ expect(value.tts).toMatchObject({+ provider: 'volcengine',+ volcengine: {+ app_id: 'tts-app',+ access_token: 'tts-token',+ cluster: 'volcano_tts',+ default_voice_id: 'BV001_streaming'+ }+ });+ });+it('recovers only the known Windows ASS drive-letter failure with the auto-video relative-path command', async () => {tempDir = mkdtempSync(join(tmpdir(), 'creator-krillin-windows-ass-'));const resourceRoot = join(tempDir, 'runtime');diff --git a/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts b/apps/daemon/test/integration/creator-stickman-real-pipeline.test.tsindex e50ec497..96aaa27f 100644--- a/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts+++ b/apps/daemon/test/integration/creator-stickman-real-pipeline.test.ts@@ -583,7 +583,7 @@ function inMemoryConfigStore(config: CreatorServicesConfig) {}function configuredTts(config: CreatorServicesConfig): {- provider: 'openai' | 'aliyun' | 'minimax';+ provider: 'openai' | 'aliyun' | 'minimax' | 'volcengine';model: string;voiceId: string;} {diff --git a/apps/daemon/test/unit/creator-krillin-capabilities.test.ts b/apps/daemon/test/unit/creator-krillin-capabilities.test.tsindex cdcee1c0..b1e72a8f 100644--- a/apps/daemon/test/unit/creator-krillin-capabilities.test.ts+++ b/apps/daemon/test/unit/creator-krillin-capabilities.test.ts@@ -6,6 +6,21 @@ describe('KrillinAI transcription capabilities', () => {const capabilities = createKrillinCreatorServicesCapabilities('darwin', 'arm64');expect(capabilities.transcription.providers).toEqual(expect.arrayContaining([+ expect.objectContaining({+ provider: 'openai',+ kind: 'cloud',+ available: true+ }),+ expect.objectContaining({+ provider: 'aliyun',+ kind: 'cloud',+ available: true+ }),+ expect.objectContaining({+ provider: 'volcengine',+ kind: 'cloud',+ available: true+ }),expect.objectContaining({provider: 'whisperkit',kind: 'local',diff --git a/apps/daemon/test/unit/creator-services-config-store.test.ts b/apps/daemon/test/unit/creator-services-config-store.test.tsindex 1d82434d..2234d7b7 100644--- a/apps/daemon/test/unit/creator-services-config-store.test.ts+++ b/apps/daemon/test/unit/creator-services-config-store.test.ts@@ -1,4 +1,4 @@-import { mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { afterEach, describe, expect, it, vi } from 'vitest';@@ -20,6 +20,7 @@ describe('CreatorServicesConfigStore', () => {afterEach(() => {if (root !== undefined) rmSync(root, { recursive: true, force: true });root = undefined;+ vi.unstubAllEnvs();});it('returns KrillinAI-compatible defaults when no credential is saved', async () => {@@ -74,6 +75,8 @@ describe('CreatorServicesConfigStore', () => {});it('stores public settings in config.toml and credentials in credentials.json', async () => {+ vi.stubEnv('VOLCENGINE_APP_ID', '');+ vi.stubEnv('VOLCENGINE_ACCESS_TOKEN', '');root = mkdtempSync(join(tmpdir(), 'opencreator-creator-services-'));const configFile = join(root, 'config.toml');const credentialsFile = join(root, 'credentials.json');@@ -98,6 +101,28 @@ describe('CreatorServicesConfigStore', () => {});});+ it('fills volcengine speech credentials from environment without persisting them', async () => {+ vi.stubEnv('VOLCENGINE_APP_ID', 'env-app-id');+ vi.stubEnv('VOLCENGINE_ACCESS_TOKEN', 'env-access-token');+ root = mkdtempSync(join(tmpdir(), 'opencreator-creator-services-'));+ const configFile = join(root, 'config.toml');+ const credentialsFile = join(root, 'credentials.json');+ const store = createOpenCreatorCreatorServicesConfigStore({+ configFile,+ credentialsFile+ });++ const config = await store.read();+ expect(config.transcription.provider).toBe('volcengine');+ expect(config.transcription.volcengine.appId).toBe('env-app-id');+ expect(config.transcription.volcengine.accessToken).toBe('env-access-token');+ expect(config.tts.provider).toBe('volcengine');+ expect(config.tts.volcengine.appId).toBe('env-app-id');+ expect(config.tts.volcengine.apiKey).toBe('env-access-token');+ expect(existsSync(configFile)).toBe(false);+ expect(existsSync(credentialsFile)).toBe(false);+ });+it('redacts and retains credentials for every image and video provider', () => {const current = createDefaultCreatorServicesConfig();current.image.openai.apiKey = 'image-openai';@@ -156,6 +181,30 @@ describe('CreatorServicesConfigStore', () => {expect(retainCreatorServicesCredentials(presented.config, current).tts).toEqual(current.tts);});+ it('redacts and retains Volcengine speech credentials', () => {+ const current = createDefaultCreatorServicesConfig();+ current.transcription.volcengine.appId = 'asr-app';+ current.transcription.volcengine.accessToken = 'asr-token';+ current.tts.volcengine.appId = 'tts-app';+ current.tts.volcengine.apiKey = 'tts-token';++ const presented = presentCreatorServicesConfig(current);+ expect(presented.configuredCredentials).toEqual(expect.arrayContaining([+ 'transcription.volcengine.appId',+ 'transcription.volcengine.accessToken',+ 'tts.volcengine.appId',+ 'tts.volcengine.apiKey'+ ]));+ expect(presented.config.transcription.volcengine.appId).toBe('');+ expect(presented.config.transcription.volcengine.accessToken).toBe('');+ expect(presented.config.tts.volcengine.appId).toBe('');+ expect(presented.config.tts.volcengine.apiKey).toBe('');+ expect(retainCreatorServicesCredentials(presented.config, current).transcription.volcengine)+ .toEqual(current.transcription.volcengine);+ expect(retainCreatorServicesCredentials(presented.config, current).tts.volcengine)+ .toEqual(current.tts.volcengine);+ });+it('uses Codex as fallback until a custom text model is configured', async () => {let persisted = createDefaultCreatorServicesConfig();const rawStore: CreatorServicesConfigStore = {diff --git a/apps/daemon/test/unit/env-local.test.ts b/apps/daemon/test/unit/env-local.test.tsnew file mode 100644index 00000000..c309f896--- /dev/null+++ b/apps/daemon/test/unit/env-local.test.ts@@ -0,0 +1,60 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, describe, expect, it } from 'vitest';+import { loadLocalEnvironment, parseEnvFile } from '../../src/env-local.js';++describe('local environment files', () => {+ let root: string | undefined;++ afterEach(() => {+ if (root !== undefined) rmSync(root, { recursive: true, force: true });+ root = undefined;+ });++ it('parses KEY=VALUE lines and ignores comments', () => {+ expect(parseEnvFile([+ '# comment',+ 'VOLCENGINE_APP_ID=123',+ 'export VOLCENGINE_ACCESS_TOKEN="token-value"',+ "VOLCENGINE_SECRET_KEY='secret-value'",+ 'INVALID',+ ''+ ].join('\n'))).toEqual({+ VOLCENGINE_APP_ID: '123',+ VOLCENGINE_ACCESS_TOKEN: 'token-value',+ VOLCENGINE_SECRET_KEY: 'secret-value'+ });+ });++ it('loads .env.local from the working directory walk-up without overriding existing values', () => {+ root = mkdtempSync(join(tmpdir(), 'opencreator-env-local-'));+ const nested = join(root, 'apps', 'daemon');+ mkdirSync(nested, { recursive: true });+ writeFileSync(join(root, '.env.local'), [+ 'VOLCENGINE_APP_ID=file-app-id',+ 'VOLCENGINE_ACCESS_TOKEN=file-token'+ ].join('\n'));++ const env: NodeJS.ProcessEnv = {+ VOLCENGINE_APP_ID: 'shell-app-id'+ };+ const loaded = loadLocalEnvironment({ cwd: nested, env, homeDir: null });++ expect(loaded).toEqual([join(root, '.env.local')]);+ expect(env.VOLCENGINE_APP_ID).toBe('shell-app-id');+ expect(env.VOLCENGINE_ACCESS_TOKEN).toBe('file-token');+ });++ it('loads an explicit OPENCREATOR_ENV_FILE when provided', () => {+ root = mkdtempSync(join(tmpdir(), 'opencreator-env-file-'));+ const file = join(root, 'secrets.env');+ writeFileSync(file, 'VOLCENGINE_SECRET_KEY=from-explicit-file\n');+ const env: NodeJS.ProcessEnv = {+ OPENCREATOR_ENV_FILE: file+ };++ expect(loadLocalEnvironment({ cwd: root, env, homeDir: null })).toEqual([file]);+ expect(env.VOLCENGINE_SECRET_KEY).toBe('from-explicit-file');+ });+});diff --git a/apps/daemon/test/unit/krillin-tts-service.test.ts b/apps/daemon/test/unit/krillin-tts-service.test.tsindex a516bf2a..5cae24eb 100644--- a/apps/daemon/test/unit/krillin-tts-service.test.ts+++ b/apps/daemon/test/unit/krillin-tts-service.test.ts@@ -248,6 +248,57 @@ describe('KrillinTtsService', () => {);});+ it('lists bundled Volcengine voices and synthesizes through the V1 HTTP API', async () => {+ const root = await temporaryRoot();+ const config = createDefaultCreatorServicesConfig();+ config.tts.provider = 'volcengine';+ config.tts.volcengine.appId = 'app-1';+ config.tts.volcengine.apiKey = 'token-1';+ const audio = Buffer.from('mp3-bytes');+ const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({+ code: 3000,+ message: 'Success',+ data: audio.toString('base64')+ }), { status: 200, headers: { 'content-type': 'application/json' } }));+ vi.stubGlobal('fetch', fetchMock);+ const executeUtility = vi.fn();+ const service = createKrillinTtsService({+ resourceRoot: join(root, 'runtime'),+ workRoot: join(root, 'work'),+ configStore: createConfigStore(config),+ executeUtility+ });++ await expect(service.listVoices('volcengine')).resolves.toMatchObject({+ provider: 'volcengine',+ model: 'volcano_tts',+ voices: expect.arrayContaining([+ expect.objectContaining({ id: 'BV001_streaming', recommended: true })+ ])+ });+ expect(executeUtility).not.toHaveBeenCalled();++ const result = await service.synthesize({+ text: '你好,火山。',+ provider: 'volcengine',+ voiceId: 'BV001_streaming',+ format: 'mp3'+ });+ expect(result.content).toEqual(audio);+ expect(fetchMock.mock.calls[0]![0]).toBe('https://openspeech.bytedance.com/api/v1/tts');+ expect((fetchMock.mock.calls[0]![1] as RequestInit).headers).toMatchObject({+ authorization: 'Bearer;token-1'+ });+ const request = JSON.parse(String((fetchMock.mock.calls[0]![1] as RequestInit).body));+ expect(request.app).toEqual({+ appid: 'app-1',+ token: 'token-1',+ cluster: 'volcano_tts'+ });+ expect(request.audio.voice_type).toBe('BV001_streaming');+ expect(request.request.operation).toBe('query');+ });+it('requires the configured provider API key before synthesis', async () => {const root = await temporaryRoot();const service = createKrillinTtsService({diff --git a/apps/daemon/test/unit/volcengine-env.test.ts b/apps/daemon/test/unit/volcengine-env.test.tsnew file mode 100644index 00000000..4c471543--- /dev/null+++ b/apps/daemon/test/unit/volcengine-env.test.ts@@ -0,0 +1,44 @@+import { describe, expect, it } from 'vitest';+import { createDefaultCreatorServicesConfig } from '@opencreator/protocol';+import { applyVolcengineEnvironmentOverrides } from '../../src/creator-services/volcengine-env.js';++describe('volcengine environment overrides', () => {+ it('fills empty volcengine speech credentials and prefers that provider', () => {+ const config = applyVolcengineEnvironmentOverrides(+ createDefaultCreatorServicesConfig(),+ {+ VOLCENGINE_APP_ID: 'env-app-id',+ VOLCENGINE_ACCESS_TOKEN: 'env-access-token'+ }+ );++ expect(config.transcription.provider).toBe('volcengine');+ expect(config.transcription.volcengine.appId).toBe('env-app-id');+ expect(config.transcription.volcengine.accessToken).toBe('env-access-token');+ expect(config.tts.provider).toBe('volcengine');+ expect(config.tts.volcengine.appId).toBe('env-app-id');+ expect(config.tts.volcengine.apiKey).toBe('env-access-token');+ });++ it('does not replace saved credentials or a non-default provider', () => {+ const current = createDefaultCreatorServicesConfig();+ current.transcription.provider = 'aliyun';+ current.transcription.volcengine.appId = 'saved-app-id';+ current.transcription.volcengine.accessToken = 'saved-token';+ current.tts.provider = 'minimax';+ current.tts.volcengine.appId = 'saved-app-id';+ current.tts.volcengine.apiKey = 'saved-token';++ const config = applyVolcengineEnvironmentOverrides(current, {+ VOLCENGINE_APP_ID: 'env-app-id',+ VOLCENGINE_ACCESS_TOKEN: 'env-access-token'+ });++ expect(config.transcription.provider).toBe('aliyun');+ expect(config.transcription.volcengine.appId).toBe('saved-app-id');+ expect(config.transcription.volcengine.accessToken).toBe('saved-token');+ expect(config.tts.provider).toBe('minimax');+ expect(config.tts.volcengine.appId).toBe('saved-app-id');+ expect(config.tts.volcengine.apiKey).toBe('saved-token');+ });+});diff --git a/apps/web/src/features/dashboard/SmartDubbingWorkspace.tsx b/apps/web/src/features/dashboard/SmartDubbingWorkspace.tsxindex b1b8f2fb..a299b6b0 100644--- a/apps/web/src/features/dashboard/SmartDubbingWorkspace.tsx+++ b/apps/web/src/features/dashboard/SmartDubbingWorkspace.tsx@@ -532,7 +532,7 @@ function readLatestResult(artifacts: CreatorArtifact[]): DubbingResult | undefinartifact,fileName: readString(metadata.fileName) || `OpenCreator-dubbing.${readFormat(metadata.format)}`,size: readNonNegativeNumber(metadata.bytes),- provider: provider === 'aliyun' || provider === 'minimax' ? provider : 'openai',+ provider: provider === 'aliyun' || provider === 'minimax' || provider === 'volcengine' ? provider : 'openai',model: readString(metadata.model),voice: readString(metadata.voiceCode),voiceName: readString(metadata.voiceName) || readString(metadata.voiceCode),@@ -549,6 +549,7 @@ function providerLabel(): string {if (provider === 'aliyun') return l('阿里云百炼', 'Alibaba Cloud Model Studio');if (provider === 'minimax') return 'MiniMax';+ if (provider === 'volcengine') return l('火山引擎', 'Volcengine');if (provider === 'edge-tts') return 'Edge TTS';return 'OpenAI TTS';}@@ -592,7 +593,7 @@ function readString(value: CreatorJson | undefined): string {}function readProvider(value: CreatorJson | undefined): CreatorTtsProvider | undefined {- return value === 'openai' || value === 'aliyun' || value === 'minimax' || value === 'edge-tts'+ return value === 'openai' || value === 'aliyun' || value === 'minimax' || value === 'edge-tts' || value === 'volcengine'? value: undefined;}diff --git a/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx b/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsxindex c24126e2..73f905f7 100644--- a/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx+++ b/apps/web/src/features/dashboard/StickmanVideoWorkspace.tsx@@ -1182,7 +1182,8 @@ function isTtsProvider(value: unknown): value is CreatorTtsProvider {return value === 'openai'|| value === 'aliyun'|| value === 'minimax'- || value === 'edge-tts';+ || value === 'edge-tts'+ || value === 'volcengine';}function cloneScriptManifest(value: ScriptManifest): ScriptManifest {diff --git a/apps/web/src/features/dashboard/VideoGenerationWorkspace.test.tsx b/apps/web/src/features/dashboard/VideoGenerationWorkspace.test.tsxindex 3c3cfe2e..7bdb51b4 100644--- a/apps/web/src/features/dashboard/VideoGenerationWorkspace.test.tsx+++ b/apps/web/src/features/dashboard/VideoGenerationWorkspace.test.tsx@@ -358,13 +358,26 @@ function createVideoConfig() {aliyun: {oss: { accessKeyId: '', accessKeySecret: '', bucket: '' },speech: { accessKeyId: '', accessKeySecret: '', appKey: '' }+ },+ volcengine: {+ appId: '',+ accessToken: '',+ resourceId: 'volc.seedasr.auc',+ baseUrl: 'https://openspeech.bytedance.com'}},tts: {provider: 'openai' as const,openai: { baseUrl: '', apiKey: '', model: '', defaultVoiceId: '' },minimax: { baseUrl: '', apiKey: '', model: '', defaultVoiceId: '' },- aliyun: { baseUrl: '', apiKey: '', model: '', defaultVoiceId: '' }+ aliyun: { baseUrl: '', apiKey: '', model: '', defaultVoiceId: '' },+ volcengine: {+ baseUrl: 'https://openspeech.bytedance.com',+ apiKey: '',+ model: 'volcano_tts',+ defaultVoiceId: 'BV001_streaming',+ appId: ''+ }},image: {provider: 'openai' as const,diff --git a/apps/web/src/features/dashboard/VideoTranslationWorkspace.tsx b/apps/web/src/features/dashboard/VideoTranslationWorkspace.tsxindex eeffbb97..283bd485 100644--- a/apps/web/src/features/dashboard/VideoTranslationWorkspace.tsx+++ b/apps/web/src/features/dashboard/VideoTranslationWorkspace.tsx@@ -243,6 +243,7 @@ function languageLabel(options: LanguageOption[], value: string) {function ttsProviderLabel(provider: CreatorTtsProvider, l: LocalizeCopy): string {if (provider === 'aliyun') return l('阿里云百炼', 'Alibaba Cloud Model Studio');if (provider === 'minimax') return 'MiniMax';+ if (provider === 'volcengine') return l('火山引擎', 'Volcengine');if (provider === 'edge-tts') return 'Edge TTS';return 'OpenAI TTS';}@@ -985,7 +986,8 @@ function isTtsProvider(value: unknown): value is CreatorTtsProvider {return value === 'openai'|| value === 'aliyun'|| value === 'minimax'- || value === 'edge-tts';+ || value === 'edge-tts'+ || value === 'volcengine';}function readSubtitleFontSetting(diff --git a/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx b/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsxindex 7bc565ff..a61d4255 100644--- a/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx+++ b/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx@@ -99,7 +99,18 @@ describe('CreatorServicesSettingsView', () => {expect(screen.getByText('语音服务')).toBeInTheDocument();expect(screen.getAllByLabelText('Access Key Secret')).toHaveLength(2);+ await user.click(screen.getByRole('combobox', { name: '语音识别服务' }));+ await user.click(screen.getByRole('option', { name: '火山引擎' }));+ expect(screen.getByLabelText('Access Token')).toBeInTheDocument();+ expect(screen.getByLabelText('资源 ID')).toHaveValue('volc.seedasr.auc');+ expect(screen.queryByText('OSS 存储')).not.toBeInTheDocument();+await user.click(screen.getByRole('tab', { name: '配音服务' }));+ await user.click(screen.getByRole('combobox', { name: '服务商' }));+ await user.click(screen.getByRole('option', { name: '火山引擎' }));+ expect(screen.getByLabelText('集群')).toHaveValue('volcano_tts');+ expect(screen.getByLabelText('Access Token')).toBeInTheDocument();+await user.click(screen.getByRole('combobox', { name: '服务商' }));await user.click(screen.getByRole('option', { name: 'Edge TTS' }));expect(screen.getByText('无需填写凭据。运行时会使用本地 Edge TTS 服务。'))@@ -378,6 +389,13 @@ function runtimeCapabilities(available: true,models: [],gpuAcceleration: false+ },+ {+ provider: 'volcengine',+ kind: 'cloud',+ available: true,+ models: [],+ gpuAcceleration: false}]}diff --git a/apps/web/src/features/settings/CreatorServicesSettingsView.tsx b/apps/web/src/features/settings/CreatorServicesSettingsView.tsxindex c17b34b8..4acec6c4 100644--- a/apps/web/src/features/settings/CreatorServicesSettingsView.tsx+++ b/apps/web/src/features/settings/CreatorServicesSettingsView.tsx@@ -11,7 +11,9 @@ import {type CreatorTranscriptionProvider,type CreatorTranscriptionProviderCapability,type KlingAiConfig,- type OpenAiCompatibleConfig+ type OpenAiCompatibleConfig,+ type VolcengineAsrConfig,+ type VolcengineTtsConfig} from '@opencreator/protocol';import {AudioLines,@@ -621,6 +623,23 @@ function TranscriptionSettings(props: SettingsGroupProps & {})}/>) : null}+ {provider === 'volcengine' ? (+ <>+ <VolcengineAsrFields+ configuredCredentials={props.configuredCredentials}+ value={props.config.transcription.volcengine}+ onChange={value => props.update(config => {+ config.transcription.volcengine = value;+ })}+ />+ <p className="creator-services-inline-note">+ {l(+ '使用豆包语音控制台的 App ID 和 Access Token。录音文件识别大模型需单独开通。',+ 'Use the App ID and Access Token from the Doubao Voice console. File recognition must be enabled separately.'+ )}+ </p>+ </>+ ) : null}{selectedCapability?.kind === 'local' && selectedCapability.available ? (<p className="creator-services-inline-note">{l(@@ -688,6 +707,7 @@ function TtsSettings(props: SettingsGroupProps) {['openai', 'OpenAI TTS'],['minimax', 'MiniMax'],['aliyun', l('阿里云百炼', 'Alibaba Cloud Model Studio')],+ ['volcengine', l('火山引擎', 'Volcengine')],['edge-tts', 'Edge TTS']]}onChange={value => props.update(config => {@@ -732,6 +752,23 @@ function TtsSettings(props: SettingsGroupProps) {})}/>) : null}+ {provider === 'volcengine' ? (+ <>+ <VolcengineTtsFields+ configuredCredentials={props.configuredCredentials}+ value={props.config.tts.volcengine}+ onChange={value => props.update(config => {+ config.tts.volcengine = { ...config.tts.volcengine, ...value };+ })}+ />+ <p className="creator-services-inline-note">+ {l(+ '使用豆包语音合成 V1 HTTP 一次性接口。App ID、Access Token 可与听写共用;2.0 音色暂不支持。',+ 'Uses the Doubao TTS V1 HTTP one-shot API. App ID and Access Token can be reused from transcription. 2.0 voices are not supported yet.'+ )}+ </p>+ </>+ ) : null}{provider !== 'edge-tts' ? (<div className="creator-services-tts-voice"><TtsVoicePicker@@ -900,6 +937,88 @@ function KlingFields(props: {);}+function VolcengineAsrFields(props: {+ configuredCredentials: ReadonlySet<CreatorServicesCredentialField>;+ value: VolcengineAsrConfig;+ onChange(value: VolcengineAsrConfig): void;+}) {+ const l = useLocalizedCopy();+ return (+ <>+ <PasswordField+ id="transcription-volcengine-app-id"+ label="App ID"+ value={props.value.appId}+ configured={props.configuredCredentials.has('transcription.volcengine.appId')}+ onChange={appId => props.onChange({ ...props.value, appId })}+ />+ <PasswordField+ id="transcription-volcengine-access-token"+ label="Access Token"+ value={props.value.accessToken}+ configured={props.configuredCredentials.has('transcription.volcengine.accessToken')}+ onChange={accessToken => props.onChange({ ...props.value, accessToken })}+ />+ <TextField+ id="transcription-volcengine-resource-id"+ label={l('资源 ID', 'Resource ID')}+ value={props.value.resourceId}+ placeholder="volc.seedasr.auc"+ onChange={resourceId => props.onChange({ ...props.value, resourceId })}+ />+ <TextField+ id="transcription-volcengine-base-url"+ label="Base URL"+ value={props.value.baseUrl}+ placeholder="https://openspeech.bytedance.com"+ onChange={baseUrl => props.onChange({ ...props.value, baseUrl })}+ wide+ />+ </>+ );+}++function VolcengineTtsFields(props: {+ configuredCredentials: ReadonlySet<CreatorServicesCredentialField>;+ value: VolcengineTtsConfig;+ onChange(value: VolcengineTtsConfig): void;+}) {+ const l = useLocalizedCopy();+ return (+ <>+ <TextField+ id="tts-volcengine-base-url"+ label="Base URL"+ value={props.value.baseUrl}+ placeholder="https://openspeech.bytedance.com"+ onChange={baseUrl => props.onChange({ ...props.value, baseUrl })}+ wide+ />+ <PasswordField+ id="tts-volcengine-app-id"+ label="App ID"+ value={props.value.appId}+ configured={props.configuredCredentials.has('tts.volcengine.appId')}+ onChange={appId => props.onChange({ ...props.value, appId })}+ />+ <PasswordField+ id="tts-volcengine-api-key"+ label="Access Token"+ value={props.value.apiKey}+ configured={props.configuredCredentials.has('tts.volcengine.apiKey')}+ onChange={apiKey => props.onChange({ ...props.value, apiKey })}+ />+ <TextField+ id="tts-volcengine-cluster"+ label={l('集群', 'Cluster')}+ value={props.value.model}+ placeholder="volcano_tts"+ onChange={model => props.onChange({ ...props.value, model })}+ />+ </>+ );+}+function AliyunFields(props: {id: string;credentialPrefix: 'transcription.aliyun';@@ -1208,6 +1327,8 @@ function selectedTranscriptionModel(config: CreatorServicesConfig): string {return config.transcription.whisperCpp.model;case 'aliyun':return '';+ case 'volcengine':+ return config.transcription.volcengine.resourceId;}}@@ -1226,6 +1347,8 @@ function transcriptionProviderLabel(return 'Whisper.cpp';case 'aliyun':return l('阿里云百炼', 'Alibaba Cloud Model Studio');+ case 'volcengine':+ return l('火山引擎', 'Volcengine');}}diff --git a/packages/protocol/src/creator-services.ts b/packages/protocol/src/creator-services.tsindex 7a2d1ef9..7e051eea 100644--- a/packages/protocol/src/creator-services.ts+++ b/packages/protocol/src/creator-services.ts@@ -1,15 +1,33 @@+import { defaultVideoGenerationModels } from './media-generation.js';+export type OpenAiCompatibleConfig = {baseUrl: string;apiKey: string;model: string;};-export type CreatorTtsProvider = 'openai' | 'aliyun' | 'edge-tts' | 'minimax';+export const creatorTtsProviders = ['openai', 'aliyun', 'edge-tts', 'minimax', 'volcengine'] as const;+export type CreatorTtsProvider = (typeof creatorTtsProviders)[number];++export function isCreatorTtsProvider(value: unknown): value is CreatorTtsProvider {+ return creatorTtsProviders.some(provider => provider === value);+}export type CreatorTtsProviderConfig = OpenAiCompatibleConfig & {defaultVoiceId: string;};+export type VolcengineTtsConfig = CreatorTtsProviderConfig & {+ appId: string;+};++export type VolcengineAsrConfig = {+ appId: string;+ accessToken: string;+ resourceId: string;+ baseUrl: string;+};+export type CreatorTtsVoice = {id: string;name: string;@@ -54,12 +72,21 @@ export type AliyunSpeechConfig = {appKey: string;};-export type CreatorTranscriptionProvider =- | 'openai'- | 'faster-whisper'- | 'whisperkit'- | 'whisper.cpp'- | 'aliyun';+export const creatorTranscriptionProviders = [+ 'openai',+ 'faster-whisper',+ 'whisperkit',+ 'whisper.cpp',+ 'aliyun',+ 'volcengine'+] as const;+export type CreatorTranscriptionProvider = (typeof creatorTranscriptionProviders)[number];++export function isCreatorTranscriptionProvider(+ value: unknown+): value is CreatorTranscriptionProvider {+ return creatorTranscriptionProviders.some(provider => provider === value);+}export type CreatorServicesConfig = {proxy: string;@@ -78,12 +105,14 @@ export type CreatorServicesConfig = {oss: AliyunOssConfig;speech: AliyunSpeechConfig;};+ volcengine: VolcengineAsrConfig;};tts: {provider: CreatorTtsProvider;openai: CreatorTtsProviderConfig;minimax: CreatorTtsProviderConfig;aliyun: CreatorTtsProviderConfig;+ volcengine: VolcengineTtsConfig;};image: {provider: 'openai' | 'jimeng' | 'kling' | 'gemini';@@ -130,9 +159,13 @@ export type CreatorServicesCredentialField =| 'transcription.aliyun.speech.accessKeyId'| 'transcription.aliyun.speech.accessKeySecret'| 'transcription.aliyun.speech.appKey'+ | 'transcription.volcengine.appId'+ | 'transcription.volcengine.accessToken'| 'tts.openai.apiKey'| 'tts.minimax.apiKey'| 'tts.aliyun.apiKey'+ | 'tts.volcengine.appId'+ | 'tts.volcengine.apiKey'| 'image.openai.apiKey'| 'image.jimeng.apiKey'| 'image.kling.accessKey'@@ -143,6 +176,11 @@ export type CreatorServicesCredentialField =| 'video.kling.secretKey'| 'video.veo.apiKey';+export const defaultVolcengineSpeechBaseUrl = 'https://openspeech.bytedance.com';+export const defaultVolcengineAsrResourceId = 'volc.seedasr.auc';+export const defaultVolcengineTtsCluster = 'volcano_tts';+export const defaultVolcengineTtsVoiceId = 'BV001_streaming';+export function createDefaultCreatorServicesConfig(): CreatorServicesConfig {return {proxy: '',@@ -167,6 +205,12 @@ export function createDefaultCreatorServicesConfig(): CreatorServicesConfig {aliyun: {oss: { accessKeyId: '', accessKeySecret: '', bucket: '' },speech: { accessKeyId: '', accessKeySecret: '', appKey: '' }+ },+ volcengine: {+ appId: '',+ accessToken: '',+ resourceId: defaultVolcengineAsrResourceId,+ baseUrl: defaultVolcengineSpeechBaseUrl}},tts: {@@ -188,6 +232,13 @@ export function createDefaultCreatorServicesConfig(): CreatorServicesConfig {apiKey: '',model: 'qwen3-tts-flash',defaultVoiceId: 'Cherry'+ },+ volcengine: {+ baseUrl: defaultVolcengineSpeechBaseUrl,+ apiKey: '',+ model: defaultVolcengineTtsCluster,+ defaultVoiceId: defaultVolcengineTtsVoiceId,+ appId: ''}},image: {@@ -235,4 +286,3 @@ export function createDefaultCreatorServicesConfig(): CreatorServicesConfig {}};}-import { defaultVideoGenerationModels } from './media-generation.js';diff --git a/packages/protocol/src/smart-dubbing.ts b/packages/protocol/src/smart-dubbing.tsindex 670d375e..635ca1f1 100644--- a/packages/protocol/src/smart-dubbing.ts+++ b/packages/protocol/src/smart-dubbing.ts@@ -25,7 +25,7 @@ export type SmartDubbingResult = {fileName: string;mime: 'audio/mpeg' | 'audio/wav';size: number;- provider: 'openai' | 'aliyun' | 'minimax';+ provider: 'openai' | 'aliyun' | 'minimax' | 'volcengine';model: string;voice: SmartDubbingVoice;voiceName?: string;diff --git a/runtime/krillinai/config/config-example.toml b/runtime/krillinai/config/config-example.tomlindex 2e20f223..0581ad1a 100644--- a/runtime/krillinai/config/config-example.toml+++ b/runtime/krillinai/config/config-example.toml@@ -24,7 +24,7 @@json = false # 所使用的llm接口是否支持json格式,如果支持请设置为true,若不知道这是什么,请保持为false[transcribe] # 视频转文本支持多种方案,配置时先填provider,再填对应的配置- provider = "openai" #语音识别,当前可选值:openai,fasterwhisper,whisperkit,whisper.cpp,aliyun。(fasterwhisper不支持macOS,whisperkit只支持M芯片)+ provider = "openai" #语音识别,当前可选值:openai,fasterwhisper,whisperkit,whisper.cpp,aliyun,volcengine。(fasterwhisper不支持macOS,whisperkit只支持M芯片)enable_gpu_acceleration = false # 给fasterwhisper进行GPU加速选项,50系显卡请务必开启,否则无法正常运行[transcribe.openai]base_url = ""@@ -45,9 +45,14 @@access_key_id = ""access_key_secret = ""app_key= ""+ [transcribe.volcengine] # 豆包录音文件识别大模型,provider选volcengine时填写+ app_id = ""+ access_token = ""+ resource_id = "volc.seedasr.auc"+ base_url = "https://openspeech.bytedance.com"[tts]- provider = "aliyun" # 可选值:openai,aliyun,edge-tts,minimax+ provider = "aliyun" # 可选值:openai,aliyun,edge-tts,minimax,volcengine[tts.openai]base_url = ""api_key = ""@@ -60,6 +65,12 @@base_url = "https://dashscope.aliyuncs.com/api/v1"api_key = ""model = "qwen3-tts-flash"+ [tts.volcengine] # 豆包语音合成 HTTP 一次性接口,provider选volcengine时填写+ app_id = ""+ access_token = ""+ cluster = "volcano_tts"+ default_voice_id = "BV001_streaming"+ base_url = "https://openspeech.bytedance.com"[dubbing]min_subtitle_duration = 2.5 # 最短配音字幕时长,短句会优先合并diff --git a/runtime/krillinai/config/config.go b/runtime/krillinai/config/config.goindex f02960c9..a84977e5 100644--- a/runtime/krillinai/config/config.go+++ b/runtime/krillinai/config/config.go@@ -62,6 +62,21 @@ type AliyunTranscribeConfig struct {Speech AliyunSpeechConfig `toml:"speech"`}+type VolcengineAsrConfig struct {+ AppId string `toml:"app_id"`+ AccessToken string `toml:"access_token"`+ ResourceId string `toml:"resource_id"`+ BaseUrl string `toml:"base_url"`+}++type VolcengineTtsConfig struct {+ AppId string `toml:"app_id"`+ AccessToken string `toml:"access_token"`+ Cluster string `toml:"cluster"`+ DefaultVoiceId string `toml:"default_voice_id"`+ BaseUrl string `toml:"base_url"`+}+type Transcribe struct {Provider string `toml:"provider"`EnableGpuAcceleration bool `toml:"enable_gpu_acceleration"`@@ -70,6 +85,7 @@ type Transcribe struct {Whisperkit LocalModelConfig `toml:"whisperkit"`Whispercpp LocalModelConfig `toml:"whispercpp"`Aliyun AliyunTranscribeConfig `toml:"aliyun"`+ Volcengine VolcengineAsrConfig `toml:"volcengine"`}type AliyunTtsConfig struct {@@ -81,10 +97,11 @@ type AliyunTtsConfig struct {}type Tts struct {- Provider string `toml:"provider"`- Openai OpenaiCompatibleConfig `toml:"openai"`- Aliyun AliyunTtsConfig `toml:"aliyun"`- Minimax OpenaiCompatibleConfig `toml:"minimax"`+ Provider string `toml:"provider"`+ Openai OpenaiCompatibleConfig `toml:"openai"`+ Aliyun AliyunTtsConfig `toml:"aliyun"`+ Minimax OpenaiCompatibleConfig `toml:"minimax"`+ Volcengine VolcengineTtsConfig `toml:"volcengine"`}type Dubbing struct {@@ -152,6 +169,10 @@ var Conf = Config{Whispercpp: LocalModelConfig{Model: "large-v2",},+ Volcengine: VolcengineAsrConfig{+ ResourceId: "volc.seedasr.auc",+ BaseUrl: "https://openspeech.bytedance.com",+ },},Tts: Tts{Provider: "openai",@@ -166,6 +187,11 @@ var Conf = Config{BaseUrl: "https://api.minimax.io",Model: "speech-2.8-hd",},+ Volcengine: VolcengineTtsConfig{+ BaseUrl: "https://openspeech.bytedance.com",+ Cluster: "volcano_tts",+ DefaultVoiceId: "BV001_streaming",+ },},Dubbing: Dubbing{MinSubtitleDuration: 2.5,@@ -219,6 +245,10 @@ func ValidateTranscriptionConfig() error {if Conf.Transcribe.Aliyun.Speech.AccessKeyId == "" || Conf.Transcribe.Aliyun.Speech.AccessKeySecret == "" || Conf.Transcribe.Aliyun.Speech.AppKey == "" {return errors.New("使用阿里云语音服务需要配置相关密钥")}+ case "volcengine":+ if Conf.Transcribe.Volcengine.AppId == "" || Conf.Transcribe.Volcengine.AccessToken == "" {+ return errors.New("使用火山引擎语音识别需要配置 App ID 和 Access Token")+ }default:return errors.New("不支持的转录提供商")}@@ -240,6 +270,10 @@ func ValidateTTSConfig() error {if Conf.Tts.Minimax.ApiKey == "" {return errors.New("使用 MiniMax 配音服务需要配置 API Key")}+ case "volcengine":+ if Conf.Tts.Volcengine.AppId == "" || Conf.Tts.Volcengine.AccessToken == "" {+ return errors.New("使用火山引擎配音服务需要配置 App ID 和 Access Token")+ }case "edge-tts":return nildefault:diff --git a/runtime/krillinai/internal/service/init.go b/runtime/krillinai/internal/service/init.goindex 273c5f56..f8f78aa4 100644--- a/runtime/krillinai/internal/service/init.go+++ b/runtime/krillinai/internal/service/init.go@@ -10,6 +10,7 @@ import (pkgimage "krillin-ai/pkg/image""krillin-ai/pkg/localtts""krillin-ai/pkg/openai"+ "krillin-ai/pkg/volcengine""krillin-ai/pkg/whisper""krillin-ai/pkg/whispercpp""krillin-ai/pkg/whisperkit"@@ -48,6 +49,14 @@ func NewService() *Service {return nil}transcriber = cc+ case "volcengine":+ transcriber = volcengine.NewAsrClient(+ config.Conf.Transcribe.Volcengine.BaseUrl,+ config.Conf.Transcribe.Volcengine.AppId,+ config.Conf.Transcribe.Volcengine.AccessToken,+ config.Conf.Transcribe.Volcengine.ResourceId,+ config.Conf.App.Proxy,+ )}log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.Transcribe.Provider))@@ -62,6 +71,8 @@ func NewService() *Service {ttsClient = localtts.NewEdgeTtsClient()case "minimax":ttsClient, _ = ttsprovider.New("minimax")+ case "volcengine":+ ttsClient, _ = ttsprovider.New("volcengine")}s := &Service{diff --git a/runtime/krillinai/internal/ttsprovider/provider.go b/runtime/krillinai/internal/ttsprovider/provider.goindex eae21648..1b86570c 100644--- a/runtime/krillinai/internal/ttsprovider/provider.go+++ b/runtime/krillinai/internal/ttsprovider/provider.go@@ -8,6 +8,7 @@ import ("krillin-ai/pkg/aliyun""krillin-ai/pkg/minimax""krillin-ai/pkg/openai"+ "krillin-ai/pkg/volcengine")func New(provider string) (types.TTSProvider, error) {@@ -32,6 +33,14 @@ func New(provider string) (types.TTSProvider, error) {config.Conf.Tts.Minimax.ApiKey,config.Conf.Tts.Minimax.Model,), nil+ case "volcengine":+ return volcengine.NewTtsClient(+ config.Conf.Tts.Volcengine.BaseUrl,+ config.Conf.Tts.Volcengine.AppId,+ config.Conf.Tts.Volcengine.AccessToken,+ config.Conf.Tts.Volcengine.Cluster,+ config.Conf.App.Proxy,+ ), nildefault:return nil, fmt.Errorf("unsupported tts provider: %s", provider)}diff --git a/runtime/krillinai/pkg/volcengine/asr.go b/runtime/krillinai/pkg/volcengine/asr.gonew file mode 100644index 00000000..4062b14b--- /dev/null+++ b/runtime/krillinai/pkg/volcengine/asr.go@@ -0,0 +1,281 @@+package volcengine++import (+ "bytes"+ "context"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "os"+ "path/filepath"+ "strings"+ "time"++ "krillin-ai/internal/types"+ "krillin-ai/log"+ "krillin-ai/pkg/util"++ "github.com/google/uuid"+ "go.uber.org/zap"+)++var processAudioFile = util.ProcessAudio++const (+ DefaultAsrBaseURL = "https://openspeech.bytedance.com"+ DefaultAsrResourceID = "volc.seedasr.auc"+ submitPath = "/api/v3/auc/bigmodel/submit"+ queryPath = "/api/v3/auc/bigmodel/query"+ statusSuccess = "20000000"+ statusProcessing = "20000001"+ statusQueued = "20000002"+)++type AsrClient struct {+ baseURL string+ appID string+ accessToken string+ resourceID string+ httpClient *http.Client+ pollInterval time.Duration+ maxPollTime time.Duration+}++func NewAsrClient(baseURL, appID, accessToken, resourceID, proxy string) *AsrClient {+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")+ if baseURL == "" {+ baseURL = DefaultAsrBaseURL+ }+ resourceID = strings.TrimSpace(resourceID)+ if resourceID == "" {+ resourceID = DefaultAsrResourceID+ }+ transport := http.DefaultTransport.(*http.Transport).Clone()+ if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {+ transport.Proxy = http.ProxyURL(proxyURL)+ }+ return &AsrClient{+ baseURL: baseURL,+ appID: strings.TrimSpace(appID),+ accessToken: strings.TrimSpace(accessToken),+ resourceID: resourceID,+ httpClient: &http.Client{+ Transport: transport,+ Timeout: 60 * time.Second,+ },+ pollInterval: 3 * time.Second,+ maxPollTime: 10 * time.Minute,+ }+}++type asrSubmitRequest struct {+ User asrUser `json:"user"`+ Audio asrAudio `json:"audio"`+ Request asrOptions `json:"request"`+}++type asrUser struct {+ UID string `json:"uid"`+}++type asrAudio struct {+ Data string `json:"data"`+ Format string `json:"format"`+}++type asrOptions struct {+ ModelName string `json:"model_name"`+ EnableITN bool `json:"enable_itn"`+ EnablePunc bool `json:"enable_punc"`+ ShowUtterances bool `json:"show_utterances"`+}++type asrQueryResponse struct {+ Result *asrResult `json:"result"`+}++type asrResult struct {+ Text string `json:"text"`+ Utterances []asrUtterance `json:"utterances"`+}++type asrUtterance struct {+ Text string `json:"text"`+ StartTime int64 `json:"start_time"`+ EndTime int64 `json:"end_time"`+ Words []asrWord `json:"words"`+}++type asrWord struct {+ Text string `json:"text"`+ StartTime int64 `json:"start_time"`+ EndTime int64 `json:"end_time"`+}++func (c *AsrClient) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {+ _ = workDir+ if c.appID == "" || c.accessToken == "" {+ return nil, fmt.Errorf("volcengine asr app id or access token is empty")+ }+ processed, err := processAudioFile(audioFile)+ if err != nil {+ return nil, err+ }+ audio, err := os.ReadFile(processed)+ if err != nil {+ return nil, fmt.Errorf("volcengine asr read audio: %w", err)+ }+ if len(audio) == 0 {+ return nil, fmt.Errorf("volcengine asr audio is empty")+ }+ requestID := uuid.NewString()+ body, err := json.Marshal(asrSubmitRequest{+ User: asrUser{UID: "opencreator"},+ Audio: asrAudio{+ Data: base64.StdEncoding.EncodeToString(audio),+ Format: audioFormat(processed),+ },+ Request: asrOptions{+ ModelName: "bigmodel",+ EnableITN: true,+ EnablePunc: true,+ ShowUtterances: true,+ },+ })+ if err != nil {+ return nil, err+ }+ submitStatus, _, err := c.post(c.baseURL+submitPath, requestID, body)+ if err != nil {+ return nil, err+ }+ if submitStatus != "" && submitStatus != statusSuccess && submitStatus != statusQueued && submitStatus != statusProcessing {+ return nil, fmt.Errorf("volcengine asr submit failed: status %s", submitStatus)+ }+ deadline := time.Now().Add(c.maxPollTime)+ for {+ if time.Now().After(deadline) {+ return nil, fmt.Errorf("volcengine asr polling timeout after %s", c.maxPollTime)+ }+ time.Sleep(c.pollInterval)+ status, payload, err := c.post(c.baseURL+queryPath, requestID, []byte("{}"))+ if err != nil {+ return nil, err+ }+ switch status {+ case "", statusQueued, statusProcessing:+ if result := parseAsrResult(payload); result != nil && result.Text != "" {+ return transcriptionFromAsr(result, language), nil+ }+ continue+ case statusSuccess:+ result := parseAsrResult(payload)+ if result == nil || strings.TrimSpace(result.Text) == "" {+ return nil, fmt.Errorf("volcengine asr returned an empty transcript")+ }+ log.GetLogger().Info("volcengine asr transcription succeeded", zap.String("request_id", requestID))+ return transcriptionFromAsr(result, language), nil+ default:+ return nil, fmt.Errorf("volcengine asr query failed: status %s", status)+ }+ }+}++func (c *AsrClient) post(endpoint, requestID string, body []byte) (string, []byte, error) {+ req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, endpoint, bytes.NewReader(body))+ if err != nil {+ return "", nil, err+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("X-Api-App-Key", c.appID)+ req.Header.Set("X-Api-Access-Key", c.accessToken)+ req.Header.Set("X-Api-Resource-Id", c.resourceID)+ req.Header.Set("X-Api-Request-Id", requestID)+ req.Header.Set("X-Api-Sequence", "-1")+ response, err := c.httpClient.Do(req)+ if err != nil {+ return "", nil, fmt.Errorf("volcengine asr request failed: %w", err)+ }+ defer response.Body.Close()+ payload, err := io.ReadAll(io.LimitReader(response.Body, 32<<20))+ if err != nil {+ return "", nil, err+ }+ if response.StatusCode != http.StatusOK {+ return "", payload, fmt.Errorf("volcengine asr returned HTTP %d: %s", response.StatusCode, boundedMessage(payload))+ }+ return response.Header.Get("X-Api-Status-Code"), payload, nil+}++func parseAsrResult(payload []byte) *asrResult {+ var decoded asrQueryResponse+ if err := json.Unmarshal(payload, &decoded); err != nil {+ return nil+ }+ return decoded.Result+}++func transcriptionFromAsr(result *asrResult, language string) *types.TranscriptionData {+ data := &types.TranscriptionData{+ Language: language,+ Text: strings.TrimSpace(result.Text),+ }+ var words []types.Word+ index := 0+ for _, utterance := range result.Utterances {+ if len(utterance.Words) > 0 {+ for _, word := range utterance.Words {+ text := strings.TrimSpace(word.Text)+ if text == "" {+ continue+ }+ words = append(words, types.Word{+ Num: index,+ Text: text,+ Start: float64(word.StartTime) / 1000,+ End: float64(word.EndTime) / 1000,+ })+ index+++ }+ continue+ }+ text := strings.TrimSpace(utterance.Text)+ if text == "" {+ continue+ }+ words = append(words, types.Word{+ Num: index,+ Text: text,+ Start: float64(utterance.StartTime) / 1000,+ End: float64(utterance.EndTime) / 1000,+ })+ index+++ }+ data.Words = words+ return data+}++func audioFormat(path string) string {+ switch strings.ToLower(filepath.Ext(path)) {+ case ".wav":+ return "wav"+ case ".ogg":+ return "ogg"+ case ".m4a":+ return "m4a"+ default:+ return "mp3"+ }+}++func boundedMessage(value []byte) string {+ const limit = 300+ message := strings.TrimSpace(string(value))+ if len(message) <= limit {+ return message+ }+ return message[:limit]+}diff --git a/runtime/krillinai/pkg/volcengine/tts.go b/runtime/krillinai/pkg/volcengine/tts.gonew file mode 100644index 00000000..839f1efc--- /dev/null+++ b/runtime/krillinai/pkg/volcengine/tts.go@@ -0,0 +1,221 @@+package volcengine++import (+ "bytes"+ "context"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "os"+ "path/filepath"+ "strings"+ "time"+ "unicode/utf8"++ "krillin-ai/internal/types"++ "github.com/google/uuid"+)++const (+ DefaultTTSBaseURL = "https://openspeech.bytedance.com"+ DefaultTTSCluster = "volcano_tts"+ DefaultTTSVoice = "BV001_streaming"+ ttsPath = "/api/v1/tts"+ ttsSuccessCode = 3000+ maxTTSTextBytes = 1024+)++type TtsClient struct {+ BaseURL string+ AppID string+ AccessToken string+ Cluster string+ httpClient *http.Client+}++func NewTtsClient(baseURL, appID, accessToken, cluster, proxy string) *TtsClient {+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")+ if baseURL == "" {+ baseURL = DefaultTTSBaseURL+ }+ cluster = strings.TrimSpace(cluster)+ if cluster == "" {+ cluster = DefaultTTSCluster+ }+ transport := http.DefaultTransport.(*http.Transport).Clone()+ if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {+ transport.Proxy = http.ProxyURL(proxyURL)+ }+ return &TtsClient{+ BaseURL: baseURL,+ AppID: strings.TrimSpace(appID),+ AccessToken: strings.TrimSpace(accessToken),+ Cluster: cluster,+ httpClient: &http.Client{+ Transport: transport,+ Timeout: 90 * time.Second,+ },+ }+}++type ttsRequest struct {+ App ttsApp `json:"app"`+ User ttsUser `json:"user"`+ Audio ttsAudio `json:"audio"`+ Request ttsPayload `json:"request"`+}++type ttsApp struct {+ AppID string `json:"appid"`+ Token string `json:"token"`+ Cluster string `json:"cluster"`+}++type ttsUser struct {+ UID string `json:"uid"`+}++type ttsAudio struct {+ VoiceType string `json:"voice_type"`+ Encoding string `json:"encoding"`+ SpeedRatio float64 `json:"speed_ratio"`+}++type ttsPayload struct {+ ReqID string `json:"reqid"`+ Text string `json:"text"`+ TextType string `json:"text_type"`+ Operation string `json:"operation"`+}++type ttsResponse struct {+ Code int `json:"code"`+ Message string `json:"message"`+ Data string `json:"data"`+}++func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: text,+ Voice: voice,+ OutputFile: outputFile,+ })+}++func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {+ if c.AppID == "" || c.AccessToken == "" {+ return fmt.Errorf("volcengine tts app id or access token is empty")+ }+ text := strings.TrimSpace(options.Text)+ if text == "" {+ return fmt.Errorf("volcengine tts text is empty")+ }+ if utf8.RuneCountInString(text) > 0 && len(text) > maxTTSTextBytes {+ return fmt.Errorf("volcengine tts text exceeds %d bytes", maxTTSTextBytes)+ }+ voice := strings.TrimSpace(options.Voice)+ if voice == "" {+ voice = DefaultTTSVoice+ }+ speed := options.Speed+ if speed <= 0 {+ speed = 1+ }+ format := speechFormat(options.Format, options.OutputFile)+ body, err := json.Marshal(ttsRequest{+ App: ttsApp{+ AppID: c.AppID,+ Token: c.AccessToken,+ Cluster: c.Cluster,+ },+ User: ttsUser{UID: "opencreator"},+ Audio: ttsAudio{+ VoiceType: voice,+ Encoding: format,+ SpeedRatio: speed,+ },+ Request: ttsPayload{+ ReqID: uuid.NewString(),+ Text: text,+ TextType: "plain",+ Operation: "query",+ },+ })+ if err != nil {+ return err+ }+ endpoint := c.BaseURL + ttsPath+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))+ if err != nil {+ return err+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", "Bearer;"+c.AccessToken)+ response, err := c.httpClient.Do(req)+ if err != nil {+ return fmt.Errorf("volcengine tts request failed: %w", err)+ }+ defer response.Body.Close()+ payload, err := io.ReadAll(io.LimitReader(response.Body, 20<<20))+ if err != nil {+ return err+ }+ if response.StatusCode != http.StatusOK {+ return fmt.Errorf("volcengine tts returned HTTP %d: %s", response.StatusCode, boundedMessage(payload))+ }+ var decoded ttsResponse+ if err := json.Unmarshal(payload, &decoded); err != nil {+ return fmt.Errorf("volcengine tts decode response: %w", err)+ }+ if decoded.Code != ttsSuccessCode {+ return fmt.Errorf("volcengine tts error %d: %s", decoded.Code, decoded.Message)+ }+ if strings.TrimSpace(decoded.Data) == "" {+ return fmt.Errorf("volcengine tts response did not include audio")+ }+ audio, err := base64.StdEncoding.DecodeString(decoded.Data)+ if err != nil {+ return fmt.Errorf("volcengine tts decode audio: %w", err)+ }+ if err := os.MkdirAll(filepath.Dir(options.OutputFile), 0755); err != nil {+ return err+ }+ return os.WriteFile(options.OutputFile, audio, 0644)+}++func (c *TtsClient) ListVoices(context.Context) ([]types.TTSVoice, error) {+ return append([]types.TTSVoice(nil), volcengineBuiltinVoices...), nil+}++func speechFormat(explicit, outputFile string) string {+ format := strings.ToLower(strings.TrimSpace(explicit))+ if format == "mp3" || format == "wav" {+ return format+ }+ if strings.EqualFold(filepath.Ext(outputFile), ".wav") {+ return "wav"+ }+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {+ return "mp3"+ }+ return "mp3"+}++var volcengineBuiltinVoices = []types.TTSVoice{+ {Code: "BV001_streaming", Name: "通用女声", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "解说配音", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}, Recommended: true},+ {Code: "BV002_streaming", Name: "通用男声", Language: "zh-CN", Gender: "male", Provider: "volcengine", Scenario: "解说配音", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV007_streaming", Name: "亲切女声", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "亲切自然", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV009_streaming", Name: "知性女声", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "知识讲解", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV019_streaming", Name: "霸气御姐", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "有声书", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV033_streaming", Name: "温柔女声", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "温柔叙事", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV056_streaming", Name: "阳光男声", Language: "zh-CN", Gender: "male", Provider: "volcengine", Scenario: "阳光解说", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV102_streaming", Name: "儒雅青年", Language: "zh-CN", Gender: "male", Provider: "volcengine", Scenario: "沉稳叙事", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV113_streaming", Name: "甜宠少御", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "甜宠", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV119_streaming", Name: "通用萌娃", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "童声", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV700_streaming", Name: "灿灿", Language: "zh-CN", Gender: "female", Provider: "volcengine", Scenario: "豆包女声", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+ {Code: "BV701_streaming", Name: "擎苍", Language: "zh-CN", Gender: "male", Provider: "volcengine", Scenario: "磁性男声", Kind: "builtin", SupportedModels: []string{DefaultTTSCluster}},+}diff --git a/runtime/krillinai/pkg/volcengine/volcengine_test.go b/runtime/krillinai/pkg/volcengine/volcengine_test.gonew file mode 100644index 00000000..a50a175b--- /dev/null+++ b/runtime/krillinai/pkg/volcengine/volcengine_test.go@@ -0,0 +1,175 @@+package volcengine++import (+ "context"+ "encoding/base64"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "krillin-ai/internal/types"+ "krillin-ai/log"+)++func TestAsrTranscriptionSubmitsBase64AudioAndMapsUtterances(t *testing.T) {+ log.InitLogger()+ originalProcess := processAudioFile+ processAudioFile = func(filePath string) (string, error) {+ return filePath, nil+ }+ t.Cleanup(func() { processAudioFile = originalProcess })++ source := filepath.Join(t.TempDir(), "clip.mp3")+ if err := os.WriteFile(source, []byte("fake-mp3"), 0644); err != nil {+ t.Fatal(err)+ }+ var requestIDs []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("X-Api-App-Key") != "app-1" {+ t.Fatalf("app key = %q", r.Header.Get("X-Api-App-Key"))+ }+ if r.Header.Get("X-Api-Access-Key") != "token-1" {+ t.Fatalf("access key = %q", r.Header.Get("X-Api-Access-Key"))+ }+ if r.Header.Get("X-Api-Resource-Id") != DefaultAsrResourceID {+ t.Fatalf("resource = %q", r.Header.Get("X-Api-Resource-Id"))+ }+ requestIDs = append(requestIDs, r.Header.Get("X-Api-Request-Id"))+ switch r.URL.Path {+ case submitPath:+ var body asrSubmitRequest+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Fatalf("decode submit: %v", err)+ }+ decoded, err := base64.StdEncoding.DecodeString(body.Audio.Data)+ if err != nil || string(decoded) != "fake-mp3" {+ t.Fatalf("audio data = %q", decoded)+ }+ if body.Audio.Format != "mp3" {+ t.Fatalf("format = %q", body.Audio.Format)+ }+ w.Header().Set("X-Api-Status-Code", statusSuccess)+ _, _ = w.Write([]byte("{}"))+ case queryPath:+ w.Header().Set("X-Api-Status-Code", statusSuccess)+ _ = json.NewEncoder(w).Encode(asrQueryResponse{+ Result: &asrResult{+ Text: "Hello world",+ Utterances: []asrUtterance{{+ Text: "Hello world",+ StartTime: 120,+ EndTime: 1800,+ Words: []asrWord{+ {Text: "Hello", StartTime: 120, EndTime: 700},+ {Text: "world", StartTime: 720, EndTime: 1800},+ },+ }},+ },+ })+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ client := NewAsrClient(server.URL, "app-1", "token-1", "", "")+ client.pollInterval = 0+ result, err := client.Transcription(source, "en", t.TempDir())+ if err != nil {+ t.Fatalf("Transcription() error = %v", err)+ }+ if result.Text != "Hello world" {+ t.Fatalf("text = %q", result.Text)+ }+ if len(result.Words) != 2 || result.Words[0].Text != "Hello" || result.Words[0].Start != 0.12 {+ t.Fatalf("words = %#v", result.Words)+ }+ if len(requestIDs) < 2 || requestIDs[0] == "" || requestIDs[0] != requestIDs[1] {+ t.Fatalf("request ids = %#v", requestIDs)+ }+}++func TestTtsSynthesizeUsesV1HTTPQueryAndWritesAudio(t *testing.T) {+ output := filepath.Join(t.TempDir(), "speech.mp3")+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != ttsPath {+ http.NotFound(w, r)+ return+ }+ if r.Header.Get("Authorization") != "Bearer;token-1" {+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))+ }+ var request ttsRequest+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Fatalf("decode tts: %v", err)+ }+ if request.App.AppID != "app-1" || request.App.Cluster != DefaultTTSCluster {+ t.Fatalf("app = %#v", request.App)+ }+ if request.Audio.VoiceType != "BV001_streaming" || request.Request.Operation != "query" {+ t.Fatalf("audio/request = %#v %#v", request.Audio, request.Request)+ }+ _ = json.NewEncoder(w).Encode(ttsResponse{+ Code: ttsSuccessCode,+ Message: "Success",+ Data: base64.StdEncoding.EncodeToString([]byte("mp3-bytes")),+ })+ }))+ defer server.Close()++ client := NewTtsClient(server.URL, "app-1", "token-1", "", "")+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: "你好,火山。",+ Voice: "BV001_streaming",+ OutputFile: output,+ Format: "mp3",+ }); err != nil {+ t.Fatalf("Synthesize() error = %v", err)+ }+ content, err := os.ReadFile(output)+ if err != nil {+ t.Fatal(err)+ }+ if string(content) != "mp3-bytes" {+ t.Fatalf("audio = %q", content)+ }+}++func TestListVoicesIncludesDefaultChineseCatalog(t *testing.T) {+ client := NewTtsClient("http://127.0.0.1:1", "unused", "unused", "", "")+ voices, err := client.ListVoices(context.Background())+ if err != nil {+ t.Fatalf("ListVoices() error = %v", err)+ }+ found := false+ for _, voice := range voices {+ if voice.Code == DefaultTTSVoice && voice.Recommended {+ found = true+ }+ }+ if !found {+ t.Fatalf("missing default voice in %#v", voices)+ }+}++func TestSpeechFormatUsesOutputExtension(t *testing.T) {+ if got := speechFormat("", "speech.wav"); got != "wav" {+ t.Fatalf("wav format = %q", got)+ }+ if got := speechFormat("", "speech.mp3"); got != "mp3" {+ t.Fatalf("mp3 format = %q", got)+ }+}++func TestAudioFormatUsesExtension(t *testing.T) {+ if got := audioFormat("a.MP3"); got != "mp3" {+ t.Fatalf("format = %q", got)+ }+ if !strings.HasPrefix(DefaultAsrBaseURL, "https://") {+ t.Fatal(DefaultAsrBaseURL)+ }+}