Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

Merge PR #335 (FunASR/SenseVoice transcription) into master

Leggett7f9e9a4ebf59
diff --cc apps/daemon/src/api/routes.creator-services.ts
index d4942965,5202a34b..8aaa6e5e
--- a/apps/daemon/src/api/routes.creator-services.ts
+++ b/apps/daemon/src/api/routes.creator-services.ts
@@@ -78,6 -78,22 +78,31 @@@ export async function registerCreatorSe
}
});
+ server.post<{ Body: unknown }>('/creator-services/transcription/test', async (request, reply) => {
+ try {
+ const body = request.body as { baseUrl?: string; apiKey?: string; model?: string; timeoutMs?: number };
+ const baseUrl = typeof body.baseUrl === 'string' ? body.baseUrl.replace(/\/+$/, '') : '';
+ const model = typeof body.model === 'string' ? body.model.trim() : '';
+ if (!/^https?:\/\//i.test(baseUrl) || !model) return reply.code(400).send(apiError('VALIDATION_FAILED', 'FunASR Base URL and model are required'));
+ const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), Math.min(Math.max(body.timeoutMs ?? 10000, 1000), 60000));
+ try {
- const response = await fetch(`${baseUrl}/models`, { headers: typeof body.apiKey === 'string' && body.apiKey ? { Authorization: `Bearer ${body.apiKey}` } : {}, signal: controller.signal });
++ const saved = await store.read();
++ const configured = saved.transcription.funasr;
++ const apiKey = typeof body.apiKey === 'string' && body.apiKey
++ ? body.apiKey
++ : configured.baseUrl.replace(/\/+$/, '') === baseUrl ? configured.apiKey : '';
++ const response = await fetch(`${baseUrl}/models`, { headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {}, signal: controller.signal });
+ const payload = await response.json().catch(() => undefined) as { data?: Array<{ id?: string }> } | undefined;
+ if (!response.ok) return reply.code(502).send(apiError('SMART_DUBBING_UPSTREAM_ERROR', `FunASR connection failed (HTTP ${response.status})`));
- return { connected: true, model, models: payload?.data?.map(item => item.id).filter((id): id is string => typeof id === 'string') ?? [], capabilities: ['audio.transcriptions'] };
++ const models = Array.isArray(payload?.data)
++ ? payload.data.map(item => item.id).filter((id): id is string => typeof id === 'string')
++ : [];
++ if (!models.includes(model)) return reply.code(502).send(apiError('SMART_DUBBING_UPSTREAM_ERROR', `FunASR model ${model} is not available`));
++ return { connected: true, model, models, capabilities: ['audio.transcriptions'] };
+ } finally { clearTimeout(timer); }
+ } catch (error) { return reply.code(502).send(apiError('SMART_DUBBING_UPSTREAM_ERROR', error instanceof Error && error.name === 'AbortError' ? 'FunASR connection timed out' : 'FunASR connection failed')); }
+ });
+
server.delete('/creator-services/config', async (_request, reply) => {
try {
return presentCreatorServicesConfig(await store.reset());
diff --cc apps/daemon/src/creator-services/config-store.ts
index b574fe28,58d3ce63..fc778704
--- a/apps/daemon/src/creator-services/config-store.ts
+++ b/apps/daemon/src/creator-services/config-store.ts
@@@ -412,6 -403,6 +415,9 @@@ export function presentCreatorServicesC
redact('transcription.volcengine.accessToken', redacted.transcription.volcengine.accessToken, () => {
redacted.transcription.volcengine.accessToken = '';
});
++ redact('transcription.funasr.apiKey', redacted.transcription.funasr.apiKey, () => {
++ redacted.transcription.funasr.apiKey = '';
++ });
redact('tts.openai.apiKey', redacted.tts.openai.apiKey, () => { redacted.tts.openai.apiKey = ''; });
redact('tts.minimax.apiKey', redacted.tts.minimax.apiKey, () => { redacted.tts.minimax.apiKey = ''; });
redact('tts.aliyun.apiKey', redacted.tts.aliyun.apiKey, () => { redacted.tts.aliyun.apiKey = ''; });
@@@ -452,6 -443,6 +458,13 @@@ export function retainCreatorServicesCr
value => { merged.transcription.volcengine.accessToken = value; },
current.transcription.volcengine.accessToken
);
++ if (merged.transcription.funasr.baseUrl.replace(/\/+$/, '') === current.transcription.funasr.baseUrl.replace(/\/+$/, '')) {
++ retainBlank(
++ () => merged.transcription.funasr.apiKey,
++ value => { merged.transcription.funasr.apiKey = value; },
++ current.transcription.funasr.apiKey
++ );
++ }
retainBlank(() => merged.tts.openai.apiKey, value => { merged.tts.openai.apiKey = value; }, current.tts.openai.apiKey);
retainBlank(() => merged.tts.minimax.apiKey, value => { merged.tts.minimax.apiKey = value; }, current.tts.minimax.apiKey);
retainBlank(() => merged.tts.aliyun.apiKey, value => { merged.tts.aliyun.apiKey = value; }, current.tts.aliyun.apiKey);
diff --cc apps/daemon/test/integration/creator-services-api.test.ts
index e8c2ba43,e8c2ba43..5815d709
--- a/apps/daemon/test/integration/creator-services-api.test.ts
+++ b/apps/daemon/test/integration/creator-services-api.test.ts
@@@ -1,4 -1,4 +1,5 @@@
import Fastify, { type FastifyInstance } from 'fastify';
++import { createServer } from 'node:http';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
createDefaultCreatorServicesConfig,
@@@ -104,6 -104,6 +105,66 @@@ describe('creator services API', () =>
}));
});
++ it('does not return the FunASR API key from settings reads or saves', async () => {
++ const config = createDefaultCreatorServicesConfig();
++ config.transcription.funasr.apiKey = 'funasr-secret';
++ vi.mocked(store.read).mockResolvedValue(config);
++
++ const read = await server.inject({ method: 'GET', url: '/creator-services/config' });
++ expect(read.statusCode).toBe(200);
++ expect(read.body).not.toContain('funasr-secret');
++ expect(read.json().configuredCredentials).toContain('transcription.funasr.apiKey');
++
++ const saved = await server.inject({
++ method: 'PATCH', url: '/creator-services/config', payload: read.json().config
++ });
++ expect(saved.statusCode).toBe(200);
++ expect(saved.body).not.toContain('funasr-secret');
++ expect(store.write).toHaveBeenCalledWith(expect.objectContaining({
++ transcription: expect.objectContaining({
++ funasr: expect.objectContaining({ apiKey: 'funasr-secret' })
++ })
++ }));
++ });
++
++ it('tests the configured FunASR model with the retained credential', async () => {
++ let authorization: string | undefined;
++ const endpoint = createServer((request, response) => {
++ authorization = request.headers.authorization;
++ response.setHeader('Content-Type', 'application/json');
++ response.end(JSON.stringify({ data: [{ id: 'sensevoice' }] }));
++ });
++ await new Promise<void>(resolve => endpoint.listen(0, '127.0.0.1', resolve));
++ try {
++ const address = endpoint.address();
++ if (address === null || typeof address === 'string') throw new Error('Missing endpoint address');
++ const baseUrl = `http://127.0.0.1:${address.port}/v1`;
++ const config = createDefaultCreatorServicesConfig();
++ config.transcription.funasr.baseUrl = baseUrl;
++ config.transcription.funasr.apiKey = 'funasr-secret';
++ vi.mocked(store.read).mockResolvedValue(config);
++
++ const payload = { baseUrl, model: 'sensevoice', apiKey: '', timeoutMs: 1000 };
++ const connected = await server.inject({ method: 'POST', url: '/creator-services/transcription/test', payload });
++ expect(connected.statusCode).toBe(200);
++ expect(connected.json().connected).toBe(true);
++ expect(authorization).toBe('Bearer funasr-secret');
++
++ const missing = await server.inject({ method: 'POST', url: '/creator-services/transcription/test',
++ payload: { ...payload, model: 'unknown-model' } });
++ expect(missing.statusCode).toBe(502);
++ expect(missing.json().error.message).toContain('unknown-model');
++
++ authorization = undefined;
++ const otherEndpoint = await server.inject({ method: 'POST', url: '/creator-services/transcription/test',
++ payload: { ...payload, baseUrl: `${baseUrl}/other` } });
++ expect(otherEndpoint.statusCode).toBe(200);
++ expect(authorization).toBeUndefined();
++ } finally {
++ endpoint.close();
++ }
++ });
++
it('accepts the default OpenAI provider with an empty API key', async () => {
vi.mocked(store.read).mockResolvedValue(createDefaultCreatorServicesConfig());
const config = createDefaultCreatorServicesConfig();
diff --cc apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts
index ca301f27,49f28588..09e7389c
--- a/apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts
+++ b/apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts
@@@ -1,4 -1,4 +1,5 @@@
import { dirname } from 'node:path';
++import { parse } from '@iarna/toml';
import { createDefaultCreatorServicesConfig } from '@opencreator/protocol';
import { describe, expect, it } from 'vitest';
import {
@@@ -127,4 -108,4 +128,22 @@@ describe('KrillinAI CLI protocol', () =
expect(createKrillinConfigToml(config)).toContain('model = "large-v3-turbo"');
});
++
++ it('passes the configured FunASR endpoint, model and timeout to KrillinAI', () => {
++ const config = createDefaultCreatorServicesConfig();
++ config.transcription.provider = 'funasr';
++ config.transcription.funasr = {
++ baseUrl: 'http://127.0.0.1:8000/v1', apiKey: 'funasr-secret', model: 'sensevoice', timeoutMs: 60000
++ };
++
++ const document = parse(createKrillinConfigToml(config)) as {
++ transcribe: { provider: string; funasr: Record<string, string | number> };
++ };
++ expect(document.transcribe).toMatchObject({
++ provider: 'funasr',
++ funasr: {
++ base_url: 'http://127.0.0.1:8000/v1', api_key: 'funasr-secret', model: 'sensevoice', timeout_ms: 60000
++ }
++ });
++ });
});
diff --cc apps/daemon/test/unit/creator-services-config-store.test.ts
index dced3fd6,015cbc16..0f3cbcf2
--- a/apps/daemon/test/unit/creator-services-config-store.test.ts
+++ b/apps/daemon/test/unit/creator-services-config-store.test.ts
@@@ -261,6 -233,6 +261,22 @@@ describe('CreatorServicesConfigStore',
.toEqual(current.tts.volcengine);
});
++ it('redacts and retains a configured FunASR API key', () => {
++ const current = createDefaultCreatorServicesConfig();
++ current.transcription.funasr.apiKey = 'funasr-secret';
++
++ const presented = presentCreatorServicesConfig(current);
++ expect(presented.configuredCredentials).toContain('transcription.funasr.apiKey');
++ expect(presented.config.transcription.funasr.apiKey).toBe('');
++ expect(retainCreatorServicesCredentials(presented.config, current).transcription.funasr.apiKey)
++ .toBe('funasr-secret');
++
++ const changedEndpoint = structuredClone(presented.config);
++ changedEndpoint.transcription.funasr.baseUrl = 'https://another-asr.example/v1';
++ expect(retainCreatorServicesCredentials(changedEndpoint, current).transcription.funasr.apiKey)
++ .toBe('');
++ });
++
it('migrates legacy tts.volcengine.apiKey into accessToken', () => {
const current = createDefaultCreatorServicesConfig();
const parsed = parseCreatorServicesConfig({
diff --cc docs/funasr-transcription.md
index 00000000,e23d2ac8..769dbdf8
mode 000000,100644..100644
--- a/docs/funasr-transcription.md
+++ b/docs/funasr-transcription.md
@@@ -1,0 -1,16 +1,16 @@@
+ # FunASR / SenseVoice transcription
+
+ OpenCreator can use a user-managed FunASR server through its OpenAI-compatible API. OpenCreator does not package Python, CUDA, model files, or a resident Python process.
+
+ Start the official server separately, for example:
+
+ ```bash
+ pip install funasr fastapi uvicorn python-multipart
+ funasr-server --host 127.0.0.1 --model sensevoice --device cpu
+ ```
+
+ In **AI services → Transcription**, choose **FunASR / SenseVoice**, set the server Base URL (normally `http://127.0.0.1:8000/v1`) and model alias (`sensevoice`, `paraformer`, or another alias exposed by the server). API Key is optional and is only sent as a Bearer credential when configured. The timeout applies to the transcription request.
+
-The connection test checks `/v1/models` and reports the reachable models and `audio.transcriptions` capability. Transcription uses `POST /v1/audio/transcriptions` with multipart `file`, `model`, and `response_format=verbose_json`; the response is normalized into the runtime transcription contract. HTTP errors, invalid responses, timeouts, and cancellation remain executor failures.
++The connection test checks `/v1/models` for the selected model; it does not execute a transcription or verify output quality. Transcription uses `POST /v1/audio/transcriptions` with multipart `file`, `model`, and `response_format=verbose_json`; the response is normalized into the runtime transcription contract. HTTP errors, invalid responses, timeouts, and cancellation remain executor failures.
+
+ See the [official FunASR OpenAI-compatible API example](https://github.com/modelscope/FunASR/tree/main/examples/openai_api) for server setup and model details.
diff --cc packages/protocol/src/creator-services.ts
index 49bc69d4,3e668de4..c0d8c278
--- a/packages/protocol/src/creator-services.ts
+++ b/packages/protocol/src/creator-services.ts
@@@ -246,9 -247,10 +249,10 @@@ export function createDefaultCreatorSer
whisperKit: { model: 'large-v2' },
whisperCpp: { model: 'tiny' },
aliyun: {
- oss: { accessKeyId: '', accessKeySecret: '', bucket: '' },
+ oss: { accessKeyId: '', accessKeySecret: '', bucket: '', region: 'cn-shanghai', endpoint: '' },
speech: { accessKeyId: '', accessKeySecret: '', appKey: '' }
},
+ funasr: { baseUrl: 'http://127.0.0.1:8000/v1', apiKey: '', model: 'sensevoice', timeoutMs: 120000 },
volcengine: {
appId: '',
accessToken: '',
diff --cc scripts/check-performance-baseline.mjs
index a52454bf,6b0cf595..6c1679d2
--- a/scripts/check-performance-baseline.mjs
+++ b/scripts/check-performance-baseline.mjs
@@@ -60,7 -60,7 +60,7 @@@ function measureAssets()
{
label: 'AI services settings JavaScript',
pattern: /^CreatorServicesSettingsView-[^.]+\.js$/,
-- maxBytes: 45_000,
++ maxBytes: 48_000,
maxGzipBytes: 14_000
},
{