Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat(creator): support Whisper large-v3-turbo

MijaelXb805df0f100b
diff --git a/apps/daemon/src/creator-services/config-store.ts b/apps/daemon/src/creator-services/config-store.ts
index 6a4c5a9a..ea6dbc58 100644
--- a/apps/daemon/src/creator-services/config-store.ts
+++ b/apps/daemon/src/creator-services/config-store.ts
@@ -127,7 +127,7 @@ export const creatorServicesConfigSchema = z.object({
openai: openAiCompatibleSchema,
fasterWhisper: z.object({ model: z.enum(['tiny', 'medium', 'large-v2']) }).strict(),
whisperKit: z.object({ model: z.literal('large-v2') }).strict(),
- whisperCpp: z.object({ model: z.enum(['tiny', 'medium', 'large-v2']) }).strict(),
+ whisperCpp: z.object({ model: z.enum(['tiny', 'medium', 'large-v2', 'large-v3-turbo']) }).strict(),
aliyun: aliyunSchema,
volcengine: volcengineAsrSchema.default(creatorServicesDefaults.transcription.volcengine)
}).strict(),
diff --git a/apps/daemon/src/creator/krillin/capabilities.ts b/apps/daemon/src/creator/krillin/capabilities.ts
index 832263a1..3bc7af40 100644
--- a/apps/daemon/src/creator/krillin/capabilities.ts
+++ b/apps/daemon/src/creator/krillin/capabilities.ts
@@ -33,10 +33,16 @@ export function createKrillinCreatorServicesCapabilities(
),
localProvider(
'whisper.cpp',
- ['tiny', 'medium', 'large-v2'],
+ ['tiny', 'medium', 'large-v2', 'large-v3-turbo'],
whisperCppAvailable,
false,
- whisperCppAvailable ? undefined : 'unsupported_platform'
+ whisperCppAvailable ? undefined : 'unsupported_platform',
+ {
+ tiny: { diskBytes: 77691713 },
+ medium: { diskBytes: 1533763059 },
+ 'large-v2': { diskBytes: 3094623691 },
+ 'large-v3-turbo': { diskBytes: 1624555275 }
+ }
),
cloudProvider('aliyun', []),
cloudProvider('volcengine', [])
@@ -63,7 +69,8 @@ function localProvider(
models: string[],
available: boolean,
gpuAcceleration: boolean,
- unavailableReason: CreatorTranscriptionProviderCapability['unavailableReason']
+ unavailableReason: CreatorTranscriptionProviderCapability['unavailableReason'],
+ modelDetails?: CreatorTranscriptionProviderCapability['modelDetails']
): CreatorTranscriptionProviderCapability {
return {
provider,
@@ -71,6 +78,7 @@ function localProvider(
available,
models,
gpuAcceleration,
+ ...(modelDetails === undefined ? {} : { modelDetails }),
...(unavailableReason === undefined ? {} : { unavailableReason })
};
}
diff --git a/apps/daemon/src/creator/krillin/cli-runner.ts b/apps/daemon/src/creator/krillin/cli-runner.ts
index d5a5f83b..a94fc14a 100644
--- a/apps/daemon/src/creator/krillin/cli-runner.ts
+++ b/apps/daemon/src/creator/krillin/cli-runner.ts
@@ -540,7 +540,7 @@ export async function prepareCliResourceRoot(input: {
dependencyRoot: string;
launcherRoot: string;
onDemandTranscriptionProvider?: 'whisperkit' | 'whisper.cpp';
- onDemandTranscriptionModel?: 'tiny' | 'medium' | 'large-v2';
+ onDemandTranscriptionModel?: 'tiny' | 'medium' | 'large-v2' | 'large-v3-turbo';
useOnDemandTranscription?: boolean;
useWhisperCpp?: boolean;
ytDlpRuntime?: YtDlpRuntime;
@@ -562,7 +562,7 @@ export async function prepareCliResourceRoot(input: {
const mountedModels = join(input.launcherRoot, 'models', 'whispercpp');
await mkdir(mountedModels, { recursive: true });
await linkDirectoryEntries(sourceModels, mountedModels);
- if (model !== 'large-v2') {
+ if (model !== 'large-v2' && model !== 'large-v3-turbo') {
await linkFile(
join(sourceModels, `ggml-${model}.bin`),
join(mountedModels, 'ggml-large-v2.bin'),
diff --git a/apps/daemon/src/creator/krillin/config-bridge.ts b/apps/daemon/src/creator/krillin/config-bridge.ts
index d1dda8f0..deba6132 100644
--- a/apps/daemon/src/creator/krillin/config-bridge.ts
+++ b/apps/daemon/src/creator/krillin/config-bridge.ts
@@ -18,9 +18,10 @@ export function createKrillinConfigToml(config: CreatorServicesConfig): string {
whisperkit: config.transcription.whisperKit,
whispercpp: {
...config.transcription.whisperCpp,
- // KrillinAI 2.1 validates this label even though whisper.cpp reads the
- // selected model from the compatible file mounted by the CLI runner.
+ // Preserve the KrillinAI 2.1 compatibility bridge for legacy models,
+ // while large-v3-turbo is supported by name end to end.
model: config.transcription.provider === 'whisper.cpp'
+ && config.transcription.whisperCpp.model !== 'large-v3-turbo'
? 'large-v2'
: config.transcription.whisperCpp.model
},
diff --git a/apps/daemon/src/creator/krillin/dependency-loader.ts b/apps/daemon/src/creator/krillin/dependency-loader.ts
index 87aeb8fc..3d8c2ba9 100644
--- a/apps/daemon/src/creator/krillin/dependency-loader.ts
+++ b/apps/daemon/src/creator/krillin/dependency-loader.ts
@@ -80,6 +80,12 @@ const whisperCppRelease = {
sha256: '9a423fe4d40c82774b6af34115b8b935f34152246eb19e80e376071d3f999487',
size: 3_094_623_691,
environment: 'OPENCREATOR_WHISPERCPP_LARGE_V2_MODEL'
+ },
+ 'large-v3-turbo': {
+ url: 'https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-large-v3-turbo.bin',
+ sha256: '1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69',
+ size: 1_624_555_275,
+ environment: 'OPENCREATOR_WHISPERCPP_LARGE_V3_TURBO_MODEL'
}
}
} as const;
@@ -361,14 +367,28 @@ async function installWhisperCppModel(
await mkdir(modelRoot, { recursive: true });
const model = join(modelRoot, `ggml-${input.model}.bin`);
const marker = join(modelRoot, `.opencreator-${input.model}.json`);
- await rm(model, { force: true });
- await promoteDependencyPath(source, model);
- await writeFile(marker, `${JSON.stringify({
- version: 1,
- model: input.model,
- sha256: release.sha256,
- size: release.size
- }, null, 2)}\n`, { mode: 0o600 });
+ const backupModel = join(staging, `previous-${input.model}.bin`);
+ const backupMarker = join(staging, `previous-${input.model}.json`);
+ let movedModel = false;
+ let movedMarker = false;
+ let wroteMarker = false;
+ try {
+ movedMarker = await moveIfExists(marker, backupMarker);
+ movedModel = await moveIfExists(model, backupModel);
+ await promoteDependencyPath(source, model);
+ wroteMarker = true;
+ await writeFile(marker, `${JSON.stringify({
+ version: 1,
+ model: input.model,
+ sha256: release.sha256,
+ size: release.size
+ }, null, 2)}\n`, { mode: 0o600 });
+ } catch (cause) {
+ if (wroteMarker) await rm(marker, { force: true });
+ if (movedModel) await promoteDependencyPath(backupModel, model);
+ if (movedMarker) await promoteDependencyPath(backupMarker, marker);
+ throw cause;
+ }
}
async function isWhisperCppInstalled(root: string, model: WhisperCppModel): Promise<boolean> {
@@ -771,6 +791,18 @@ export async function promoteDependencyPath(
}
}
+async function moveIfExists(source: string, destination: string): Promise<boolean> {
+ try {
+ await promoteDependencyPath(source, destination);
+ return true;
+ } catch (error) {
+ if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT') {
+ return false;
+ }
+ throw error;
+ }
+}
+
function isTransientRenameError(error: unknown): boolean {
if (!(error instanceof Error) || !('code' in error)) return false;
return ['EACCES', 'EBUSY', 'EPERM'].includes(String(error.code));
diff --git a/apps/daemon/test/unit/creator-krillin-capabilities.test.ts b/apps/daemon/test/unit/creator-krillin-capabilities.test.ts
index b1e72a8f..e10213db 100644
--- a/apps/daemon/test/unit/creator-krillin-capabilities.test.ts
+++ b/apps/daemon/test/unit/creator-krillin-capabilities.test.ts
@@ -55,7 +55,10 @@ describe('KrillinAI transcription capabilities', () => {
provider: 'whisper.cpp',
available: true,
gpuAcceleration: false,
- models: ['tiny', 'medium', 'large-v2']
+ models: ['tiny', 'medium', 'large-v2', 'large-v3-turbo'],
+ modelDetails: expect.objectContaining({
+ 'large-v3-turbo': { diskBytes: 1624555275 }
+ })
}),
expect.objectContaining({
provider: 'whisperkit',
diff --git a/apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts b/apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts
index 722f41e7..49f28588 100644
--- a/apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts
+++ b/apps/daemon/test/unit/creator-krillin-cli-protocol.test.ts
@@ -1,4 +1,5 @@
import { dirname } from 'node:path';
+import { createDefaultCreatorServicesConfig } from '@opencreator/protocol';
import { describe, expect, it } from 'vitest';
import {
buildKrillinCliCommandArguments,
@@ -7,6 +8,7 @@ import {
parseKrillinCliProgressFrame,
resolveKrillinCliSource
} from '../../src/creator/krillin/cli-runner.js';
+import { createKrillinConfigToml } from '../../src/creator/krillin/config-bridge.js';
describe('KrillinAI CLI protocol', () => {
it.each(['source_subtitle', 'target_subtitle'])('passes %s as an explicit CLI input', kind => {
@@ -98,4 +100,12 @@ describe('KrillinAI CLI protocol', () => {
'vertical_subtitle'
]);
});
+
+ it('passes large-v3-turbo to the Whisper.cpp runtime without remapping', () => {
+ const config = createDefaultCreatorServicesConfig();
+ config.transcription.provider = 'whisper.cpp';
+ config.transcription.whisperCpp.model = 'large-v3-turbo';
+
+ expect(createKrillinConfigToml(config)).toContain('model = "large-v3-turbo"');
+ });
});
diff --git a/apps/daemon/test/unit/creator-krillin-dependency-loader.test.ts b/apps/daemon/test/unit/creator-krillin-dependency-loader.test.ts
index 67f9d80f..97cafd30 100644
--- a/apps/daemon/test/unit/creator-krillin-dependency-loader.test.ts
+++ b/apps/daemon/test/unit/creator-krillin-dependency-loader.test.ts
@@ -189,6 +189,47 @@ describe('KrillinAI on-demand dependency loader', () => {
expect(phases).toContain('model');
});
+ it('passes large-v3-turbo through the dependency loader', async () => {
+ let installed = false;
+ let selectedModel = '';
+ const loader = createKrillinDependencyLoader({
+ root: 'C:\\opencreator-test-dependencies', platform: 'win32', arch: 'x64',
+ whisperCppInstaller: {
+ async isInstalled() { return installed; },
+ async install(input) {
+ selectedModel = input.model;
+ installed = true;
+ }
+ }
+ });
+ const config = createDefaultCreatorServicesConfig();
+ config.transcription.provider = 'whisper.cpp';
+ config.transcription.whisperCpp.model = 'large-v3-turbo';
+
+ await loader.ensure({ config, signal: new AbortController().signal, reportProgress() {} });
+
+ expect(selectedModel).toBe('large-v3-turbo');
+ });
+
+ it('does not accept a large-v3-turbo install that fails post-install verification', async () => {
+ const loader = createKrillinDependencyLoader({
+ root: 'C:\\opencreator-test-dependencies', platform: 'win32', arch: 'x64',
+ whisperCppInstaller: {
+ async isInstalled() { return false; },
+ async install() {}
+ }
+ });
+ const config = createDefaultCreatorServicesConfig();
+ config.transcription.provider = 'whisper.cpp';
+ config.transcription.whisperCpp.model = 'large-v3-turbo';
+
+ await expect(loader.ensure({
+ config,
+ signal: new AbortController().signal,
+ reportProgress() {}
+ })).rejects.toThrow('dependency verification failed after installation');
+ });
+
it('rejects Whisper.cpp outside Windows x64 without installing anything', async () => {
let installs = 0;
const loader = createKrillinDependencyLoader({
diff --git a/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx b/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx
index dde09932..396bd45a 100644
--- a/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx
+++ b/apps/web/src/features/settings/CreatorServicesSettingsView.test.tsx
@@ -314,6 +314,7 @@ describe('CreatorServicesSettingsView', () => {
await user.click(screen.getByRole('button', { name: '本地 Whisper' }));
expect(screen.getByRole('combobox', { name: '语音识别服务' })).toHaveTextContent('Whisper.cpp');
expect(screen.getByText('tiny')).toBeInTheDocument();
+ expect(screen.getByText(/预计占用磁盘 74 MiB/)).toBeInTheDocument();
expect(screen.queryByText('WhisperKit')).not.toBeInTheDocument();
expect(screen.queryByText('FasterWhisper')).not.toBeInTheDocument();
await user.click(screen.getByRole('button', { name: '保存配置' }));
@@ -538,7 +539,13 @@ function runtimeCapabilities(
provider: 'whisper.cpp',
kind: 'local',
available: whisperCppAvailable,
- models: ['tiny', 'medium', 'large-v2'],
+ models: ['tiny', 'medium', 'large-v2', 'large-v3-turbo'],
+ modelDetails: {
+ tiny: { diskBytes: 77691713 },
+ medium: { diskBytes: 1533763059 },
+ 'large-v2': { diskBytes: 3094623691 },
+ 'large-v3-turbo': { diskBytes: 1624555275 }
+ },
gpuAcceleration: false,
...(whisperCppAvailable ? {} : { unavailableReason: 'unsupported_platform' as const })
},
diff --git a/apps/web/src/features/settings/CreatorServicesSettingsView.tsx b/apps/web/src/features/settings/CreatorServicesSettingsView.tsx
index 3cc7eb90..ccf13307 100644
--- a/apps/web/src/features/settings/CreatorServicesSettingsView.tsx
+++ b/apps/web/src/features/settings/CreatorServicesSettingsView.tsx
@@ -741,7 +741,7 @@ function TranscriptionSettings(props: SettingsGroupProps & {
value={props.config.transcription.whisperCpp.model}
options={(selectedCapability?.models ?? []).map(model => [model, model])}
onChange={value => props.update(config => {
- config.transcription.whisperCpp.model = value as 'tiny' | 'medium' | 'large-v2';
+ config.transcription.whisperCpp.model = value as 'tiny' | 'medium' | 'large-v2' | 'large-v3-turbo';
})}
/>
) : null}
@@ -785,6 +785,9 @@ function TranscriptionSettings(props: SettingsGroupProps & {
)}
</p>
) : null}
+ {selectedCapability?.kind === 'local' && selectedCapability.available
+ ? <TranscriptionModelDetails capability={selectedCapability} model={selectedTranscriptionModel(props.config)} />
+ : null}
</SettingsFieldset>
);
}
@@ -1511,6 +1514,25 @@ function ReadonlyModelField(props: { label: string; value: string }) {
);
}
+function TranscriptionModelDetails(props: {
+ capability: CreatorTranscriptionProviderCapability;
+ model: string;
+}) {
+ const l = useLocalizedCopy();
+ const diskBytes = props.capability.modelDetails?.[props.model]?.diskBytes;
+ if (diskBytes === undefined) return null;
+ return (
+ <p className="creator-services-inline-note">
+ {l(`本地执行 · 预计占用磁盘 ${formatDiskSize(diskBytes)}`, `Local execution · estimated disk usage ${formatDiskSize(diskBytes)}`)}
+ </p>
+ );
+}
+
+function formatDiskSize(bytes: number): string {
+ const gibibytes = bytes / (1024 ** 3);
+ return gibibytes >= 1 ? `${gibibytes.toFixed(2)} GiB` : `${(bytes / (1024 ** 2)).toFixed(0)} MiB`;
+}
+
function ToggleField(props: {
label: string;
description?: string;
diff --git a/packages/protocol/src/creator-services.ts b/packages/protocol/src/creator-services.ts
index c0887c6a..a4d98d08 100644
--- a/packages/protocol/src/creator-services.ts
+++ b/packages/protocol/src/creator-services.ts
@@ -111,7 +111,7 @@ export type CreatorServicesConfig = {
openai: OpenAiCompatibleConfig;
fasterWhisper: { model: 'tiny' | 'medium' | 'large-v2' };
whisperKit: { model: 'large-v2' };
- whisperCpp: { model: 'tiny' | 'medium' | 'large-v2' };
+ whisperCpp: { model: 'tiny' | 'medium' | 'large-v2' | 'large-v3-turbo' };
aliyun: {
oss: AliyunOssConfig;
speech: AliyunSpeechConfig;
@@ -150,6 +150,7 @@ export type CreatorTranscriptionProviderCapability = {
kind: 'cloud' | 'local';
available: boolean;
models: string[];
+ modelDetails?: Record<string, { diskBytes: number }>;
gpuAcceleration: boolean;
unavailableReason?: 'unsupported_platform' | 'installer_unavailable';
};
diff --git a/runtime/krillinai/config/config-example.toml b/runtime/krillinai/config/config-example.toml
index c3d3b9b7..0e352f71 100644
--- a/runtime/krillinai/config/config-example.toml
+++ b/runtime/krillinai/config/config-example.toml
@@ -35,7 +35,7 @@
[transcribe.whisperkit]
model = "large-v2" # whisperkit的本地模型可选值:large-v2
[transcribe.whispercpp]
- model = "tiny" # whispercpp的本地模型可选值:tiny,medium,large-v2
+ model = "tiny" # whispercpp的本地模型可选值:tiny,medium,large-v2,large-v3-turbo
[transcribe.aliyun] # provider选aliyun这块就都要填
[transcribe.aliyun.oss]
access_key_id = ""
diff --git a/runtime/krillinai/config/config.go b/runtime/krillinai/config/config.go
index fcf4480e..5524d0d0 100644
--- a/runtime/krillinai/config/config.go
+++ b/runtime/krillinai/config/config.go
@@ -239,11 +239,11 @@ func ValidateTranscriptionConfig() error {
return errors.New("检测到开启了whisperkit,但模型选型配置不正确,请检查配置")
}
case "whispercpp":
- if runtime.GOOS != "windows" { // 当前先仅支持win,模型仅支持large-v2,最小化产品
+ if runtime.GOOS != "windows" { // 当前先仅支持Windows,模型由受控 capability catalog 提供
log.GetLogger().Error("whispercpp only support windows", zap.String("current os", runtime.GOOS))
return fmt.Errorf("whispercpp only support windows")
}
- if Conf.Transcribe.Whispercpp.Model != "tiny" && Conf.Transcribe.Whispercpp.Model != "medium" && Conf.Transcribe.Whispercpp.Model != "large-v2" {
+ if Conf.Transcribe.Whispercpp.Model != "tiny" && Conf.Transcribe.Whispercpp.Model != "medium" && Conf.Transcribe.Whispercpp.Model != "large-v2" && Conf.Transcribe.Whispercpp.Model != "large-v3-turbo" {
return errors.New("检测到开启了whisper.cpp,但模型选型配置不正确,请检查配置")
}
case "aliyun":
diff --git a/runtime/krillinai/config/config_validation_test.go b/runtime/krillinai/config/config_validation_test.go
index 49007542..10320366 100644
--- a/runtime/krillinai/config/config_validation_test.go
+++ b/runtime/krillinai/config/config_validation_test.go
@@ -37,3 +37,19 @@ func TestValidateTranscriptionConfigAcceptsTinyWhisperCpp(t *testing.T) {
t.Fatalf("ValidateTranscriptionConfig() error = %v, want platform error", err)
}
}
+
+func TestValidateTranscriptionConfigAcceptsLargeV3TurboWhisperCpp(t *testing.T) {
+ previous := Conf
+ t.Cleanup(func() { Conf = previous })
+
+ Conf.Transcribe.Provider = "whispercpp"
+ Conf.Transcribe.Whispercpp.Model = "large-v3-turbo"
+
+ err := ValidateTranscriptionConfig()
+ if runtime.GOOS == "windows" && err != nil {
+ t.Fatalf("ValidateTranscriptionConfig() error = %v", err)
+ }
+ if runtime.GOOS != "windows" && (err == nil || !strings.Contains(err.Error(), "only support windows")) {
+ t.Fatalf("ValidateTranscriptionConfig() error = %v, want platform error", err)
+ }
+}