Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: add whisperkit support

沈浩c6008ae5bbbf
diff --git a/.DS_Store b/.DS_Store
deleted file mode 100644
index 4015a0e3..00000000
Binary files a/.DS_Store and /dev/null differ
diff --git a/config/config-example.toml b/config/config-example.toml
index a3026ae7..5891113c 100644
--- a/config/config-example.toml
+++ b/config/config-example.toml
@@ -2,7 +2,7 @@
segment_duration = 5 # 音频切分处理间隔,单位:分钟,建议值:5-10,如果视频中话语较少可以适当提高
translate_parallel_num = 5 # 并发进行模型转录和翻译的数量上限,建议值:5,如果使用了本地模型,该项自动不生效
proxy = "" # 网络代理地址,格式如http://127.0.0.1:7890,可不填
- transcribe_provider = "openai" # 语音识别,当前可选值:openai,fasterwhisper,whisperkit,aliyun。fasterwhisper不支持macOS
+ transcribe_provider = "openai" # 语音识别,当前可选值:openai,fasterwhisper,whisperkit,aliyun。(fasterwhisper不支持macOS,whisperkit只支持macOS)
llm_provider = "openai" # LLM,当前可选值:openai,aliyun
[server]
@@ -11,7 +11,7 @@
# 下方的配置非必填,请结合上方的选项和文档说明进行配置
[local_model]
- whisperkit = "large-v2" # fasterwhisper的本地模型选择,可选值:tiny,medium,large-v2。建议medium及以上
+ whisperkit = "medium" # fasterwhisper的本地模型可选值:tiny,medium,large-v2。whisperkit的本地模型可选值:large-v2,建议medium及以上
[openai]
base_url = "" # OpenAI API 自定义base url,可配合转发站密钥使用,留空为默认API地址
diff --git a/config/config.go b/config/config.go
index a44a02fa..9d5ec29d 100644
--- a/config/config.go
+++ b/config/config.go
@@ -2,10 +2,13 @@ package config
import (
"errors"
+ "fmt"
"github.com/BurntSushi/toml"
+ "go.uber.org/zap"
"krillin-ai/log"
"net/url"
"os"
+ "runtime"
"strconv"
)
@@ -24,7 +27,7 @@ type Server struct {
}
type LocalModel struct {
- WhisperKit string `toml:"whisperkit"`
+ Whisper string `toml:"whisper"`
}
type OpenAiWhisper struct {
@@ -81,7 +84,7 @@ var Conf = Config{
Port: 8888,
},
LocalModel: LocalModel{
- WhisperKit: "large-v2",
+ Whisper: "large-v2",
},
}
@@ -119,8 +122,8 @@ func loadFromEnv() {
}
// LocalModel 配置
- if v := os.Getenv("KRILLIN_FASTER_WHISPER"); v != "" {
- Conf.LocalModel.WhisperKit = v
+ if v := os.Getenv("KRILLIN_LOCAL_WHISPER"); v != "" {
+ Conf.LocalModel.Whisper = v
}
// OpenAI 配置
@@ -178,8 +181,16 @@ func validateConfig() error {
if Conf.Openai.Whisper.ApiKey == "" {
return errors.New("使用OpenAI转写服务需要配置 OpenAI API Key")
}
+ case "fasterwhisper":
+ if Conf.LocalModel.Whisper != "tiny" && Conf.LocalModel.Whisper != "medium" && Conf.LocalModel.Whisper != "large-v2" {
+ return errors.New("检测到开启了fasterwhisper,但模型选型配置不正确,请检查配置")
+ }
case "whisperkit":
- if Conf.LocalModel.WhisperKit != "tiny" && Conf.LocalModel.WhisperKit != "medium" && Conf.LocalModel.WhisperKit != "large-v2" {
+ if runtime.GOOS != "darwin" {
+ log.GetLogger().Error("whisperkit只支持macos", zap.String("当前系统", runtime.GOOS))
+ return fmt.Errorf("whisperkit只支持macos")
+ }
+ if Conf.LocalModel.Whisper != "large-v2" {
return errors.New("检测到开启了whisperkit,但模型选型配置不正确,请检查配置")
}
case "aliyun":
@@ -225,7 +236,7 @@ func LoadConfig() error {
}
// 本地模型不并发
- if Conf.App.TranscribeProvider == "whisperkit" {
+ if Conf.App.TranscribeProvider == "fasterwhisper" || Conf.App.TranscribeProvider == "whisperkit" {
Conf.App.TranslateParallelNum = 1
}
diff --git a/docs/docker.md b/docs/docker.md
index 13382171..33f7d1e0 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -39,7 +39,7 @@ KrillinAI 支持通过环境变量来代替配置文件。所有环境变量都
- `KRILLIN_SERVER_PORT`: 服务器监听端口(整数,默认值: 8888)
### 本地模型配置
-- `KRILLIN_FASTER_WHISPER`: Faster Whisper 所使用的模型(当 transcribe_provider 为 fasterwhisper 时有效,默认值: medium,可选: tiny/medium/large-v2)
+- `KRILLIN_LOCAL_WHISPER`: Local Whisper 所使用的模型(当 transcribe_provider 为 fasterwhisper 或 whisperkit 时有效,默认值: medium,可选: tiny/medium/large-v2)
### OpenAI 配置
- `KRILLIN_OPENAI_BASE_URL`: OpenAI API 基础 URL(可选,默认值: 官方 API 地址)
diff --git a/internal/deps/checker.go b/internal/deps/checker.go
index ded65f00..ef4194e2 100644
--- a/internal/deps/checker.go
+++ b/internal/deps/checker.go
@@ -29,8 +29,28 @@ func CheckDependency() error {
log.GetLogger().Error("yt-dlp环境准备失败", zap.Error(err))
return err
}
+ if config.Conf.App.TranscribeProvider == "fasterwhisper" {
+ err = checkFasterWhisper()
+ if err != nil {
+ log.GetLogger().Error("fasterwhisper环境准备失败", zap.Error(err))
+ return err
+ }
+ err = checkModel("fasterwhisper")
+ if err != nil {
+ log.GetLogger().Error("本地模型环境准备失败", zap.Error(err))
+ return err
+ }
+ }
if config.Conf.App.TranscribeProvider == "whisperkit" {
- checkWhisperKit()
+ if err = checkWhisperKit(); err != nil {
+ log.GetLogger().Error("whisperkit环境准备失败", zap.Error(err))
+ return err
+ }
+ err = checkModel("whisperkit")
+ if err != nil {
+ log.GetLogger().Error("本地模型环境准备失败", zap.Error(err))
+ return err
+ }
}
return nil
@@ -234,8 +254,126 @@ func checkAndDownloadYtDlp() error {
}
// 检测faster whisper
+func checkFasterWhisper() error {
+ var (
+ filePath string
+ err error
+ )
+ if runtime.GOOS == "windows" {
+ filePath = "./bin/faster-whisper/Faster-Whisper-XXL/faster-whisper-xxl.exe"
+ } else if runtime.GOOS == "linux" {
+ filePath = "./bin/faster-whisper/Whisper-Faster-XXL/whisper-faster-xxl"
+ } else {
+ return fmt.Errorf("fasterwhisper不支持你当前的操作系统: %s,请选择其它transcription provider", runtime.GOOS)
+ }
+ if _, err = os.Stat(filePath); os.IsNotExist(err) {
+ log.GetLogger().Info("没有找到faster-whisper,即将开始自动下载,文件较大请耐心等待")
+ err = os.MkdirAll("./bin", 0755)
+ if err != nil {
+ log.GetLogger().Error("创建./bin目录失败", zap.Error(err))
+ return err
+ }
+ var downloadUrl string
+ if runtime.GOOS == "windows" {
+ downloadUrl = "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/Faster-Whisper-XXL_r194.5_windows.zip"
+ } else {
+ downloadUrl = "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/Faster-Whisper-XXL_r192.3.1_linux.zip"
+ }
+ err = util.DownloadFile(downloadUrl, "./bin/faster-whisper.zip", config.Conf.App.Proxy)
+ if err != nil {
+ log.GetLogger().Error("下载faster-whisper失败", zap.Error(err))
+ return err
+ }
+ log.GetLogger().Info("开始解压faster-whisper")
+ err = util.Unzip("./bin/faster-whisper.zip", "./bin/faster-whisper/")
+ if err != nil {
+ log.GetLogger().Error("解压faster-whisper失败", zap.Error(err))
+ return err
+ }
+ }
+ if runtime.GOOS != "windows" {
+ err = os.Chmod(filePath, 0755)
+ if err != nil {
+ log.GetLogger().Error("设置文件权限失败", zap.Error(err))
+ return err
+ }
+ }
+ storage.FasterwhisperPath = filePath
+ log.GetLogger().Info("faster-whisper检查完成", zap.String("路径", filePath))
+ return nil
+}
+
+// 检测本地模型
+func checkModel(whisperType string) error {
+ var err error
+ if _, err = os.Stat("./models/whisperkit"); os.IsNotExist(err) {
+ err = os.MkdirAll("./models/whisperkit", 0755)
+ if err != nil {
+ log.GetLogger().Error("创建./models目录失败", zap.Error(err))
+ return err
+ }
+ }
+ // 模型文件
+ model := config.Conf.LocalModel.Whisper
+ var modelPath string // cli中使用的model path
+ switch whisperType {
+ case "fasterwhisper":
+ modelPath = fmt.Sprintf("./models/faster-whisper-%s/model.bin", model)
+ if _, err = os.Stat(modelPath); os.IsNotExist(err) {
+ // 下载
+ log.GetLogger().Info(fmt.Sprintf("没有找到模型文件%s,即将开始自动下载", modelPath))
+ downloadUrl := fmt.Sprintf("https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/faster-whisper-%s.zip", model)
+ err = util.DownloadFile(downloadUrl, fmt.Sprintf("./models/faster-whisper-%s.zip", model), config.Conf.App.Proxy)
+ if err != nil {
+ log.GetLogger().Error("下载fasterwhisper模型失败", zap.Error(err))
+ return err
+ }
+ err = util.Unzip(fmt.Sprintf("./models/faster-whisper-%s.zip", model), fmt.Sprintf("./models/faster-whisper-%s/", model))
+ if err != nil {
+ log.GetLogger().Error("解压模型失败", zap.Error(err))
+ return err
+ }
+ log.GetLogger().Info("模型下载完成", zap.String("路径", modelPath))
+ }
+ case "whisperkit":
+ modelPath = "./models/whisperkit/openai_whisper-large-v2"
+ files, _ := os.ReadDir(modelPath)
+ if len(files) == 0 {
+ log.GetLogger().Info("没有找到whisperkit模型,即将开始自动下载")
+ downloadUrl := "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/whisperkit-large-v2.zip"
+ err = util.DownloadFile(downloadUrl, "./models/whisperkit/openai_whisper-large-v2.zip", config.Conf.App.Proxy)
+ if err != nil {
+ log.GetLogger().Info("下载whisperkit模型失败", zap.Error(err))
+ return err
+ }
+ err = util.Unzip("./models/whisperkit/openai_whisper-large-v2.zip", "./models/whisperkit/")
+ if err != nil {
+ log.GetLogger().Error("解压whisperkit模型失败", zap.Error(err))
+ return err
+ }
+ log.GetLogger().Info("whisperkit模型下载完成", zap.String("路径", modelPath))
+ }
+ }
+
+ log.GetLogger().Info("模型检查完成", zap.String("路径", modelPath))
+ return nil
+}
+
+// 检测whisperkit
func checkWhisperKit() error {
+ cmd := exec.Command("which", "whisperkit-cli")
+ err := cmd.Run()
+ if err != nil {
+ log.GetLogger().Info("没有找到whisperkit-cli,即将开始自动安装")
+ cmd = exec.Command("brew", "install", "whisperkit-cli")
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ log.GetLogger().Error("whisperkit-cli 安装失败", zap.String("info", string(output)), zap.Error(err))
+ return err
+ }
+ log.GetLogger().Info("whisperkit-cli 安装成功")
+ }
storage.WhisperKitPath = "whisperkit-cli"
-
+ log.GetLogger().Info("检测到whisperkit-cli已安装")
return nil
}
diff --git a/internal/service/init.go b/internal/service/init.go
index 5fe25180..180eaa04 100644
--- a/internal/service/init.go
+++ b/internal/service/init.go
@@ -6,9 +6,10 @@ import (
"krillin-ai/internal/types"
"krillin-ai/log"
"krillin-ai/pkg/aliyun"
- "krillin-ai/pkg/whisperkit"
+ "krillin-ai/pkg/fasterwhisper"
"krillin-ai/pkg/openai"
"krillin-ai/pkg/whisper"
+ "krillin-ai/pkg/whisperkit"
)
type Service struct {
@@ -28,8 +29,10 @@ func NewService() *Service {
transcriber = whisper.NewClient(config.Conf.Openai.Whisper.BaseUrl, config.Conf.Openai.Whisper.ApiKey, config.Conf.App.Proxy)
case "aliyun":
transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)
+ case "fasterwhisper":
+ transcriber = fasterwhisper.NewFastwhisperProcessor(config.Conf.LocalModel.Whisper)
case "whisperkit":
- transcriber = whisperkit.NewWhisperKitProcessor(config.Conf.LocalModel.WhisperKit)
+ transcriber = whisperkit.NewWhisperKitProcessor(config.Conf.LocalModel.Whisper)
}
log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.App.TranscribeProvider))
diff --git a/internal/storage/bin.go b/internal/storage/bin.go
index ddcb4bae..0ea6c84a 100644
--- a/internal/storage/bin.go
+++ b/internal/storage/bin.go
@@ -4,5 +4,6 @@ var (
FfmpegPath string
FfprobePath string
YtdlpPath string
- WhisperKitPath string
+ FasterwhisperPath string
+ WhisperKitPath string
)
diff --git a/pkg/fasterwhisper/init.go b/pkg/fasterwhisper/init.go
index 6db7a01d..a3328721 100644
--- a/pkg/fasterwhisper/init.go
+++ b/pkg/fasterwhisper/init.go
@@ -1 +1,12 @@
package fasterwhisper
+
+type FastwhisperProcessor struct {
+ WorkDir string // 生成中间文件的目录
+ Model string
+}
+
+func NewFastwhisperProcessor(model string) *FastwhisperProcessor {
+ return &FastwhisperProcessor{
+ Model: model,
+ }
+}
diff --git a/pkg/fasterwhisper/transcription.go b/pkg/fasterwhisper/transcription.go
index 6db7a01d..23b151ee 100644
--- a/pkg/fasterwhisper/transcription.go
+++ b/pkg/fasterwhisper/transcription.go
@@ -1 +1,86 @@
package fasterwhisper
+
+import (
+ "encoding/json"
+ "go.uber.org/zap"
+ "krillin-ai/internal/storage"
+ "krillin-ai/internal/types"
+ "krillin-ai/log"
+ "krillin-ai/pkg/util"
+ "os"
+ "os/exec"
+ "strings"
+)
+
+func (c *FastwhisperProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
+ cmdArgs := []string{
+ "--model_dir", "./models/",
+ "--model", c.Model,
+ "--one_word", "2",
+ "--output_format", "json",
+ "--language", language,
+ "--output_dir", workDir,
+ audioFile,
+ }
+ cmd := exec.Command(storage.FasterwhisperPath, cmdArgs...)
+ log.GetLogger().Info("FastwhisperProcessor转录开始", zap.String("cmd", cmd.String()))
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ log.GetLogger().Error("FastwhisperProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))
+ return nil, err
+ }
+ log.GetLogger().Info("FastwhisperProcessor转录json生成完毕", zap.String("audio file", audioFile))
+
+ var result types.FasterWhisperOutput
+ fileData, err := os.Open(util.ChangeFileExtension(audioFile, ".json"))
+ if err != nil {
+ log.GetLogger().Error("FastwhisperProcessor 打开json文件失败", zap.Error(err))
+ return nil, err
+ }
+ defer fileData.Close()
+ decoder := json.NewDecoder(fileData)
+ if err = decoder.Decode(&result); err != nil {
+ log.GetLogger().Error("FastwhisperProcessor 解析json文件失败", zap.Error(err))
+ return nil, err
+ }
+
+ var (
+ transcriptionData types.TranscriptionData
+ num int
+ )
+ for _, segment := range result.Segments {
+ transcriptionData.Text += strings.ReplaceAll(segment.Text, "—", " ") // 连字符处理,因为模型存在很多错误添加到连字符
+ for _, word := range segment.Words {
+ if strings.Contains(word.Word, "—") {
+ // 对称切分
+ mid := (word.Start + word.End) / 2
+ seperatedWords := strings.Split(word.Word, "—")
+ transcriptionData.Words = append(transcriptionData.Words, []types.Word{
+ {
+ Num: num,
+ Text: util.CleanPunction(strings.TrimSpace(seperatedWords[0])),
+ Start: word.Start,
+ End: mid,
+ },
+ {
+ Num: num + 1,
+ Text: util.CleanPunction(strings.TrimSpace(seperatedWords[1])),
+ Start: mid,
+ End: word.End,
+ },
+ }...)
+ num += 2
+ } else {
+ transcriptionData.Words = append(transcriptionData.Words, types.Word{
+ Num: num,
+ Text: util.CleanPunction(strings.TrimSpace(word.Word)),
+ Start: word.Start,
+ End: word.End,
+ })
+ num++
+ }
+ }
+ }
+ log.GetLogger().Info("FastwhisperProcessor转录成功")
+ return &transcriptionData, nil
+}
diff --git a/pkg/whisperkit/transcription.go b/pkg/whisperkit/transcription.go
index 81378831..7dd8236c 100644
--- a/pkg/whisperkit/transcription.go
+++ b/pkg/whisperkit/transcription.go
@@ -16,7 +16,7 @@ import (
func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
cmdArgs := []string{
"transcribe",
- "--model-path", "/Users/outisli/AI_Models/whisperkit-coreml/openai_whisper-large-v2",
+ "--model-path", "./models/whisperkit/openai_whisper-large-v2",
"--audio-encoder-compute-units", "all",
"--text-decoder-compute-units", "all",
"--language", language,