Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

20250304-support mac using WhisperKit

OutisLi256821363f1f
diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 00000000..4015a0e3
Binary files /dev/null and b/.DS_Store differ
diff --git a/config/config.go b/config/config.go
index 1dde6bac..a44a02fa 100644
--- a/config/config.go
+++ b/config/config.go
@@ -24,7 +24,7 @@ type Server struct {
}
type LocalModel struct {
- FasterWhisper string `toml:"faster_whisper"`
+ WhisperKit string `toml:"whisperkit"`
}
type OpenAiWhisper struct {
@@ -81,7 +81,7 @@ var Conf = Config{
Port: 8888,
},
LocalModel: LocalModel{
- FasterWhisper: "medium",
+ WhisperKit: "large-v2",
},
}
@@ -120,7 +120,7 @@ func loadFromEnv() {
// LocalModel 配置
if v := os.Getenv("KRILLIN_FASTER_WHISPER"); v != "" {
- Conf.LocalModel.FasterWhisper = v
+ Conf.LocalModel.WhisperKit = v
}
// OpenAI 配置
@@ -178,9 +178,9 @@ func validateConfig() error {
if Conf.Openai.Whisper.ApiKey == "" {
return errors.New("使用OpenAI转写服务需要配置 OpenAI API Key")
}
- case "fasterwhisper":
- if Conf.LocalModel.FasterWhisper != "tiny" && Conf.LocalModel.FasterWhisper != "medium" && Conf.LocalModel.FasterWhisper != "large-v2" {
- return errors.New("检测到开启了fasterwhisper,但模型选型配置不正确,请检查配置")
+ case "whisperkit":
+ if Conf.LocalModel.WhisperKit != "tiny" && Conf.LocalModel.WhisperKit != "medium" && Conf.LocalModel.WhisperKit != "large-v2" {
+ return errors.New("检测到开启了whisperkit,但模型选型配置不正确,请检查配置")
}
case "aliyun":
if Conf.Aliyun.Speech.AccessKeyId == "" || Conf.Aliyun.Speech.AccessKeySecret == "" || Conf.Aliyun.Speech.AppKey == "" {
@@ -225,7 +225,7 @@ func LoadConfig() error {
}
// 本地模型不并发
- if Conf.App.TranscribeProvider == "fasterwhisper" {
+ if Conf.App.TranscribeProvider == "whisperkit" {
Conf.App.TranslateParallelNum = 1
}
diff --git a/internal/deps/checker.go b/internal/deps/checker.go
index 8833aced..ded65f00 100644
--- a/internal/deps/checker.go
+++ b/internal/deps/checker.go
@@ -2,7 +2,6 @@ package deps
import (
"fmt"
- "go.uber.org/zap"
"krillin-ai/config"
"krillin-ai/internal/storage"
"krillin-ai/log"
@@ -10,6 +9,8 @@ import (
"os"
"os/exec"
"runtime"
+
+ "go.uber.org/zap"
)
func CheckDependency() error {
@@ -28,17 +29,8 @@ func CheckDependency() error {
log.GetLogger().Error("yt-dlp环境准备失败", zap.Error(err))
return err
}
- if config.Conf.App.TranscribeProvider == "fasterwhisper" {
- err = checkFasterWhisper()
- if err != nil {
- log.GetLogger().Error("fasterwhisper环境准备失败", zap.Error(err))
- return err
- }
- err = checkModel()
- if err != nil {
- log.GetLogger().Error("本地模型环境准备失败", zap.Error(err))
- return err
- }
+ if config.Conf.App.TranscribeProvider == "whisperkit" {
+ checkWhisperKit()
}
return nil
@@ -242,84 +234,8 @@ func checkAndDownloadYtDlp() error {
}
// 检测faster whisper
-func checkFasterWhisper() error {
- var (
- filePath string
- err error
- )
- if runtime.GOOS == "windows" {
- filePath = "./bin/faster-whisper/Faster-Whisper-XXL/faster-whisper-xxl.exe"
- } else if runtime.GOOS == "linux" {
- filePath = "./bin/faster-whisper/Whisper-Faster-XXL/whisper-faster-xxl"
- } else {
- return fmt.Errorf("fasterwhisper不支持你当前的操作系统: %s,请选择其它transcription provider", runtime.GOOS)
- }
- if _, err = os.Stat(filePath); os.IsNotExist(err) {
- log.GetLogger().Info("没有找到faster-whisper,即将开始自动下载,文件较大请耐心等待")
- err = os.MkdirAll("./bin", 0755)
- if err != nil {
- log.GetLogger().Error("创建./bin目录失败", zap.Error(err))
- return err
- }
- var downloadUrl string
- if runtime.GOOS == "windows" {
- downloadUrl = "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/Faster-Whisper-XXL_r194.5_windows.zip"
- } else {
- downloadUrl = "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/Faster-Whisper-XXL_r192.3.1_linux.zip"
- }
- err = util.DownloadFile(downloadUrl, "./bin/faster-whisper.zip", config.Conf.App.Proxy)
- if err != nil {
- log.GetLogger().Error("下载faster-whisper失败", zap.Error(err))
- return err
- }
- log.GetLogger().Info("开始解压faster-whisper")
- err = util.Unzip("./bin/faster-whisper.zip", "./bin/faster-whisper/")
- if err != nil {
- log.GetLogger().Error("解压faster-whisper失败", zap.Error(err))
- return err
- }
- }
- if runtime.GOOS != "windows" {
- err = os.Chmod(filePath, 0755)
- if err != nil {
- log.GetLogger().Error("设置文件权限失败", zap.Error(err))
- return err
- }
- }
- storage.FasterwhisperPath = filePath
- log.GetLogger().Info("faster-whisper检查完成", zap.String("路径", filePath))
- return nil
-}
+func checkWhisperKit() error {
+ storage.WhisperKitPath = "whisperkit-cli"
-// 检测本地模型
-func checkModel() error {
- var err error
- if _, err = os.Stat("./models"); os.IsNotExist(err) {
- err = os.MkdirAll("./models", 0755)
- if err != nil {
- log.GetLogger().Error("创建./models目录失败", zap.Error(err))
- return err
- }
- }
- // 模型文件
- model := config.Conf.LocalModel.FasterWhisper
- modelFile := fmt.Sprintf("./models/faster-whisper-%s/model.bin", model)
- if _, err = os.Stat(modelFile); os.IsNotExist(err) {
- // 下载
- log.GetLogger().Info(fmt.Sprintf("没有找到模型文件%s,即将开始自动下载", modelFile))
- downloadUrl := fmt.Sprintf("https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/faster-whisper-%s.zip", model)
- err = util.DownloadFile(downloadUrl, fmt.Sprintf("./models/faster-whisper-%s.zip", model), config.Conf.App.Proxy)
- if err != nil {
- log.GetLogger().Error("下载模型失败", zap.Error(err))
- return err
- }
- err = util.Unzip(fmt.Sprintf("./models/faster-whisper-%s.zip", model), fmt.Sprintf("./models/faster-whisper-%s/", model))
- if err != nil {
- log.GetLogger().Error("解压模型失败", zap.Error(err))
- return err
- }
- log.GetLogger().Info("模型下载完成", zap.String("路径", modelFile))
- }
- log.GetLogger().Info("模型检查完成", zap.String("路径", modelFile))
return nil
}
diff --git a/internal/service/init.go b/internal/service/init.go
index 79e283f5..5fe25180 100644
--- a/internal/service/init.go
+++ b/internal/service/init.go
@@ -6,7 +6,7 @@ import (
"krillin-ai/internal/types"
"krillin-ai/log"
"krillin-ai/pkg/aliyun"
- "krillin-ai/pkg/fasterwhisper"
+ "krillin-ai/pkg/whisperkit"
"krillin-ai/pkg/openai"
"krillin-ai/pkg/whisper"
)
@@ -28,8 +28,8 @@ func NewService() *Service {
transcriber = whisper.NewClient(config.Conf.Openai.Whisper.BaseUrl, config.Conf.Openai.Whisper.ApiKey, config.Conf.App.Proxy)
case "aliyun":
transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)
- case "fasterwhisper":
- transcriber = fasterwhisper.NewFastwhisperProcessor(config.Conf.LocalModel.FasterWhisper)
+ case "whisperkit":
+ transcriber = whisperkit.NewWhisperKitProcessor(config.Conf.LocalModel.WhisperKit)
}
log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.App.TranscribeProvider))
diff --git a/internal/storage/bin.go b/internal/storage/bin.go
index de217c42..ddcb4bae 100644
--- a/internal/storage/bin.go
+++ b/internal/storage/bin.go
@@ -4,5 +4,5 @@ var (
FfmpegPath string
FfprobePath string
YtdlpPath string
- FasterwhisperPath string
+ WhisperKitPath string
)
diff --git a/internal/types/whisperkit.go b/internal/types/whisperkit.go
new file mode 100644
index 00000000..4375af17
--- /dev/null
+++ b/internal/types/whisperkit.go
@@ -0,0 +1,26 @@
+package types
+
+type WhisperKitOutput struct {
+ Text string `json:"text"`
+ Language string `json:"language"`
+ Segments []struct {
+ Seek int `json:"seek"`
+ Tokens []int `json:"tokens"`
+ CompressionRatio float64 `json:"compressionRatio"`
+ Temperature float64 `json:"temperature"`
+ AvgLogprob float64 `json:"avgLogprob"`
+ NoSpeechProb float64 `json:"noSpeechProb"`
+ Id int `json:"id"`
+ TokenLogProbs []map[string]float64 `json:"tokenLogProbs"`
+ Start float64 `json:"start"`
+ Words []struct {
+ Start float64 `json:"start"`
+ End float64 `json:"end"`
+ Word string `json:"word"`
+ Probability float64 `json:"probability"`
+ Tokens []int `json:"tokens"`
+ } `json:"words"`
+ Text string `json:"text"`
+ End float64 `json:"end"`
+ } `json:"segments"`
+}
diff --git a/pkg/fasterwhisper/init.go b/pkg/fasterwhisper/init.go
deleted file mode 100644
index a3328721..00000000
--- a/pkg/fasterwhisper/init.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package fasterwhisper
-
-type FastwhisperProcessor struct {
- WorkDir string // 生成中间文件的目录
- Model string
-}
-
-func NewFastwhisperProcessor(model string) *FastwhisperProcessor {
- return &FastwhisperProcessor{
- Model: model,
- }
-}
diff --git a/pkg/whisperkit/init.go b/pkg/whisperkit/init.go
new file mode 100644
index 00000000..8a0df018
--- /dev/null
+++ b/pkg/whisperkit/init.go
@@ -0,0 +1,12 @@
+package whisperkit
+
+type WhisperKitProcessor struct {
+ WorkDir string // 生成中间文件的目录
+ Model string
+}
+
+func NewWhisperKitProcessor(model string) *WhisperKitProcessor {
+ return &WhisperKitProcessor{
+ Model: model,
+ }
+}
diff --git a/pkg/fasterwhisper/transcription.go b/pkg/whisperkit/transcription.go
similarity index 60%
rename from pkg/fasterwhisper/transcription.go
rename to pkg/whisperkit/transcription.go
index 23b151ee..81378831 100644
--- a/pkg/fasterwhisper/transcription.go
+++ b/pkg/whisperkit/transcription.go
@@ -1,8 +1,7 @@
-package fasterwhisper
+package whisperkit
import (
"encoding/json"
- "go.uber.org/zap"
"krillin-ai/internal/storage"
"krillin-ai/internal/types"
"krillin-ai/log"
@@ -10,37 +9,42 @@ import (
"os"
"os/exec"
"strings"
+
+ "go.uber.org/zap"
)
-func (c *FastwhisperProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
+func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
cmdArgs := []string{
- "--model_dir", "./models/",
- "--model", c.Model,
- "--one_word", "2",
- "--output_format", "json",
+ "transcribe",
+ "--model-path", "/Users/outisli/AI_Models/whisperkit-coreml/openai_whisper-large-v2",
+ "--audio-encoder-compute-units", "all",
+ "--text-decoder-compute-units", "all",
"--language", language,
- "--output_dir", workDir,
- audioFile,
+ "--report",
+ "--report-path", workDir,
+ "--word-timestamps",
+ "--skip-special-tokens",
+ "--audio-path", audioFile,
}
- cmd := exec.Command(storage.FasterwhisperPath, cmdArgs...)
- log.GetLogger().Info("FastwhisperProcessor转录开始", zap.String("cmd", cmd.String()))
+ cmd := exec.Command(storage.WhisperKitPath, cmdArgs...)
+ log.GetLogger().Info("WhisperKitProcessor转录开始", zap.String("cmd", cmd.String()))
output, err := cmd.CombinedOutput()
if err != nil {
- log.GetLogger().Error("FastwhisperProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))
+ log.GetLogger().Error("WhisperKitProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))
return nil, err
}
- log.GetLogger().Info("FastwhisperProcessor转录json生成完毕", zap.String("audio file", audioFile))
+ log.GetLogger().Info("WhisperKitProcessor转录json生成完毕", zap.String("audio file", audioFile))
- var result types.FasterWhisperOutput
+ var result types.WhisperKitOutput
fileData, err := os.Open(util.ChangeFileExtension(audioFile, ".json"))
if err != nil {
- log.GetLogger().Error("FastwhisperProcessor 打开json文件失败", zap.Error(err))
+ log.GetLogger().Error("WhisperKitProcessor 打开json文件失败", zap.Error(err))
return nil, err
}
defer fileData.Close()
decoder := json.NewDecoder(fileData)
if err = decoder.Decode(&result); err != nil {
- log.GetLogger().Error("FastwhisperProcessor 解析json文件失败", zap.Error(err))
+ log.GetLogger().Error("WhisperKitProcessor 解析json文件失败", zap.Error(err))
return nil, err
}
@@ -81,6 +85,6 @@ func (c *FastwhisperProcessor) Transcription(audioFile, language, workDir string
}
}
}
- log.GetLogger().Info("FastwhisperProcessor转录成功")
+ log.GetLogger().Info("WhisperKitProcessor转录成功")
return &transcriptionData, nil
}