Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
20250304-support mac using WhisperKit
OutisLi256821363f1f
diff --git a/.DS_Store b/.DS_Storenew file mode 100644index 00000000..4015a0e3Binary files /dev/null and b/.DS_Store differdiff --git a/config/config.go b/config/config.goindex 1dde6bac..a44a02fa 100644--- a/config/config.go+++ b/config/config.go@@ -24,7 +24,7 @@ type Server struct {}type LocalModel struct {- FasterWhisper string `toml:"faster_whisper"`+ WhisperKit string `toml:"whisperkit"`}type OpenAiWhisper struct {@@ -81,7 +81,7 @@ var Conf = Config{Port: 8888,},LocalModel: LocalModel{- FasterWhisper: "medium",+ WhisperKit: "large-v2",},}@@ -120,7 +120,7 @@ func loadFromEnv() {// LocalModel 配置if v := os.Getenv("KRILLIN_FASTER_WHISPER"); v != "" {- Conf.LocalModel.FasterWhisper = v+ Conf.LocalModel.WhisperKit = v}// OpenAI 配置@@ -178,9 +178,9 @@ func validateConfig() error {if Conf.Openai.Whisper.ApiKey == "" {return errors.New("使用OpenAI转写服务需要配置 OpenAI API Key")}- case "fasterwhisper":- if Conf.LocalModel.FasterWhisper != "tiny" && Conf.LocalModel.FasterWhisper != "medium" && Conf.LocalModel.FasterWhisper != "large-v2" {- return errors.New("检测到开启了fasterwhisper,但模型选型配置不正确,请检查配置")+ case "whisperkit":+ if Conf.LocalModel.WhisperKit != "tiny" && Conf.LocalModel.WhisperKit != "medium" && Conf.LocalModel.WhisperKit != "large-v2" {+ return errors.New("检测到开启了whisperkit,但模型选型配置不正确,请检查配置")}case "aliyun":if Conf.Aliyun.Speech.AccessKeyId == "" || Conf.Aliyun.Speech.AccessKeySecret == "" || Conf.Aliyun.Speech.AppKey == "" {@@ -225,7 +225,7 @@ func LoadConfig() error {}// 本地模型不并发- if Conf.App.TranscribeProvider == "fasterwhisper" {+ if Conf.App.TranscribeProvider == "whisperkit" {Conf.App.TranslateParallelNum = 1}diff --git a/internal/deps/checker.go b/internal/deps/checker.goindex 8833aced..ded65f00 100644--- a/internal/deps/checker.go+++ b/internal/deps/checker.go@@ -2,7 +2,6 @@ package depsimport ("fmt"- "go.uber.org/zap""krillin-ai/config""krillin-ai/internal/storage""krillin-ai/log"@@ -10,6 +9,8 @@ import ("os""os/exec""runtime"++ "go.uber.org/zap")func CheckDependency() error {@@ -28,17 +29,8 @@ func CheckDependency() error {log.GetLogger().Error("yt-dlp环境准备失败", zap.Error(err))return err}- if config.Conf.App.TranscribeProvider == "fasterwhisper" {- err = checkFasterWhisper()- if err != nil {- log.GetLogger().Error("fasterwhisper环境准备失败", zap.Error(err))- return err- }- err = checkModel()- if err != nil {- log.GetLogger().Error("本地模型环境准备失败", zap.Error(err))- return err- }+ if config.Conf.App.TranscribeProvider == "whisperkit" {+ checkWhisperKit()}return nil@@ -242,84 +234,8 @@ func checkAndDownloadYtDlp() error {}// 检测faster whisper-func checkFasterWhisper() error {- var (- filePath string- err error- )- if runtime.GOOS == "windows" {- filePath = "./bin/faster-whisper/Faster-Whisper-XXL/faster-whisper-xxl.exe"- } else if runtime.GOOS == "linux" {- filePath = "./bin/faster-whisper/Whisper-Faster-XXL/whisper-faster-xxl"- } else {- return fmt.Errorf("fasterwhisper不支持你当前的操作系统: %s,请选择其它transcription provider", runtime.GOOS)- }- if _, err = os.Stat(filePath); os.IsNotExist(err) {- log.GetLogger().Info("没有找到faster-whisper,即将开始自动下载,文件较大请耐心等待")- err = os.MkdirAll("./bin", 0755)- if err != nil {- log.GetLogger().Error("创建./bin目录失败", zap.Error(err))- return err- }- var downloadUrl string- if runtime.GOOS == "windows" {- downloadUrl = "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/Faster-Whisper-XXL_r194.5_windows.zip"- } else {- downloadUrl = "https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/Faster-Whisper-XXL_r192.3.1_linux.zip"- }- err = util.DownloadFile(downloadUrl, "./bin/faster-whisper.zip", config.Conf.App.Proxy)- if err != nil {- log.GetLogger().Error("下载faster-whisper失败", zap.Error(err))- return err- }- log.GetLogger().Info("开始解压faster-whisper")- err = util.Unzip("./bin/faster-whisper.zip", "./bin/faster-whisper/")- if err != nil {- log.GetLogger().Error("解压faster-whisper失败", zap.Error(err))- return err- }- }- if runtime.GOOS != "windows" {- err = os.Chmod(filePath, 0755)- if err != nil {- log.GetLogger().Error("设置文件权限失败", zap.Error(err))- return err- }- }- storage.FasterwhisperPath = filePath- log.GetLogger().Info("faster-whisper检查完成", zap.String("路径", filePath))- return nil-}+func checkWhisperKit() error {+ storage.WhisperKitPath = "whisperkit-cli"-// 检测本地模型-func checkModel() error {- var err error- if _, err = os.Stat("./models"); os.IsNotExist(err) {- err = os.MkdirAll("./models", 0755)- if err != nil {- log.GetLogger().Error("创建./models目录失败", zap.Error(err))- return err- }- }- // 模型文件- model := config.Conf.LocalModel.FasterWhisper- modelFile := fmt.Sprintf("./models/faster-whisper-%s/model.bin", model)- if _, err = os.Stat(modelFile); os.IsNotExist(err) {- // 下载- log.GetLogger().Info(fmt.Sprintf("没有找到模型文件%s,即将开始自动下载", modelFile))- downloadUrl := fmt.Sprintf("https://modelscope.cn/models/Maranello/KrillinAI_dependency_cn/resolve/master/faster-whisper-%s.zip", model)- err = util.DownloadFile(downloadUrl, fmt.Sprintf("./models/faster-whisper-%s.zip", model), config.Conf.App.Proxy)- if err != nil {- log.GetLogger().Error("下载模型失败", zap.Error(err))- return err- }- err = util.Unzip(fmt.Sprintf("./models/faster-whisper-%s.zip", model), fmt.Sprintf("./models/faster-whisper-%s/", model))- if err != nil {- log.GetLogger().Error("解压模型失败", zap.Error(err))- return err- }- log.GetLogger().Info("模型下载完成", zap.String("路径", modelFile))- }- log.GetLogger().Info("模型检查完成", zap.String("路径", modelFile))return nil}diff --git a/internal/service/init.go b/internal/service/init.goindex 79e283f5..5fe25180 100644--- a/internal/service/init.go+++ b/internal/service/init.go@@ -6,7 +6,7 @@ import ("krillin-ai/internal/types""krillin-ai/log""krillin-ai/pkg/aliyun"- "krillin-ai/pkg/fasterwhisper"+ "krillin-ai/pkg/whisperkit""krillin-ai/pkg/openai""krillin-ai/pkg/whisper")@@ -28,8 +28,8 @@ func NewService() *Service {transcriber = whisper.NewClient(config.Conf.Openai.Whisper.BaseUrl, config.Conf.Openai.Whisper.ApiKey, config.Conf.App.Proxy)case "aliyun":transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)- case "fasterwhisper":- transcriber = fasterwhisper.NewFastwhisperProcessor(config.Conf.LocalModel.FasterWhisper)+ case "whisperkit":+ transcriber = whisperkit.NewWhisperKitProcessor(config.Conf.LocalModel.WhisperKit)}log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.App.TranscribeProvider))diff --git a/internal/storage/bin.go b/internal/storage/bin.goindex de217c42..ddcb4bae 100644--- a/internal/storage/bin.go+++ b/internal/storage/bin.go@@ -4,5 +4,5 @@ var (FfmpegPath stringFfprobePath stringYtdlpPath string- FasterwhisperPath string+ WhisperKitPath string)diff --git a/internal/types/whisperkit.go b/internal/types/whisperkit.gonew file mode 100644index 00000000..4375af17--- /dev/null+++ b/internal/types/whisperkit.go@@ -0,0 +1,26 @@+package types++type WhisperKitOutput struct {+ Text string `json:"text"`+ Language string `json:"language"`+ Segments []struct {+ Seek int `json:"seek"`+ Tokens []int `json:"tokens"`+ CompressionRatio float64 `json:"compressionRatio"`+ Temperature float64 `json:"temperature"`+ AvgLogprob float64 `json:"avgLogprob"`+ NoSpeechProb float64 `json:"noSpeechProb"`+ Id int `json:"id"`+ TokenLogProbs []map[string]float64 `json:"tokenLogProbs"`+ Start float64 `json:"start"`+ Words []struct {+ Start float64 `json:"start"`+ End float64 `json:"end"`+ Word string `json:"word"`+ Probability float64 `json:"probability"`+ Tokens []int `json:"tokens"`+ } `json:"words"`+ Text string `json:"text"`+ End float64 `json:"end"`+ } `json:"segments"`+}diff --git a/pkg/fasterwhisper/init.go b/pkg/fasterwhisper/init.godeleted file mode 100644index a3328721..00000000--- a/pkg/fasterwhisper/init.go+++ /dev/null@@ -1,12 +0,0 @@-package fasterwhisper--type FastwhisperProcessor struct {- WorkDir string // 生成中间文件的目录- Model string-}--func NewFastwhisperProcessor(model string) *FastwhisperProcessor {- return &FastwhisperProcessor{- Model: model,- }-}diff --git a/pkg/whisperkit/init.go b/pkg/whisperkit/init.gonew file mode 100644index 00000000..8a0df018--- /dev/null+++ b/pkg/whisperkit/init.go@@ -0,0 +1,12 @@+package whisperkit++type WhisperKitProcessor struct {+ WorkDir string // 生成中间文件的目录+ Model string+}++func NewWhisperKitProcessor(model string) *WhisperKitProcessor {+ return &WhisperKitProcessor{+ Model: model,+ }+}diff --git a/pkg/fasterwhisper/transcription.go b/pkg/whisperkit/transcription.gosimilarity index 60%rename from pkg/fasterwhisper/transcription.gorename to pkg/whisperkit/transcription.goindex 23b151ee..81378831 100644--- a/pkg/fasterwhisper/transcription.go+++ b/pkg/whisperkit/transcription.go@@ -1,8 +1,7 @@-package fasterwhisper+package whisperkitimport ("encoding/json"- "go.uber.org/zap""krillin-ai/internal/storage""krillin-ai/internal/types""krillin-ai/log"@@ -10,37 +9,42 @@ import ("os""os/exec""strings"++ "go.uber.org/zap")-func (c *FastwhisperProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {+func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {cmdArgs := []string{- "--model_dir", "./models/",- "--model", c.Model,- "--one_word", "2",- "--output_format", "json",+ "transcribe",+ "--model-path", "/Users/outisli/AI_Models/whisperkit-coreml/openai_whisper-large-v2",+ "--audio-encoder-compute-units", "all",+ "--text-decoder-compute-units", "all","--language", language,- "--output_dir", workDir,- audioFile,+ "--report",+ "--report-path", workDir,+ "--word-timestamps",+ "--skip-special-tokens",+ "--audio-path", audioFile,}- cmd := exec.Command(storage.FasterwhisperPath, cmdArgs...)- log.GetLogger().Info("FastwhisperProcessor转录开始", zap.String("cmd", cmd.String()))+ cmd := exec.Command(storage.WhisperKitPath, cmdArgs...)+ log.GetLogger().Info("WhisperKitProcessor转录开始", zap.String("cmd", cmd.String()))output, err := cmd.CombinedOutput()if err != nil {- log.GetLogger().Error("FastwhisperProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))+ log.GetLogger().Error("WhisperKitProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))return nil, err}- log.GetLogger().Info("FastwhisperProcessor转录json生成完毕", zap.String("audio file", audioFile))+ log.GetLogger().Info("WhisperKitProcessor转录json生成完毕", zap.String("audio file", audioFile))- var result types.FasterWhisperOutput+ var result types.WhisperKitOutputfileData, err := os.Open(util.ChangeFileExtension(audioFile, ".json"))if err != nil {- log.GetLogger().Error("FastwhisperProcessor 打开json文件失败", zap.Error(err))+ log.GetLogger().Error("WhisperKitProcessor 打开json文件失败", zap.Error(err))return nil, err}defer fileData.Close()decoder := json.NewDecoder(fileData)if err = decoder.Decode(&result); err != nil {- log.GetLogger().Error("FastwhisperProcessor 解析json文件失败", zap.Error(err))+ log.GetLogger().Error("WhisperKitProcessor 解析json文件失败", zap.Error(err))return nil, err}@@ -81,6 +85,6 @@ func (c *FastwhisperProcessor) Transcription(audioFile, language, workDir string}}}- log.GetLogger().Info("FastwhisperProcessor转录成功")+ log.GetLogger().Info("WhisperKitProcessor转录成功")return &transcriptionData, nil}