Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: support use custom llm when use whisper api
沈浩25becb7a05f1
diff --git a/config/config-example.toml b/config/config-example.tomlindex 7221174d..3dc67dec 100644--- a/config/config-example.toml+++ b/config/config-example.toml@@ -17,6 +17,9 @@base_url = "" # OpenAI API 自定义base url,可配合转发站密钥使用,留空为默认API地址model = "" # 指定模型名,可通过此字段结合base_url使用外部任何与OpenAI API兼容的大模型服务,留空默认为gpt-4o-miniapi_key = "sk-XXX" # OpenAI API密钥+ [openai.whisper] # 由于使用whisperAPI进行语音识别时,上方可能配置使用了OpenAI格式兼容的其它厂商的模型,所以此处需要独立填入openai的配置信息+ base_url = ""+ api_key = ""[aliyun] # 具体请参考文档中的“阿里云配置说明”[aliyun.oss]diff --git a/config/config.go b/config/config.goindex 10413b9e..1dde6bac 100644--- a/config/config.go+++ b/config/config.go@@ -27,12 +27,18 @@ type LocalModel struct {FasterWhisper string `toml:"faster_whisper"`}-type Openai struct {+type OpenAiWhisper struct {BaseUrl string `toml:"base_url"`- Model string `toml:"model"`ApiKey string `toml:"api_key"`}+type Openai struct {+ BaseUrl string `toml:"base_url"`+ Model string `toml:"model"`+ ApiKey string `toml:"api_key"`+ Whisper OpenAiWhisper `toml:"whisper"`+}+type AliyunOss struct {AccessKeyId string `toml:"access_key_id"`AccessKeySecret string `toml:"access_key_secret"`@@ -128,6 +134,14 @@ func loadFromEnv() {Conf.Openai.ApiKey = v}+ // Whisper配置+ if v := os.Getenv("KRILLIN_OPENAI_WHISPER_BASE_URL"); v != "" {+ Conf.Openai.Whisper.BaseUrl = v+ }+ if v := os.Getenv("KRILLIN_OPENAI_WHISPER_API_KEY"); v != "" {+ Conf.Openai.Whisper.ApiKey = v+ }+// Aliyun OSS 配置if v := os.Getenv("KRILLIN_ALIYUN_OSS_ACCESS_KEY_ID"); v != "" {Conf.Aliyun.Oss.AccessKeyId = v@@ -161,7 +175,7 @@ func validateConfig() error {// 检查转写服务提供商配置switch Conf.App.TranscribeProvider {case "openai":- if Conf.Openai.ApiKey == "" {+ if Conf.Openai.Whisper.ApiKey == "" {return errors.New("使用OpenAI转写服务需要配置 OpenAI API Key")}case "fasterwhisper":diff --git a/internal/service/init.go b/internal/service/init.goindex d7a52f6e..79e283f5 100644--- a/internal/service/init.go+++ b/internal/service/init.go@@ -8,6 +8,7 @@ import ("krillin-ai/pkg/aliyun""krillin-ai/pkg/fasterwhisper""krillin-ai/pkg/openai"+ "krillin-ai/pkg/whisper")type Service struct {@@ -24,7 +25,7 @@ func NewService() *Service {switch config.Conf.App.TranscribeProvider {case "openai":- transcriber = openai.NewClient(config.Conf.Openai.BaseUrl, config.Conf.Openai.ApiKey, config.Conf.App.Proxy)+ transcriber = whisper.NewClient(config.Conf.Openai.Whisper.BaseUrl, config.Conf.Openai.Whisper.ApiKey, config.Conf.App.Proxy)case "aliyun":transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)case "fasterwhisper":diff --git a/pkg/openai/openai.go b/pkg/openai/openai.goindex 9b39ec5e..d3afdf3b 100644--- a/pkg/openai/openai.go+++ b/pkg/openai/openai.go@@ -5,9 +5,7 @@ import (openai "github.com/sashabaranov/go-openai""go.uber.org/zap""krillin-ai/config"- "krillin-ai/internal/types""krillin-ai/log"- "strings")func (c *Client) ChatCompletion(query string) (string, error) {@@ -38,61 +36,3 @@ func (c *Client) ChatCompletion(query string) (string, error) {return resContent, nil}--func (c *Client) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {- resp, err := c.client.CreateTranscription(- context.Background(),- openai.AudioRequest{- Model: openai.Whisper1,- FilePath: audioFile,- Format: openai.AudioResponseFormatVerboseJSON,- TimestampGranularities: []openai.TranscriptionTimestampGranularity{- openai.TranscriptionTimestampGranularityWord,- },- Language: language,- },- )- if err != nil {- log.GetLogger().Error("openai create transcription failed", zap.Error(err))- return nil, err- }-- transcriptionData := &types.TranscriptionData{- Language: resp.Language,- Text: strings.ReplaceAll(resp.Text, "-", " "), // 连字符处理,因为模型存在很多错误添加到连字符- Words: make([]types.Word, 0),- }- num := 0- for _, word := range resp.Words {- if strings.Contains(word.Word, "—") {- // 对称切分- mid := (word.Start + word.End) / 2- seperatedWords := strings.Split(word.Word, "—")- transcriptionData.Words = append(transcriptionData.Words, []types.Word{- {- Num: num,- Text: seperatedWords[0],- Start: word.Start,- End: mid,- },- {- Num: num + 1,- Text: seperatedWords[1],- Start: mid,- End: word.End,- },- }...)- num += 2- } else {- transcriptionData.Words = append(transcriptionData.Words, types.Word{- Num: num,- Text: word.Word,- Start: word.Start,- End: word.End,- })- num++- }- }-- return transcriptionData, nil-}diff --git a/pkg/whisper/init.go b/pkg/whisper/init.gonew file mode 100644index 00000000..eff725e7--- /dev/null+++ b/pkg/whisper/init.go@@ -0,0 +1,30 @@+package whisper++import (+ "github.com/sashabaranov/go-openai"+ "krillin-ai/config"+ "net/http"+)++type Client struct {+ client *openai.Client+}++func NewClient(baseUrl, apiKey, proxyAddr string) *Client {+ cfg := openai.DefaultConfig(apiKey)+ if baseUrl != "" {+ cfg.BaseURL = baseUrl+ }++ if proxyAddr != "" {+ transport := &http.Transport{+ Proxy: http.ProxyURL(config.Conf.App.ParsedProxy),+ }+ cfg.HTTPClient = &http.Client{+ Transport: transport,+ }+ }++ client := openai.NewClientWithConfig(cfg)+ return &Client{client: client}+}diff --git a/pkg/whisper/whisper.go b/pkg/whisper/whisper.gonew file mode 100644index 00000000..58205b73--- /dev/null+++ b/pkg/whisper/whisper.go@@ -0,0 +1,68 @@+package whisper++import (+ "context"+ "github.com/sashabaranov/go-openai"+ "go.uber.org/zap"+ "krillin-ai/internal/types"+ "krillin-ai/log"+ "strings"+)++func (c *Client) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {+ resp, err := c.client.CreateTranscription(+ context.Background(),+ openai.AudioRequest{+ Model: openai.Whisper1,+ FilePath: audioFile,+ Format: openai.AudioResponseFormatVerboseJSON,+ TimestampGranularities: []openai.TranscriptionTimestampGranularity{+ openai.TranscriptionTimestampGranularityWord,+ },+ Language: language,+ },+ )+ if err != nil {+ log.GetLogger().Error("openai create transcription failed", zap.Error(err))+ return nil, err+ }++ transcriptionData := &types.TranscriptionData{+ Language: resp.Language,+ Text: strings.ReplaceAll(resp.Text, "-", " "), // 连字符处理,因为模型存在很多错误添加到连字符+ Words: make([]types.Word, 0),+ }+ num := 0+ for _, word := range resp.Words {+ if strings.Contains(word.Word, "—") {+ // 对称切分+ mid := (word.Start + word.End) / 2+ seperatedWords := strings.Split(word.Word, "—")+ transcriptionData.Words = append(transcriptionData.Words, []types.Word{+ {+ Num: num,+ Text: seperatedWords[0],+ Start: word.Start,+ End: mid,+ },+ {+ Num: num + 1,+ Text: seperatedWords[1],+ Start: mid,+ End: word.End,+ },+ }...)+ num += 2+ } else {+ transcriptionData.Words = append(transcriptionData.Words, types.Word{+ Num: num,+ Text: word.Word,+ Start: word.Start,+ End: word.End,+ })+ num+++ }+ }++ return transcriptionData, nil+}