Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: support use custom llm when use whisper api

沈浩25becb7a05f1
diff --git a/config/config-example.toml b/config/config-example.toml
index 7221174d..3dc67dec 100644
--- a/config/config-example.toml
+++ b/config/config-example.toml
@@ -17,6 +17,9 @@
base_url = "" # OpenAI API 自定义base url,可配合转发站密钥使用,留空为默认API地址
model = "" # 指定模型名,可通过此字段结合base_url使用外部任何与OpenAI API兼容的大模型服务,留空默认为gpt-4o-mini
api_key = "sk-XXX" # OpenAI API密钥
+ [openai.whisper] # 由于使用whisperAPI进行语音识别时,上方可能配置使用了OpenAI格式兼容的其它厂商的模型,所以此处需要独立填入openai的配置信息
+ base_url = ""
+ api_key = ""
[aliyun] # 具体请参考文档中的“阿里云配置说明”
[aliyun.oss]
diff --git a/config/config.go b/config/config.go
index 10413b9e..1dde6bac 100644
--- a/config/config.go
+++ b/config/config.go
@@ -27,12 +27,18 @@ type LocalModel struct {
FasterWhisper string `toml:"faster_whisper"`
}
-type Openai struct {
+type OpenAiWhisper struct {
BaseUrl string `toml:"base_url"`
- Model string `toml:"model"`
ApiKey string `toml:"api_key"`
}
+type Openai struct {
+ BaseUrl string `toml:"base_url"`
+ Model string `toml:"model"`
+ ApiKey string `toml:"api_key"`
+ Whisper OpenAiWhisper `toml:"whisper"`
+}
+
type AliyunOss struct {
AccessKeyId string `toml:"access_key_id"`
AccessKeySecret string `toml:"access_key_secret"`
@@ -128,6 +134,14 @@ func loadFromEnv() {
Conf.Openai.ApiKey = v
}
+ // Whisper配置
+ if v := os.Getenv("KRILLIN_OPENAI_WHISPER_BASE_URL"); v != "" {
+ Conf.Openai.Whisper.BaseUrl = v
+ }
+ if v := os.Getenv("KRILLIN_OPENAI_WHISPER_API_KEY"); v != "" {
+ Conf.Openai.Whisper.ApiKey = v
+ }
+
// Aliyun OSS 配置
if v := os.Getenv("KRILLIN_ALIYUN_OSS_ACCESS_KEY_ID"); v != "" {
Conf.Aliyun.Oss.AccessKeyId = v
@@ -161,7 +175,7 @@ func validateConfig() error {
// 检查转写服务提供商配置
switch Conf.App.TranscribeProvider {
case "openai":
- if Conf.Openai.ApiKey == "" {
+ if Conf.Openai.Whisper.ApiKey == "" {
return errors.New("使用OpenAI转写服务需要配置 OpenAI API Key")
}
case "fasterwhisper":
diff --git a/internal/service/init.go b/internal/service/init.go
index d7a52f6e..79e283f5 100644
--- a/internal/service/init.go
+++ b/internal/service/init.go
@@ -8,6 +8,7 @@ import (
"krillin-ai/pkg/aliyun"
"krillin-ai/pkg/fasterwhisper"
"krillin-ai/pkg/openai"
+ "krillin-ai/pkg/whisper"
)
type Service struct {
@@ -24,7 +25,7 @@ func NewService() *Service {
switch config.Conf.App.TranscribeProvider {
case "openai":
- transcriber = openai.NewClient(config.Conf.Openai.BaseUrl, config.Conf.Openai.ApiKey, config.Conf.App.Proxy)
+ transcriber = whisper.NewClient(config.Conf.Openai.Whisper.BaseUrl, config.Conf.Openai.Whisper.ApiKey, config.Conf.App.Proxy)
case "aliyun":
transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)
case "fasterwhisper":
diff --git a/pkg/openai/openai.go b/pkg/openai/openai.go
index 9b39ec5e..d3afdf3b 100644
--- a/pkg/openai/openai.go
+++ b/pkg/openai/openai.go
@@ -5,9 +5,7 @@ import (
openai "github.com/sashabaranov/go-openai"
"go.uber.org/zap"
"krillin-ai/config"
- "krillin-ai/internal/types"
"krillin-ai/log"
- "strings"
)
func (c *Client) ChatCompletion(query string) (string, error) {
@@ -38,61 +36,3 @@ func (c *Client) ChatCompletion(query string) (string, error) {
return resContent, nil
}
-
-func (c *Client) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
- resp, err := c.client.CreateTranscription(
- context.Background(),
- openai.AudioRequest{
- Model: openai.Whisper1,
- FilePath: audioFile,
- Format: openai.AudioResponseFormatVerboseJSON,
- TimestampGranularities: []openai.TranscriptionTimestampGranularity{
- openai.TranscriptionTimestampGranularityWord,
- },
- Language: language,
- },
- )
- if err != nil {
- log.GetLogger().Error("openai create transcription failed", zap.Error(err))
- return nil, err
- }
-
- transcriptionData := &types.TranscriptionData{
- Language: resp.Language,
- Text: strings.ReplaceAll(resp.Text, "-", " "), // 连字符处理,因为模型存在很多错误添加到连字符
- Words: make([]types.Word, 0),
- }
- num := 0
- for _, word := range resp.Words {
- if strings.Contains(word.Word, "—") {
- // 对称切分
- mid := (word.Start + word.End) / 2
- seperatedWords := strings.Split(word.Word, "—")
- transcriptionData.Words = append(transcriptionData.Words, []types.Word{
- {
- Num: num,
- Text: seperatedWords[0],
- Start: word.Start,
- End: mid,
- },
- {
- Num: num + 1,
- Text: seperatedWords[1],
- Start: mid,
- End: word.End,
- },
- }...)
- num += 2
- } else {
- transcriptionData.Words = append(transcriptionData.Words, types.Word{
- Num: num,
- Text: word.Word,
- Start: word.Start,
- End: word.End,
- })
- num++
- }
- }
-
- return transcriptionData, nil
-}
diff --git a/pkg/whisper/init.go b/pkg/whisper/init.go
new file mode 100644
index 00000000..eff725e7
--- /dev/null
+++ b/pkg/whisper/init.go
@@ -0,0 +1,30 @@
+package whisper
+
+import (
+ "github.com/sashabaranov/go-openai"
+ "krillin-ai/config"
+ "net/http"
+)
+
+type Client struct {
+ client *openai.Client
+}
+
+func NewClient(baseUrl, apiKey, proxyAddr string) *Client {
+ cfg := openai.DefaultConfig(apiKey)
+ if baseUrl != "" {
+ cfg.BaseURL = baseUrl
+ }
+
+ if proxyAddr != "" {
+ transport := &http.Transport{
+ Proxy: http.ProxyURL(config.Conf.App.ParsedProxy),
+ }
+ cfg.HTTPClient = &http.Client{
+ Transport: transport,
+ }
+ }
+
+ client := openai.NewClientWithConfig(cfg)
+ return &Client{client: client}
+}
diff --git a/pkg/whisper/whisper.go b/pkg/whisper/whisper.go
new file mode 100644
index 00000000..58205b73
--- /dev/null
+++ b/pkg/whisper/whisper.go
@@ -0,0 +1,68 @@
+package whisper
+
+import (
+ "context"
+ "github.com/sashabaranov/go-openai"
+ "go.uber.org/zap"
+ "krillin-ai/internal/types"
+ "krillin-ai/log"
+ "strings"
+)
+
+func (c *Client) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
+ resp, err := c.client.CreateTranscription(
+ context.Background(),
+ openai.AudioRequest{
+ Model: openai.Whisper1,
+ FilePath: audioFile,
+ Format: openai.AudioResponseFormatVerboseJSON,
+ TimestampGranularities: []openai.TranscriptionTimestampGranularity{
+ openai.TranscriptionTimestampGranularityWord,
+ },
+ Language: language,
+ },
+ )
+ if err != nil {
+ log.GetLogger().Error("openai create transcription failed", zap.Error(err))
+ return nil, err
+ }
+
+ transcriptionData := &types.TranscriptionData{
+ Language: resp.Language,
+ Text: strings.ReplaceAll(resp.Text, "-", " "), // 连字符处理,因为模型存在很多错误添加到连字符
+ Words: make([]types.Word, 0),
+ }
+ num := 0
+ for _, word := range resp.Words {
+ if strings.Contains(word.Word, "—") {
+ // 对称切分
+ mid := (word.Start + word.End) / 2
+ seperatedWords := strings.Split(word.Word, "—")
+ transcriptionData.Words = append(transcriptionData.Words, []types.Word{
+ {
+ Num: num,
+ Text: seperatedWords[0],
+ Start: word.Start,
+ End: mid,
+ },
+ {
+ Num: num + 1,
+ Text: seperatedWords[1],
+ Start: mid,
+ End: word.End,
+ },
+ }...)
+ num += 2
+ } else {
+ transcriptionData.Words = append(transcriptionData.Words, types.Word{
+ Num: num,
+ Text: word.Word,
+ Start: word.Start,
+ End: word.End,
+ })
+ num++
+ }
+ }
+
+ return transcriptionData, nil
+}