Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: support openai tts & refactor speech

沈浩636757c4a359
diff --git a/cmd/desktop/main.go b/cmd/desktop/main.go
index 227375cf..02b8ad3c 100644
--- a/cmd/desktop/main.go
+++ b/cmd/desktop/main.go
@@ -13,11 +13,8 @@ func main() {
log.InitLogger()
defer log.GetLogger().Sync()
- config.LoadConfig()
- if config.Conf.App.TranscribeProvider == "" || config.Conf.App.LlmProvider == "" {
+ if !config.LoadConfig() {
// 确保有最基础的配置
- config.Conf.App.TranscribeProvider = "openai"
- config.Conf.App.LlmProvider = "openai"
err := config.SaveConfig()
if err != nil {
log.GetLogger().Error("保存配置失败", zap.Error(err))
diff --git a/cmd/server/main.go b/cmd/server/main.go
index 4496861b..b5f3bd45 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -14,7 +14,9 @@ func main() {
defer log.GetLogger().Sync()
var err error
- config.LoadConfig()
+ if !config.LoadConfig() {
+ return
+ }
if err = config.CheckConfig(); err != nil {
log.GetLogger().Error("加载配置失败", zap.Error(err))
diff --git a/config/config-example.toml b/config/config-example.toml
index dbb7eadc..56e812f8 100644
--- a/config/config-example.toml
+++ b/config/config-example.toml
@@ -5,35 +5,54 @@
transcribe_max_attempts = 3 # 转录最大尝试次数,建议值:3
translate_max_attempts = 5 # 翻译最大尝试次数,建议值:5,如果模型参数量较少或翻译失败率较高可以适当调高
proxy = "" # 网络代理地址,格式如http://127.0.0.1:7890,可不填
- transcribe_provider = "openai" # 语音识别,当前可选值:openai,fasterwhisper,whisperkit,whisper.cpp,aliyun。(fasterwhisper不支持macOS,whisperkit只支持M芯片)
- llm_provider = "openai" # LLM,当前可选值:openai,aliyun
[server]
host = "127.0.0.1"
port = 8888
-# 下方的配置非必填,请结合上方的选项和文档说明进行配置
-[local_model]
- fasterwhisper = "large-v2" # fasterwhisper的本地模型可选值:tiny,medium,large-v2,建议medium及以上
- whisperkit = "large-v2" # whisperkit的本地模型可选值:large-v2
- whispercpp = "large-v2" # whisper.cpp的本地模型
+# 下方的配置不是都要填,请结合文档说明进行配置
-[openai]
- base_url = "" # OpenAI API 自定义base url,可配合转发站密钥使用,留空为默认API地址
+[llm] #支持openai,deepseek,通义千问等所有兼容openai请求格式的模型服务
+ base_url = "" # 自定义base url,可配合转发站密钥使用,留空为openai官方api
+ api_key = "" # API密钥
model = "" # 指定模型名,可通过此字段结合base_url使用外部任何与OpenAI API兼容的大模型服务,留空默认为gpt-4o-mini
- api_key = "sk-XXX" # OpenAI API密钥
- [openai.whisper] # 由于使用whisperAPI进行语音识别时,上方可能配置使用了OpenAI格式兼容的其它厂商的模型,所以此处需要独立填入openai的配置信息
+
+[transcribe] # 视频转文本支持多种方案,配置时先填provider,再填对应的配置
+ [transcribe.provider]
+ name = "openai" #语音识别,当前可选值:openai,fasterwhisper,whisperkit,whisper.cpp,aliyun。(fasterwhisper不支持macOS,whisperkit只支持M芯片)
+ [transcribe.openai]
base_url = ""
api_key = ""
+ model = "whisper-1"
+ [transcribe.fasterwhisper]
+ model = "medium" # fasterwhisper的本地模型可选值:tiny,medium,large-v2。建议medium及以上
+ [transcribe.whisperkit]
+ model = "large-v2" # whisperkit的本地模型可选值:large-v2
+ [transcribe.whispercpp]
+ model = "large-v2" # whispercpp的本地模型可选值:large-v2
+ [transcribe.aliyun]
+ [tts.aliyun.oss]
+ access_key_id = ""
+ access_key_secret = ""
+ bucket = ""
+ [transcribe.aliyun.speech]
+ access_key_id = ""
+ access_key_secret = ""
+ app_key= ""
-[aliyun] # 具体请参考文档中的“阿里云配置说明”
- [aliyun.oss]
- access_key_id = ""
- access_key_secret = ""
- bucket = ""
- [aliyun.speech]
- access_key_id = ""
- access_key_secret = ""
- app_key= ""
- [aliyun.bailian]
- api_key = ""
\ No newline at end of file
+[tts]
+ [tts.provider]
+ name = "aliyun" # 可选值:openai,aliyun
+ [tts.openai]
+ base_url = ""
+ api_key = ""
+ model = "" # gpt-4o-mini-tts, tts-1, tts-1-hd
+ [tts.aliyun]
+ [tts.aliyun.oss]
+ access_key_id = ""
+ access_key_secret = ""
+ bucket = ""
+ [tts.aliyun.speech]
+ access_key_id = ""
+ access_key_secret = ""
+ app_key= ""
\ No newline at end of file
diff --git a/config/config.go b/config/config.go
index 8293506c..dc3a6870 100644
--- a/config/config.go
+++ b/config/config.go
@@ -23,8 +23,6 @@ type App struct {
TranslateMaxAttempts int `toml:"translate_max_attempts"`
Proxy string `toml:"proxy"`
ParsedProxy *url.URL `toml:"-"`
- TranscribeProvider string `toml:"transcribe_provider"`
- LlmProvider string `toml:"llm_provider"`
}
type Server struct {
@@ -32,52 +30,64 @@ type Server struct {
Port int `toml:"port"`
}
-type LocalModel struct {
- Fasterwhisper string `toml:"fasterwhisper"`
- Whisperkit string `toml:"whisperkit"`
- Whispercpp string `toml:"whispercpp"`
-}
-
-type OpenAiWhisper struct {
+type OpenaiCompatibleConfig struct {
BaseUrl string `toml:"base_url"`
ApiKey string `toml:"api_key"`
+ Model string `toml:"model"`
}
-type Openai struct {
- BaseUrl string `toml:"base_url"`
- Model string `toml:"model"`
- ApiKey string `toml:"api_key"`
- Whisper OpenAiWhisper `toml:"whisper"`
+type LocalModelConfig struct {
+ Model string `toml:"model"`
}
-type AliyunOss struct {
+type AliyunSpeechConfig struct {
AccessKeyId string `toml:"access_key_id"`
AccessKeySecret string `toml:"access_key_secret"`
- Bucket string `toml:"bucket"`
+ AppKey string `toml:"app_key"`
}
-type AliyunSpeech struct {
+type AliyunOssConfig struct {
AccessKeyId string `toml:"access_key_id"`
AccessKeySecret string `toml:"access_key_secret"`
- AppKey string `toml:"app_key"`
+ Bucket string `toml:"bucket"`
}
-type AliyunBailian struct {
- ApiKey string `toml:"api_key"`
+type AliyunTranscribeConfig struct {
+ Oss AliyunOssConfig `toml:"oss"`
+ Speech AliyunSpeechConfig `toml:"speech"`
}
-type Aliyun struct {
- Oss AliyunOss `toml:"oss"`
- Speech AliyunSpeech `toml:"speech"`
- Bailian AliyunBailian `toml:"bailian"`
+type Transcribe struct {
+ Provider string `toml:"provider"`
+ Openai OpenaiCompatibleConfig `toml:"openai"`
+ Fasterwhisper LocalModelConfig `toml:"fasterwhisper"`
+ Whisperkit LocalModelConfig `toml:"whisperkit"`
+ Whispercpp LocalModelConfig `toml:"whispercpp"`
+ Aliyun AliyunTranscribeConfig `toml:"aliyun"`
+}
+
+type AliyunTtsConfig struct {
+ Oss AliyunOssConfig `toml:"oss"`
+ Speech AliyunSpeechConfig `toml:"speech"`
+}
+
+type Tts struct {
+ Provider string `toml:"provider"`
+ Openai OpenaiCompatibleConfig `toml:"openai"`
+ Aliyun AliyunTtsConfig `toml:"aliyun"`
+}
+
+type OpenAiWhisper struct {
+ BaseUrl string `toml:"base_url"`
+ ApiKey string `toml:"api_key"`
}
type Config struct {
- App App `toml:"app"`
- Server Server `toml:"server"`
- LocalModel LocalModel `toml:"local_model"`
- Openai Openai `toml:"openai"`
- Aliyun Aliyun `toml:"aliyun"`
+ App App `toml:"app"`
+ Server Server `toml:"server"`
+ Llm OpenaiCompatibleConfig `toml:"llm"`
+ Transcribe Transcribe `toml:"transcribe"`
+ Tts Tts `toml:"tts"`
}
var Conf = Config{
@@ -87,30 +97,47 @@ var Conf = Config{
TranscribeParallelNum: 1,
TranscribeMaxAttempts: 3,
TranslateMaxAttempts: 3,
- TranscribeProvider: "openai",
- LlmProvider: "openai",
},
Server: Server{
Host: "127.0.0.1",
Port: 8888,
},
- LocalModel: LocalModel{
- Fasterwhisper: "large-v2",
- Whisperkit: "large-v2",
- Whispercpp: "large-v2",
+ Llm: OpenaiCompatibleConfig{
+ Model: "gpt-4o-mini",
+ },
+ Transcribe: Transcribe{
+ Provider: "openai",
+ Openai: OpenaiCompatibleConfig{
+ Model: "whisper-1",
+ },
+ Fasterwhisper: LocalModelConfig{
+ Model: "large-v2",
+ },
+ Whisperkit: LocalModelConfig{
+ Model: "large-v2",
+ },
+ Whispercpp: LocalModelConfig{
+ Model: "large-v2",
+ },
+ },
+ Tts: Tts{
+ Provider: "openai",
+ Openai: OpenaiCompatibleConfig{
+ Model: "gpt-4o-mini-tts",
+ },
},
}
// 检查必要的配置是否完整
func validateConfig() error {
// 检查转写服务提供商配置
- switch Conf.App.TranscribeProvider {
+ switch Conf.Transcribe.Provider {
case "openai":
- if Conf.Openai.Whisper.ApiKey == "" {
- return errors.New("使用OpenAI转写服务需要配置 OpenAI API Key")
+ if Conf.Transcribe.Openai.ApiKey == "" {
+ return errors.New("使用OpenAI转录服务需要配置 OpenAI API Key")
}
case "fasterwhisper":
- if Conf.LocalModel.Fasterwhisper != "tiny" && Conf.LocalModel.Fasterwhisper != "medium" && Conf.LocalModel.Fasterwhisper != "large-v2" {
+ if Conf.Transcribe.Fasterwhisper.Model != "tiny" && Conf.Transcribe.Fasterwhisper.Model != "medium" && Conf.Transcribe.Fasterwhisper.Model != "large-v2" {
return errors.New("检测到开启了fasterwhisper,但模型选型配置不正确,请检查配置")
}
case "whisperkit":
@@ -118,7 +145,7 @@ func validateConfig() error {
log.GetLogger().Error("whisperkit只支持macos", zap.String("当前系统", runtime.GOOS))
return fmt.Errorf("whisperkit只支持macos")
}
- if Conf.LocalModel.Whisperkit != "large-v2" {
+ if Conf.Transcribe.Whisperkit.Model != "large-v2" {
return errors.New("检测到开启了whisperkit,但模型选型配置不正确,请检查配置")
}
case "whispercpp":
@@ -126,45 +153,33 @@ func validateConfig() error {
log.GetLogger().Error("whispercpp only support windows", zap.String("current os", runtime.GOOS))
return fmt.Errorf("whispercpp only support windows")
}
- if Conf.LocalModel.Whispercpp != "large-v2" {
+ if Conf.Transcribe.Whispercpp.Model != "large-v2" {
return errors.New("检测到开启了whisper.cpp,但模型选型配置不正确,请检查配置")
}
case "aliyun":
- if Conf.Aliyun.Speech.AccessKeyId == "" || Conf.Aliyun.Speech.AccessKeySecret == "" || Conf.Aliyun.Speech.AppKey == "" {
+ if Conf.Transcribe.Aliyun.Speech.AccessKeyId == "" || Conf.Transcribe.Aliyun.Speech.AccessKeySecret == "" || Conf.Transcribe.Aliyun.Speech.AppKey == "" {
return errors.New("使用阿里云语音服务需要配置相关密钥")
}
default:
return errors.New("不支持的转录提供商")
}
- // 检查LLM提供商配置
- switch Conf.App.LlmProvider {
- case "openai":
- if Conf.Openai.ApiKey == "" {
- return errors.New("使用OpenAI LLM服务需要配置 OpenAI API Key")
- }
- case "aliyun":
- if Conf.Aliyun.Bailian.ApiKey == "" {
- return errors.New("使用阿里云百炼服务需要配置 API Key")
- }
- default:
- return errors.New("不支持的LLM提供商")
- }
-
return nil
}
-func LoadConfig() {
+func LoadConfig() bool {
var err error
configPath := "./config/config.toml"
if _, err = os.Stat(configPath); os.IsNotExist(err) {
- return
+ log.GetLogger().Info("未找到配置文件")
+ return false
} else {
log.GetLogger().Info("已找到配置文件,从配置文件中加载配置")
if _, err = toml.DecodeFile(configPath, &Conf); err != nil {
log.GetLogger().Error("加载配置文件失败", zap.Error(err))
- return
+ return false
}
+ return true
}
}
diff --git a/go.mod b/go.mod
index 43b7aefb..0a66d3f8 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
require (
fyne.io/systray v1.11.0 // indirect
+ github.com/aliyun/alibabacloud-nls-go-sdk v1.1.1 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
@@ -58,6 +59,7 @@ require (
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rymdport/portal v0.3.0 // indirect
+ github.com/satori/go.uuid v1.2.0 // indirect
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect
github.com/stretchr/testify v1.9.0 // indirect
diff --git a/go.sum b/go.sum
index 92764684..b542a0e7 100644
--- a/go.sum
+++ b/go.sum
@@ -47,8 +47,11 @@ github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/aliyun/alibaba-cloud-sdk-go v1.61.1376/go.mod h1:9CMdKNL3ynIGPpfTcdwTvIm8SGuAZYYC4jFVSSvE1YQ=
github.com/aliyun/alibaba-cloud-sdk-go v1.63.72 h1:HvFZUzEbNvfe8F2Mg0wBGv90bPhWDxgVtDHR5zoBOU0=
github.com/aliyun/alibaba-cloud-sdk-go v1.63.72/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ=
+github.com/aliyun/alibabacloud-nls-go-sdk v1.1.1 h1:LjItoNZuu5xHlsByFo+kr3nGa4LRIESCGWhfurayxBg=
+github.com/aliyun/alibabacloud-nls-go-sdk v1.1.1/go.mod h1:4BDMUKpEaP/Ct79w0ozR0nbnEj49g1k3mrgX/IKG5I4=
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.1.3 h1:grJyLSdRJtfxKKhCTWSeJhnOQsp2WoLNdK8XA5FE9oo=
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.1.3/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -214,6 +217,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
github.com/gopherjs/gopherjs v0.0.0-20211219123610-ec9572f70e60/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/goxjs/gl v0.0.0-20210104184919-e3fafc6f8f2a/go.mod h1:dy/f2gjY09hwVfIyATps4G2ai7/hLwLkc5TrPqONuXY=
@@ -243,10 +247,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 h1:Po+wkNdMmN+Zj1tDsJQy7mJlPlwGNQd9JZoPjObagf8=
github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49/go.mod h1:YiutDnxPRLk5DLUFj6Rw4pRBBURZY07GFr54NdV9mQg=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -330,6 +336,8 @@ github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/sashabaranov/go-openai v1.36.0 h1:fcSrn8uGuorzPWCBp8L0aCR95Zjb/Dd+ZSML0YZy9EI=
github.com/sashabaranov/go-openai v1.36.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
@@ -766,6 +774,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/internal/api/subtitle.go b/internal/api/subtitle.go
index 02be1cfe..48584038 100644
--- a/internal/api/subtitle.go
+++ b/internal/api/subtitle.go
@@ -22,7 +22,7 @@ type SubtitleTask struct {
Bilingual int `json:"bilingual"` // 是否双语 1:是 2:否
TranslationSubtitlePos int `json:"translation_subtitle_pos"` // 翻译字幕位置 1:上方 2:下方
TTS int `json:"tts"` // 是否配音 1:是 2:否
- TTSVoiceCode int `json:"tts_voice_code,omitempty"` // 配音声音代码 1:女声 2:男声
+ TTSVoiceCode string `json:"tts_voice_code,omitempty"` // 配音声音代码
TTSVoiceCloneSrcFileURL string `json:"tts_voice_clone_src_file_url,omitempty"` // 音色克隆源文件URL
ModalFilter int `json:"modal_filter"` // 是否过滤语气词 1:是 2:否
Replace []string `json:"replace,omitempty"` // 词汇替换列表
diff --git a/internal/deps/checker.go b/internal/deps/checker.go
index 582f3c9d..84515e76 100644
--- a/internal/deps/checker.go
+++ b/internal/deps/checker.go
@@ -30,7 +30,7 @@ func CheckDependency() error {
log.GetLogger().Error("yt-dlp环境准备失败", zap.Error(err))
return err
}
- if config.Conf.App.TranscribeProvider == "fasterwhisper" {
+ if config.Conf.Transcribe.Provider == "fasterwhisper" {
err = checkFasterWhisper()
if err != nil {
log.GetLogger().Error("fasterwhisper环境准备失败", zap.Error(err))
@@ -42,7 +42,7 @@ func CheckDependency() error {
return err
}
}
- if config.Conf.App.TranscribeProvider == "whisperkit" {
+ if config.Conf.Transcribe.Provider == "whisperkit" {
if err = checkWhisperKit(); err != nil {
log.GetLogger().Error("whisperkit环境准备失败", zap.Error(err))
return err
@@ -53,7 +53,7 @@ func CheckDependency() error {
return err
}
}
- if config.Conf.App.TranscribeProvider == "whispercpp" {
+ if config.Conf.Transcribe.Provider == "whispercpp" {
if err = checkWhispercpp(); err != nil {
log.GetLogger().Error("whispercpp环境准备失败", zap.Error(err))
return err
@@ -396,7 +396,7 @@ func checkModel(whisperType string) error {
var modelPath string // cli中使用的model path
switch whisperType {
case "fasterwhisper":
- model = config.Conf.LocalModel.Fasterwhisper
+ model = config.Conf.Transcribe.Fasterwhisper.Model
modelPath = fmt.Sprintf("./models/faster-whisper-%s/model.bin", model)
if _, err = os.Stat(modelPath); os.IsNotExist(err) {
// 下载
@@ -415,7 +415,7 @@ func checkModel(whisperType string) error {
log.GetLogger().Info("模型下载完成", zap.String("路径", modelPath))
}
case "whispercpp":
- model = config.Conf.LocalModel.Whispercpp
+ model = config.Conf.Transcribe.Whispercpp.Model
modelPath = fmt.Sprintf("./models/whispercpp/ggml-%s.bin", model)
if _, err = os.Stat(modelPath); os.IsNotExist(err) {
log.GetLogger().Info(fmt.Sprintf("没有找到whisper.cpp模型%s,即将开始自动下载", modelPath))
@@ -428,7 +428,7 @@ func checkModel(whisperType string) error {
log.GetLogger().Info("whisper.cpp模型下载完成", zap.String("路径", modelPath))
}
case "whisperkit":
- model = config.Conf.LocalModel.Whisperkit
+ model = config.Conf.Transcribe.Whisperkit.Model
modelPath = fmt.Sprintf("./models/whisperkit/openai_whisper-%s", model)
files, _ := os.ReadDir(modelPath)
if len(files) == 0 {
diff --git a/internal/desktop/subtitle.go b/internal/desktop/subtitle.go
index 87f6df13..2f82fc19 100644
--- a/internal/desktop/subtitle.go
+++ b/internal/desktop/subtitle.go
@@ -38,7 +38,7 @@ type SubtitleManager struct {
bilingualEnabled bool
bilingualPosition int
voiceoverEnabled bool
- voiceoverGender int // 1-女声,2-男声
+ ttsVoiceCode string // 声音代码
fillerFilter bool
wordReplacements []api.WordReplacement
embedSubtitle string // none, horizontal, vertical, all
@@ -72,7 +72,7 @@ func NewSubtitleManager(window fyne.Window) *SubtitleManager {
bilingualPosition: 1,
fillerFilter: true,
voiceoverEnabled: false,
- voiceoverGender: 2,
+ ttsVoiceCode: "",
embedSubtitle: "none",
downloadContainer: container.NewVBox(),
tipsLabel: widget.NewLabel(""),
@@ -414,9 +414,9 @@ func (sm *SubtitleManager) SetVoiceoverEnabled(enabled bool) {
sm.voiceoverEnabled = enabled
}
-// SetVoiceoverGender 设置配音性别
-func (sm *SubtitleManager) SetVoiceoverGender(gender int) {
- sm.voiceoverGender = gender
+// SetTtsVoiceCode 设置配音性别
+func (sm *SubtitleManager) SetTtsVoiceCode(code string) {
+ sm.ttsVoiceCode = code
}
// SetEmbedSubtitle 设置字幕嵌入方式
@@ -485,7 +485,7 @@ func (sm *SubtitleManager) StartTask() error {
Bilingual: boolToInt(sm.bilingualEnabled),
TranslationSubtitlePos: sm.bilingualPosition,
TTS: boolToInt(sm.voiceoverEnabled),
- TTSVoiceCode: sm.voiceoverGender,
+ TTSVoiceCode: sm.ttsVoiceCode,
TTSVoiceCloneSrcFileURL: sm.voiceoverAudioPath,
ModalFilter: boolToInt(sm.fillerFilter),
EmbedSubtitleVideoType: sm.embedSubtitle,
@@ -576,7 +576,7 @@ func (sm *SubtitleManager) processMultipleVideos() {
Bilingual: boolToInt(sm.bilingualEnabled),
TranslationSubtitlePos: sm.bilingualPosition,
TTS: boolToInt(sm.voiceoverEnabled),
- TTSVoiceCode: sm.voiceoverGender,
+ TTSVoiceCode: sm.ttsVoiceCode,
TTSVoiceCloneSrcFileURL: sm.voiceoverAudioPath,
ModalFilter: boolToInt(sm.fillerFilter),
EmbedSubtitleVideoType: sm.embedSubtitle,
diff --git a/internal/desktop/ui.go b/internal/desktop/ui.go
index 1b70b5b9..811257f4 100644
--- a/internal/desktop/ui.go
+++ b/internal/desktop/ui.go
@@ -28,15 +28,11 @@ func CreateConfigTab(window fyne.Window) fyne.CanvasObject {
// 创建页面标题
pageTitle := TitleText("应用配置")
- // app 配置
appGroup := createAppConfigGroup()
serverGroup := createServerConfigGroup()
- localModelGroup := createLocalModelGroup()
- openaiGroup := createOpenAIConfigGroup()
- whisperGroup := createWhisperConfigGroup()
- aliyunOssGroup := createAliyunOSSConfigGroup()
- aliyunSpeechGroup := createAliyunSpeechConfigGroup()
- aliyunBailianGroup := createAliyunBailianConfigGroup()
+ llmGroup := createLlmConfigGroup()
+ transcribeGroup := createTranscribeConfigGroup()
+ ttsGroup := createTtsConfigGroup()
// 创建一个背景效果
background := canvas.NewRectangle(color.NRGBA{R: 248, G: 250, B: 253, A: 255})
@@ -47,18 +43,14 @@ func CreateConfigTab(window fyne.Window) fyne.CanvasObject {
spacer2 := canvas.NewRectangle(color.NRGBA{R: 0, G: 0, B: 0, A: 0})
spacer2.SetMinSize(fyne.NewSize(0, 10))
- // 创建滚动容器
configContainer := container.NewVBox(
container.NewPadded(pageTitle),
spacer1,
container.NewPadded(appGroup),
container.NewPadded(serverGroup),
- container.NewPadded(localModelGroup),
- container.NewPadded(openaiGroup),
- container.NewPadded(whisperGroup),
- container.NewPadded(aliyunOssGroup),
- container.NewPadded(aliyunSpeechGroup),
- container.NewPadded(aliyunBailianGroup),
+ container.NewPadded(llmGroup),
+ container.NewPadded(transcribeGroup),
+ container.NewPadded(ttsGroup),
spacer2,
)
@@ -205,14 +197,9 @@ func createAppConfigGroup() *fyne.Container {
appProxyEntry.Bind(binding.BindString(&config.Conf.App.Proxy))
appTranscribeProviderEntry := StyledSelect([]string{"openai", "fasterwhisper", "whispercpp", "whisperkit", "aliyun"}, func(s string) {
- config.Conf.App.TranscribeProvider = s
+ config.Conf.Transcribe.Provider = s
})
- appTranscribeProviderEntry.SetSelected(config.Conf.App.TranscribeProvider)
-
- appLlmProviderEntry := StyledSelect([]string{"openai", "aliyun"}, func(s string) {
- config.Conf.App.LlmProvider = s
- })
- appLlmProviderEntry.SetSelected(config.Conf.App.LlmProvider)
+ appTranscribeProviderEntry.SetSelected(config.Conf.Transcribe.Provider)
form := widget.NewForm(
widget.NewFormItem("字幕分段处理时长(分钟) Segment duration (minutes)", appSegmentDurationEntry),
@@ -222,7 +209,6 @@ func createAppConfigGroup() *fyne.Container {
widget.NewFormItem("翻译最大尝试次数 Translate max attempts", appTranslateMaxAttemptsEntry),
widget.NewFormItem("网络代理地址 proxy", appProxyEntry),
widget.NewFormItem("语音识别服务源 Transcriber provider", appTranscribeProviderEntry),
- widget.NewFormItem("LLM服务源 Llm provider", appLlmProviderEntry),
)
return GlassCard("应用配置 App Config", "基本参数 Basic config", form)
@@ -254,30 +240,131 @@ func createServerConfigGroup() *fyne.Container {
return GlassCard("服务器配置 Server Config", "API服务器设置 API server settings", form)
}
-// 创建本地模型配置组
-func createLocalModelGroup() *fyne.Container {
- localModelFasterwhisperEntry := StyledSelect([]string{"tiny", "medium", "large-v2"}, func(s string) {
- config.Conf.LocalModel.Fasterwhisper = s
- })
- localModelFasterwhisperEntry.SetSelected(config.Conf.LocalModel.Fasterwhisper)
+func createLlmConfigGroup() *fyne.Container {
+ baseUrlEntry := StyledEntry("API Base URL")
+ baseUrlEntry.Bind(binding.BindString(&config.Conf.Llm.BaseUrl))
+
+ apiKeyEntry := StyledPasswordEntry("API Key")
+ apiKeyEntry.Bind(binding.BindString(&config.Conf.Llm.ApiKey))
+
+ modelEntry := StyledEntry("模型名称 Model name")
+ modelEntry.Bind(binding.BindString(&config.Conf.Llm.Model))
+
+ form := widget.NewForm(
+ widget.NewFormItem("API Base URL", baseUrlEntry),
+ widget.NewFormItem("API Key", apiKeyEntry),
+ widget.NewFormItem("模型名称 Model name", modelEntry),
+ )
+ return GlassCard("LLM 配置 LLM Config", "LLM配置 LLM config", form)
+}
- localModelWhisperkitEntry := StyledSelect([]string{"large-v2"}, func(s string) {
- config.Conf.LocalModel.Whisperkit = s
+func createTranscribeConfigGroup() *fyne.Container {
+ providerOptions := []string{"openai", "fasterwhisper", "whisperkit", "whispercpp", "aliyun"}
+ providerSelect := widget.NewSelect(providerOptions, func(value string) {
+ config.Conf.Transcribe.Provider = value
})
- localModelWhisperkitEntry.SetSelected(config.Conf.LocalModel.Whisperkit)
+ providerSelect.SetSelected(config.Conf.Transcribe.Provider)
+
+ openaiBaseUrlEntry := StyledEntry("API Base URL")
+ openaiBaseUrlEntry.Bind(binding.BindString(&config.Conf.Transcribe.Openai.BaseUrl))
+ openaiApiKeyEntry := StyledPasswordEntry("API Key")
+ openaiApiKeyEntry.Bind(binding.BindString(&config.Conf.Transcribe.Openai.ApiKey))
+ openaiModelEntry := StyledEntry("模型名称 Model name")
+ openaiModelEntry.Bind(binding.BindString(&config.Conf.Transcribe.Openai.Model))
+
+ fasterWhisperModelEntry := StyledEntry("模型名称 Model name")
+ fasterWhisperModelEntry.Bind(binding.BindString(&config.Conf.Transcribe.Fasterwhisper.Model))
+
+ whisperKitModelEntry := StyledEntry("模型名称 Model name")
+ whisperKitModelEntry.Bind(binding.BindString(&config.Conf.Transcribe.Whisperkit.Model))
+
+ whisperCppModelEntry := StyledEntry("模型名称 Model name")
+ whisperCppModelEntry.Bind(binding.BindString(&config.Conf.Transcribe.Whispercpp.Model))
+
+ aliyunOssKeyIdEntry := StyledEntry("阿里云 Aliyun Access Key ID")
+ aliyunOssKeyIdEntry.Bind(binding.BindString(&config.Conf.Transcribe.Aliyun.Oss.AccessKeyId))
+ aliyunOssKeySecretEntry := StyledPasswordEntry("阿里云 Aliyun Access Key Secret")
+ aliyunOssKeySecretEntry.Bind(binding.BindString(&config.Conf.Transcribe.Aliyun.Oss.AccessKeySecret))
+ aliyunOssBucketEntry := StyledEntry("阿里云 Aliyun OSS Bucket名称")
+ aliyunOssBucketEntry.Bind(binding.BindString(&config.Conf.Transcribe.Aliyun.Oss.Bucket))
+
+ aliyunSpeechKeyIdEntry := StyledEntry("阿里云 Aliyun Speech Access Key ID")
+ aliyunSpeechKeyIdEntry.Bind(binding.BindString(&config.Conf.Transcribe.Aliyun.Speech.AccessKeyId))
+ aliyunSpeechKeySecretEntry := StyledPasswordEntry("阿里云 Aliyun Speech Access Key Secret")
+ aliyunSpeechKeySecretEntry.Bind(binding.BindString(&config.Conf.Transcribe.Aliyun.Speech.AccessKeySecret))
+ aliyunSpeechAppKeyEntry := StyledEntry("阿里云 Aliyun Speech App Key")
+ aliyunSpeechAppKeyEntry.Bind(binding.BindString(&config.Conf.Transcribe.Aliyun.Speech.AppKey))
+
+ form := widget.NewForm(
+ widget.NewFormItem("提供商 Provider", providerSelect),
+
+ widget.NewFormItem("OpenAI Base URL", openaiBaseUrlEntry),
+ widget.NewFormItem("OpenAI API Key", openaiApiKeyEntry),
+ widget.NewFormItem("OpenAI 模型 Model", openaiModelEntry),
+
+ widget.NewFormItem("FasterWhisper 模型 Model", fasterWhisperModelEntry),
+
+ widget.NewFormItem("WhisperKit 模型 Model", whisperKitModelEntry),
+
+ widget.NewFormItem("WhisperCpp 模型 Model", whisperCppModelEntry),
+
+ widget.NewFormItem("阿里云 Aliyun OSS Access Key ID", aliyunOssKeyIdEntry),
+ widget.NewFormItem("阿里云 Aliyun OSS Access Key Secret", aliyunOssKeySecretEntry),
+ widget.NewFormItem("阿里云 Aliyun OSS Bucket Name", aliyunOssBucketEntry),
- localModelWhispercppEntry := StyledSelect([]string{"large-v2"}, func(s string) {
- config.Conf.LocalModel.Whisperkit = s
+ widget.NewFormItem("阿里云语音 Aliyun Speech Access Key ID", aliyunSpeechKeyIdEntry),
+ widget.NewFormItem("阿里云语音 Aliyun Speech Access Key Secret", aliyunSpeechKeySecretEntry),
+ widget.NewFormItem("阿里云语音 Aliyun Speech App Key", aliyunSpeechAppKeyEntry),
+ )
+
+ return GlassCard("语音识别配置 Transcribe Config", "语音识别配置 Transcribe config", form)
+}
+
+func createTtsConfigGroup() *fyne.Container {
+ providerOptions := []string{"openai", "aliyun"}
+ providerSelect := widget.NewSelect(providerOptions, func(value string) {
+ config.Conf.Tts.Provider = value
})
- localModelWhispercppEntry.SetSelected(config.Conf.LocalModel.Whispercpp)
+ providerSelect.SetSelected(config.Conf.Tts.Provider)
+
+ openaiBaseUrlEntry := StyledEntry("API Base URL")
+ openaiBaseUrlEntry.Bind(binding.BindString(&config.Conf.Tts.Openai.BaseUrl))
+ openaiApiKeyEntry := StyledPasswordEntry("API Key")
+ openaiApiKeyEntry.Bind(binding.BindString(&config.Conf.Tts.Openai.ApiKey))
+ openaiModelEntry := StyledEntry("模型名称 Model name")
+ openaiModelEntry.Bind(binding.BindString(&config.Conf.Tts.Openai.Model))
+
+ aliyunOssKeyIdEntry := StyledEntry("阿里云 Aliyun Access Key ID")
+ aliyunOssKeyIdEntry.Bind(binding.BindString(&config.Conf.Tts.Aliyun.Oss.AccessKeyId))
+ aliyunOssKeySecretEntry := StyledPasswordEntry("阿里云 Aliyun Access Key Secret")
+ aliyunOssKeySecretEntry.Bind(binding.BindString(&config.Conf.Tts.Aliyun.Oss.AccessKeySecret))
+ aliyunOssBucketEntry := StyledEntry("阿里云 Aliyun OSS Bucket名称")
+ aliyunOssBucketEntry.Bind(binding.BindString(&config.Conf.Tts.Aliyun.Oss.Bucket))
+
+ aliyunSpeechKeyIdEntry := StyledEntry("阿里云 Aliyun Speech Access Key ID")
+ aliyunSpeechKeyIdEntry.Bind(binding.BindString(&config.Conf.Tts.Aliyun.Speech.AccessKeyId))
+ aliyunSpeechKeySecretEntry := StyledPasswordEntry("阿里云 Aliyun Speech Access Key Secret")
+ aliyunSpeechKeySecretEntry.Bind(binding.BindString(&config.Conf.Tts.Aliyun.Speech.AccessKeySecret))
+ aliyunSpeechAppKeyEntry := StyledEntry("阿里云 Aliyun Speech App Key")
+ aliyunSpeechAppKeyEntry.Bind(binding.BindString(&config.Conf.Tts.Aliyun.Speech.AppKey))
form := widget.NewForm(
- widget.NewFormItem("Fasterwhisper模型 Model", localModelFasterwhisperEntry),
- widget.NewFormItem("Whispercpp模型 Model", localModelWhispercppEntry),
- widget.NewFormItem("Whisperkit模型 Model", localModelWhisperkitEntry),
+ widget.NewFormItem("提供商 Provider", providerSelect),
+
+ widget.NewFormItem("OpenAI Base URL", openaiBaseUrlEntry),
+ widget.NewFormItem("OpenAI API Key", openaiApiKeyEntry),
+ widget.NewFormItem("OpenAI 模型 Model", openaiModelEntry),
+
+ widget.NewFormItem("阿里云 Aliyun OSS Access Key ID", aliyunOssKeyIdEntry),
+ widget.NewFormItem("阿里云 Aliyun OSS Access Key Secret", aliyunOssKeySecretEntry),
+ widget.NewFormItem("阿里云 Aliyun OSS Bucket", aliyunOssBucketEntry),
+
+ widget.NewFormItem("阿里云 Aliyun Speech Access Key ID", aliyunSpeechKeyIdEntry),
+ widget.NewFormItem("阿里云 Aliyun Speech Access Key Secret", aliyunSpeechKeySecretEntry),
+ widget.NewFormItem("阿里云 Aliyun Speech App Key", aliyunSpeechAppKeyEntry),
)
- return StyledCard("本地模型配置 Local model setting", form)
+ return GlassCard("文本转语音配置 TTS Config", "文本转语音配置 TTS config", form)
}
// 创建视频输入容器
@@ -444,33 +531,31 @@ func createSubtitleSettingsCard(sm *SubtitleManager) *fyne.Container {
// 创建配音设置卡片
func createVoiceSettingsCard(sm *SubtitleManager) *fyne.Container {
- genderSelect := StyledSelect([]string{"男声 Male", "女声 Female"}, func(value string) {
- if value == "男声 Male" {
- sm.SetVoiceoverGender(2)
- } else {
- sm.SetVoiceoverGender(1)
- }
- })
- genderSelect.SetSelected("男声 Male")
- genderSelect.Disable()
+ voiceCodeEntry := widget.NewEntry()
+ voiceCodeEntry.SetPlaceHolder("输入声音代码 Enter voice code")
+ voiceCodeEntry.OnChanged = func(text string) {
+ sm.SetTtsVoiceCode(text)
+ }
+ voiceCodeEntry.Disable()
+ // todo 限制为仅阿里云
audioSampleButton := SecondaryButton("选择音色克隆样本 Choose voice clone sample", theme.MediaMusicIcon(), sm.ShowAudioFileDialog)
audioSampleButton.Disable()
voiceoverCheck := widget.NewCheck("启用配音 Enable dubbing", func(checked bool) {
sm.SetVoiceoverEnabled(checked)
if checked {
- genderSelect.Enable()
+ voiceCodeEntry.Enable()
audioSampleButton.Enable()
} else {
- genderSelect.Disable()
+ voiceCodeEntry.Disable()
audioSampleButton.Disable()
}
})
grid := container.NewVBox(
container.NewHBox(voiceoverCheck),
- container.NewHBox(genderSelect, audioSampleButton),
+ container.NewHBox(container.NewBorder(voiceCodeEntry, nil, nil, audioSampleButton)),
)
return StyledCard("3. 配音设置 Dubbing setting", grid)
@@ -756,91 +841,3 @@ func createStartButton(window fyne.Window, sm *SubtitleManager, videoInputContai
return btn
}
-
-// 创建OpenAI配置组
-func createOpenAIConfigGroup() *fyne.Container {
- openaiBaseUrlEntry := StyledEntry("OpenAI API base url")
- openaiBaseUrlEntry.Bind(binding.BindString(&config.Conf.Openai.BaseUrl))
-
- openaiModelEntry := StyledEntry("OpenAI模型名称 Model name")
- openaiModelEntry.Bind(binding.BindString(&config.Conf.Openai.Model))
-
- openaiApiKeyEntry := StyledPasswordEntry("OpenAI API密钥 Key")
- openaiApiKeyEntry.Bind(binding.BindString(&config.Conf.Openai.ApiKey))
-
- form := widget.NewForm(
- widget.NewFormItem("API base url", openaiBaseUrlEntry),
- widget.NewFormItem("模型名称 Model name", openaiModelEntry),
- widget.NewFormItem("API密钥 key", openaiApiKeyEntry),
- )
-
- return StyledCard("OpenAI配置 Config", form)
-}
-
-// 创建Whisper配置组
-func createWhisperConfigGroup() *fyne.Container {
- whisperBaseUrlEntry := StyledEntry("Whisper API base url")
- whisperBaseUrlEntry.Bind(binding.BindString(&config.Conf.Openai.Whisper.BaseUrl))
-
- whisperApiKeyEntry := StyledPasswordEntry("Whisper API密钥")
- whisperApiKeyEntry.Bind(binding.BindString(&config.Conf.Openai.Whisper.ApiKey))
-
- form := widget.NewForm(
- widget.NewFormItem("API base url", whisperBaseUrlEntry),
- widget.NewFormItem("API密钥 Key", whisperApiKeyEntry),
- )
-
- return StyledCard("Whisper配置 Config", form)
-}
-
-// 创建阿里云OSS配置组
-func createAliyunOSSConfigGroup() *fyne.Container {
- ossAccessKeyIdEntry := StyledEntry("阿里云AccessKey ID")
- ossAccessKeyIdEntry.Bind(binding.BindString(&config.Conf.Aliyun.Oss.AccessKeyId))
-
- ossAccessKeySecretEntry := StyledPasswordEntry("阿里云AccessKey Secret")
- ossAccessKeySecretEntry.Bind(binding.BindString(&config.Conf.Aliyun.Oss.AccessKeySecret))
-
- ossBucketEntry := StyledEntry("OSS Bucket名称 ")
- ossBucketEntry.Bind(binding.BindString(&config.Conf.Aliyun.Oss.Bucket))
-
- form := widget.NewForm(
- widget.NewFormItem("AccessKey ID", ossAccessKeyIdEntry),
- widget.NewFormItem("AccessKey Secret", ossAccessKeySecretEntry),
- widget.NewFormItem("Bucket名称 Name", ossBucketEntry),
- )
-
- return GlassCard("阿里云OSS配置 Aliyun OSS Config", "对象存储服务OSS service", form)
-}
-
-// 创建阿里云语音配置组
-func createAliyunSpeechConfigGroup() *fyne.Container {
- ossAccessKeyIdEntry := StyledEntry("阿里云 AccessKey ID")
- ossAccessKeyIdEntry.Bind(binding.BindString(&config.Conf.Aliyun.Speech.AccessKeyId))
-
- ossAccessKeySecretEntry := StyledPasswordEntry("阿里云 AccessKey Secret")
- ossAccessKeySecretEntry.Bind(binding.BindString(&config.Conf.Aliyun.Speech.AccessKeySecret))
-
- speechAppKeyEntry := StyledEntry("阿里云语音服务 AppKey")
- speechAppKeyEntry.Bind(binding.BindString(&config.Conf.Aliyun.Speech.AppKey))
-
- form := widget.NewForm(
- widget.NewFormItem("AccessKey ID", ossAccessKeyIdEntry),
- widget.NewFormItem("AccessKey Secret", ossAccessKeySecretEntry),
- widget.NewFormItem("AppKey", speechAppKeyEntry),
- )
-
- return StyledCard("阿里云语音配置 Aliyun Speech config", form)
-}
-
-// 创建阿里云百炼配置组
-func createAliyunBailianConfigGroup() *fyne.Container {
- bailianApiKeyEntry := StyledPasswordEntry("阿里云百炼API密钥 Aliyun bailian api key")
- bailianApiKeyEntry.Bind(binding.BindString(&config.Conf.Aliyun.Bailian.ApiKey))
-
- form := widget.NewForm(
- widget.NewFormItem("API密钥 key", bailianApiKeyEntry),
- )
-
- return StyledCard("阿里云百炼配置 Aliyun bailian config", form)
-}
diff --git a/internal/dto/subtitle_task.go b/internal/dto/subtitle_task.go
index ec108f0c..da56b246 100644
--- a/internal/dto/subtitle_task.go
+++ b/internal/dto/subtitle_task.go
@@ -1,21 +1,21 @@
package dto
type StartVideoSubtitleTaskReq struct {
- AppId uint32 `json:"app_id"`
- Url string `json:"url"`
- OriginLanguage string `json:"origin_lang"`
- TargetLang string `json:"target_lang"`
- Bilingual uint8 `json:"bilingual"`
- TranslationSubtitlePos uint8 `json:"translation_subtitle_pos"`
- ModalFilter uint8 `json:"modal_filter"`
- Tts uint8 `json:"tts"`
- TtsVoiceCode uint8 `json:"tts_voice_code"`
- TtsVoiceCloneSrcFileUrl string `json:"tts_voice_clone_src_file_url"`
- Replace []string `json:"replace"`
- Language string `json:"language"`
- EmbedSubtitleVideoType string `json:"embed_subtitle_video_type"`
- VerticalMajorTitle string `json:"vertical_major_title"`
- VerticalMinorTitle string `json:"vertical_minor_title"`
+ AppId uint32 `json:"app_id"`
+ Url string `json:"url"`
+ OriginLanguage string `json:"origin_lang"`
+ TargetLang string `json:"target_lang"`
+ Bilingual uint8 `json:"bilingual"`
+ TranslationSubtitlePos uint8 `json:"translation_subtitle_pos"`
+ ModalFilter uint8 `json:"modal_filter"`
+ Tts uint8 `json:"tts"`
+ TtsVoiceCode string `json:"tts_voice_code"`
+ TtsVoiceCloneSrcFileUrl string `json:"tts_voice_clone_src_file_url"`
+ Replace []string `json:"replace"`
+ Language string `json:"language"`
+ EmbedSubtitleVideoType string `json:"embed_subtitle_video_type"`
+ VerticalMajorTitle string `json:"vertical_major_title"`
+ VerticalMinorTitle string `json:"vertical_minor_title"`
OriginLanguageWordOneLine int `json:"origin_language_word_one_line"`
}
diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.go
index 811706d0..2a6831f4 100644
--- a/internal/service/audio2subtitle.go
+++ b/internal/service/audio2subtitle.go
@@ -771,7 +771,7 @@ func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int, []types.Word)
}
func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segmentIdx int, originLanguage types.StandardLanguageCode, words []types.Word, resultType types.SubtitleResultType, maxWordOneLine int) error {
- if len(srtBlocks) == 0 {
+ if len(srtBlocks) == 0 || len(words) == 0 {
return nil
}
diff --git a/internal/service/init.go b/internal/service/init.go
index 7b42006b..47150403 100644
--- a/internal/service/init.go
+++ b/internal/service/init.go
@@ -6,9 +6,9 @@ import (
"krillin-ai/log"
"krillin-ai/pkg/aliyun"
"krillin-ai/pkg/fasterwhisper"
- "krillin-ai/pkg/whispercpp"
"krillin-ai/pkg/openai"
"krillin-ai/pkg/whisper"
+ "krillin-ai/pkg/whispercpp"
"krillin-ai/pkg/whisperkit"
"go.uber.org/zap"
@@ -17,7 +17,7 @@ import (
type Service struct {
Transcriber types.Transcriber
ChatCompleter types.ChatCompleter
- TtsClient *aliyun.TtsClient
+ TtsClient types.Ttser
OssClient *aliyun.OssClient
VoiceCloneClient *aliyun.VoiceCloneClient
}
@@ -25,34 +25,41 @@ type Service struct {
func NewService() *Service {
var transcriber types.Transcriber
var chatCompleter types.ChatCompleter
+ var ttsClient types.Ttser
- switch config.Conf.App.TranscribeProvider {
+ switch config.Conf.Transcribe.Provider {
case "openai":
- transcriber = whisper.NewClient(config.Conf.Openai.Whisper.BaseUrl, config.Conf.Openai.Whisper.ApiKey, config.Conf.App.Proxy)
- case "aliyun":
- transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)
+ transcriber = whisper.NewClient(config.Conf.Transcribe.Openai.BaseUrl, config.Conf.Transcribe.Openai.ApiKey, config.Conf.App.Proxy)
case "fasterwhisper":
- transcriber = fasterwhisper.NewFastwhisperProcessor(config.Conf.LocalModel.Fasterwhisper)
+ transcriber = fasterwhisper.NewFastwhisperProcessor(config.Conf.Transcribe.Fasterwhisper.Model)
case "whispercpp":
- transcriber = whispercpp.NewWhispercppProcessor(config.Conf.LocalModel.Whispercpp)
+ transcriber = whispercpp.NewWhispercppProcessor(config.Conf.Transcribe.Whispercpp.Model)
case "whisperkit":
- transcriber = whisperkit.NewWhisperKitProcessor(config.Conf.LocalModel.Whisperkit)
+ transcriber = whisperkit.NewWhisperKitProcessor(config.Conf.Transcribe.Whisperkit.Model)
+ case "aliyun":
+ cc, err := aliyun.NewAsrClient(config.Conf.Transcribe.Aliyun.Speech.AccessKeyId, config.Conf.Transcribe.Aliyun.Speech.AccessKeySecret, config.Conf.Transcribe.Aliyun.Speech.AppKey, true)
+ if err != nil {
+ log.GetLogger().Error("创建阿里云语音识别客户端失败: ", zap.Error(err))
+ return nil
+ }
+ transcriber = cc
}
- log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.App.TranscribeProvider))
+ log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.Transcribe.Provider))
+
+ chatCompleter = openai.NewClient(config.Conf.Llm.BaseUrl, config.Conf.Llm.ApiKey, config.Conf.App.Proxy)
- switch config.Conf.App.LlmProvider {
+ switch config.Conf.Tts.Provider {
case "openai":
- chatCompleter = openai.NewClient(config.Conf.Openai.BaseUrl, config.Conf.Openai.ApiKey, config.Conf.App.Proxy)
+ ttsClient = openai.NewClient(config.Conf.Tts.Openai.BaseUrl, config.Conf.Tts.Openai.ApiKey, config.Conf.App.Proxy)
case "aliyun":
- chatCompleter = aliyun.NewChatClient(config.Conf.Aliyun.Bailian.ApiKey)
+ ttsClient = aliyun.NewTtsClient(config.Conf.Tts.Aliyun.Speech.AccessKeyId, config.Conf.Tts.Aliyun.Speech.AccessKeySecret, config.Conf.Tts.Aliyun.Speech.AppKey)
}
- log.GetLogger().Info("当前选择的LLM源: ", zap.String("llm", config.Conf.App.LlmProvider))
return &Service{
Transcriber: transcriber,
ChatCompleter: chatCompleter,
- TtsClient: aliyun.NewTtsClient(config.Conf.Aliyun.Speech.AccessKeyId, config.Conf.Aliyun.Speech.AccessKeySecret, config.Conf.Aliyun.Speech.AppKey),
- OssClient: aliyun.NewOssClient(config.Conf.Aliyun.Oss.AccessKeyId, config.Conf.Aliyun.Oss.AccessKeySecret, config.Conf.Aliyun.Oss.Bucket),
- VoiceCloneClient: aliyun.NewVoiceCloneClient(config.Conf.Aliyun.Speech.AccessKeyId, config.Conf.Aliyun.Speech.AccessKeySecret, config.Conf.Aliyun.Speech.AppKey),
+ TtsClient: ttsClient,
+ OssClient: aliyun.NewOssClient(config.Conf.Transcribe.Aliyun.Oss.AccessKeyId, config.Conf.Transcribe.Aliyun.Oss.AccessKeySecret, config.Conf.Transcribe.Aliyun.Oss.Bucket),
+ VoiceCloneClient: aliyun.NewVoiceCloneClient(config.Conf.Tts.Aliyun.Speech.AccessKeyId, config.Conf.Tts.Aliyun.Speech.AccessKeySecret, config.Conf.Tts.Aliyun.Speech.AppKey),
}
}
diff --git a/internal/service/subtitle_service.go b/internal/service/subtitle_service.go
index 3b1369dc..21c5e174 100644
--- a/internal/service/subtitle_service.go
+++ b/internal/service/subtitle_service.go
@@ -81,13 +81,6 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.Star
}
storage.SubtitleTasks.Store(taskId, taskPtr)
- var ttsVoiceCode string
- if req.TtsVoiceCode == types.SubtitleTaskTtsVoiceCodeLongyu {
- ttsVoiceCode = "longyu"
- } else {
- ttsVoiceCode = "longchen"
- }
-
// 处理声音克隆源
var voiceCloneAudioUrl string
if req.TtsVoiceCloneSrcFileUrl != "" {
@@ -110,7 +103,7 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.Star
SubtitleResultType: resultType,
EnableModalFilter: req.ModalFilter == types.SubtitleTaskModalFilterYes,
EnableTts: req.Tts == types.SubtitleTaskTtsYes,
- TtsVoiceCode: ttsVoiceCode,
+ TtsVoiceCode: req.TtsVoiceCode,
VoiceCloneAudioUrl: voiceCloneAudioUrl,
ReplaceWordsMap: replaceWordsMap,
OriginLanguage: types.StandardLanguageCode(req.OriginLanguage),
diff --git a/internal/types/interface.go b/internal/types/interface.go
index 6d6de42c..203c0a33 100644
--- a/internal/types/interface.go
+++ b/internal/types/interface.go
@@ -7,3 +7,7 @@ type ChatCompleter interface {
type Transcriber interface {
Transcription(audioFile, language, wordDir string) (*TranscriptionData, error)
}
+
+type Ttser interface {
+ Text2Speech(text string, voice string, outputFile string) error
+}
diff --git a/pkg/aliyun/asr.go b/pkg/aliyun/asr.go
index c071a465..8cc6b642 100644
--- a/pkg/aliyun/asr.go
+++ b/pkg/aliyun/asr.go
@@ -1,346 +1,232 @@
package aliyun
import (
+ "context"
"encoding/json"
+ "errors"
"fmt"
- "github.com/google/uuid"
- "github.com/gorilla/websocket"
+ "github.com/aliyun/alibaba-cloud-sdk-go/sdk"
+ "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
+ "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"go.uber.org/zap"
- "io"
- "krillin-ai/internal/storage"
+ "krillin-ai/config"
"krillin-ai/internal/types"
"krillin-ai/log"
- "net/http"
- "os"
- "os/exec"
+ "krillin-ai/pkg/util"
"path/filepath"
"strings"
"time"
)
type AsrClient struct {
- BailianApiKey string
-}
-
-func NewAsrClient(bailianApiKey string) *AsrClient {
- return &AsrClient{
- BailianApiKey: bailianApiKey,
+ client *sdk.Client
+ appKey string
+ regionID string
+ domain string
+ apiVersion string
+ product string
+ enableWords bool
+ pollInterval time.Duration
+ maxPollTime time.Duration
+ ossClient *OssClient
+}
+
+func NewAsrClient(accessKeyID, accessKeySecret, appKey string, enableWords bool) (*AsrClient, error) {
+ const (
+ regionID = "cn-shanghai"
+ domain = "filetrans.cn-shanghai.aliyuncs.com"
+ apiVersion = "2018-08-17"
+ product = "nls-filetrans"
+ pollInterval = 10 * time.Second
+ maxPollTime = 10 * time.Minute
+ )
+
+ clientcConfig := sdk.NewConfig()
+ credential := credentials.NewAccessKeyCredential(accessKeyID, accessKeySecret)
+ client, err := sdk.NewClientWithOptions(regionID, clientcConfig, credential)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create alibaba cloud client: %v", err)
}
-}
-const (
- wsURL = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/" // WebSocket服务器地址
-)
-
-var dialer = websocket.DefaultDialer
+ return &AsrClient{
+ client: client,
+ appKey: appKey,
+ regionID: regionID,
+ domain: domain,
+ apiVersion: apiVersion,
+ product: product,
+ enableWords: enableWords,
+ pollInterval: pollInterval,
+ maxPollTime: maxPollTime,
+ ossClient: NewOssClient(config.Conf.Transcribe.Aliyun.Oss.AccessKeyId, config.Conf.Transcribe.Aliyun.Oss.AccessKeySecret, config.Conf.Transcribe.Aliyun.Oss.Bucket),
+ }, nil
+}
+
+type Word struct {
+ Word string `json:"word"`
+ BeginTime float64 `json:"beginTime"`
+ EndTime float64 `json:"endTime"`
+ Confidence float64 `json:"confidence"`
+}
+
+type Sentence struct {
+ Text string `json:"Text"`
+ BeginTime float64 `json:"BeginTime"`
+ EndTime float64 `json:"EndTime"`
+}
+
+type RecognitionResult struct {
+ Sentences []Sentence `json:"Sentences"`
+ Words []Word `json:"Words,omitempty"`
+}
+
+type TaskResponse struct {
+ TaskId string `json:"TaskId"`
+ StatusText string `json:"StatusText"`
+ RequestId string `json:"RequestId"`
+ Result *RecognitionResult `json:"Result,omitempty"`
+ StatusCode int `json:"StatusCode"`
+ BizDuration float64 `json:"BizDuration"`
+ pollInterval time.Duration
+ maxPollTime time.Duration
+}
+
+func (c *AsrClient) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
+ const (
+ postRequestAction = "SubmitTask"
+ getRequestAction = "GetTaskResult"
+ keyTask = "Task"
+ keyTaskId = "TaskId"
+ statusSuccess = "SUCCESS"
+ statusRunning = "RUNNING"
+ statusQueueing = "QUEUEING"
+ )
-func (c AsrClient) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
// 处理音频
- processedAudioFile, err := processAudio(audioFile)
+ processedAudioFile, err := util.ProcessAudio(audioFile)
if err != nil {
log.GetLogger().Error("处理音频失败", zap.Error(err), zap.String("audio file", audioFile))
return nil, err
}
- // 连接WebSocket服务
- conn, err := connectWebSocket(c.BailianApiKey)
+ // 上传音频文件
+ fileKey := util.GenerateRandStringWithUpperLowerNum(5) + filepath.Ext(audioFile)
+ err = c.ossClient.UploadFile(context.Background(), fileKey, processedAudioFile, c.ossClient.Bucket)
if err != nil {
- log.GetLogger().Error("连接WebSocket失败", zap.Error(err), zap.String("audio file", audioFile))
- return nil, err
+ log.GetLogger().Error("StartVideoSubtitleTask UploadFile err", zap.Any("audio file", audioFile), zap.Error(err))
+ return nil, errors.New("上传声音克隆源失败")
}
- defer closeConnection(conn)
-
- // 启动一个goroutine来接收结果
- taskStarted := make(chan bool)
- taskDone := make(chan bool)
+ audioUrl := fmt.Sprintf("https://%s.oss-cn-shanghai.aliyuncs.com/%s", c.ossClient.Bucket, fileKey)
+ log.GetLogger().Info("上传待转录音频到阿里云oss成功", zap.String("local file name", audioFile), zap.String("oss url", audioUrl))
- words := make([]types.Word, 0)
- text := ""
- startResultReceiver(conn, &words, &text, taskStarted, taskDone)
+ // 提交识别任务
+ taskParams := map[string]string{
+ "appkey": c.appKey,
+ "file_link": audioUrl,
+ "version": "4.0",
+ "enable_words": fmt.Sprintf("%v", c.enableWords),
+ }
- // 发送run-task指令
- taskID, err := sendRunTaskCmd(conn, language)
+ task, err := json.Marshal(taskParams)
if err != nil {
- log.GetLogger().Error("发送run-task指令失败", zap.Error(err), zap.String("audio file", audioFile))
+ return nil, fmt.Errorf("failed to marshal task params: %v", err)
}
- // 等待task-started事件
- waitForTaskStarted(taskStarted)
+ postRequest := requests.NewCommonRequest()
+ postRequest.Domain = c.domain
+ postRequest.Version = c.apiVersion
+ postRequest.Product = c.product
+ postRequest.ApiName = postRequestAction
+ postRequest.Method = "POST"
+ postRequest.FormParams[keyTask] = string(task)
- // 发送待识别音频文件流
- if err := sendAudioData(conn, processedAudioFile); err != nil {
- log.GetLogger().Error("发送音频数据失败", zap.Error(err))
+ postResponse, err := c.client.ProcessCommonRequest(postRequest)
+ if err != nil {
+ return nil, fmt.Errorf("failed to submit task: %v", err)
}
- // 发送finish-task指令
- if err := sendFinishTaskCmd(conn, taskID); err != nil {
- log.GetLogger().Error("发送finish-task指令失败", zap.Error(err), zap.String("audio file", audioFile))
+ if postResponse.GetHttpStatus() != 200 {
+ return nil, fmt.Errorf("recognition request failed, HTTP error code: %d", postResponse.GetHttpStatus())
}
- // 等待任务完成或失败
- <-taskDone
-
- if len(words) == 0 {
- log.GetLogger().Info("识别结果为空", zap.String("audio file", audioFile))
+ var postResult TaskResponse
+ if err := json.Unmarshal([]byte(postResponse.GetHttpContentString()), &postResult); err != nil {
+ return nil, fmt.Errorf("failed to parse response: %v", err)
}
- log.GetLogger().Debug("识别结果", zap.Any("words", words), zap.String("text", text), zap.String("audio file", audioFile))
- transcriptionData := &types.TranscriptionData{
- Text: text,
- Words: words,
+ if postResult.StatusText != statusSuccess {
+ return nil, fmt.Errorf("recognition request failed with status: %s (code: %d)",
+ postResult.StatusText, postResult.StatusCode)
}
- return transcriptionData, nil
-}
-
-// 定义结构体来表示JSON数据
-type AsrHeader struct {
- Action string `json:"action"`
- TaskID string `json:"task_id"`
- Streaming string `json:"streaming"`
- Event string `json:"event"`
- ErrorCode string `json:"error_code,omitempty"`
- ErrorMessage string `json:"error_message,omitempty"`
- Attributes map[string]interface{} `json:"attributes"`
-}
-
-type Output struct {
- Sentence struct {
- BeginTime int64 `json:"begin_time"`
- EndTime *int64 `json:"end_time"`
- Text string `json:"text"`
- Words []struct {
- BeginTime int64 `json:"begin_time"`
- EndTime *int64 `json:"end_time"`
- Text string `json:"text"`
- Punctuation string `json:"punctuation"`
- } `json:"words"`
- } `json:"sentence"`
- Usage interface{} `json:"usage"`
-}
-
-type Payload struct {
- TaskGroup string `json:"task_group"`
- Task string `json:"task"`
- Function string `json:"function"`
- Model string `json:"model"`
- Parameters Params `json:"parameters"`
- Resources []Resource `json:"resources"`
- Input Input `json:"input"`
- Output Output `json:"output,omitempty"`
-}
+ taskID := postResult.TaskId
+ if taskID == "" {
+ return nil, fmt.Errorf("empty task ID in response")
+ }
-type Params struct {
- Format string `json:"format"`
- SampleRate int `json:"sample_rate"`
- VocabularyID string `json:"vocabulary_id"`
- DisfluencyRemovalEnabled bool `json:"disfluency_removal_enabled"`
- LanguageHints []string `json:"language_hints"`
-}
+ // 查询识别结果
+ getRequest := requests.NewCommonRequest()
+ getRequest.Domain = c.domain
+ getRequest.Version = c.apiVersion
+ getRequest.Product = c.product
+ getRequest.ApiName = getRequestAction
+ getRequest.Method = "GET"
+ getRequest.QueryParams[keyTaskId] = taskID
-type Resource struct {
- ResourceID string `json:"resource_id"`
- ResourceType string `json:"resource_type"`
-}
+ startTime := time.Now()
+ var resultData *types.TranscriptionData
-type Input struct {
-}
+ for {
+ if time.Since(startTime) > c.maxPollTime {
+ return nil, fmt.Errorf("polling timeout after %v", c.maxPollTime)
+ }
-type Event struct {
- Header AsrHeader `json:"header"`
- Payload Payload `json:"payload"`
-}
+ getResponse, err := c.client.ProcessCommonRequest(getRequest)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get task result: %v", err)
+ }
-// 把音频处理成单声道、16k采样率
-func processAudio(filePath string) (string, error) {
- dest := strings.ReplaceAll(filePath, filepath.Ext(filePath), "_mono_16K.mp3")
- cmdArgs := []string{"-i", filePath, "-ac", "1", "-ar", "16000", "-b:a", "192k", dest}
- cmd := exec.Command(storage.FfmpegPath, cmdArgs...)
- output, err := cmd.CombinedOutput()
- if err != nil {
- log.GetLogger().Error("处理音频失败", zap.Error(err), zap.String("audio file", filePath), zap.String("output", string(output)))
- return "", err
- }
- return dest, nil
-}
+ if getResponse.GetHttpStatus() != 200 {
+ return nil, fmt.Errorf("result query request failed, HTTP error code: %d", getResponse.GetHttpStatus())
+ }
-// 连接WebSocket服务
-func connectWebSocket(apiKey string) (*websocket.Conn, error) {
- header := make(http.Header)
- header.Add("X-DashScope-DataInspection", "enable")
- header.Add("Authorization", fmt.Sprintf("bearer %s", apiKey))
- conn, _, err := dialer.Dial(wsURL, header)
- return conn, err
-}
+ var getResult TaskResponse
+ if err := json.Unmarshal([]byte(getResponse.GetHttpContentString()), &getResult); err != nil {
+ return nil, fmt.Errorf("failed to parse result: %v", err)
+ }
-// 启动一个goroutine异步接收WebSocket消息
-func startResultReceiver(conn *websocket.Conn, words *[]types.Word, text *string, taskStarted chan<- bool, taskDone chan<- bool) {
- go func() {
- for {
- _, message, err := conn.ReadMessage()
- if err != nil {
- log.GetLogger().Error("解析服务器消息失败:", zap.Error(err))
- continue
+ switch getResult.StatusText {
+ case statusRunning, statusQueueing:
+ time.Sleep(c.pollInterval)
+ continue
+ case statusSuccess:
+ if getResult.Result == nil || len(getResult.Result.Sentences) == 0 {
+ return nil, fmt.Errorf("empty recognition result")
}
- currentEvent := Event{}
- err = json.Unmarshal(message, &currentEvent)
- if err != nil {
- log.GetLogger().Error("解析服务器消息失败:", zap.Error(err))
- continue
+
+ // 构建返回结果
+ resultData = &types.TranscriptionData{
+ Language: language,
+ Text: getResult.Result.Sentences[0].Text,
}
- if currentEvent.Payload.Output.Sentence.EndTime != nil {
- // 本句结束,添加当前的words和text
- *text += currentEvent.Payload.Output.Sentence.Text
- currentNum := 0
- if len(*words) > 0 {
- currentNum = (*words)[len(*words)-1].Num + 1
- }
- for _, word := range currentEvent.Payload.Output.Sentence.Words {
- *words = append(*words, types.Word{
- Num: currentNum,
- Text: strings.TrimSpace(word.Text), // 阿里云这边的word后面会有空格
- Start: float64(word.BeginTime) / 1000,
- End: float64(*word.EndTime) / 1000,
+
+ var words []types.Word
+ if c.enableWords && getResult.Result.Words != nil {
+ for i, v := range getResult.Result.Words {
+ words = append(words, types.Word{
+ Num: i,
+ Text: strings.TrimSpace(v.Word), // 阿里云这边的word后面会有空格
+ Start: v.BeginTime / 1000,
+ End: v.EndTime / 1000,
})
- currentNum++
}
}
- if handleEvent(conn, &currentEvent, taskStarted, taskDone) {
- return
- }
- }
- }()
-}
-
-// 发送run-task指令
-func sendRunTaskCmd(conn *websocket.Conn, language string) (string, error) {
- runTaskCmd, taskID, err := generateRunTaskCmd(language)
- if err != nil {
- return "", err
- }
- err = conn.WriteMessage(websocket.TextMessage, []byte(runTaskCmd))
- return taskID, err
-}
-
-// 生成run-task指令
-func generateRunTaskCmd(language string) (string, string, error) {
- taskID := uuid.New().String()
- runTaskCmd := Event{
- Header: AsrHeader{
- Action: "run-task",
- TaskID: taskID,
- Streaming: "duplex",
- },
- Payload: Payload{
- TaskGroup: "audio",
- Task: "asr",
- Function: "recognition",
- Model: "paraformer-realtime-v2",
- Parameters: Params{
- Format: "mp3",
- SampleRate: 16000,
- LanguageHints: []string{language},
- },
- Input: Input{},
- },
- }
- runTaskCmdJSON, err := json.Marshal(runTaskCmd)
- return string(runTaskCmdJSON), taskID, err
-}
-
-// 等待task-started事件
-func waitForTaskStarted(taskStarted chan bool) {
- select {
- case <-taskStarted:
- log.GetLogger().Info("阿里云语音识别任务开启成功")
- case <-time.After(10 * time.Second):
- log.GetLogger().Error("等待task-started超时,任务开启失败")
- }
-}
+ resultData.Words = words
-// 发送音频数据
-func sendAudioData(conn *websocket.Conn, filePath string) error {
- file, err := os.Open(filePath)
- if err != nil {
- return err
- }
- defer file.Close()
-
- buf := make([]byte, 1024) // 100ms的音频大约1024字节
- for {
- n, err := file.Read(buf)
- if n == 0 {
- break
+ return resultData, nil
+ default:
+ return nil, fmt.Errorf("aliyun asr recognition failed with status: %s (code: %d)",
+ getResult.StatusText, getResult.StatusCode)
}
- if err != nil && err != io.EOF {
- return err
- }
- err = conn.WriteMessage(websocket.BinaryMessage, buf[:n])
- if err != nil {
- return err
- }
- time.Sleep(100 * time.Millisecond)
- }
- return nil
-}
-
-// 发送finish-task指令
-func sendFinishTaskCmd(conn *websocket.Conn, taskID string) error {
- finishTaskCmd, err := generateFinishTaskCmd(taskID)
- if err != nil {
- return err
- }
- err = conn.WriteMessage(websocket.TextMessage, []byte(finishTaskCmd))
- return err
-}
-
-// 生成finish-task指令
-func generateFinishTaskCmd(taskID string) (string, error) {
- finishTaskCmd := Event{
- Header: AsrHeader{
- Action: "finish-task",
- TaskID: taskID,
- Streaming: "duplex",
- },
- Payload: Payload{
- Input: Input{},
- },
- }
- finishTaskCmdJSON, err := json.Marshal(finishTaskCmd)
- return string(finishTaskCmdJSON), err
-}
-
-// 处理事件
-func handleEvent(conn *websocket.Conn, event *Event, taskStarted chan<- bool, taskDone chan<- bool) bool {
- switch event.Header.Event {
- case "task-started":
- log.GetLogger().Info("收到task-started事件", zap.String("taskID", event.Header.TaskID))
- taskStarted <- true
- case "result-generated":
- log.GetLogger().Info("收到result-generated事件", zap.String("当前text", event.Payload.Output.Sentence.Text))
- case "task-finished":
- log.GetLogger().Info("收到task-finished事件,任务完成", zap.String("taskID", event.Header.TaskID))
- taskDone <- true
- return true
- case "task-failed":
- log.GetLogger().Info("收到task-failed事件", zap.String("taskID", event.Header.TaskID))
- handleTaskFailed(event, conn)
- taskDone <- true
- return true
- default:
- log.GetLogger().Info("未知事件:", zap.String("event", event.Header.Event))
- }
- return false
-}
-
-// 处理任务失败事件
-func handleTaskFailed(event *Event, conn *websocket.Conn) {
- log.GetLogger().Error("任务失败:", zap.String("error", event.Header.ErrorMessage))
-}
-
-// 关闭连接
-func closeConnection(conn *websocket.Conn) {
- if conn != nil {
- conn.Close()
}
}
diff --git a/pkg/openai/openai.go b/pkg/openai/openai.go
index 29eaa224..c55035f6 100644
--- a/pkg/openai/openai.go
+++ b/pkg/openai/openai.go
@@ -2,16 +2,20 @@ package openai
import (
"context"
+ "fmt"
openai "github.com/sashabaranov/go-openai"
"go.uber.org/zap"
"io"
"krillin-ai/config"
"krillin-ai/log"
+ "net/http"
+ "os"
+ "strings"
)
func (c *Client) ChatCompletion(query string) (string, error) {
req := openai.ChatCompletionRequest{
- Model: openai.GPT4oMini20240718,
+ Model: config.Conf.Llm.Model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
@@ -25,9 +29,6 @@ func (c *Client) ChatCompletion(query string) (string, error) {
Stream: true,
MaxTokens: 8192,
}
- if config.Conf.Openai.Model != "" {
- req.Model = config.Conf.Openai.Model
- }
stream, err := c.client.CreateChatCompletionStream(context.Background(), req)
if err != nil {
@@ -56,3 +57,53 @@ func (c *Client) ChatCompletion(query string) (string, error) {
return resContent, nil
}
+
+func (c *Client) Text2Speech(text, voice string, outputFile string) error {
+ baseUrl := config.Conf.Tts.Openai.BaseUrl
+ if baseUrl == "" {
+ baseUrl = "https://api.openai.com/v1"
+ }
+ url := baseUrl + "/audio/speech"
+
+ // 创建HTTP请求
+ reqBody := fmt.Sprintf(`{
+ "model": "tts-1",
+ "input": "%s",
+ "voice":"%s",
+ "response_format": "wav"
+ }`, text, voice)
+ req, err := http.NewRequest("POST", url, strings.NewReader(reqBody))
+ if err != nil {
+ return err
+ }
+
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", config.Conf.Tts.Openai.ApiKey))
+
+ // 发送HTTP请求
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ log.GetLogger().Error("openai tts failed", zap.Int("status_code", resp.StatusCode), zap.String("body", string(body)))
+ return fmt.Errorf("openai tts none-200 status code: %d", resp.StatusCode)
+ }
+
+ file, err := os.Create(outputFile)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ _, err = io.Copy(file, resp.Body)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/pkg/util/audio.go b/pkg/util/audio.go
new file mode 100644
index 00000000..29b5ae53
--- /dev/null
+++ b/pkg/util/audio.go
@@ -0,0 +1,23 @@
+package util
+
+import (
+ "go.uber.org/zap"
+ "krillin-ai/internal/storage"
+ "krillin-ai/log"
+ "os/exec"
+ "path/filepath"
+ "strings"
+)
+
+// 把音频处理成单声道、16k采样率
+func ProcessAudio(filePath string) (string, error) {
+ dest := strings.ReplaceAll(filePath, filepath.Ext(filePath), "_mono_16K.mp3")
+ cmdArgs := []string{"-i", filePath, "-ac", "1", "-ar", "16000", "-b:a", "192k", dest}
+ cmd := exec.Command(storage.FfmpegPath, cmdArgs...)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ log.GetLogger().Error("处理音频失败", zap.Error(err), zap.String("audio file", filePath), zap.String("output", string(output)))
+ return "", err
+ }
+ return dest, nil
+}