Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: add voice clone

沈浩51279e268ac5
diff --git a/config/config-example.toml b/config/config-example.toml
index dc669116..c4e12a44 100644
--- a/config/config-example.toml
+++ b/config/config-example.toml
@@ -13,9 +13,13 @@
api_key = "sk-XXX" # OpenAI API密钥
[aliyun]
- [aliyun.tts] # tts配置可以先忽略
+ [aliyun.oss] # 使用音色克隆必须配置该项
+ access_key_id = ""
+ access_key_secret = ""
+ bucket = ""
+ [aliyun.speech] # 使用配音必须配置该项
access_key_id = ""
access_key_secret = ""
app_key= ""
- [aliyun.bailian] # 阿里云百炼大模型平台api key
+ [aliyun.bailian] # llm_provider为aliyun时必须配置该项
api_key = ""
\ No newline at end of file
diff --git a/config/config.go b/config/config.go
index 5e8b50ec..562225df 100644
--- a/config/config.go
+++ b/config/config.go
@@ -23,7 +23,13 @@ type Openai struct {
ApiKey string `toml:"api_key"`
}
-type AliyunTts struct {
+type AliyunOss struct {
+ AccessKeyId string `toml:"access_key_id"`
+ AccessKeySecret string `toml:"access_key_secret"`
+ Bucket string `toml:"bucket"`
+}
+
+type AliyunSpeech struct {
AccessKeyId string `toml:"access_key_id"`
AccessKeySecret string `toml:"access_key_secret"`
AppKey string `toml:"app_key"`
@@ -34,7 +40,8 @@ type AliyunBailian struct {
}
type Aliyun struct {
- Tts AliyunTts `toml:"tts"`
+ Oss AliyunOss `toml:"oss"`
+ Speech AliyunSpeech `toml:"speech"`
Bailian AliyunBailian `toml:"bailian"`
}
diff --git a/go.mod b/go.mod
index 894d6a1c..129f86c1 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ toolchain go1.22.10
require (
github.com/BurntSushi/toml v1.3.2
github.com/aliyun/alibaba-cloud-sdk-go v1.63.72
+ github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.1.3
github.com/gin-gonic/gin v1.10.0
github.com/go-resty/resty/v2 v2.7.0
github.com/google/uuid v1.4.0
@@ -49,6 +50,7 @@ require (
golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.20.0 // indirect
+ golang.org/x/time v0.4.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
index c8045081..a8cfc57c 100644
--- a/go.sum
+++ b/go.sum
@@ -6,6 +6,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXY
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/aliyun/alibaba-cloud-sdk-go v1.63.72 h1:HvFZUzEbNvfe8F2Mg0wBGv90bPhWDxgVtDHR5zoBOU0=
github.com/aliyun/alibaba-cloud-sdk-go v1.63.72/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ=
+github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.1.3 h1:grJyLSdRJtfxKKhCTWSeJhnOQsp2WoLNdK8XA5FE9oo=
+github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.1.3/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
@@ -163,6 +165,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
+golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY=
+golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/internal/dto/subtitle_task.go b/internal/dto/subtitle_task.go
index 23028eba..abf5ee87 100644
--- a/internal/dto/subtitle_task.go
+++ b/internal/dto/subtitle_task.go
@@ -1,17 +1,18 @@
package dto
type StartVideoSubtitleTaskReq struct {
- AppId uint32 `json:"app_id"`
- Url string `json:"url"`
- OriginLanguage string `json:"origin_lang"`
- TargetLang string `json:"target_lang"`
- Bilingual uint8 `json:"bilingual"`
- TranslationSubtitlePos uint8 `json:"translation_subtitle_pos"`
- ModalFilter uint8 `json:"modal_filter"`
- Tts uint8 `json:"tts"`
- TtsVoiceCode uint8 `json:"tts_voice_code"`
- Replace []string `json:"replace"`
- Language string `json:"language"`
+ AppId uint32 `json:"app_id"`
+ Url string `json:"url"`
+ OriginLanguage string `json:"origin_lang"`
+ TargetLang string `json:"target_lang"`
+ Bilingual uint8 `json:"bilingual"`
+ TranslationSubtitlePos uint8 `json:"translation_subtitle_pos"`
+ ModalFilter uint8 `json:"modal_filter"`
+ Tts uint8 `json:"tts"`
+ TtsVoiceCode uint8 `json:"tts_voice_code"`
+ TtsVoiceCloneSrcFileUrl string `json:"tts_voice_clone_src_file_url"`
+ Replace []string `json:"replace"`
+ Language string `json:"language"`
}
type StartVideoSubtitleTaskResData struct {
diff --git a/internal/service/init.go b/internal/service/init.go
index 7b676942..042cdc2b 100644
--- a/internal/service/init.go
+++ b/internal/service/init.go
@@ -10,11 +10,11 @@ import (
)
type Service struct {
- OpenaiClient *openai.Client
- CosyCloneClient *aliyun.Client
-
- Transcriber types.Transcriber
- ChatCompleter types.ChatCompleter
+ Transcriber types.Transcriber
+ ChatCompleter types.ChatCompleter
+ TtsClient *aliyun.TtsClient
+ OssClient *aliyun.OssClient
+ VoiceCloneClient *aliyun.VoiceCloneClient
}
func NewService() *Service {
@@ -25,7 +25,7 @@ func NewService() *Service {
case "openai":
transcriber = openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.Proxy)
case "aliyun":
- transcriber = aliyun.NewClient()
+ transcriber = aliyun.NewAsrClient(config.Conf.Aliyun.Bailian.ApiKey)
}
log.GetLogger().Info("当前选择的转录源: ", zap.String("transcriber", config.Conf.App.TranscribeProvider))
@@ -38,10 +38,10 @@ func NewService() *Service {
log.GetLogger().Info("当前选择的LLM源: ", zap.String("llm", config.Conf.App.LlmProvider))
return &Service{
- OpenaiClient: openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.Proxy),
- CosyCloneClient: aliyun.NewClient(),
-
- Transcriber: transcriber,
- ChatCompleter: chatCompleter,
+ Transcriber: transcriber,
+ ChatCompleter: chatCompleter,
+ TtsClient: aliyun.NewTtsClient(config.Conf.Aliyun.Speech.AccessKeyId, config.Conf.Aliyun.Speech.AccessKeySecret, config.Conf.Aliyun.Speech.AppKey),
+ OssClient: aliyun.NewOssClient(config.Conf.Aliyun.Oss.AccessKeyId, config.Conf.Aliyun.Oss.AccessKeySecret, config.Conf.Aliyun.Oss.Bucket),
+ VoiceCloneClient: aliyun.NewVoiceCloneClient(config.Conf.Aliyun.Speech.AccessKeyId, config.Conf.Aliyun.Speech.AccessKeySecret, config.Conf.Aliyun.Speech.AppKey),
}
}
diff --git a/internal/service/subtitle_service.go b/internal/service/subtitle_service.go
index ae3806db..b8f927df 100644
--- a/internal/service/subtitle_service.go
+++ b/internal/service/subtitle_service.go
@@ -94,6 +94,21 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.Star
} else {
ttsVoiceCode = "sicheng"
}
+
+ // 处理声音克隆源
+ var voiceCloneAudioUrl string
+ if req.TtsVoiceCloneSrcFileUrl != "" {
+ localFileUrl := strings.TrimPrefix(req.TtsVoiceCloneSrcFileUrl, "local:")
+ fileKey := util.GenerateRandStringWithUpperLowerNum(5) + filepath.Ext(localFileUrl) // 防止url encode的问题,这里统一处理
+ err = s.OssClient.UploadFile(context.Background(), fileKey, localFileUrl, s.OssClient.Bucket)
+ if err != nil {
+ log.GetLogger().Error("StartVideoSubtitleTask UploadFile err", zap.Any("req", req), zap.Error(err))
+ return nil, errors.New("上传声音克隆源失败")
+ }
+ voiceCloneAudioUrl = fmt.Sprintf("https://%s.oss-cn-shanghai.aliyuncs.com/%s", s.OssClient.Bucket, fileKey)
+ log.GetLogger().Info("StartVideoSubtitleTask 上传声音克隆源成功", zap.Any("oss url", voiceCloneAudioUrl))
+ }
+
stepParam := types.SubtitleTaskStepParam{
TaskId: taskId,
TaskBasePath: taskBasePath,
@@ -102,6 +117,7 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.Star
EnableModalFilter: req.ModalFilter == types.SubtitleTaskModalFilterYes,
EnableTts: req.Tts == types.SubtitleTaskTtsYes,
TtsVoiceCode: ttsVoiceCode,
+ VoiceCloneAudioUrl: voiceCloneAudioUrl,
ReplaceWordsMap: replaceWordsMap,
OriginLanguage: types.StandardLanguageName(req.OriginLanguage),
TargetLanguage: types.StandardLanguageName(req.TargetLang),
@@ -298,7 +314,7 @@ func (s Service) getVideoInfo(ctx context.Context, stepParam *types.SubtitleTask
log.GetLogger().Debug("getVideoInfo title and description", zap.String("title", title), zap.String("description", description))
// 翻译
var result string
- result, err = s.OpenaiClient.ChatCompletion(fmt.Sprintf(types.TranslateVideoTitleAndDescriptionPrompt, types.GetStandardLanguageName(stepParam.TargetLanguage), title+"####"+description))
+ result, err = s.ChatCompleter.ChatCompletion(fmt.Sprintf(types.TranslateVideoTitleAndDescriptionPrompt, types.GetStandardLanguageName(stepParam.TargetLanguage), title+"####"+description))
if err != nil {
log.GetLogger().Error("getVideoInfo openai chat completion error", zap.Any("stepParam", stepParam), zap.Error(err))
}
@@ -961,8 +977,6 @@ func (s Service) srtFileToSpeech(ctx context.Context, stepParam *types.SubtitleT
return err
}
- // 创建临时目录存储音频文件
- aliyunClient := s.CosyCloneClient
var audioFiles []string
var currentTime time.Time
@@ -974,10 +988,22 @@ func (s Service) srtFileToSpeech(ctx context.Context, stepParam *types.SubtitleT
}
defer durationDetailFile.Close()
- // Step 2: 使用 Google TTS 生成音频
+ // Step 2: 使用 阿里云TTS
+ // 判断是否使用音色克隆
+ voiceCode := stepParam.TtsVoiceCode
+ if stepParam.VoiceCloneAudioUrl != "" {
+ var code string
+ code, err = s.VoiceCloneClient.CosyVoiceClone("krillinai", stepParam.VoiceCloneAudioUrl)
+ if err != nil {
+ log.GetLogger().Error("generateAudioSubtitles.srtFileToSpeech.VoiceCloneClient.CosyVoiceClone err", zap.Any("stepParam", stepParam), zap.Error(err))
+ return err
+ }
+ voiceCode = code
+ }
+
for i, sub := range subtitles {
outputFile := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf("subtitle_%d.wav", i+1))
- err = aliyunClient.Text2Speech(sub.Text, stepParam.TtsVoiceCode, outputFile)
+ err = s.TtsClient.Text2Speech(sub.Text, voiceCode, outputFile)
if err != nil {
log.GetLogger().Error("generateAudioSubtitles.srtFileToSpeech.Text2Speech err", zap.Any("taskId", stepParam.TaskId), zap.Any("num", i+1), zap.Error(err))
return err
@@ -1119,7 +1145,7 @@ func adjustAudioDuration(inputFile, outputFile, taskBasePath string, subtitleDur
}
silenceAudioDuration, _ := getAudioDuration(silenceFile)
- fmt.Println("silenceDuration:", silenceAudioDuration)
+ log.GetLogger().Debug("adjustAudioDuration", zap.Any("silenceDuration", silenceAudioDuration))
// 拼接音频和静音
concatFile := filepath.Join(taskBasePath, "concat.txt")
@@ -1144,7 +1170,7 @@ func adjustAudioDuration(inputFile, outputFile, taskBasePath string, subtitleDur
}
concatFileDuration, _ := getAudioDuration(outputFile)
- fmt.Println("concatFileDuration:", concatFileDuration)
+ log.GetLogger().Debug("adjustAudioDuration", zap.Any("concatFileDuration", concatFileDuration))
return nil
}
diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.go
index 6e5ff952..54d2cc6f 100644
--- a/internal/types/subtitle_task.go
+++ b/internal/types/subtitle_task.go
@@ -220,6 +220,7 @@ type SubtitleTaskStepParam struct {
EnableModalFilter bool
EnableTts bool
TtsVoiceCode string // 人声语音编码
+ VoiceCloneAudioUrl string // 音色克隆的源音频oss地址
ReplaceWordsMap map[string]string
OriginLanguage StandardLanguageName // 视频源语言
TargetLanguage StandardLanguageName // 用户希望的目标翻译语言
diff --git a/pkg/aliyun/transcription.go b/pkg/aliyun/asr.go
similarity index 95%
rename from pkg/aliyun/transcription.go
rename to pkg/aliyun/asr.go
index c54db64e..2dcdd95c 100644
--- a/pkg/aliyun/transcription.go
+++ b/pkg/aliyun/asr.go
@@ -17,13 +17,23 @@ import (
"time"
)
+type AsrClient struct {
+ BailianApiKey string
+}
+
+func NewAsrClient(bailianApiKey string) *AsrClient {
+ return &AsrClient{
+ BailianApiKey: bailianApiKey,
+ }
+}
+
const (
wsURL = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/" // WebSocket服务器地址
)
var dialer = websocket.DefaultDialer
-func (c *Client) Transcription(audioFile, language string) (*types.TranscriptionData, error) {
+func (c AsrClient) Transcription(audioFile, language string) (*types.TranscriptionData, error) {
// 处理音频
processedAudioFile, err := processAudio(audioFile)
if err != nil {
@@ -32,7 +42,7 @@ func (c *Client) Transcription(audioFile, language string) (*types.Transcription
}
// 连接WebSocket服务
- conn, err := connectWebSocket(c.bailianApiKey)
+ conn, err := connectWebSocket(c.BailianApiKey)
if err != nil {
log.GetLogger().Error("连接WebSocket失败", zap.Error(err), zap.String("audio file", audioFile))
return nil, err
@@ -241,7 +251,7 @@ func generateRunTaskCmd(language string) (string, string, error) {
func waitForTaskStarted(taskStarted chan bool) {
select {
case <-taskStarted:
- fmt.Println("任务开启成功")
+ log.GetLogger().Info("阿里云语音识别任务开启成功")
case <-time.After(10 * time.Second):
log.GetLogger().Error("等待task-started超时,任务开启失败")
}
@@ -308,7 +318,7 @@ func handleEvent(conn *websocket.Conn, event *Event, taskStarted chan<- bool, ta
case "result-generated":
log.GetLogger().Info("收到result-generated事件", zap.String("当前text", event.Payload.Output.Sentence.Text))
case "task-finished":
- fmt.Println("任务完成")
+ log.GetLogger().Info("收到task-finished事件,任务完成", zap.String("taskID", event.Header.TaskID))
taskDone <- true
return true
case "task-failed":
diff --git a/pkg/aliyun/token.go b/pkg/aliyun/base.go
similarity index 82%
rename from pkg/aliyun/token.go
rename to pkg/aliyun/base.go
index 4879299f..afce8082 100644
--- a/pkg/aliyun/token.go
+++ b/pkg/aliyun/base.go
@@ -5,7 +5,6 @@ import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"go.uber.org/zap"
- "krillin-ai/config"
"krillin-ai/log"
)
@@ -18,8 +17,8 @@ type TokenResult struct {
}
}
-func CreateToken() (string, error) {
- client, err := sdk.NewClientWithAccessKey("cn-shanghai", config.Conf.Aliyun.Tts.AccessKeyId, config.Conf.Aliyun.Tts.AccessKeySecret)
+func CreateToken(ak, sk string) (string, error) {
+ client, err := sdk.NewClientWithAccessKey("cn-shanghai", ak, sk)
if err != nil {
return "", err
}
diff --git a/pkg/aliyun/oss.go b/pkg/aliyun/oss.go
new file mode 100644
index 00000000..bfd34440
--- /dev/null
+++ b/pkg/aliyun/oss.go
@@ -0,0 +1,46 @@
+package aliyun
+
+import (
+ "context"
+ "fmt"
+ "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
+ "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
+ "os"
+)
+
+type OssClient struct {
+ *oss.Client
+ Bucket string
+}
+
+func NewOssClient(accessKeyID, accessKeySecret, bucket string) *OssClient {
+ credProvider := credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret)
+
+ cfg := oss.LoadDefaultConfig().
+ WithCredentialsProvider(credProvider).
+ WithRegion("cn-shanghai")
+
+ client := oss.NewClient(cfg)
+
+ return &OssClient{client, bucket}
+}
+
+func (o *OssClient) UploadFile(ctx context.Context, objectKey, filePath, bucket string) error {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return fmt.Errorf("failed to open file: %v", err)
+ }
+ defer file.Close()
+
+ _, err = o.PutObject(ctx, &oss.PutObjectRequest{
+ Bucket: &bucket,
+ Key: &objectKey,
+ Body: file,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to upload file to OSS: %v", err)
+ }
+
+ fmt.Printf("File %s uploaded successfully to bucket %s as %s\n", filePath, bucket, objectKey)
+ return nil
+}
diff --git a/pkg/aliyun/speech_client.go b/pkg/aliyun/speech_client.go
deleted file mode 100644
index a63d46d7..00000000
--- a/pkg/aliyun/speech_client.go
+++ /dev/null
@@ -1,162 +0,0 @@
-package aliyun
-
-import (
- "encoding/json"
- "fmt"
- "go.uber.org/zap"
- "krillin-ai/log"
- "strings"
- "time"
-
- "github.com/google/uuid"
- "github.com/gorilla/websocket"
-)
-
-type SpeechClient struct {
- conn *websocket.Conn
- appkey string
- taskID string
- synthesisStarted chan struct{}
- synthesisComplete chan struct{}
-}
-
-type TtsHeader struct {
- Appkey string `json:"appkey"`
- MessageID string `json:"message_id"`
- TaskID string `json:"task_id"`
- Namespace string `json:"namespace"`
- Name string `json:"name"`
-}
-
-type StartSynthesisPayload struct {
- Voice string `json:"voice,omitempty"`
- Format string `json:"format,omitempty"`
- SampleRate int `json:"sample_rate,omitempty"`
- Volume int `json:"volume,omitempty"`
- SpeechRate int `json:"speech_rate,omitempty"`
- PitchRate int `json:"pitch_rate,omitempty"`
- EnableSubtitle bool `json:"enable_subtitle,omitempty"`
- EnablePhonemeTimestamp bool `json:"enable_phoneme_timestamp,omitempty"`
-}
-
-type RunSynthesisPayload struct {
- Text string `json:"text"`
-}
-
-type Message struct {
- Header TtsHeader `json:"header"`
- Payload interface{} `json:"payload,omitempty"`
-}
-
-func NewSpeechClient(appkey string, onTextMessage func(string), onBinaryMessage func([]byte)) (*SpeechClient, error) {
- token, _ := CreateToken()
- fullURL := "wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=" + token
- dialer := websocket.DefaultDialer
- dialer.HandshakeTimeout = 10 * time.Second
- conn, _, err := dialer.Dial(fullURL, nil)
- if err != nil {
- return nil, err
- }
- conn.SetReadDeadline(time.Now().Add(60 * time.Second))
- client := &SpeechClient{
- conn: conn,
- appkey: appkey,
- taskID: generateID(),
- synthesisComplete: make(chan struct{}),
- synthesisStarted: make(chan struct{}),
- }
-
- go client.receiveMessages(onTextMessage, onBinaryMessage)
-
- return client, nil
-}
-
-func generateID() string {
- return strings.ReplaceAll(uuid.New().String(), "-", "")
-}
-
-func (c *SpeechClient) sendMessage(name string, payload interface{}) error {
- message := Message{
- Header: TtsHeader{
- Appkey: c.appkey,
- MessageID: generateID(),
- TaskID: c.taskID,
- Namespace: "FlowingSpeechSynthesizer",
- Name: name,
- },
- Payload: payload,
- }
- jsonData, _ := json.Marshal(message)
- fmt.Println(string(jsonData))
- log.GetLogger().Debug("SpeechClient sendMessage", zap.String("message", string(jsonData)))
- return c.conn.WriteJSON(message)
-}
-
-func (c *SpeechClient) StartSynthesis(payload StartSynthesisPayload) error {
- err := c.sendMessage("StartSynthesis", payload)
- if err != nil {
- return err
- }
-
- // 阻塞等待 SynthesisStarted 事件
- <-c.synthesisStarted
-
- return nil
-}
-
-func (c *SpeechClient) RunSynthesis(text string) error {
- return c.sendMessage("RunSynthesis", RunSynthesisPayload{Text: text})
-}
-
-func (c *SpeechClient) StopSynthesis() error {
- err := c.sendMessage("StopSynthesis", nil)
- if err != nil {
- return err
- }
-
- // 阻塞等待 SynthesisCompleted 事件
- <-c.synthesisComplete
-
- return nil
-}
-
-func (c *SpeechClient) Close() error {
- err := c.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
- if err != nil {
- return err
- }
- return c.conn.Close()
-}
-
-func (c *SpeechClient) receiveMessages(onTextMessage func(string), onBinaryMessage func([]byte)) {
- defer close(c.synthesisComplete)
- for {
- messageType, message, err := c.conn.ReadMessage()
- if err != nil {
- if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
- log.GetLogger().Error("SpeechClient receiveMessages websocket非正常关闭", zap.Error(err))
- return
- }
- return
- }
- if messageType == websocket.TextMessage {
- var msg Message
- if err := json.Unmarshal(message, &msg); err != nil {
- log.GetLogger().Error("SpeechClient receiveMessages json解析失败", zap.Error(err))
- return
- }
- if msg.Header.Name == "SynthesisCompleted" {
- log.GetLogger().Info("SynthesisCompleted event received")
- // 收到结束消息退出
- break
- } else if msg.Header.Name == "SynthesisStarted" {
- log.GetLogger().Info("SynthesisStarted event received")
- close(c.synthesisStarted)
- } else {
- onTextMessage(string(message))
- }
- } else if messageType == websocket.BinaryMessage {
- onBinaryMessage(message)
- }
- }
-}
diff --git a/pkg/aliyun/tts.go b/pkg/aliyun/tts.go
new file mode 100644
index 00000000..3bc7f1e8
--- /dev/null
+++ b/pkg/aliyun/tts.go
@@ -0,0 +1,200 @@
+package aliyun
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/gorilla/websocket"
+ "go.uber.org/zap"
+ "krillin-ai/log"
+ "krillin-ai/pkg/util"
+ "os"
+ "time"
+)
+
+type TtsClient struct {
+ AccessKeyID string
+ AccessKeySecret string
+ Appkey string
+}
+
+type TtsHeader struct {
+ Appkey string `json:"appkey"`
+ MessageID string `json:"message_id"`
+ TaskID string `json:"task_id"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+}
+
+type StartSynthesisPayload struct {
+ Voice string `json:"voice,omitempty"`
+ Format string `json:"format,omitempty"`
+ SampleRate int `json:"sample_rate,omitempty"`
+ Volume int `json:"volume,omitempty"`
+ SpeechRate int `json:"speech_rate,omitempty"`
+ PitchRate int `json:"pitch_rate,omitempty"`
+ EnableSubtitle bool `json:"enable_subtitle,omitempty"`
+ EnablePhonemeTimestamp bool `json:"enable_phoneme_timestamp,omitempty"`
+}
+
+type RunSynthesisPayload struct {
+ Text string `json:"text"`
+}
+
+type Message struct {
+ Header TtsHeader `json:"header"`
+ Payload interface{} `json:"payload,omitempty"`
+}
+
+func NewTtsClient(accessKeyId, accessKeySecret, appkey string) *TtsClient {
+ return &TtsClient{
+ AccessKeyID: accessKeyId,
+ AccessKeySecret: accessKeySecret,
+ Appkey: appkey,
+ }
+}
+
+func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {
+ file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0666)
+ if err != nil {
+ return fmt.Errorf("failed to create file: %w", err)
+ }
+ defer file.Close()
+
+ var conn *websocket.Conn
+ token, _ := CreateToken(c.AccessKeyID, c.AccessKeySecret)
+ fullURL := "wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=" + token
+ dialer := websocket.DefaultDialer
+ dialer.HandshakeTimeout = 10 * time.Second
+ conn, _, err = dialer.Dial(fullURL, nil)
+ if err != nil {
+ return err
+ }
+ _ = conn.SetReadDeadline(time.Now().Add(time.Second * 60))
+ defer c.Close(conn)
+
+ onTextMessage := func(message string) {
+ log.GetLogger().Info("Received text message", zap.String("Message", message))
+ }
+
+ onBinaryMessage := func(data []byte) {
+ if _, err := file.Write(data); err != nil {
+ log.GetLogger().Error("Failed to write data to file", zap.Error(err))
+ }
+ }
+
+ var (
+ synthesisStarted = make(chan struct{})
+ synthesisComplete = make(chan struct{})
+ )
+
+ startPayload := StartSynthesisPayload{
+ Voice: voice,
+ Format: "wav",
+ SampleRate: 44100,
+ Volume: 50,
+ SpeechRate: 0,
+ PitchRate: 0,
+ }
+
+ go c.receiveMessages(conn, onTextMessage, onBinaryMessage, synthesisStarted, synthesisComplete)
+
+ taskId := util.GenerateID()
+ if err := c.StartSynthesis(conn, taskId, startPayload, synthesisStarted); err != nil {
+ return fmt.Errorf("failed to start synthesis: %w", err)
+ }
+
+ if err := c.RunSynthesis(conn, taskId, text); err != nil {
+ return fmt.Errorf("failed to run synthesis: %w", err)
+ }
+
+ if err := c.StopSynthesis(conn, taskId, synthesisComplete); err != nil {
+ return fmt.Errorf("failed to stop synthesis: %w", err)
+ }
+
+ return nil
+}
+
+func (c *TtsClient) sendMessage(conn *websocket.Conn, taskId, name string, payload interface{}) error {
+ message := Message{
+ Header: TtsHeader{
+ Appkey: c.Appkey,
+ MessageID: util.GenerateID(),
+ TaskID: taskId,
+ Namespace: "FlowingSpeechSynthesizer",
+ Name: name,
+ },
+ Payload: payload,
+ }
+ jsonData, _ := json.Marshal(message)
+ log.GetLogger().Debug("SpeechClient sendMessage", zap.String("message", string(jsonData)))
+ return conn.WriteJSON(message)
+}
+
+func (c *TtsClient) StartSynthesis(conn *websocket.Conn, taskId string, payload StartSynthesisPayload, synthesisStarted chan struct{}) error {
+ err := c.sendMessage(conn, taskId, "StartSynthesis", payload)
+ if err != nil {
+ return err
+ }
+
+ // 阻塞等待 SynthesisStarted 事件
+ <-synthesisStarted
+
+ return nil
+}
+
+func (c *TtsClient) RunSynthesis(conn *websocket.Conn, taskId, text string) error {
+ return c.sendMessage(conn, taskId, "RunSynthesis", RunSynthesisPayload{Text: text})
+}
+
+func (c *TtsClient) StopSynthesis(conn *websocket.Conn, taskId string, synthesisComplete chan struct{}) error {
+ err := c.sendMessage(conn, taskId, "StopSynthesis", nil)
+ if err != nil {
+ return err
+ }
+
+ // 阻塞等待 SynthesisCompleted 事件
+ <-synthesisComplete
+
+ return nil
+}
+
+func (c *TtsClient) Close(conn *websocket.Conn) error {
+ err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
+ if err != nil {
+ return err
+ }
+ return conn.Close()
+}
+
+func (c *TtsClient) receiveMessages(conn *websocket.Conn, onTextMessage func(string), onBinaryMessage func([]byte), synthesisStarted, synthesisComplete chan struct{}) {
+ defer close(synthesisComplete)
+ for {
+ messageType, message, err := conn.ReadMessage()
+ if err != nil {
+ if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
+ log.GetLogger().Error("SpeechClient receiveMessages websocket非正常关闭", zap.Error(err))
+ return
+ }
+ return
+ }
+ if messageType == websocket.TextMessage {
+ var msg Message
+ if err := json.Unmarshal(message, &msg); err != nil {
+ log.GetLogger().Error("SpeechClient receiveMessages json解析失败", zap.Error(err))
+ return
+ }
+ if msg.Header.Name == "SynthesisCompleted" {
+ log.GetLogger().Info("SynthesisCompleted event received")
+ // 收到结束消息退出
+ break
+ } else if msg.Header.Name == "SynthesisStarted" {
+ log.GetLogger().Info("SynthesisStarted event received")
+ close(synthesisStarted)
+ } else {
+ onTextMessage(string(message))
+ }
+ } else if messageType == websocket.BinaryMessage {
+ onBinaryMessage(message)
+ }
+ }
+}
diff --git a/pkg/aliyun/speech.go b/pkg/aliyun/voice_clone.go
similarity index 58%
rename from pkg/aliyun/speech.go
rename to pkg/aliyun/voice_clone.go
index 3657e7bb..1004e179 100644
--- a/pkg/aliyun/speech.go
+++ b/pkg/aliyun/voice_clone.go
@@ -6,10 +6,8 @@ import (
"encoding/base64"
"fmt"
"go.uber.org/zap"
- "krillin-ai/config"
"krillin-ai/log"
"net/url"
- "os"
"sort"
"strings"
"time"
@@ -53,26 +51,31 @@ func GenerateSignature(secret, stringToSign string) string {
return _encodeText(signature)
}
-type Client struct {
+type VoiceCloneResp struct {
+ RequestId string `json:"RequestId"`
+ Message string `json:"Message"`
+ Code int `json:"Code"`
+ VoiceName string `json:"VoiceName"`
+}
+
+type VoiceCloneClient struct {
restyClient *resty.Client
accessKeyID string
accessKeySecret string
appkey string
- bailianApiKey string
}
-func NewClient() *Client {
- cli := resty.New()
- return &Client{
- restyClient: cli,
- accessKeyID: config.Conf.Aliyun.Tts.AccessKeyId,
- accessKeySecret: config.Conf.Aliyun.Tts.AccessKeySecret,
- appkey: config.Conf.Aliyun.Tts.AppKey,
- bailianApiKey: config.Conf.Aliyun.Bailian.ApiKey,
+func NewVoiceCloneClient(accessKeyID, accessKeySecret, appkey string) *VoiceCloneClient {
+ return &VoiceCloneClient{
+ restyClient: resty.New(),
+ accessKeyID: accessKeyID,
+ accessKeySecret: accessKeySecret,
+ appkey: appkey,
}
}
-func (c *Client) CosyVoiceClone(voicePrefix, audioURL string) {
+func (c *VoiceCloneClient) CosyVoiceClone(voicePrefix, audioURL string) (string, error) {
+ log.GetLogger().Info("CosyVoiceClone请求开始", zap.String("voicePrefix", voicePrefix), zap.String("audioURL", audioURL))
parameters := map[string]string{
"AccessKeyId": c.accessKeyID,
"Action": "CosyVoiceClone",
@@ -88,27 +91,29 @@ func (c *Client) CosyVoiceClone(voicePrefix, audioURL string) {
}
queryString := _encodeDict(parameters)
- log.GetLogger().Debug("规范化的请求字符串:", zap.String("value", queryString))
stringToSign := "POST" + "&" + _encodeText("/") + "&" + _encodeText(queryString)
- log.GetLogger().Debug("待签名的字符串:", zap.String("value", stringToSign))
signature := GenerateSignature(c.accessKeySecret, stringToSign)
- log.GetLogger().Debug("签名:", zap.String("value", signature))
fullURL := fmt.Sprintf("https://nls-slp.cn-shanghai.aliyuncs.com/?Signature=%s&%s", signature, queryString)
- log.GetLogger().Debug("fullURL", zap.String("value", fullURL))
values := url.Values{}
for key, value := range parameters {
values.Add(key, value)
}
- resp, err := c.restyClient.R().Post(fullURL)
+ var res VoiceCloneResp
+ resp, err := c.restyClient.R().SetResult(&res).Post(fullURL)
if err != nil {
log.GetLogger().Error("CosyVoiceClone请求失败", zap.Error(err))
- return
+ return "", err
}
- fmt.Println("Response:", resp.String())
+ log.GetLogger().Info("CosyVoiceClone请求完毕", zap.String("Response", resp.String()))
+ if res.Message != "SUCCESS" {
+ log.GetLogger().Error("CosyVoiceClone请求响应错误", zap.String("Message", res.Message))
+ return "", fmt.Errorf("CosyVoiceClone请求响应错误: %s", res.Message)
+ }
+ return res.VoiceName, nil
}
-func (c *Client) CosyCloneList(voicePrefix string, pageIndex, pageSize int) {
+func (c *VoiceCloneClient) CosyCloneList(voicePrefix string, pageIndex, pageSize int) {
parameters := map[string]string{
"AccessKeyId": c.accessKeyID,
"Action": "ListCosyVoice",
@@ -125,70 +130,18 @@ func (c *Client) CosyCloneList(voicePrefix string, pageIndex, pageSize int) {
}
queryString := _encodeDict(parameters)
- fmt.Println("规范化的请求字符串:", queryString)
stringToSign := "POST" + "&" + _encodeText("/") + "&" + _encodeText(queryString)
- fmt.Println("待签名的字符串:", stringToSign)
signature := GenerateSignature(c.accessKeySecret, stringToSign)
- fmt.Println("URL编码后的签名:", signature)
fullURL := fmt.Sprintf("https://nls-slp.cn-shanghai.aliyuncs.com/?Signature=%s&%s", signature, queryString)
- fmt.Printf("url: %s\n", fullURL)
- // Make the POST request using resty
values := url.Values{}
for key, value := range parameters {
values.Add(key, value)
}
resp, err := c.restyClient.R().Post(fullURL)
if err != nil {
- fmt.Println("Error:", err)
+ log.GetLogger().Error("CosyCloneList请求失败", zap.Error(err))
return
}
- fmt.Println("Response:", resp.String())
-}
-
-func (c *Client) Text2Speech(text, voice, outputFile string) error {
- file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0666)
- if err != nil {
- return fmt.Errorf("failed to create file: %w", err)
- }
- defer file.Close()
-
- onTextMessage := func(message string) {
- fmt.Println("Received text message:", message)
- }
-
- onBinaryMessage := func(data []byte) {
- if _, err := file.Write(data); err != nil {
-
- fmt.Println("Failed to write data to file:", err)
- }
- }
-
- speechClient, err := NewSpeechClient(c.appkey, onTextMessage, onBinaryMessage)
- if err != nil {
- return fmt.Errorf("failed to create speech c: %w", err)
- }
- defer speechClient.Close()
-
- startPayload := StartSynthesisPayload{
- Voice: voice,
- Format: "wav",
- SampleRate: 44100,
- Volume: 50,
- SpeechRate: 0,
- PitchRate: 0,
- }
- if err := speechClient.StartSynthesis(startPayload); err != nil {
- return fmt.Errorf("failed to start synthesis: %w", err)
- }
-
- if err := speechClient.RunSynthesis(text); err != nil {
- return fmt.Errorf("failed to run synthesis: %w", err)
- }
-
- if err := speechClient.StopSynthesis(); err != nil {
- return fmt.Errorf("failed to stop synthesis: %w", err)
- }
-
- return nil
+ log.GetLogger().Info("CosyCloneList请求成功", zap.String("Response", resp.String()))
}
diff --git a/pkg/util/base.go b/pkg/util/base.go
index 06f1cad0..769e97be 100644
--- a/pkg/util/base.go
+++ b/pkg/util/base.go
@@ -3,6 +3,7 @@ package util
import (
"archive/zip"
"fmt"
+ "github.com/google/uuid"
"go.uber.org/zap"
"io"
"krillin-ai/config"
@@ -316,3 +317,7 @@ func CheckAndDownloadYtDlp() error {
return nil
}
+
+func GenerateID() string {
+ return strings.ReplaceAll(uuid.New().String(), "-", "")
+}