Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: add aliyun transcription support
沈浩0854725577c5
diff --git a/config/config-example.toml b/config/config-example.tomlindex 1ccb1d94..c125b44c 100644--- a/config/config-example.toml+++ b/config/config-example.toml@@ -1,7 +1,9 @@[app]segment_duration = 5 # 音频分割间隔,单位:分钟translate_parallel_num = 5 # 并发进行大模型翻译的数量上限- proxy = "" # 网络代理+ proxy = "" # 网络代理地址,格式如http://127.0.0.1:7890+ transcribe_provider = "openai" # 语音识别供应商,当前可选值:openai,aliyun。强烈建议openai,选择后需要在下方配置供应商密钥等信息,下同+ llm_provider = "openai" # LLM供应商,当前可选值:openai[server]host = "127.0.0.1"@@ -10,8 +12,10 @@[openai]api_key = "sk-XXX" # OpenAI API密钥-[aliyun] # 此处配置可以先忽略- access_key_id = ""- access_key_secret = ""- app_key = ""- cosy_voice_ws_addr = ""\ No newline at end of file+[aliyun]+ [aliyun.tts] # tts配置可以先忽略+ access_key_id = ""+ access_key_secret = ""+ app_key= ""+ [aliyun.asr]+ api_key = ""\ No newline at end of filediff --git a/config/config.go b/config/config.goindex 93506dd3..5e8b50ec 100644--- a/config/config.go+++ b/config/config.go@@ -10,6 +10,8 @@ type App struct {TranslateParallelNum int `toml:"translate_parallel_num"`Proxy string `toml:"proxy"`ParsedProxy *url.URL+ TranscribeProvider string `toml:"transcribe_provider"`+ LlmProvider string `toml:"llm_provider"`}type Server struct {@@ -21,11 +23,19 @@ type Openai struct {ApiKey string `toml:"api_key"`}-type Aliyun struct {+type AliyunTts struct {AccessKeyId string `toml:"access_key_id"`AccessKeySecret string `toml:"access_key_secret"`AppKey string `toml:"app_key"`- CosyVoiceWsAddr string `toml:"cosy_voice_ws_addr"`+}++type AliyunBailian struct {+ ApiKey string `toml:"api_key"`+}++type Aliyun struct {+ Tts AliyunTts `toml:"tts"`+ Bailian AliyunBailian `toml:"bailian"`}type Config struct {diff --git a/internal/service/init.go b/internal/service/init.goindex ec898a96..bd69201d 100644--- a/internal/service/init.go+++ b/internal/service/init.go@@ -2,18 +2,32 @@ package serviceimport ("krillin-ai/config"+ "krillin-ai/internal/types""krillin-ai/pkg/aliyun""krillin-ai/pkg/openai")type Service struct {OpenaiClient *openai.Client- CosyCloneClient *aliyun.CosyCloneClient+ CosyCloneClient *aliyun.AliyunClient++ Transcriber types.Transcriber}func NewService() *Service {+ var transcriber types.Transcriber++ switch config.Conf.App.TranscribeProvider {+ case "openai":+ transcriber = openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.ParsedProxy)+ case "aliyun":+ transcriber = aliyun.NewClient()+ }+return &Service{OpenaiClient: openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.ParsedProxy),- CosyCloneClient: aliyun.NewCosyCloneClient(),+ CosyCloneClient: aliyun.NewClient(),++ Transcriber: transcriber,}}diff --git a/internal/service/subtitle_service.go b/internal/service/subtitle_service.goindex 1409a1d4..8336b4e0 100644--- a/internal/service/subtitle_service.go+++ b/internal/service/subtitle_service.go@@ -14,7 +14,6 @@ import ("krillin-ai/internal/storage""krillin-ai/internal/types""krillin-ai/log"- "krillin-ai/pkg/openai""krillin-ai/pkg/util""os""os/exec"@@ -188,7 +187,7 @@ func (s Service) GetTaskStatus(req dto.GetVideoSubtitleTaskReq) (*dto.GetVideoSu}, nil}-// 新版流程:链接->本地音频文件->扣费->视频信息获取(若有)->本地字幕文件->cos上的字幕信息+// 新版流程:链接->本地音频文件->视频信息获取(若有)->本地字幕文件->cos上的字幕信息func (s Service) linkToAudioFile(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {var (@@ -439,13 +438,13 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt<-parallelControlChan}()// 语音转文字- var transcriptionData *openai.TranscriptionData+ var transcriptionData *types.TranscriptionDatafor i := 0; i < 3; i++ {language := string(stepParam.OriginLanguage)if language == "zh_cn" {language = "zh" // 切换一下}- transcriptionData, err = s.OpenaiClient.Transcription(audioFile.AudioFile, language)+ transcriptionData, err = s.Transcriber.Transcription(audioFile.AudioFile, language)if err == nil {break}@@ -629,7 +628,7 @@ func (s Service) splitSrt(ctx context.Context, stepParam *types.SubtitleTaskStepreturn nil}-func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64, language types.StandardLanguageName) (types.SrtSentence, float64, error) {+func getSentenceTimestamps(words []types.Word, sentence string, lastTs float64, language types.StandardLanguageName) (types.SrtSentence, float64, error) {var srtSt types.SrtSentencevar sentenceWordList []stringif language == types.LanguageNameEnglish || language == types.LanguageNameGerman { // 处理方式不同@@ -638,7 +637,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,return srtSt, 0, fmt.Errorf("sentence is empty")}- sentenceWords := make([]openai.Word, 0)+ sentenceWords := make([]types.Word, 0)thisLastTs := lastTssentenceWordIndex := 0@@ -663,7 +662,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,}if sentenceWordIndex >= len(words) {- sentenceWords = append(sentenceWords, openai.Word{+ sentenceWords = append(sentenceWords, types.Word{Text: sentenceWord,})sentenceWordIndex = 0@@ -726,7 +725,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,return srtSt, 0, fmt.Errorf("sentence is empty")}- sentenceWords := make([]openai.Word, 0)+ sentenceWords := make([]types.Word, 0)thisLastTs := lastTssentenceWordIndex := 0@@ -768,7 +767,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,}// 找到 Num 值递增的最大连续子数组-func findMaxIncreasingSubArray(words []openai.Word) (int, int) {+func findMaxIncreasingSubArray(words []types.Word) (int, int) {if len(words) == 0 {return 0, 0}@@ -805,7 +804,7 @@ func findMaxIncreasingSubArray(words []openai.Word) (int, int) {}// 跳跃(非连续)找到 Num 值递增的最大子数组-func jumpFindMaxIncreasingSubArray(words []openai.Word) (int, int) {+func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int) {if len(words) == 0 {return -1, -1}diff --git a/internal/types/interface.go b/internal/types/interface.gonew file mode 100644index 00000000..5cc6b891--- /dev/null+++ b/internal/types/interface.go@@ -0,0 +1,9 @@+package types++type ChatCompleter interface {+ ChatCompletion(query string) (string, error)+}++type Transcriber interface {+ Transcription(audioFile, language string) (*TranscriptionData, error)+}diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.goindex d0c9ef61..6e5ff952 100644--- a/internal/types/subtitle_task.go+++ b/internal/types/subtitle_task.go@@ -1,9 +1,5 @@package types-import (- "krillin-ai/pkg/openai"-)-var SplitTextPrompt = `你是一个语言处理专家,擅长翻译和处理文本,按下面的要求,根据句意和标点对给出的内容进行拆分并翻译:- 将原句子翻译成 %s@@ -76,7 +72,7 @@ var TranslateVideoTitleAndDescriptionPrompt = `你是一个专业的翻译专家type SmallAudio struct {AudioFile stringNum int- TranscriptionData *openai.TranscriptionData+ TranscriptionData *TranscriptionDataSrtNoTsFile string}@@ -140,6 +136,10 @@ const (TtsResultAudioFileName = "tts_final_audio.wav")+const (+ AsrMono16kAudioFileName = "mono_16k_audio.mp3"+)+type StandardLanguageName stringconst (@@ -269,8 +269,20 @@ type SubtitleTask struct {SrtNum int `json:"srt_num" gorm:"column:srt_num"` // 字幕数量SubtitleInfos []SubtitleInfo `gorm:"foreignKey:TaskId;references:TaskId"`Cover string `json:"cover" gorm:"column:cover"` // 封面- Cost uint32 `json:"cost" gorm:"column:cost"` // 消耗的额度SpeechDownloadUrl string `json:"speech_download_url" gorm:"column:speech_download_url"` // 语音文件下载地址CreateTime int64 `json:"create_time" gorm:"column:create_time;autoCreateTime"` // 创建时间UpdateTime int64 `json:"update_time" gorm:"column:update_time;autoUpdateTime"` // 更新时间}++type Word struct {+ Num int+ Text string+ Start float64+ End float64+}++type TranscriptionData struct {+ Language string+ Text string+ Words []Word+}diff --git a/main.go b/main.goindex 4853deca..dea38fd5 100644--- a/main.go+++ b/main.go@@ -35,6 +35,7 @@ func main() {err = util.CheckAndDownloadFfprobe()if err != nil {log.GetLogger().Error("ffprobe环境准备失败", zap.Error(err))+ return}err = util.CheckAndDownloadYtDlp()if err != nil {diff --git a/pkg/aliyun/speech.go b/pkg/aliyun/speech.goindex 09e1f9bf..2fd71ea7 100644--- a/pkg/aliyun/speech.go+++ b/pkg/aliyun/speech.go@@ -5,7 +5,9 @@ import ("crypto/sha1""encoding/base64""fmt"+ "go.uber.org/zap""krillin-ai/config"+ "krillin-ai/log""net/url""os""sort"@@ -16,8 +18,6 @@ import ("github.com/google/uuid")-// type CosyClone struct{}-// _encodeText URL-编码文本,保证符合规范func _encodeText(text string) string {encoded := url.QueryEscape(text)@@ -53,28 +53,28 @@ func GenerateSignature(secret, stringToSign string) string {return _encodeText(signature)}-type CosyCloneClient struct {+type AliyunClient struct {restyClient *resty.ClientaccessKeyID stringaccessKeySecret stringappkey string- wsUrl string+ bailianApiKey string}-func NewCosyCloneClient() *CosyCloneClient {+func NewClient() *AliyunClient {cli := resty.New()- return &CosyCloneClient{+ return &AliyunClient{restyClient: cli,- accessKeyID: config.Conf.Aliyun.AccessKeyId,- accessKeySecret: config.Conf.Aliyun.AccessKeySecret,- appkey: config.Conf.Aliyun.AppKey,- wsUrl: config.Conf.Aliyun.CosyVoiceWsAddr,+ accessKeyID: config.Conf.Aliyun.Tts.AccessKeyId,+ accessKeySecret: config.Conf.Aliyun.Tts.AccessKeySecret,+ appkey: config.Conf.Aliyun.Tts.AppKey,+ bailianApiKey: config.Conf.Aliyun.Bailian.ApiKey,}}-func (client *CosyCloneClient) CosyVoiceClone(voicePrefix, audioURL string) {+func (c *AliyunClient) CosyVoiceClone(voicePrefix, audioURL string) {parameters := map[string]string{- "AccessKeyId": client.accessKeyID,+ "AccessKeyId": c.accessKeyID,"Action": "CosyVoiceClone","Format": "JSON","RegionId": "cn-shanghai",@@ -88,31 +88,29 @@ func (client *CosyCloneClient) CosyVoiceClone(voicePrefix, audioURL string) {}queryString := _encodeDict(parameters)- fmt.Println("规范化的请求字符串:", queryString)+ log.GetLogger().Debug("规范化的请求字符串:", zap.String("value", queryString))stringToSign := "POST" + "&" + _encodeText("/") + "&" + _encodeText(queryString)- fmt.Println("待签名的字符串:", stringToSign)- signature := GenerateSignature(client.accessKeySecret, stringToSign)- fmt.Println("URL编码后的签名:", signature)+ log.GetLogger().Debug("待签名的字符串:", zap.String("value", stringToSign))+ signature := GenerateSignature(c.accessKeySecret, stringToSign)+ log.GetLogger().Debug("签名:", zap.String("value", signature))fullURL := fmt.Sprintf("https://nls-slp.cn-shanghai.aliyuncs.com/?Signature=%s&%s", signature, queryString)- fmt.Printf("url: %s\n", fullURL)+ log.GetLogger().Debug("fullURL", zap.String("value", fullURL))- // Make the POST request using restyvalues := url.Values{}for key, value := range parameters {values.Add(key, value)}- // resp, err := client.restyClient.R().SetQueryParam("Signature", signature).SetQueryParamsFromValues(values).Post(fullURL)- resp, err := client.restyClient.R().Post(fullURL)+ resp, err := c.restyClient.R().Post(fullURL)if err != nil {- fmt.Println("Error:", err)+ log.GetLogger().Error("CosyVoiceClone请求失败", zap.Error(err))return}fmt.Println("Response:", resp.String())}-func (client *CosyCloneClient) CosyCloneList(voicePrefix string, pageIndex, pageSize int) {+func (c *AliyunClient) CosyCloneList(voicePrefix string, pageIndex, pageSize int) {parameters := map[string]string{- "AccessKeyId": client.accessKeyID,+ "AccessKeyId": c.accessKeyID,"Action": "ListCosyVoice","Format": "JSON","RegionId": "cn-shanghai",@@ -130,7 +128,7 @@ func (client *CosyCloneClient) CosyCloneList(voicePrefix string, pageIndex, pagefmt.Println("规范化的请求字符串:", queryString)stringToSign := "POST" + "&" + _encodeText("/") + "&" + _encodeText(queryString)fmt.Println("待签名的字符串:", stringToSign)- signature := GenerateSignature(client.accessKeySecret, stringToSign)+ signature := GenerateSignature(c.accessKeySecret, stringToSign)fmt.Println("URL编码后的签名:", signature)fullURL := fmt.Sprintf("https://nls-slp.cn-shanghai.aliyuncs.com/?Signature=%s&%s", signature, queryString)fmt.Printf("url: %s\n", fullURL)@@ -140,7 +138,7 @@ func (client *CosyCloneClient) CosyCloneList(voicePrefix string, pageIndex, pagefor key, value := range parameters {values.Add(key, value)}- resp, err := client.restyClient.R().Post(fullURL)+ resp, err := c.restyClient.R().Post(fullURL)if err != nil {fmt.Println("Error:", err)return@@ -148,7 +146,7 @@ func (client *CosyCloneClient) CosyCloneList(voicePrefix string, pageIndex, pagefmt.Println("Response:", resp.String())}-func (client *CosyCloneClient) Text2Speech(text, voice, outputFile string) error {+func (c *AliyunClient) Text2Speech(text, voice, outputFile string) error {file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0666)if err != nil {return fmt.Errorf("failed to create file: %w", err)@@ -161,13 +159,14 @@ func (client *CosyCloneClient) Text2Speech(text, voice, outputFile string) erroronBinaryMessage := func(data []byte) {if _, err := file.Write(data); err != nil {+fmt.Println("Failed to write data to file:", err)}}- speechClient, err := NewSpeechClient(client.wsUrl, client.appkey, onTextMessage, onBinaryMessage)+ speechClient, err := NewSpeechClient(c.appkey, onTextMessage, onBinaryMessage)if err != nil {- return fmt.Errorf("failed to create speech client: %w", err)+ return fmt.Errorf("failed to create speech c: %w", err)}defer speechClient.Close()diff --git a/pkg/aliyun/speech_client.go b/pkg/aliyun/speech_client.goindex e9f427d4..a63d46d7 100644--- a/pkg/aliyun/speech_client.go+++ b/pkg/aliyun/speech_client.go@@ -3,7 +3,8 @@ package aliyunimport ("encoding/json""fmt"- "log"+ "go.uber.org/zap"+ "krillin-ai/log""strings""time"@@ -19,7 +20,7 @@ type SpeechClient struct {synthesisComplete chan struct{}}-type Header struct {+type TtsHeader struct {Appkey string `json:"appkey"`MessageID string `json:"message_id"`TaskID string `json:"task_id"`@@ -43,11 +44,11 @@ type RunSynthesisPayload struct {}type Message struct {- Header Header `json:"header"`+ Header TtsHeader `json:"header"`Payload interface{} `json:"payload,omitempty"`}-func NewSpeechClient(url, appkey string, onTextMessage func(string), onBinaryMessage func([]byte)) (*SpeechClient, error) {+func NewSpeechClient(appkey string, onTextMessage func(string), onBinaryMessage func([]byte)) (*SpeechClient, error) {token, _ := CreateToken()fullURL := "wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=" + tokendialer := websocket.DefaultDialer@@ -76,7 +77,7 @@ func generateID() string {func (c *SpeechClient) sendMessage(name string, payload interface{}) error {message := Message{- Header: Header{+ Header: TtsHeader{Appkey: c.appkey,MessageID: generateID(),TaskID: c.taskID,@@ -85,10 +86,9 @@ func (c *SpeechClient) sendMessage(name string, payload interface{}) error {},Payload: payload,}- //打印message,转成json字符串格式jsonData, _ := json.Marshal(message)- // 打印 JSON 字符串fmt.Println(string(jsonData))+ log.GetLogger().Debug("SpeechClient sendMessage", zap.String("message", string(jsonData)))return c.conn.WriteJSON(message)}@@ -123,7 +123,6 @@ func (c *SpeechClient) StopSynthesis() error {func (c *SpeechClient) Close() error {err := c.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))if err != nil {- log.Println("write close:", err)return err}return c.conn.Close()@@ -135,24 +134,23 @@ func (c *SpeechClient) receiveMessages(onTextMessage func(string), onBinaryMessamessageType, message, err := c.conn.ReadMessage()if err != nil {if websocket.IsCloseError(err, websocket.CloseNormalClosure) {- log.Println("connection closed normally")+ log.GetLogger().Error("SpeechClient receiveMessages websocket非正常关闭", zap.Error(err))return}- fmt.Println("read:", err)return}if messageType == websocket.TextMessage {var msg Messageif err := json.Unmarshal(message, &msg); err != nil {- fmt.Println("failed to unmarshal message:", err)+ log.GetLogger().Error("SpeechClient receiveMessages json解析失败", zap.Error(err))return}if msg.Header.Name == "SynthesisCompleted" {- fmt.Println("SynthesisCompleted event received")+ log.GetLogger().Info("SynthesisCompleted event received")// 收到结束消息退出break} else if msg.Header.Name == "SynthesisStarted" {- fmt.Println("SynthesisStarted event received")+ log.GetLogger().Info("SynthesisStarted event received")close(c.synthesisStarted)} else {onTextMessage(string(message))diff --git a/pkg/aliyun/token.go b/pkg/aliyun/token.goindex 2cfef4c7..4879299f 100644--- a/pkg/aliyun/token.go+++ b/pkg/aliyun/token.go@@ -19,7 +19,7 @@ type TokenResult struct {}func CreateToken() (string, error) {- client, err := sdk.NewClientWithAccessKey("cn-shanghai", config.Conf.Aliyun.AccessKeyId, config.Conf.Aliyun.AccessKeySecret)+ client, err := sdk.NewClientWithAccessKey("cn-shanghai", config.Conf.Aliyun.Tts.AccessKeyId, config.Conf.Aliyun.Tts.AccessKeySecret)if err != nil {return "", err}diff --git a/pkg/aliyun/transcription.go b/pkg/aliyun/transcription.gonew file mode 100644index 00000000..de36e3ad--- /dev/null+++ b/pkg/aliyun/transcription.go@@ -0,0 +1,335 @@+package aliyun++import (+ "encoding/json"+ "fmt"+ "github.com/google/uuid"+ "github.com/gorilla/websocket"+ "go.uber.org/zap"+ "io"+ "krillin-ai/internal/types"+ "krillin-ai/log"+ "net/http"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "time"+)++const (+ wsURL = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/" // WebSocket服务器地址+)++var dialer = websocket.DefaultDialer++func (c *AliyunClient) Transcription(audioFile, language string) (*types.TranscriptionData, error) {+ // 处理音频+ processedAudioFile, err := processAudio(audioFile)+ if err != nil {+ log.GetLogger().Error("处理音频失败", zap.Error(err), zap.String("audio file", audioFile))+ return nil, err+ }++ // 连接WebSocket服务+ conn, err := connectWebSocket(c.bailianApiKey)+ if err != nil {+ log.GetLogger().Error("连接WebSocket失败", zap.Error(err), zap.String("audio file", audioFile))+ return nil, err+ }+ defer closeConnection(conn)++ // 启动一个goroutine来接收结果+ taskStarted := make(chan bool)+ taskDone := make(chan bool)++ words := make([]types.Word, 0)+ text := ""+ startResultReceiver(conn, &words, &text, taskStarted, taskDone)++ // 发送run-task指令+ taskID, err := sendRunTaskCmd(conn, language)+ if err != nil {+ log.GetLogger().Error("发送run-task指令失败", zap.Error(err), zap.String("audio file", audioFile))+ }++ // 等待task-started事件+ waitForTaskStarted(taskStarted)++ // 发送待识别音频文件流+ if err := sendAudioData(conn, processedAudioFile); err != nil {+ log.GetLogger().Error("发送音频数据失败", zap.Error(err))+ }++ // 发送finish-task指令+ if err := sendFinishTaskCmd(conn, taskID); err != nil {+ log.GetLogger().Error("发送finish-task指令失败", zap.Error(err), zap.String("audio file", audioFile))+ }++ // 等待任务完成或失败+ <-taskDone++ if len(words) == 0 {+ log.GetLogger().Info("识别结果为空", zap.String("audio file", audioFile))+ }+ log.GetLogger().Debug("识别结果", zap.Any("words", words), zap.String("text", text), zap.String("audio file", audioFile))++ transcriptionData := &types.TranscriptionData{+ Text: text,+ Words: words,+ }++ return transcriptionData, nil+}++// 定义结构体来表示JSON数据+type AsrHeader struct {+ Action string `json:"action"`+ TaskID string `json:"task_id"`+ Streaming string `json:"streaming"`+ Event string `json:"event"`+ ErrorCode string `json:"error_code,omitempty"`+ ErrorMessage string `json:"error_message,omitempty"`+ Attributes map[string]interface{} `json:"attributes"`+}++type Output struct {+ Sentence struct {+ BeginTime int64 `json:"begin_time"`+ EndTime *int64 `json:"end_time"`+ Text string `json:"text"`+ Words []struct {+ BeginTime int64 `json:"begin_time"`+ EndTime *int64 `json:"end_time"`+ Text string `json:"text"`+ Punctuation string `json:"punctuation"`+ } `json:"words"`+ } `json:"sentence"`+ Usage interface{} `json:"usage"`+}++type Payload struct {+ TaskGroup string `json:"task_group"`+ Task string `json:"task"`+ Function string `json:"function"`+ Model string `json:"model"`+ Parameters Params `json:"parameters"`+ Resources []Resource `json:"resources"`+ Input Input `json:"input"`+ Output Output `json:"output,omitempty"`+}++type Params struct {+ Format string `json:"format"`+ SampleRate int `json:"sample_rate"`+ VocabularyID string `json:"vocabulary_id"`+ DisfluencyRemovalEnabled bool `json:"disfluency_removal_enabled"`+ LanguageHints []string `json:"language_hints"`+}++type Resource struct {+ ResourceID string `json:"resource_id"`+ ResourceType string `json:"resource_type"`+}++type Input struct {+}++type Event struct {+ Header AsrHeader `json:"header"`+ Payload Payload `json:"payload"`+}++// 把音频处理成单声道、16k采样率+func processAudio(filePath string) (string, error) {+ dest := strings.ReplaceAll(filePath, filepath.Ext(filePath), "_mono_16K.mp3")+ cmdArgs := []string{"-i", filePath, "-ac", "1", "-ar", "16000", "-b:a", "192k", dest}+ cmd := exec.Command("ffmpeg", cmdArgs...)+ output, err := cmd.CombinedOutput()+ if err != nil {+ log.GetLogger().Error("处理音频失败", zap.Error(err), zap.String("audio file", filePath), zap.String("output", string(output)))+ return "", err+ }+ return dest, nil+}++// 连接WebSocket服务+func connectWebSocket(apiKey string) (*websocket.Conn, error) {+ header := make(http.Header)+ header.Add("X-DashScope-DataInspection", "enable")+ header.Add("Authorization", fmt.Sprintf("bearer %s", apiKey))+ conn, _, err := dialer.Dial(wsURL, header)+ return conn, err+}++// 启动一个goroutine异步接收WebSocket消息+func startResultReceiver(conn *websocket.Conn, words *[]types.Word, text *string, taskStarted chan<- bool, taskDone chan<- bool) {+ go func() {+ for {+ _, message, err := conn.ReadMessage()+ if err != nil {+ log.GetLogger().Error("解析服务器消息失败:", zap.Error(err))+ continue+ }+ currentEvent := Event{}+ err = json.Unmarshal(message, ¤tEvent)+ if err != nil {+ log.GetLogger().Error("解析服务器消息失败:", zap.Error(err))+ continue+ }+ if currentEvent.Payload.Output.Sentence.EndTime != nil {+ // 本句结束,添加当前的words和text+ *text += currentEvent.Payload.Output.Sentence.Text+ currentNum := 0+ if len(*words) > 0 {+ currentNum = (*words)[len(*words)-1].Num + 1+ }+ for _, word := range currentEvent.Payload.Output.Sentence.Words {+ *words = append(*words, types.Word{+ Num: currentNum,+ Text: strings.TrimSpace(word.Text), // 阿里云这边的word后面会有空格+ Start: float64(word.BeginTime) / 1000,+ End: float64(*word.EndTime) / 1000,+ })+ currentNum+++ }+ }+ if handleEvent(conn, ¤tEvent, taskStarted, taskDone) {+ return+ }+ }+ }()+}++// 发送run-task指令+func sendRunTaskCmd(conn *websocket.Conn, language string) (string, error) {+ runTaskCmd, taskID, err := generateRunTaskCmd(language)+ if err != nil {+ return "", err+ }+ err = conn.WriteMessage(websocket.TextMessage, []byte(runTaskCmd))+ return taskID, err+}++// 生成run-task指令+func generateRunTaskCmd(language string) (string, string, error) {+ taskID := uuid.New().String()+ runTaskCmd := Event{+ Header: AsrHeader{+ Action: "run-task",+ TaskID: taskID,+ Streaming: "duplex",+ },+ Payload: Payload{+ TaskGroup: "audio",+ Task: "asr",+ Function: "recognition",+ Model: "paraformer-realtime-v2",+ Parameters: Params{+ Format: "mp3",+ SampleRate: 16000,+ LanguageHints: []string{language},+ },+ Input: Input{},+ },+ }+ runTaskCmdJSON, err := json.Marshal(runTaskCmd)+ return string(runTaskCmdJSON), taskID, err+}++// 等待task-started事件+func waitForTaskStarted(taskStarted chan bool) {+ select {+ case <-taskStarted:+ fmt.Println("任务开启成功")+ case <-time.After(10 * time.Second):+ log.GetLogger().Error("等待task-started超时,任务开启失败")+ }+}++// 发送音频数据+func sendAudioData(conn *websocket.Conn, filePath string) error {+ file, err := os.Open(filePath)+ if err != nil {+ return err+ }+ defer file.Close()++ buf := make([]byte, 1024) // 100ms的音频大约1024字节+ for {+ n, err := file.Read(buf)+ if n == 0 {+ break+ }+ if err != nil && err != io.EOF {+ return err+ }+ err = conn.WriteMessage(websocket.BinaryMessage, buf[:n])+ if err != nil {+ return err+ }+ time.Sleep(100 * time.Millisecond)+ }+ return nil+}++// 发送finish-task指令+func sendFinishTaskCmd(conn *websocket.Conn, taskID string) error {+ finishTaskCmd, err := generateFinishTaskCmd(taskID)+ if err != nil {+ return err+ }+ err = conn.WriteMessage(websocket.TextMessage, []byte(finishTaskCmd))+ return err+}++// 生成finish-task指令+func generateFinishTaskCmd(taskID string) (string, error) {+ finishTaskCmd := Event{+ Header: AsrHeader{+ Action: "finish-task",+ TaskID: taskID,+ Streaming: "duplex",+ },+ Payload: Payload{+ Input: Input{},+ },+ }+ finishTaskCmdJSON, err := json.Marshal(finishTaskCmd)+ return string(finishTaskCmdJSON), err+}++// 处理事件+func handleEvent(conn *websocket.Conn, event *Event, taskStarted chan<- bool, taskDone chan<- bool) bool {+ switch event.Header.Event {+ case "task-started":+ log.GetLogger().Info("收到task-started事件", zap.String("taskID", event.Header.TaskID))+ taskStarted <- true+ case "result-generated":+ log.GetLogger().Info("收到result-generated事件", zap.String("当前text", event.Payload.Output.Sentence.Text))+ case "task-finished":+ fmt.Println("任务完成")+ taskDone <- true+ return true+ case "task-failed":+ log.GetLogger().Info("收到task-failed事件", zap.String("taskID", event.Header.TaskID))+ handleTaskFailed(event, conn)+ taskDone <- true+ return true+ default:+ log.GetLogger().Info("未知事件:", zap.String("event", event.Header.Event))+ }+ return false+}++// 处理任务失败事件+func handleTaskFailed(event *Event, conn *websocket.Conn) {+ log.GetLogger().Error("任务失败:", zap.String("error", event.Header.ErrorMessage))+}++// 关闭连接+func closeConnection(conn *websocket.Conn) {+ if conn != nil {+ conn.Close()+ }+}diff --git a/pkg/openai/openai.go b/pkg/openai/openai.goindex a89fbe4f..d5f0b93d 100644--- a/pkg/openai/openai.go+++ b/pkg/openai/openai.go@@ -4,22 +4,10 @@ import ("context"openai "github.com/sashabaranov/go-openai""go.uber.org/zap"+ "krillin-ai/internal/types""krillin-ai/log")-type Word struct {- Num int- Text string- Start float64- End float64-}--type TranscriptionData struct {- Language string- Text string- Words []Word-}-func (c *Client) ChatCompletion(query string) (string, error) {req := openai.ChatCompletionRequest{Model: openai.GPT4oMini20240718,@@ -46,7 +34,7 @@ func (c *Client) ChatCompletion(query string) (string, error) {return resContent, nil}-func (c *Client) Transcription(audioFile, language string) (*TranscriptionData, error) {+func (c *Client) Transcription(audioFile, language string) (*types.TranscriptionData, error) {resp, err := c.client.CreateTranscription(context.Background(),openai.AudioRequest{@@ -64,14 +52,14 @@ func (c *Client) Transcription(audioFile, language string) (*TranscriptionData,return nil, err}- transcriptionData := &TranscriptionData{+ transcriptionData := &types.TranscriptionData{Language: resp.Language,Text: resp.Text,- Words: make([]Word, 0),+ Words: make([]types.Word, 0),}num := 0for _, word := range resp.Words {- wordItem := Word{+ wordItem := types.Word{Num: num,Text: word.Word,Start: word.Start,