Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
refactor&feat: simultaneous transcription and translation (#186)
Wenhao Zhu361a71c95d6b
diff --git a/config/config-example.toml b/config/config-example.tomlindex 5b081656..dbb7eadc 100644--- a/config/config-example.toml+++ b/config/config-example.toml@@ -1,6 +1,9 @@[app]segment_duration = 5 # 音频切分处理间隔,单位:分钟,建议值:5-10,如果视频中话语较少可以适当提高- translate_parallel_num = 5 # 并发进行模型转录和翻译的数量上限,建议值:5,如果使用了本地模型,该项自动不生效+ transcribe_parallel_num = 5 # 并发进行转录的数量上限,建议值:5,如果使用了本地模型,可以适当调低+ translate_parallel_num = 10 # 并发进行翻译的数量上限,建议值:2倍于转录的并发量,如果使用TPM限制严格的API,可以适当调低+ transcribe_max_attempts = 3 # 转录最大尝试次数,建议值:3+ translate_max_attempts = 5 # 翻译最大尝试次数,建议值:5,如果模型参数量较少或翻译失败率较高可以适当调高proxy = "" # 网络代理地址,格式如http://127.0.0.1:7890,可不填transcribe_provider = "openai" # 语音识别,当前可选值:openai,fasterwhisper,whisperkit,whisper.cpp,aliyun。(fasterwhisper不支持macOS,whisperkit只支持M芯片)llm_provider = "openai" # LLM,当前可选值:openai,aliyundiff --git a/config/config.go b/config/config.goindex c1825410..571e2ef0 100644--- a/config/config.go+++ b/config/config.go@@ -3,22 +3,26 @@ package configimport ("errors""fmt"- "github.com/BurntSushi/toml"- "go.uber.org/zap""krillin-ai/log""net/url""os""path/filepath""runtime"++ "github.com/BurntSushi/toml"+ "go.uber.org/zap")type App struct {- SegmentDuration int `toml:"segment_duration"`- TranslateParallelNum int `toml:"translate_parallel_num"`- Proxy string `toml:"proxy"`- ParsedProxy *url.URL `toml:"-"`- TranscribeProvider string `toml:"transcribe_provider"`- LlmProvider string `toml:"llm_provider"`+ SegmentDuration int `toml:"segment_duration"`+ TranscribeParallelNum int `toml:"transcribe_parallel_num"`+ TranslateParallelNum int `toml:"translate_parallel_num"`+ TranscribeMaxAttempts int `toml:"transcribe_max_attempts"`+ TranslateMaxAttempts int `toml:"translate_max_attempts"`+ Proxy string `toml:"proxy"`+ ParsedProxy *url.URL `toml:"-"`+ TranscribeProvider string `toml:"transcribe_provider"`+ LlmProvider string `toml:"llm_provider"`}type Server struct {@@ -76,10 +80,13 @@ type Config struct {var Conf = Config{App: App{- SegmentDuration: 5,- TranslateParallelNum: 5,- TranscribeProvider: "openai",- LlmProvider: "openai",+ SegmentDuration: 5,+ TranslateParallelNum: 5,+ TranscribeParallelNum: 10,+ TranscribeMaxAttempts: 3,+ TranslateMaxAttempts: 3,+ TranscribeProvider: "openai",+ LlmProvider: "openai",},Server: Server{Host: "127.0.0.1",@@ -105,7 +112,6 @@ func validateConfig() error {return errors.New("检测到开启了fasterwhisper,但模型选型配置不正确,请检查配置")}case "whisperkit":- Conf.App.TranslateParallelNum = 1if runtime.GOOS != "darwin" {log.GetLogger().Error("whisperkit只支持macos", zap.String("当前系统", runtime.GOOS))return fmt.Errorf("whisperkit只支持macos")diff --git a/internal/desktop/ui.go b/internal/desktop/ui.goindex 8e41dc64..ed2366ec 100644--- a/internal/desktop/ui.go+++ b/internal/desktop/ui.go@@ -152,9 +152,35 @@ func createAppConfigGroup() *fyne.Container {return nil}+ appTranscribeParallelNumEntry := StyledEntry("转录并行数量")+ appTranscribeParallelNumEntry.Bind(binding.IntToString(binding.BindInt(&config.Conf.App.TranscribeParallelNum)))+ appTranscribeParallelNumEntry.Validator = func(s string) error {+ val, err := strconv.Atoi(s)+ if err != nil {+ return fmt.Errorf("请输入数字")+ }+ if val < 1 || val > 10 {+ return fmt.Errorf("请输入1-10之间的数字")+ }+ return nil+ }+appTranslateParallelNumEntry := StyledEntry("翻译并行数量")appTranslateParallelNumEntry.Bind(binding.IntToString(binding.BindInt(&config.Conf.App.TranslateParallelNum)))appTranslateParallelNumEntry.Validator = func(s string) error {+ val, err := strconv.Atoi(s)+ if err != nil {+ return fmt.Errorf("请输入数字")+ }+ if val < 1 || val > 20 {+ return fmt.Errorf("请输入1-20之间的数字")+ }+ return nil+ }++ appTranscribeMaxAttemptsEntry := StyledEntry("转录最大尝试次数")+ appTranscribeMaxAttemptsEntry.Bind(binding.IntToString(binding.BindInt(&config.Conf.App.TranscribeMaxAttempts)))+ appTranscribeMaxAttemptsEntry.Validator = func(s string) error {val, err := strconv.Atoi(s)if err != nil {return fmt.Errorf("请输入数字")@@ -165,6 +191,19 @@ func createAppConfigGroup() *fyne.Container {return nil}+ appTranslateMaxAttemptsEntry := StyledEntry("翻译最大尝试次数")+ appTranslateMaxAttemptsEntry.Bind(binding.IntToString(binding.BindInt(&config.Conf.App.TranslateMaxAttempts)))+ appTranslateMaxAttemptsEntry.Validator = func(s string) error {+ val, err := strconv.Atoi(s)+ if err != nil {+ return fmt.Errorf("请输入数字")+ }+ if val < 1 || val > 20 {+ return fmt.Errorf("请输入1-20之间的数字")+ }+ return nil+ }+appProxyEntry := StyledEntry("网络代理地址")appProxyEntry.Bind(binding.BindString(&config.Conf.App.Proxy))@@ -181,7 +220,10 @@ func createAppConfigGroup() *fyne.Container {// 格式化表单项以使其更美观form := widget.NewForm(widget.NewFormItem("字幕分段处理时长(分钟) Segment duration (minutes)", appSegmentDurationEntry),+ widget.NewFormItem("转录并行数量 Transcribe parallel num", appTranscribeParallelNumEntry),widget.NewFormItem("翻译并行数量 Translate parallel num", appTranslateParallelNumEntry),+ widget.NewFormItem("转录最大尝试次数 Transcribe max attempts", appTranscribeMaxAttemptsEntry),+ widget.NewFormItem("翻译最大尝试次数 Translate max attempts", appTranslateMaxAttemptsEntry),widget.NewFormItem("网络代理地址 proxy", appProxyEntry),widget.NewFormItem("语音识别服务源 Transcriber provider", appTranscribeProviderEntry),widget.NewFormItem("LLM服务源 Llm provider", appLlmProviderEntry),diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.goindex 135a75d4..2926faa4 100644--- a/internal/service/audio2subtitle.go+++ b/internal/service/audio2subtitle.go@@ -10,7 +10,6 @@ import ("krillin-ai/internal/types""krillin-ai/log""krillin-ai/pkg/util"- "math""os""os/exec""path/filepath"@@ -24,9 +23,15 @@ import ("golang.org/x/sync/errgroup")+// 翻译结果数据结构+type TranslatedItem struct {+ OriginText string+ TranslatedText string+}+func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {var err error- err = s.splitAudio(ctx, stepParam)+ err = splitAudio(stepParam)if err != nil {return fmt.Errorf("audioToSubtitle splitAudio error: %w", err)}@@ -34,7 +39,7 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTif err != nil {return fmt.Errorf("audioToSubtitle audioToSrt error: %w", err)}- err = s.splitSrt(ctx, stepParam)+ err = splitSrt(stepParam)if err != nil {return fmt.Errorf("audioToSubtitle splitSrt error: %w", err)}@@ -43,7 +48,7 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTreturn nil}-func (s Service) splitAudio(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {+func splitAudio(stepParam *types.SubtitleTaskStepParam) error {log.GetLogger().Info("audioToSubtitle.splitAudio start", zap.String("task id", stepParam.TaskId))var err error// 使用ffmpeg分割音频@@ -76,13 +81,10 @@ func (s Service) splitAudio(ctx context.Context, stepParam *types.SubtitleTaskStreturn errors.New("audioToSubtitle splitAudio no audio files found")}- num := 1for _, audioFile := range audioFiles {stepParam.SmallAudios = append(stepParam.SmallAudios, &types.SmallAudio{AudioFile: audioFile,- Num: num,})- num++}// 更新字幕任务信息@@ -92,93 +94,243 @@ func (s Service) splitAudio(ctx context.Context, stepParam *types.SubtitleTaskStreturn nil}+func (s Service) transcribeAudio(audioFilePath string, language string, taskBasePath string) (*types.TranscriptionData, error) {+ defer func() {+ if r := recover(); r != nil {+ log.GetLogger().Error("audioToSubtitle transcribeAudio panic recovered", zap.Any("panic", r), zap.String("stack", string(debug.Stack())))+ }+ }()++ if language == "zh_cn" {+ language = "zh" // 切换一下+ }+ transcriptionData, err := s.Transcriber.Transcription(audioFilePath, language, taskBasePath)++ if err != nil {+ return nil, fmt.Errorf("audioToSubtitle transcribeAudio Transcription err: %w", err)+ }++ if transcriptionData.Text == "" {+ log.GetLogger().Info("audioToSubtitle transcribeAudio TranscriptionData.Text is empty", zap.Any("audioFilePath", audioFilePath), zap.Any("taskBasePath", taskBasePath))+ }+ return transcriptionData, nil+}++func (s Service) splitTextAndTranslate(inputText string, targetLanguage string, enableModalFilter bool) ([]TranslatedItem, error) {+ var prompt string++ // 选择提示词+ if enableModalFilter {+ prompt = fmt.Sprintf(types.SplitTextPromptWithModalFilter, targetLanguage)+ } else {+ prompt = fmt.Sprintf(types.SplitTextPrompt, targetLanguage)+ }++ // 如果输入文本为空,则返回空结果+ if inputText == "" {+ return []TranslatedItem{}, nil+ }++ textResult, err := s.ChatCompleter.ChatCompletion(prompt + inputText)+ if err != nil {+ return nil, fmt.Errorf("audioToSubtitle splitTextAndTranslate ChatCompletion error: %w", err)+ }+ re := regexp.MustCompile(`^\s*<think>.*?</think>`)+ textResult = strings.TrimSpace(re.ReplaceAllString(textResult, ""))++ results, err := parseAndCheckContent(textResult, inputText)+ if err != nil {+ return nil, fmt.Errorf("audioToSubtitle splitTextAndTranslate error: %w", err)+ }+ return results, nil+}+func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {- log.GetLogger().Info("audioToSubtitle.audioToSrt start", zap.Any("taskId", stepParam.TaskId))+ defer func() {+ if r := recover(); r != nil {+ log.GetLogger().Error("audioToSubtitle audioToSrt panic recovered", zap.Any("panic", r), zap.String("stack", string(debug.Stack())))+ }+ }()++ type DataWithId[T any] struct {+ Data T+ Id int+ }+var (- cancel context.CancelFunc- stepNum = 0- parallelControlChan = make(chan struct{}, config.Conf.App.TranslateParallelNum)- eg *errgroup.Group- stepNumMu sync.Mutex+ // 待翻译的文本队列+ pendingTranslationQueue = make(chan DataWithId[string], len(stepParam.SmallAudios))+ // 翻译结果队列+ translatedQueue = make(chan DataWithId[[]TranslatedItem], len(stepParam.SmallAudios))+ // 待转录的音频文件队列+ pendingTranscriptionQueue = make(chan DataWithId[string], len(stepParam.SmallAudios))+ // 转录结果队列+ transcribedQueue = make(chan DataWithId[*types.TranscriptionData], len(stepParam.SmallAudios))+ eg = errgroup.Group{})- ctx, cancel = context.WithCancel(ctx)++ log.GetLogger().Info("audioToSubtitle.audioToSrt start", zap.Any("taskId", stepParam.TaskId))+ ctx, cancel := context.WithCancel(ctx)defer cancel()- eg, ctx = errgroup.WithContext(ctx)- for _, audioFileItem := range stepParam.SmallAudios {- parallelControlChan <- struct{}{}- audioFile := audioFileItem++ // 并发启动协程处理翻译+ for range config.Conf.App.TranslateParallelNum {eg.Go(func() error {- defer func() {- <-parallelControlChan- if r := recover(); r != nil {- log.GetLogger().Error("audioToSubtitle.audioToSrt panic recovered", zap.Any("panic", r), zap.String("stack", string(debug.Stack())))- }- }()- var err error- select {- case <-ctx.Done():- return ctx.Err()- default:- }- // 语音转文字- var transcriptionData *types.TranscriptionData- for i := 0; i < 3; i++ {- language := string(stepParam.OriginLanguage)- if language == "zh_cn" {- language = "zh" // 切换一下- }- transcriptionData, err = s.Transcriber.Transcription(audioFile.AudioFile, language, stepParam.TaskBasePath)- if err == nil {- break+ for {+ select {+ case <-ctx.Done():+ return nil+ case translateItem, ok := <-pendingTranslationQueue:+ if !ok {+ return nil+ }+ var translatedResults []TranslatedItem+ var err error+ // 翻译文本+ log.GetLogger().Info("Begin to translate", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))+ for range config.Conf.App.TranslateMaxAttempts {+ translatedResults, err = s.splitTextAndTranslate(translateItem.Data, types.GetStandardLanguageName(stepParam.TargetLanguage), stepParam.EnableModalFilter)+ if err == nil {+ break+ }+ }+ if err != nil {+ cancel()+ return fmt.Errorf("audioToSubtitle audioToSrt splitTextAndTranslate err: %w", err)+ }+ log.GetLogger().Info("Translate completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))+ // 发送翻译结果+ translatedQueue <- DataWithId[[]TranslatedItem]{+ Data: translatedResults,+ Id: translateItem.Id,+ }}}- if err != nil {- cancel()- log.GetLogger().Error("audioToSubtitle audioToSrt Transcription err", zap.Any("stepParam", stepParam), zap.String("audio file", audioFile.AudioFile), zap.Error(err))- return fmt.Errorf("audioToSubtitle audioToSrt Transcription err: %w", err)- }-- if transcriptionData.Text == "" {- log.GetLogger().Info("audioToSubtitle audioToSrt TranscriptionData.Text is empty", zap.Any("stepParam", stepParam), zap.String("audio file", audioFile.AudioFile))- }+ })+ }- audioFile.TranscriptionData = transcriptionData-- // 更新字幕任务信息- stepNumMu.Lock()- stepNum++- processPct := uint8(20 + 70*stepNum/(len(stepParam.SmallAudios)*2))- stepParam.TaskPtr.ProcessPct = processPct- stepNumMu.Unlock()-- // 拆分字幕并翻译- err = s.splitTextAndTranslate(stepParam.TaskId, stepParam.TaskBasePath, stepParam.TargetLanguage, stepParam.EnableModalFilter, audioFile)- if err != nil {- cancel()- log.GetLogger().Error("audioToSubtitle audioToSrt splitTextAndTranslate err", zap.Any("stepParam", stepParam), zap.String("audio file", audioFile.AudioFile), zap.Error(err))- return fmt.Errorf("audioToSubtitle audioToSrt err: %w", err)- }+ // 并发启动协程处理音频转录+ for range config.Conf.App.TranscribeParallelNum {+ eg.Go(func() error {+ for {+ select {+ case <-ctx.Done():+ return nil+ case audioFileItem, ok := <-pendingTranscriptionQueue:+ if !ok {+ return nil+ }+ var (+ err error+ transcriptionData *types.TranscriptionData+ )+ log.GetLogger().Info("Begin transcribe", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", audioFileItem.Id))+ // 语音转文字+ for range config.Conf.App.TranscribeMaxAttempts {+ transcriptionData, err = s.transcribeAudio(audioFileItem.Data, string(stepParam.OriginLanguage), stepParam.TaskBasePath)+ if err == nil {+ break+ }+ }+ if err != nil {+ cancel()+ return fmt.Errorf("audioToSubtitle audioToSrt Transcription err: %w", err)+ }+ log.GetLogger().Info("Transcribe completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", audioFileItem.Id))+ pendingTranslationQueue <- DataWithId[string]{+ Data: transcriptionData.Text,+ Id: audioFileItem.Id,+ }+ // 发送转录结果+ transcribedQueue <- DataWithId[*types.TranscriptionData]{+ Data: transcriptionData,+ Id: audioFileItem.Id,+ }- stepNumMu.Lock()- stepNum++- processPct = uint8(20 + 70*stepNum/(len(stepParam.SmallAudios)*2))- stepParam.TaskPtr.ProcessPct = processPct- stepNumMu.Unlock()-- // 生成时间戳- err = s.generateTimestamps(stepParam.TaskId, stepParam.TaskBasePath, stepParam.OriginLanguage, stepParam.SubtitleResultType, audioFile, stepParam.MaxWordOneLine)- if err != nil {- cancel()- log.GetLogger().Error("audioToSubtitle audioToSrt generateTimestamps err", zap.Any("stepParam", stepParam), zap.String("audio file", audioFile.AudioFile), zap.Error(err))- return fmt.Errorf("audioToSubtitle audioToSrt err: %w", err)+ }}- return nil})}+ // 输入音频文件到转录队列,开始流程+ for i, audioFileItem := range stepParam.SmallAudios {+ pendingTranscriptionQueue <- DataWithId[string]{+ Data: audioFileItem.AudioFile,+ Id: i,+ }+ }- if err := eg.Wait(); err != nil {- log.GetLogger().Error("audioToSubtitle audioToSrt eg.Wait err", zap.Any("taskId", stepParam.TaskId), zap.Error(err))- return fmt.Errorf("audioToSubtitle audioToSrt eg.Wait err: %w", err)+ // 处理转录和翻译结果+ var mu sync.Mutex+ eg.Go(func() error {+ stepNum := 0+ for {+ select {+ case <-ctx.Done():+ return nil+ case transcribedItem := <-transcribedQueue:+ mu.Lock()+ stepNum+++ // 更新字幕任务信息+ processPct := uint8(20 + 70*stepNum/len(stepParam.SmallAudios)/2)+ stepParam.TaskPtr.ProcessPct = processPct+ mu.Unlock()+ // 处理转录结果+ stepParam.SmallAudios[transcribedItem.Id].TranscriptionData = transcribedItem.Data+ case translatedItems := <-translatedQueue:+ mu.Lock()+ stepNum+++ // 更新字幕任务信息+ processPct := uint8(20 + 70*stepNum/len(stepParam.SmallAudios)/2)+ stepParam.TaskPtr.ProcessPct = processPct+ mu.Unlock()+ // 处理翻译结果,保存不带时间戳的原始字幕+ originNoTsSrtFileName := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitSrtNoTimestampFileNamePattern, translatedItems.Id))+ originNoTsSrtFile, err := os.Create(originNoTsSrtFileName)+ if err != nil {+ return fmt.Errorf("audioToSubtitle audioToSrt create srt file err: %w", err)+ }+ // 保存不带时间戳的原始字幕+ for i, translatedItem := range translatedItems.Data {+ originNoTsSrtFile.WriteString(fmt.Sprintf("%d\n", i+1))+ originNoTsSrtFile.WriteString(fmt.Sprintf("[%s]\n", translatedItem.TranslatedText))+ originNoTsSrtFile.WriteString(fmt.Sprintf("[%s]\n\n", translatedItem.OriginText))+ }+ originNoTsSrtFile.Sync()+ originNoTsSrtFile.Close()+ smallAudioPtr := stepParam.SmallAudios[translatedItems.Id]+ smallAudioPtr.SrtNoTsFile = originNoTsSrtFileName+ // 生成时间戳+ srtBlocks := []*util.SrtBlock{}+ for i, translatedItem := range translatedItems.Data {+ srtBlocks = append(srtBlocks, &util.SrtBlock{+ Index: i + 1,+ Timestamp: "",+ OriginLanguageSentence: translatedItem.OriginText,+ TargetLanguageSentence: translatedItem.TranslatedText,+ })+ }++ err = generateSrtWithTimestamps(srtBlocks, stepParam.TaskBasePath, translatedItems.Id, stepParam.OriginLanguage, smallAudioPtr.TranscriptionData.Words, stepParam.SubtitleResultType, stepParam.MaxWordOneLine)+ if err != nil {+ cancel()+ return fmt.Errorf("audioToSubtitle audioToSrt generateTimestamps err: %w", err)+ }+ // 转录、翻译任务全部完成+ if stepNum >= len(stepParam.SmallAudios)*2 {+ close(pendingTranscriptionQueue)+ close(transcribedQueue)+ close(pendingTranslationQueue)+ close(translatedQueue)+ return nil+ }+ }+ }+ })++ err := eg.Wait()+ if err != nil {+ log.GetLogger().Error("audioToSubtitle audioToSrt errgroup wait err", zap.Any("stepParam", stepParam), zap.Error(err))+ return fmt.Errorf("audioToSubtitle audioToSrt errgroup wait err: %w", err)}// 合并文件@@ -186,7 +338,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskStbilingualFiles := make([]string, 0)shortOriginMixedFiles := make([]string, 0)shortOriginFiles := make([]string, 0)- for i := 1; i <= len(stepParam.SmallAudios); i++ {+ for i := range stepParam.SmallAudios {splitOriginNoTsFile := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitSrtNoTimestampFileNamePattern, i))originNoTsFiles = append(originNoTsFiles, splitOriginNoTsFile)splitBilingualFile := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitBilingualSrtFileNamePattern, i))@@ -199,7 +351,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt// 合并原始无时间戳字幕originNoTsFile := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, types.SubtitleTaskSrtNoTimestampFileName)- err := util.MergeFile(originNoTsFile, originNoTsFiles...)+ err = util.MergeFile(originNoTsFile, originNoTsFiles...)if err != nil {log.GetLogger().Error("audioToSubtitle audioToSrt merge originNoTsFile err",zap.Any("stepParam", stepParam), zap.Error(err))@@ -245,7 +397,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskStreturn nil}-func (s Service) splitSrt(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {+func splitSrt(stepParam *types.SubtitleTaskStepParam) error {log.GetLogger().Info("audioToSubtitle.splitSrt start", zap.Any("task id", stepParam.TaskId))originLanguageSrtFilePath := filepath.Join(stepParam.TaskBasePath, types.SubtitleTaskOriginLanguageSrtFileName)@@ -402,7 +554,6 @@ func getSentenceTimestamps(words []types.Word, sentence string, lastTs float64,sentenceWords = append(sentenceWords, wordNow)sentenceWordIndex = 0}-beginWordIndex, endWordIndex := findMaxIncreasingSubArray(sentenceWords)if (endWordIndex - beginWordIndex) == 0 {return srtSt, sentenceWords, 0, errors.New("getSentenceTimestamps no valid sentence")@@ -479,7 +630,7 @@ func getSentenceTimestamps(words []types.Word, sentence string, lastTs float64,}// 这里的sentence words不是字面上连续的,而是可能有重复,可读连续的用下面的readable- readableSentenceWords := make([]types.Word, 0)+ var readableSentenceWords []types.WordthisLastTs := lastTssentenceWordIndex := 0wordNow := words[sentenceWordIndex]@@ -572,7 +723,7 @@ func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int, []types.Word)prev := make([]int, len(words))// 初始化,所有的 dp[i] 都是 1,因为每个元素本身就是一个长度为 1 的子数组- for i := 0; i < len(words); i++ {+ for i := range len(words) {dp[i] = 1prev[i] = -1}@@ -625,30 +776,7 @@ func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int, []types.Word)return startIdx, endIdx, result}-func (s Service) generateTimestamps(taskId, basePath string, originLanguage types.StandardLanguageCode,- resultType types.SubtitleResultType, audioFile *types.SmallAudio, originLanguageWordOneLine int) error {- // 判断有没有文本- if len(audioFile.TranscriptionData.Words) == 0 {- return nil- }- srtNoTsFile, err := os.Open(audioFile.SrtNoTsFile)- if err != nil {- log.GetLogger().Error("audioToSubtitle generateTimestamps open SrtNoTsFile error", zap.String("taskId", taskId), zap.Error(err))- return fmt.Errorf("audioToSubtitle generateTimestamps open SrtNoTsFile error: %w", err)- }- scanner := bufio.NewScanner(srtNoTsFile)- if scanner.Scan() {- if strings.Contains(scanner.Text(), "[无文本]") {- return nil- }- }- srtNoTsFile.Close()- // 获取原始无时间戳字幕内容- srtBlocks, err := util.ParseSrtNoTsToSrtBlock(audioFile.SrtNoTsFile)- if err != nil {- log.GetLogger().Error("audioToSubtitle generateTimestamps read SrtBlocks error", zap.String("taskId", taskId), zap.Error(err))- return fmt.Errorf("audioToSubtitle generateTimestamps read SrtBlocks error: %w", err)- }+func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segmentIdx int, originLanguage types.StandardLanguageCode, words []types.Word, resultType types.SubtitleResultType, maxWordOneLine int) error {if len(srtBlocks) == 0 {return nil}@@ -660,12 +788,11 @@ func (s Service) generateTimestamps(taskId, basePath string, originLanguage typeif srtBlock.OriginLanguageSentence == "" {continue}- sentenceTs, sentenceWords, ts, err := getSentenceTimestamps(audioFile.TranscriptionData.Words, srtBlock.OriginLanguageSentence, lastTs, originLanguage)+ sentenceTs, sentenceWords, ts, err := getSentenceTimestamps(words, srtBlock.OriginLanguageSentence, lastTs, originLanguage)if err != nil || ts < lastTs {continue}-- tsOffset := float64(config.Conf.App.SegmentDuration) * 60 * float64(audioFile.Num-1)+ tsOffset := float64(config.Conf.App.SegmentDuration) * 60 * float64(segmentIdx)srtBlock.Timestamp = fmt.Sprintf("%s --> %s", util.FormatTime(float32(sentenceTs.Start+tsOffset)), util.FormatTime(float32(sentenceTs.End+tsOffset)))// 生成短句子的英文字幕@@ -675,7 +802,7 @@ func (s Service) generateTimestamps(taskId, basePath string, originLanguage typeendWord types.Word)- if len(sentenceWords) <= originLanguageWordOneLine {+ if len(sentenceWords) <= maxWordOneLine {shortOriginSrtMap[srtBlock.Index] = append(shortOriginSrtMap[srtBlock.Index], util.SrtBlock{Index: srtBlock.Index,Timestamp: fmt.Sprintf("%s --> %s", util.FormatTime(float32(sentenceTs.Start+tsOffset)), util.FormatTime(float32(sentenceTs.End+tsOffset))),@@ -685,14 +812,14 @@ func (s Service) generateTimestamps(taskId, basePath string, originLanguage typecontinue}- thisLineWord := originLanguageWordOneLine- if len(sentenceWords) > originLanguageWordOneLine && len(sentenceWords) <= 2*originLanguageWordOneLine {+ thisLineWord := maxWordOneLine+ if len(sentenceWords) > maxWordOneLine && len(sentenceWords) <= 2*maxWordOneLine {thisLineWord = len(sentenceWords)/2 + 1- } else if len(sentenceWords) > 2*originLanguageWordOneLine && len(sentenceWords) <= 3*originLanguageWordOneLine {+ } else if len(sentenceWords) > 2*maxWordOneLine && len(sentenceWords) <= 3*maxWordOneLine {thisLineWord = len(sentenceWords)/3 + 1- } else if len(sentenceWords) > 3*originLanguageWordOneLine && len(sentenceWords) <= 4*originLanguageWordOneLine {+ } else if len(sentenceWords) > 3*maxWordOneLine && len(sentenceWords) <= 4*maxWordOneLine {thisLineWord = len(sentenceWords)/4 + 1- } else if len(sentenceWords) > 4*originLanguageWordOneLine && len(sentenceWords) <= 5*originLanguageWordOneLine {+ } else if len(sentenceWords) > 4*maxWordOneLine && len(sentenceWords) <= 5*maxWordOneLine {thisLineWord = len(sentenceWords)/5 + 1}@@ -755,10 +882,9 @@ func (s Service) generateTimestamps(taskId, basePath string, originLanguage type}// 保存带时间戳的原始字幕- finalBilingualSrtFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitBilingualSrtFileNamePattern, audioFile.Num))+ finalBilingualSrtFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitBilingualSrtFileNamePattern, segmentIdx))finalBilingualSrtFile, err := os.Create(finalBilingualSrtFileName)if err != nil {- log.GetLogger().Error("audioToSubtitle generateTimestamps create bilingual srt file error", zap.String("taskId", taskId), zap.Error(err))return fmt.Errorf("audioToSubtitle generateTimestamps create bilingual srt file error: %w", err)}defer finalBilingualSrtFile.Close()@@ -778,19 +904,17 @@ func (s Service) generateTimestamps(taskId, basePath string, originLanguage type}// 保存带时间戳的字幕,长中文+短英文(示意,也支持其他语言)- srtShortOriginMixedFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginMixedSrtFileNamePattern, audioFile.Num))+ srtShortOriginMixedFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginMixedSrtFileNamePattern, segmentIdx))srtShortOriginMixedFile, err := os.Create(srtShortOriginMixedFileName)if err != nil {- log.GetLogger().Error("audioToSubtitle generateTimestamps create srtShortOriginMixedFile err", zap.String("taskId", taskId), zap.Error(err))return fmt.Errorf("audioToSubtitle generateTimestamps create srtShortOriginMixedFile err: %w", err)}defer srtShortOriginMixedFile.Close()// 保存带时间戳的短英文字幕- srtShortOriginFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginSrtFileNamePattern, audioFile.Num))+ srtShortOriginFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginSrtFileNamePattern, segmentIdx))srtShortOriginFile, err := os.Create(srtShortOriginFileName)if err != nil {- log.GetLogger().Error("audioToSubtitle generateTimestamps create srtShortOriginFile err", zap.String("taskId", taskId), zap.Error(err))return fmt.Errorf("audioToSubtitle generateTimestamps create srtShortOriginFile err: %w", err)}defer srtShortOriginMixedFile.Close()@@ -820,89 +944,35 @@ func (s Service) generateTimestamps(taskId, basePath string, originLanguage typereturn nil}-func (s Service) splitTextAndTranslate(taskId, baseTaskPath string, targetLanguage types.StandardLanguageCode, enableModalFilter bool, audioFile *types.SmallAudio) error {- var (- splitContent string- splitPrompt string- err error- )- if enableModalFilter {- splitPrompt = fmt.Sprintf(types.SplitTextPromptWithModalFilter, types.GetStandardLanguageName(targetLanguage))- } else {- splitPrompt = fmt.Sprintf(types.SplitTextPrompt, types.GetStandardLanguageName(targetLanguage))- }- if audioFile.TranscriptionData.Text == "" {- splitContent = ""- } else {- // 最多尝试4次获取有效的翻译结果- for i := range 4 {- splitContent, err = s.ChatCompleter.ChatCompletion(splitPrompt + audioFile.TranscriptionData.Text)- re := regexp.MustCompile(`(?s)<think>.*?</think>`)- splitContent = strings.TrimSpace(re.ReplaceAllString(splitContent, ""))- if err != nil {- log.GetLogger().Warn("audioToSubtitle splitTextAndTranslate ChatCompletion error, retrying...",- zap.Any("taskId", taskId), zap.Int("attempt", i+1), zap.Error(err))- continue- }-- // 验证返回内容的格式和原文匹配度- if isValidSplitContent(splitContent, audioFile.TranscriptionData.Text) {- log.GetLogger().Info("audioToSubtitle splitTextAndTranslate split content check success")- break- }-- originTextFileName := filepath.Join(baseTaskPath, "error_output.txt")- os.WriteFile(originTextFileName, []byte(splitContent), 0644)+func parseAndCheckContent(splitContent, originalText string) ([]TranslatedItem, error) {+ result := []TranslatedItem{}- log.GetLogger().Warn("audioToSubtitle splitTextAndTranslate invalid response format or content mismatch, retrying...",- zap.Any("taskId", taskId), zap.Int("attempt", i+1))- //err = fmt.Errorf("invalid split content format or content mismatch, audio file num: %d", audioFile.Num)- }-- if err != nil {- log.GetLogger().Error("audioToSubtitle splitTextAndTranslate failed after retries", zap.Any("taskId", taskId), zap.Error(err))- return fmt.Errorf("audioToSubtitle splitTextAndTranslate error: %w", err)- }- }-- // 保存不带时间戳的原始字幕- originNoTsSrtFileName := filepath.Join(baseTaskPath, fmt.Sprintf(types.SubtitleTaskSplitSrtNoTimestampFileNamePattern, audioFile.Num))- originNoTsSrtFile, err := os.Create(originNoTsSrtFileName)- if err != nil {- log.GetLogger().Error("audioToSubtitle splitTextAndTranslate create srt file err", zap.Any("taskId", taskId), zap.Error(err))- return fmt.Errorf("audioToSubtitle splitTextAndTranslate create srt file err: %w", err)- }- defer originNoTsSrtFile.Sync()- defer originNoTsSrtFile.Close()- _, err = originNoTsSrtFile.WriteString(splitContent)- if err != nil {- log.GetLogger().Error("audioToSubtitle splitTextAndTranslate write originNoTsSrtFile err", zap.Any("taskId", taskId), zap.Error(err))- return fmt.Errorf("audioToSubtitle splitTextAndTranslate write originNoTsSrtFile err: %w", err)- }- audioFile.SrtNoTsFile = originNoTsSrtFileName- return nil-}--// isValidSplitContent 验证分割后的内容是否符合格式要求,并检查原文字数是否与输入文本相近-func isValidSplitContent(splitContent, originalText string) bool {// 处理空内容情况if splitContent == "" || originalText == "" {- return splitContent == "" && originalText == ""+ if splitContent == originalText {+ return result, nil+ } else if splitContent == "" {+ return nil, errors.New("splitContent is empty but originalText is not")+ } else {+ return nil, errors.New("originalText is empty but splitContent is not")+ }}// 处理无文本标记if strings.Contains(splitContent, "[无文本]") {- return originalText == "" || len(strings.TrimSpace(originalText)) < 10+ if len(strings.TrimSpace(originalText)) < 10 {+ return result, nil+ } else {+ return nil, errors.New("originalText is not empty but splitContent contains [无文本]")+ }}lines := strings.Split(splitContent, "\n")if len(lines) < 3 { // 至少需要一个完整的块- //log.GetLogger().Warn("audioToSubtitle invaild Format, not enough lines", zap.Any("splitContent", splitContent))- return false+ log.GetLogger().Error("audioToSubtitle invaild Format, not enough lines", zap.Any("splitContent", splitContent))+ return nil, fmt.Errorf("audioToSubtitle invaild Format, not enough lines")}- var originalLines []string-// 验证格式并提取原文for i := 0; i < len(lines); i++ {line := strings.TrimSpace(lines[i])@@ -911,30 +981,43 @@ func isValidSplitContent(splitContent, originalText string) bool {}// 检查是否为序号行- if _, err := strconv.Atoi(line); err == nil {- if i+2 >= len(lines) {- log.GetLogger().Warn("audioToSubtitle invaild Format, block is not complete", zap.Any("splitContent", splitContent), zap.Any("line", line))- return false- }- // 收集原文行(第三行),并去除方括号- originalLine := strings.TrimSpace(lines[i+2])- originalLine = strings.TrimPrefix(originalLine, "[")- originalLine = strings.TrimSuffix(originalLine, "]")- originalLines = append(originalLines, originalLine)- i += 2 // 跳过翻译行和原文行+ if _, err := strconv.Atoi(line); err != nil {+ continue}- }- if len(originalLines) == 0 {- log.GetLogger().Warn("audioToSubtitle invaild Format, original line misiing", zap.Any("splitContent", splitContent))- return false+ if i+2 >= len(lines) {+ log.GetLogger().Error("audioToSubtitle invaild Format, block is not complete", zap.Any("splitContent", splitContent), zap.Any("line", line))+ return nil, fmt.Errorf("audioToSubtitle invaild Format, block is not complete")+ }+ // 获取翻译行和原文行+ translatedLine := strings.TrimSpace(lines[i+1])+ translatedLine = strings.TrimPrefix(translatedLine, "[")+ translatedLine = strings.TrimSuffix(translatedLine, "]")+ originalLine := strings.TrimSpace(lines[i+2])+ originalLine = strings.TrimPrefix(originalLine, "[")+ originalLine = strings.TrimSuffix(originalLine, "]")+ result = append(result, TranslatedItem{+ OriginText: originalLine,+ TranslatedText: translatedLine,+ })+ i += 2 // 跳过翻译行和原文行}// 合并原文并比较字数- combinedOriginal := strings.Join(originalLines, "")+ combinedLength := 0+ for _, translatedItem := range result {+ combinedLength += len(strings.TrimSpace(translatedItem.OriginText))+ }originalTextLength := len(strings.TrimSpace(originalText))- combinedLength := len(strings.TrimSpace(combinedOriginal))- // 允许300字符的误差- return math.Abs(float64(originalTextLength-combinedLength)) <= 300+ lenError := originalTextLength - combinedLength+ if lenError < 0 {+ lenError = -lenError+ }++ if lenError > len(originalText)/10 {+ // return nil, fmt.Errorf("audioToSubtitle invaild Format, originalText and splitContent length not match", zap.Any("splitContent", splitContent), zap.Any("originalText", originalText))+ log.GetLogger().Warn("audioToSubtitle invaild Format, originalText and splitContent length not match", zap.Any("splitContent", splitContent), zap.Any("originalText", originalText))+ }+ return result, nil}diff --git a/internal/service/audio2subtitle_test.go b/internal/service/audio2subtitle_test.goindex eff8499a..22b48b0a 100644--- a/internal/service/audio2subtitle_test.go+++ b/internal/service/audio2subtitle_test.go@@ -23,7 +23,7 @@ func Test_isValidSplitContent(t *testing.T) {}// 执行测试- if got := isValidSplitContent(string(splitContent), string(originalText)); !got {- t.Errorf("isValidSplitContent() = %v, want true", got)+ if _, err := parseAndCheckContent(string(splitContent), string(originalText)); err != nil {+ t.Errorf("parseAndCheckContent() error = %v, want nil", err)}}diff --git a/internal/service/srt_embed.go b/internal/service/srt_embed.goindex 004e7bdc..1daddb7c 100644--- a/internal/service/srt_embed.go+++ b/internal/service/srt_embed.go@@ -96,7 +96,7 @@ func splitMajorTextInHorizontal(text string, language types.StandardLanguageCodecurrentWidth := 0splitIndex := 0- for i, _ := range segments {+ for i := range segments {currentWidth++// 当达到 2/5 宽度时,设置拆分点diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.goindex f78e29db..30ccdded 100644--- a/internal/types/subtitle_task.go+++ b/internal/types/subtitle_task.go@@ -85,7 +85,6 @@ var TranslateVideoTitleAndDescriptionPrompt = `你是一个专业的翻译专家type SmallAudio struct {AudioFile string- Num intTranscriptionData *TranscriptionDataSrtNoTsFile string}diff --git a/pkg/util/subtitle.go b/pkg/util/subtitle.goindex c2d700f4..73955db9 100644--- a/pkg/util/subtitle.go+++ b/pkg/util/subtitle.go@@ -104,60 +104,6 @@ func TrimString(s string) string {return s}-func ParseSrtNoTsToSrtBlock(srtNoTsFile string) ([]*SrtBlock, error) {- file, err := os.Open(srtNoTsFile)- if err != nil {- return nil, err- }- defer file.Close()-- var blocks []*SrtBlock- var currentBlock SrtBlock- scanner := bufio.NewScanner(file)- start := true-- for scanner.Scan() {- line := TrimString(scanner.Text())- // 去掉最开始的描述- if start && !IsNumber(line) {- continue- } else {- start = false- }- if line == "" { // 空行表示一个块的结束- if currentBlock.Index != 0 {- cur := currentBlock- blocks = append(blocks, &cur)- currentBlock = SrtBlock{} // 重置- }- continue- }-- if currentBlock.Index == 0 { // 按文件内容依次赋值- var index int- _, err = fmt.Sscanf(line, "%d", &index)- if err != nil {- return blocks, nil- } // 可能是空语音等,直接忽略- currentBlock.Index = index- } else if currentBlock.TargetLanguageSentence == "" {- currentBlock.TargetLanguageSentence = line- } else if currentBlock.OriginLanguageSentence == "" {- currentBlock.OriginLanguageSentence = line- }- }- // 最后的块- if currentBlock.Index != 0 {- cur := currentBlock- blocks = append(blocks, &cur)- }-- if err = scanner.Err(); err != nil {- return nil, err- }- return blocks, nil-}-func SplitSentence(sentence string) []string {// 使用正则表达式移除标点符号和特殊字符(保留各语言字母、数字和空格)re := regexp.MustCompile(`[^\p{L}\p{N}\s']+`)