Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: improve audio splitting logic (#209)

Wenhao Zhu2d32c0880638
diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.go
index a2d4d7f0..4eb86fbe 100644
--- a/internal/service/audio2subtitle.go
+++ b/internal/service/audio2subtitle.go
@@ -7,14 +7,13 @@ import (
"errors"
"fmt"
"krillin-ai/config"
- "krillin-ai/internal/storage"
"krillin-ai/internal/types"
"krillin-ai/log"
"krillin-ai/pkg/util"
"os"
- "os/exec"
"path/filepath"
"regexp"
+ "runtime"
"runtime/debug"
"strconv"
"strings"
@@ -31,10 +30,6 @@ type TranslatedItem struct {
func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {
var err error
- err = splitAudio(stepParam)
- if err != nil {
- return fmt.Errorf("audioToSubtitle splitAudio error: %w", err)
- }
err = s.audioToSrt(ctx, stepParam) // 这里进度更新到90%了
if err != nil {
return fmt.Errorf("audioToSubtitle audioToSrt error: %w", err)
@@ -48,63 +43,63 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleT
return nil
}
-func splitAudio(stepParam *types.SubtitleTaskStepParam) error {
- log.GetLogger().Info("audioToSubtitle.splitAudio start", zap.String("task id", stepParam.TaskId))
- var err error
- // 使用ffmpeg分割音频
- outputPattern := filepath.Join(stepParam.TaskBasePath, types.SubtitleTaskSplitAudioFileNamePattern) // 输出文件格式
- segmentDuration := config.Conf.App.SegmentDuration * 60
-
- cmd := exec.Command(
- storage.FfmpegPath,
- "-i", stepParam.AudioFilePath, // 输入
- "-f", "segment", // 输出文件格式为分段
- "-segment_time", fmt.Sprintf("%d", segmentDuration), // 每段时长(以秒为单位)
- "-reset_timestamps", "1", // 重置每段时间戳
- "-y", // 覆盖输出文件
- outputPattern,
- )
- err = cmd.Run()
- if err != nil {
- log.GetLogger().Error("audioToSubtitle splitAudio ffmpeg err", zap.Any("stepParam", stepParam), zap.Error(err))
- return fmt.Errorf("audioToSubtitle splitAudio ffmpeg err: %w", err)
- }
-
- // 获取分割后的文件列表
- audioFiles, err := filepath.Glob(filepath.Join(stepParam.TaskBasePath, fmt.Sprintf("%s_*.mp3", types.SubtitleTaskSplitAudioFileNamePrefix)))
- if err != nil {
- log.GetLogger().Error("audioToSubtitle splitAudio filepath.Glob err", zap.Any("stepParam", stepParam), zap.Error(err))
- return fmt.Errorf("audioToSubtitle splitAudio filepath.Glob err: %w", err)
- }
- if len(audioFiles) == 0 {
- log.GetLogger().Error("audioToSubtitle splitAudio no audio files found", zap.Any("stepParam", stepParam))
- return errors.New("audioToSubtitle splitAudio no audio files found")
- }
-
- for _, audioFile := range audioFiles {
- stepParam.SmallAudios = append(stepParam.SmallAudios, &types.SmallAudio{
- AudioFile: audioFile,
- })
- }
-
- // 更新字幕任务信息
- stepParam.TaskPtr.ProcessPct = 20
-
- log.GetLogger().Info("audioToSubtitle.splitAudio end", zap.String("task id", stepParam.TaskId))
- return nil
-}
-
-func (s Service) transcribeAudio(id int, audioFilePath string, language string, taskBasePath string) (*types.TranscriptionData, error) {
+//func splitAudio(stepParam *types.SubtitleTaskStepParam) error {
+// log.GetLogger().Info("audioToSubtitle.splitAudio start", zap.String("task id", stepParam.TaskId))
+// var err error
+// // 使用ffmpeg分割音频
+// outputPattern := filepath.Join(stepParam.TaskBasePath, types.SubtitleTaskSplitAudioFileNamePattern) // 输出文件格式
+// segmentDuration := config.Conf.App.SegmentDuration * 60
+//
+// cmd := exec.Command(
+// storage.FfmpegPath,
+// "-i", stepParam.AudioFilePath, // 输入
+// "-f", "segment", // 输出文件格式为分段
+// "-segment_time", fmt.Sprintf("%d", segmentDuration), // 每段时长(以秒为单位)
+// "-reset_timestamps", "1", // 重置每段时间戳
+// "-y", // 覆盖输出文件
+// outputPattern,
+// )
+// err = cmd.Run()
+// if err != nil {
+// log.GetLogger().Error("audioToSubtitle splitAudio ffmpeg err", zap.Any("stepParam", stepParam), zap.Error(err))
+// return fmt.Errorf("audioToSubtitle splitAudio ffmpeg err: %w", err)
+// }
+//
+// // 获取分割后的文件列表
+// audioFiles, err := filepath.Glob(filepath.Join(stepParam.TaskBasePath, fmt.Sprintf("%s_*.mp3", types.SubtitleTaskSplitAudioFileNamePrefix)))
+// if err != nil {
+// log.GetLogger().Error("audioToSubtitle splitAudio filepath.Glob err", zap.Any("stepParam", stepParam), zap.Error(err))
+// return fmt.Errorf("audioToSubtitle splitAudio filepath.Glob err: %w", err)
+// }
+// if len(audioFiles) == 0 {
+// log.GetLogger().Error("audioToSubtitle splitAudio no audio files found", zap.Any("stepParam", stepParam))
+// return errors.New("audioToSubtitle splitAudio no audio files found")
+// }
+//
+// for _, audioFile := range audioFiles {
+// stepParam.SmallAudios = append(stepParam.SmallAudios, &types.SmallAudio{
+// AudioFile: audioFile,
+// })
+// }
+//
+// // 更新字幕任务信息
+// stepParam.TaskPtr.ProcessPct = 20
+//
+// log.GetLogger().Info("audioToSubtitle.splitAudio end", zap.String("task id", stepParam.TaskId))
+// return nil
+//}
+
+func (s Service) transcribeAudio(id int, audioFilePath string, language string, taskBasePath string) (transcriptionData *types.TranscriptionData, err error) {
defer func() {
if r := recover(); r != nil {
- log.GetLogger().Error("audioToSubtitle transcribeAudio panic recovered", zap.Any("panic", r), zap.String("stack", string(debug.Stack())))
+ err = fmt.Errorf("audioToSubtitle transcribeAudio panic recovered: %v", r)
}
}()
if language == "zh_cn" {
language = "zh" // 切换一下
}
- transcriptionData, err := s.Transcriber.Transcription(audioFilePath, language, taskBasePath)
+ transcriptionData, err = s.Transcriber.Transcription(audioFilePath, language, taskBasePath)
if err != nil {
return nil, fmt.Errorf("audioToSubtitle transcribeAudio Transcription err: %w", err)
@@ -164,82 +159,96 @@ func (s Service) splitTextAndTranslate(inputText string, targetLanguage string,
return results, nil
}
-func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {
+func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskStepParam) (err error) {
defer func() {
if r := recover(); r != nil {
log.GetLogger().Error("audioToSubtitle audioToSrt panic recovered", zap.Any("panic", r), zap.String("stack", string(debug.Stack())))
+ err = fmt.Errorf("audioToSubtitle audioToSrt panic recovered: %v", r)
}
}()
+ log.GetLogger().Info("audioToSubtitle.audioToSrt start", zap.Any("taskId", stepParam.TaskId))
+ timePoints, err := GetSplitPoints(stepParam.AudioFilePath, float64(config.Conf.App.SegmentDuration)*60)
+ if err != nil {
+ log.GetLogger().Error("audioToSubtitle audioToSrt GetSplitPoints err", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
+ return fmt.Errorf("audioToSubtitle audioToSrt GetSplitPoints err: %w", err)
+ }
+ log.GetLogger().Info("audioToSubtitle audioToSrt GetSplitPoints completed", zap.Any("taskId", stepParam.TaskId), zap.Any("timePoints", timePoints))
+
+ // 更新字幕任务信息
+ stepParam.TaskPtr.ProcessPct = 15
+ segmentNum := len(timePoints) - 1
+
type DataWithId[T any] struct {
Data T
Id int
}
var (
- // 待翻译的文本队列
- pendingTranslationQueue = make(chan DataWithId[string], len(stepParam.SmallAudios))
- // 翻译结果队列
- translatedQueue = make(chan DataWithId[[]TranslatedItem], len(stepParam.SmallAudios))
+ // 待剪辑的音频片段队列
+ pendingSplitQueue = make(chan DataWithId[[2]float64], segmentNum)
+ // 剪辑结果队列
+ splitResultQueue = make(chan DataWithId[string], segmentNum)
// 待转录的音频文件队列
- pendingTranscriptionQueue = make(chan DataWithId[string], len(stepParam.SmallAudios))
+ pendingTranscriptionQueue = make(chan DataWithId[string], segmentNum)
// 转录结果队列
- transcribedQueue = make(chan DataWithId[*types.TranscriptionData], len(stepParam.SmallAudios))
- eg = errgroup.Group{}
+ transcribedQueue = make(chan DataWithId[*types.TranscriptionData], segmentNum)
+ // 待翻译的文本队列
+ pendingTranslationQueue = make(chan DataWithId[string], segmentNum)
+ // 翻译结果队列
+ translatedQueue = make(chan DataWithId[[]TranslatedItem], segmentNum)
)
+ eg, ctx := errgroup.WithContext(ctx)
log.GetLogger().Info("audioToSubtitle.audioToSrt start", zap.Any("taskId", stepParam.TaskId))
- ctx, cancel := context.WithCancel(ctx)
- defer cancel()
- // 并发启动协程处理翻译
- for range config.Conf.App.TranslateParallelNum {
+ // 构造长度为segmentNum的音频片段切片
+ type AudioSegment struct {
+ AudioFile string
+ TranscriptionData *types.TranscriptionData
+ SrtNoTsFile string
+ }
+ audioSegments := make([]AudioSegment, segmentNum)
+
+ // 输入音频文件到分割队列
+ for i := range segmentNum {
+ pendingSplitQueue <- DataWithId[[2]float64]{
+ Data: [2]float64{timePoints[i], timePoints[i+1]},
+ Id: i,
+ }
+ }
+
+ // 分割音频
+ for range runtime.NumCPU() {
eg.Go(func() error {
for {
select {
case <-ctx.Done():
return nil
- case translateItem, ok := <-pendingTranslationQueue:
+ case splitItem, ok := <-pendingSplitQueue:
if !ok {
return nil
}
- var translatedResults []TranslatedItem
- var err error
- // 翻译文本
- log.GetLogger().Info("Begin to translate", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))
- for range config.Conf.App.TranslateMaxAttempts {
- translatedResults, err = s.splitTextAndTranslate(translateItem.Data, types.GetStandardLanguageName(stepParam.TargetLanguage), stepParam.EnableModalFilter)
- if err == nil {
- break
- }
- }
+ log.GetLogger().Info("Begin split audio", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", splitItem.Id))
+ // 分割音频
+ outputFileName := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitAudioFileNamePattern, splitItem.Id))
+ err := ClipAudio(stepParam.AudioFilePath, outputFileName, splitItem.Data[0], splitItem.Data[1])
if err != nil {
- cancel()
- return fmt.Errorf("audioToSubtitle audioToSrt splitTextAndTranslate err: %w", err)
+ return fmt.Errorf("audioToSubtitle audioToSrt ClipAudio err: %w", err)
}
- _ = util.SaveToDisk(translatedResults, filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskTranslationDataPersistenceFileNamePattern, translateItem.Id)))
- log.GetLogger().Info("Translate completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))
- // 分割长句
- splitResults, err := s.splitTranslateItem(translatedResults)
- if err != nil {
- // 不中断
- log.GetLogger().Error("audioToSubtitle audioToSrt splitTranslateItem err", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id), zap.Error(err))
- translatedQueue <- DataWithId[[]TranslatedItem]{
- Data: translatedResults,
- Id: translateItem.Id,
- }
- } else {
- translatedQueue <- DataWithId[[]TranslatedItem]{
- Data: splitResults,
- Id: translateItem.Id,
- }
+ log.GetLogger().Info("Split audio completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", splitItem.Id))
+
+ // 发送分割结果
+ splitResultQueue <- DataWithId[string]{
+ Data: outputFileName,
+ Id: splitItem.Id,
}
}
}
})
}
- // 并发启动协程处理音频转录
+ // 音频转录
for range config.Conf.App.TranscribeParallelNum {
eg.Go(func() error {
for {
@@ -263,51 +272,109 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
}
}
if err != nil {
- cancel()
return fmt.Errorf("audioToSubtitle audioToSrt Transcription err: %w", err)
}
log.GetLogger().Info("Transcribe completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", audioFileItem.Id))
- pendingTranslationQueue <- DataWithId[string]{
- Data: transcriptionData.Text,
- Id: audioFileItem.Id,
- }
+
// 发送转录结果
transcribedQueue <- DataWithId[*types.TranscriptionData]{
Data: transcriptionData,
Id: audioFileItem.Id,
}
-
}
}
})
}
- // 输入音频文件到转录队列,开始流程
- for i, audioFileItem := range stepParam.SmallAudios {
- pendingTranscriptionQueue <- DataWithId[string]{
- Data: audioFileItem.AudioFile,
- Id: i,
- }
+
+ // 分句翻译
+ for range config.Conf.App.TranslateParallelNum {
+ eg.Go(func() error {
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case translateItem, ok := <-pendingTranslationQueue:
+ if !ok {
+ return nil
+ }
+ var translatedResults []TranslatedItem
+ var err error
+ // 翻译文本
+ log.GetLogger().Info("Begin to translate", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))
+ for range config.Conf.App.TranslateMaxAttempts {
+ translatedResults, err = s.splitTextAndTranslate(translateItem.Data, types.GetStandardLanguageName(stepParam.TargetLanguage), stepParam.EnableModalFilter)
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ return fmt.Errorf("audioToSubtitle audioToSrt splitTextAndTranslate err: %w", err)
+ }
+ _ = util.SaveToDisk(translatedResults, filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskTranslationDataPersistenceFileNamePattern, translateItem.Id)))
+ log.GetLogger().Info("Translate completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))
+ // 分割长句
+ splitResults, err := s.splitTranslateItem(translatedResults)
+ if err != nil {
+ // 不中断
+ log.GetLogger().Error("audioToSubtitle audioToSrt splitTranslateItem err", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id), zap.Error(err))
+ translatedQueue <- DataWithId[[]TranslatedItem]{
+ Data: translatedResults,
+ Id: translateItem.Id,
+ }
+ } else {
+ translatedQueue <- DataWithId[[]TranslatedItem]{
+ Data: splitResults,
+ Id: translateItem.Id,
+ }
+ }
+ }
+ }
+ })
}
- // 处理转录和翻译结果
+ // 处理结果,更新字幕任务信息
eg.Go(func() error {
- stepNum := 0
+ // SPLIT_WEIGHT + TRANSCRIBE_WEIGHT + TRANSLATE_WEIGHT == 1
+ const (
+ SPLIT_WEIGHT = 0.1
+ TRANSCRIBE_WEIGHT = 0.4
+ TRANSLATE_WEIGHT = 0.5
+ )
+ // 总体任务在进度条中的占比
+ taskWeight := (90 - 15) / float64(segmentNum)
+ processPct := 15.0
+ // 完成的任务数量
+ completedTasks := 0
for {
select {
case <-ctx.Done():
return nil
+ case splitResultItem := <-splitResultQueue:
+ // 更新字幕任务信息
+ processPct += taskWeight * SPLIT_WEIGHT
+ stepParam.TaskPtr.ProcessPct = uint8(processPct)
+ // 处理分割结果
+ audioSegments[splitResultItem.Id].AudioFile = splitResultItem.Data
+ // 发送转录任务
+ pendingTranscriptionQueue <- DataWithId[string]{
+ Data: splitResultItem.Data,
+ Id: splitResultItem.Id,
+ }
case transcribedItem := <-transcribedQueue:
- stepNum++
// 更新字幕任务信息
- processPct := uint8(20 + 70*stepNum/len(stepParam.SmallAudios)/2)
- stepParam.TaskPtr.ProcessPct = processPct
+ processPct += taskWeight * TRANSCRIBE_WEIGHT
+ stepParam.TaskPtr.ProcessPct = uint8(processPct)
// 处理转录结果
- stepParam.SmallAudios[transcribedItem.Id].TranscriptionData = transcribedItem.Data
+ audioSegments[transcribedItem.Id].TranscriptionData = transcribedItem.Data
+ // 发送翻译任务
+ pendingTranslationQueue <- DataWithId[string]{
+ Data: transcribedItem.Data.Text,
+ Id: transcribedItem.Id,
+ }
case translatedItems := <-translatedQueue:
- stepNum++
// 更新字幕任务信息
- processPct := uint8(20 + 70*stepNum/len(stepParam.SmallAudios)/2)
- stepParam.TaskPtr.ProcessPct = processPct
+ processPct += taskWeight * TRANSLATE_WEIGHT
+ stepParam.TaskPtr.ProcessPct = uint8(processPct)
// 处理翻译结果,保存不带时间戳的原始字幕
originNoTsSrtFileName := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitSrtNoTimestampFileNamePattern, translatedItems.Id))
originNoTsSrtFile, err := os.Create(originNoTsSrtFileName)
@@ -320,10 +387,11 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
originNoTsSrtFile.WriteString(fmt.Sprintf("[%s]\n", translatedItem.TranslatedText))
originNoTsSrtFile.WriteString(fmt.Sprintf("[%s]\n\n", translatedItem.OriginText))
}
+
+ // 此处是为了修复一个未知原因的文件不创建的问题
originNoTsSrtFile.Sync()
originNoTsSrtFile.Close()
- smallAudioPtr := stepParam.SmallAudios[translatedItems.Id]
- smallAudioPtr.SrtNoTsFile = originNoTsSrtFileName
+ audioSegments[translatedItems.Id].SrtNoTsFile = originNoTsSrtFileName
// 生成时间戳
srtBlocks := []*util.SrtBlock{}
for i, translatedItem := range translatedItems.Data {
@@ -335,13 +403,17 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
})
}
- err = generateSrtWithTimestamps(srtBlocks, stepParam.TaskBasePath, translatedItems.Id, stepParam.OriginLanguage, smallAudioPtr.TranscriptionData.Words, stepParam.SubtitleResultType, stepParam.MaxWordOneLine)
+ segmentIdx := translatedItems.Id
+
+ err = generateSrtWithTimestamps(srtBlocks, timePoints[segmentIdx], audioSegments[segmentIdx].TranscriptionData.Words, segmentIdx, stepParam)
if err != nil {
- cancel()
return fmt.Errorf("audioToSubtitle audioToSrt generateTimestamps err: %w", err)
}
- // 转录、翻译任务全部完成
- if stepNum >= len(stepParam.SmallAudios)*2 {
+ completedTasks++
+ // 拆分、转录、翻译任务全部完成
+ if completedTasks >= segmentNum {
+ close(pendingSplitQueue)
+ close(splitResultQueue)
close(pendingTranscriptionQueue)
close(transcribedQueue)
close(pendingTranslationQueue)
@@ -352,9 +424,8 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
}
})
- err := eg.Wait()
- if err != nil {
- log.GetLogger().Error("audioToSubtitle audioToSrt errgroup wait err", zap.Any("stepParam", stepParam), zap.Error(err))
+ if err := eg.Wait(); err != nil {
+ log.GetLogger().Error("audioToSubtitle audioToSrt errgroup wait err", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle audioToSrt errgroup wait err: %w", err)
}
@@ -363,7 +434,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
bilingualFiles := make([]string, 0)
shortOriginMixedFiles := make([]string, 0)
shortOriginFiles := make([]string, 0)
- for i := range stepParam.SmallAudios {
+ for i := range segmentNum {
splitOriginNoTsFile := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitSrtNoTimestampFileNamePattern, i))
originNoTsFiles = append(originNoTsFiles, splitOriginNoTsFile)
splitBilingualFile := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitBilingualSrtFileNamePattern, i))
@@ -379,7 +450,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
err = util.MergeFile(originNoTsFile, originNoTsFiles...)
if err != nil {
log.GetLogger().Error("audioToSubtitle audioToSrt merge originNoTsFile err",
- zap.Any("stepParam", stepParam), zap.Error(err))
+ zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle audioToSrt merge originNoTsFile err: %w", err)
}
@@ -388,7 +459,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
err = util.MergeSrtFiles(bilingualFile, bilingualFiles...)
if err != nil {
log.GetLogger().Error("audioToSubtitle audioToSrt merge BilingualFile err",
- zap.Any("stepParam", stepParam), zap.Error(err))
+ zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle audioToSrt merge BilingualFile err: %w", err)
}
@@ -397,7 +468,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
err = util.MergeSrtFiles(shortOriginMixedFile, shortOriginMixedFiles...)
if err != nil {
log.GetLogger().Error("audioToSubtitle audioToSrt merge shortOriginMixedFile err",
- zap.Any("stepParam", stepParam), zap.Error(err))
+ zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSrt merge shortOriginMixedFile err: %w", err)
}
stepParam.ShortOriginMixedSrtFilePath = shortOriginMixedFile
@@ -407,7 +478,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
err = util.MergeSrtFiles(shortOriginFile, shortOriginFiles...)
if err != nil {
log.GetLogger().Error("audioToSubtitle audioToSrt mergeShortOriginFile err",
- zap.Any("stepParam", stepParam), zap.Error(err))
+ zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSrt mergeShortOriginFile err: %w", err)
}
@@ -432,7 +503,7 @@ func splitSrt(stepParam *types.SubtitleTaskStepParam) error {
// 打开双语字幕文件
file, err := os.Open(stepParam.BilingualSrtFilePath)
if err != nil {
- log.GetLogger().Error("audioToSubtitle splitSrt open bilingual srt file error", zap.Any("stepParam", stepParam), zap.Error(err))
+ log.GetLogger().Error("audioToSubtitle splitSrt open bilingual srt file error", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle splitSrt open bilingual srt file error: %w", err)
}
defer file.Close()
@@ -440,28 +511,28 @@ func splitSrt(stepParam *types.SubtitleTaskStepParam) error {
// 打开输出字幕和文稿文件
originLanguageSrtFile, err := os.Create(originLanguageSrtFilePath)
if err != nil {
- log.GetLogger().Error("audioToSubtitle splitSrt create originLanguageSrtFile error", zap.Any("stepParam", stepParam), zap.Error(err))
+ log.GetLogger().Error("audioToSubtitle splitSrt create originLanguageSrtFile error", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle splitSrt create originLanguageSrtFile error: %w", err)
}
defer originLanguageSrtFile.Close()
originLanguageTextFile, err := os.Create(originLanguageTextFilePath)
if err != nil {
- log.GetLogger().Error("audioToSubtitle splitSrt create originLanguageTextFile error", zap.Any("stepParam", stepParam), zap.Error(err))
+ log.GetLogger().Error("audioToSubtitle splitSrt create originLanguageTextFile error", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle splitSrt create originLanguageTextFile error: %w", err)
}
defer originLanguageTextFile.Close()
targetLanguageSrtFile, err := os.Create(targetLanguageSrtFilePath)
if err != nil {
- log.GetLogger().Error("audioToSubtitle.splitSrt create targetLanguageSrtFile error", zap.Any("stepParam", stepParam), zap.Error(err))
+ log.GetLogger().Error("audioToSubtitle.splitSrt create targetLanguageSrtFile error", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle.splitSrt create targetLanguageSrtFile error: %w", err)
}
defer targetLanguageSrtFile.Close()
targetLanguageTextFile, err := os.Create(targetLanguageTextFilePath)
if err != nil {
- log.GetLogger().Error("audioToSubtitle.splitSrt create targetLanguageTextFile error", zap.Any("stepParam", stepParam), zap.Error(err))
+ log.GetLogger().Error("audioToSubtitle.splitSrt create targetLanguageTextFile error", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle.splitSrt create targetLanguageTextFile error: %w", err)
}
defer targetLanguageTextFile.Close()
@@ -489,7 +560,7 @@ func splitSrt(stepParam *types.SubtitleTaskStepParam) error {
}
if err = scanner.Err(); err != nil {
- log.GetLogger().Error("audioToSubtitle splitSrt scan bilingual srt file error", zap.Any("stepParam", stepParam), zap.Error(err))
+ log.GetLogger().Error("audioToSubtitle splitSrt scan bilingual srt file error", zap.Any("taskId", stepParam.TaskId), zap.Error(err))
return fmt.Errorf("audioToSubtitle splitSrt scan bilingual srt file error: %w", err)
}
// 添加原语言单语字幕
@@ -811,7 +882,7 @@ func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int, []types.Word)
return startIdx, endIdx, result
}
-func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segmentIdx int, originLanguage types.StandardLanguageCode, words []types.Word, resultType types.SubtitleResultType, maxWordOneLine int) error {
+func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, tsOffset float64, words []types.Word, segmentIdx int, stepParam *types.SubtitleTaskStepParam) error {
if len(srtBlocks) == 0 || len(words) == 0 {
return nil
}
@@ -823,11 +894,10 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
if srtBlock.OriginLanguageSentence == "" {
continue
}
- sentenceTs, sentenceWords, ts, err := getSentenceTimestamps(words, srtBlock.OriginLanguageSentence, lastTs, originLanguage)
+ sentenceTs, sentenceWords, ts, err := getSentenceTimestamps(words, srtBlock.OriginLanguageSentence, lastTs, stepParam.OriginLanguage)
if err != nil || ts < lastTs {
continue
}
- tsOffset := float64(config.Conf.App.SegmentDuration) * 60 * float64(segmentIdx)
srtBlock.Timestamp = fmt.Sprintf("%s --> %s", util.FormatTime(float32(sentenceTs.Start+tsOffset)), util.FormatTime(float32(sentenceTs.End+tsOffset)))
// 生成短句子的英文字幕
@@ -837,7 +907,7 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
endWord types.Word
)
- if len(sentenceWords) <= maxWordOneLine {
+ if len(sentenceWords) <= stepParam.MaxWordOneLine {
shortOriginSrtMap[srtBlock.Index] = append(shortOriginSrtMap[srtBlock.Index], util.SrtBlock{
Index: srtBlock.Index,
Timestamp: fmt.Sprintf("%s --> %s", util.FormatTime(float32(sentenceTs.Start+tsOffset)), util.FormatTime(float32(sentenceTs.End+tsOffset))),
@@ -847,14 +917,14 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
continue
}
- thisLineWord := maxWordOneLine
- if len(sentenceWords) > maxWordOneLine && len(sentenceWords) <= 2*maxWordOneLine {
+ thisLineWord := stepParam.MaxWordOneLine
+ if len(sentenceWords) > stepParam.MaxWordOneLine && len(sentenceWords) <= 2*stepParam.MaxWordOneLine {
thisLineWord = len(sentenceWords)/2 + 1
- } else if len(sentenceWords) > 2*maxWordOneLine && len(sentenceWords) <= 3*maxWordOneLine {
+ } else if len(sentenceWords) > 2*stepParam.MaxWordOneLine && len(sentenceWords) <= 3*stepParam.MaxWordOneLine {
thisLineWord = len(sentenceWords)/3 + 1
- } else if len(sentenceWords) > 3*maxWordOneLine && len(sentenceWords) <= 4*maxWordOneLine {
+ } else if len(sentenceWords) > 3*stepParam.MaxWordOneLine && len(sentenceWords) <= 4*stepParam.MaxWordOneLine {
thisLineWord = len(sentenceWords)/4 + 1
- } else if len(sentenceWords) > 4*maxWordOneLine && len(sentenceWords) <= 5*maxWordOneLine {
+ } else if len(sentenceWords) > 4*stepParam.MaxWordOneLine && len(sentenceWords) <= 5*stepParam.MaxWordOneLine {
thisLineWord = len(sentenceWords)/5 + 1
}
@@ -917,7 +987,7 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
}
// 保存带时间戳的原始字幕
- finalBilingualSrtFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitBilingualSrtFileNamePattern, segmentIdx))
+ finalBilingualSrtFileName := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitBilingualSrtFileNamePattern, segmentIdx))
finalBilingualSrtFile, err := os.Create(finalBilingualSrtFileName)
if err != nil {
return fmt.Errorf("audioToSubtitle generateTimestamps create bilingual srt file error: %w", err)
@@ -928,7 +998,7 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
for _, srtBlock := range srtBlocks {
_, _ = finalBilingualSrtFile.WriteString(fmt.Sprintf("%d\n", srtBlock.Index))
_, _ = finalBilingualSrtFile.WriteString(srtBlock.Timestamp + "\n")
- if resultType == types.SubtitleResultTypeBilingualTranslationOnTop {
+ if stepParam.SubtitleResultType == types.SubtitleResultTypeBilingualTranslationOnTop {
_, _ = finalBilingualSrtFile.WriteString(srtBlock.TargetLanguageSentence + "\n")
_, _ = finalBilingualSrtFile.WriteString(srtBlock.OriginLanguageSentence + "\n\n")
} else {
@@ -939,7 +1009,7 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
}
// 保存带时间戳的字幕,长中文+短英文(示意,也支持其他语言)
- srtShortOriginMixedFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginMixedSrtFileNamePattern, segmentIdx))
+ srtShortOriginMixedFileName := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginMixedSrtFileNamePattern, segmentIdx))
srtShortOriginMixedFile, err := os.Create(srtShortOriginMixedFileName)
if err != nil {
return fmt.Errorf("audioToSubtitle generateTimestamps create srtShortOriginMixedFile err: %w", err)
@@ -947,7 +1017,7 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, basePath string, segm
defer srtShortOriginMixedFile.Close()
// 保存带时间戳的短英文字幕
- srtShortOriginFileName := fmt.Sprintf("%s/%s", basePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginSrtFileNamePattern, segmentIdx))
+ srtShortOriginFileName := fmt.Sprintf("%s/%s", stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitShortOriginSrtFileNamePattern, segmentIdx))
srtShortOriginFile, err := os.Create(srtShortOriginFileName)
if err != nil {
return fmt.Errorf("audioToSubtitle generateTimestamps create srtShortOriginFile err: %w", err)
diff --git a/internal/service/split_audio.go b/internal/service/split_audio.go
new file mode 100644
index 00000000..89f15c66
--- /dev/null
+++ b/internal/service/split_audio.go
@@ -0,0 +1,162 @@
+package service
+
+import (
+ "fmt"
+ "io"
+ "krillin-ai/internal/storage"
+ "krillin-ai/pkg/util"
+ "math"
+ "os/exec"
+ "runtime"
+
+ "golang.org/x/sync/errgroup"
+)
+
+// MIN_SEGMENT_DURATION >= MIN_DURATION + TOLERANCE_DURATION * 2 > ENERGY_WINDOW_DURATION > 0
+const (
+ SAMPLE_RATE = 3000
+ ENERGY_WINDOW_DURATION = 1.5 // 计算音频能量的时间长度
+ TOLERANCE_DURATION = 8 // 容忍的时间误差
+ MIN_DURATION = 10 // 最小音频时长
+ MIN_SEGMENT_DURATION = 20 // 最小分割时长
+)
+
+func buildFFmpegCmd(input string, start, end float64) (*exec.Cmd, error) {
+ if start < 0 || end <= start {
+ return nil, fmt.Errorf("invalid start or end time: start=%f, end=%f", start, end)
+ }
+ cmd := exec.Command(
+ storage.FfmpegPath,
+ "-y",
+ "-ss", fmt.Sprintf("%.3f", start), // 起始时间
+ "-to", fmt.Sprintf("%.3f", end), // 结束时间
+ "-i", input,
+
+ "-f", "s16le",
+ "-ar", fmt.Sprintf("%d", SAMPLE_RATE),
+ "-ac", "1",
+ "-af", "lowpass=f=3000,highpass=f=300",
+ "pipe:1",
+ )
+ return cmd, nil
+}
+
+func getQuietestTimePoint(input string, start, end float64) (second float64, err error) {
+ cmd, err := buildFFmpegCmd(input, start, end)
+ if err != nil {
+ return 0, fmt.Errorf("failed to build ffmpeg command: %w", err)
+ }
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return 0, fmt.Errorf("failed to create stdout pipe: %w", err)
+ }
+
+ if err := cmd.Start(); err != nil {
+ return 0, fmt.Errorf("failed to start ffmpeg command: [%s] %w", cmd.String(), err)
+ }
+
+ originBuffer := make([]byte, 1024)
+ headBuffer := [2]byte{}
+ circularQueue := util.NewCircularQueue[float32](SAMPLE_RATE * ENERGY_WINDOW_DURATION)
+ currentEnergy := float32(0)
+ index := 0
+ var (
+ minEnergy float32 = math.MaxFloat32
+ minEnergyIndex int
+ )
+ for {
+ n, err := stdout.Read(originBuffer)
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return 0, fmt.Errorf("error reading from stdout: [%s] %w", cmd.String(), err)
+ }
+ for i := range n {
+ if i%2 == 0 {
+ headBuffer[0] = originBuffer[i]
+ continue
+ }
+ headBuffer[1] = originBuffer[i]
+ index++
+ sample := int16(headBuffer[0]) | int16(headBuffer[1])<<8
+ sampleEnergy := float32(sample) * float32(sample)
+ if !circularQueue.IsFull() {
+ circularQueue.Enqueue(sampleEnergy)
+ currentEnergy += sampleEnergy
+ continue
+ }
+ earliestEnergy, _ := circularQueue.Dequeue()
+ currentEnergy -= earliestEnergy
+ circularQueue.Enqueue(sampleEnergy)
+ currentEnergy += sampleEnergy
+
+ if currentEnergy <= minEnergy {
+ minEnergy = currentEnergy
+ minEnergyIndex = index - SAMPLE_RATE*ENERGY_WINDOW_DURATION/2
+ }
+ }
+ }
+ if err := cmd.Wait(); err != nil {
+ return 0, fmt.Errorf("ffmpeg command run failed: [%s] %w", cmd.String(), err)
+ }
+ return float64(minEnergyIndex)/SAMPLE_RATE + start, nil
+}
+
+func GetSplitPoints(input string, segmentDuration float64) ([]float64, error) {
+ if segmentDuration < MIN_SEGMENT_DURATION {
+ return nil, fmt.Errorf("segment duration must be greater than %v seconds", MIN_SEGMENT_DURATION)
+ }
+
+ audioDuration, err := util.GetAudioDuration(input)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get audio duration: %w", err)
+ }
+ segmentNum := int(math.Ceil(audioDuration / segmentDuration))
+ timePoints := make([]float64, segmentNum+1)
+ for i := range segmentNum {
+ timePoints[i] = float64(i) * segmentDuration
+ }
+ eg := errgroup.Group{}
+ eg.SetLimit(runtime.NumCPU())
+ for i := 1; i < segmentNum; i++ {
+ i := i
+ eg.Go(func() error {
+ start := timePoints[i] - TOLERANCE_DURATION
+ end := timePoints[i] + TOLERANCE_DURATION
+ timePoint, err := getQuietestTimePoint(input, start, end)
+ if err != nil {
+ return fmt.Errorf("failed to get quietest time point: %w", err)
+ }
+ timePoints[i] = timePoint
+ return nil
+ })
+ }
+ if err := eg.Wait(); err != nil {
+ return nil, fmt.Errorf("failed to get quietest time points: %w", err)
+ }
+ // 如果最后一个片段短于最小分割时长,则将其合并到前一个片段
+ if audioDuration-timePoints[segmentNum-1] < MIN_DURATION {
+ timePoints = timePoints[:segmentNum]
+ }
+ timePoints[len(timePoints)-1] = audioDuration
+ return timePoints, nil
+}
+
+func ClipAudio(input, output string, start, end float64) error {
+ if start < 0 || end <= start {
+ return fmt.Errorf("invalid start or end time: start=%f, end=%f", start, end)
+ }
+ cmd := exec.Command(
+ storage.FfmpegPath,
+ "-y",
+ "-ss", fmt.Sprintf("%.3f", start), // 起始时间
+ "-to", fmt.Sprintf("%.3f", end), // 结束时间
+ "-i", input,
+ output,
+ )
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf("failed to clip audio: [%s] %w", cmd.String(), err)
+ }
+ return nil
+}
diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.go
index 33451220..eb861642 100644
--- a/internal/types/subtitle_task.go
+++ b/internal/types/subtitle_task.go
@@ -123,9 +123,10 @@ var SplitLongSentencePrompt = `请将以下原文和译文分割成2-3个部分
译文:%s
要求:
-1. 分割后的句子必须保持语义完整,避免切断完整概念,避免切分后的句子不符合语法规范
-2. 确保原文和译文的分割部分一一对应
-3. 务必返回JSON格式,包含origin_part和translated_part数组,例如:
+1. 分割后的句子必须保持语义完整,避免切断完整概念
+2. 切分后的句子需要符合语法规范,可添加连词等保证阅读时语言自然
+3. 确保原文和译文的分割部分一一对应
+4. 务必返回JSON格式,包含origin_part和translated_part数组,例如:
{"align":[{"origin_part":"原文部分1","translated_part":"译文部分1"},{"origin_part":"原文部分2","translated_part":"译文部分2"}]}`
type SmallAudio struct {
@@ -223,7 +224,6 @@ type SubtitleTaskStepParam struct {
TaskBasePath string
Link string
AudioFilePath string
- SmallAudios []*SmallAudio
SubtitleResultType SubtitleResultType
EnableModalFilter bool
EnableTts bool
diff --git a/pkg/util/queue.go b/pkg/util/queue.go
new file mode 100644
index 00000000..974b4e02
--- /dev/null
+++ b/pkg/util/queue.go
@@ -0,0 +1,60 @@
+package util
+
+type Queue[T any] interface {
+ Enqueue(item T) bool
+ Dequeue() (T, bool)
+ IsEmpty() bool
+ IsFull() bool
+ Size() int
+ Peek() (T, bool) // 可选,根据需求添加
+}
+
+type CircularQueue[T any] struct {
+ data []T
+ front int // 队首指针
+ rear int // 队尾指针
+ count int // 跟踪元素数量
+}
+
+// 创建一个新的循环队列
+func NewCircularQueue[T any](maxSize int) *CircularQueue[T] {
+ return &CircularQueue[T]{
+ data: make([]T, maxSize),
+ front: 0,
+ rear: 0,
+ count: 0,
+ }
+}
+
+func (q *CircularQueue[T]) IsEmpty() bool { return q.count == 0 }
+func (q *CircularQueue[T]) IsFull() bool { return q.count == len(q.data) }
+func (q *CircularQueue[T]) Size() int { return q.count }
+
+func (q *CircularQueue[T]) Enqueue(item T) bool {
+ if q.IsFull() {
+ return false
+ }
+ q.data[q.rear] = item
+ q.rear = (q.rear + 1) % len(q.data)
+ q.count++
+ return true
+}
+
+func (q *CircularQueue[T]) Dequeue() (T, bool) {
+ var zero T
+ if q.IsEmpty() {
+ return zero, false
+ }
+ item := q.data[q.front]
+ q.front = (q.front + 1) % len(q.data)
+ q.count--
+ return item, true
+}
+
+func (q *CircularQueue[T]) Peek() (T, bool) {
+ var zero T
+ if q.IsEmpty() {
+ return zero, false
+ }
+ return q.data[q.front], true
+}