Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

fix: timestamps wrong in some cases

沈浩626181832d20
diff --git a/go.mod b/go.mod
index 0a66d3f8..4aa0bafc 100644
--- a/go.mod
+++ b/go.mod
@@ -63,6 +63,7 @@ require (
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect
github.com/stretchr/testify v1.9.0 // indirect
+ github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/yuin/goldmark v1.7.1 // indirect
diff --git a/go.sum b/go.sum
index b542a0e7..55a5de9f 100644
--- a/go.sum
+++ b/go.sum
@@ -372,6 +372,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U=
+github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.go
index 2a6831f4..30fcadb5 100644
--- a/internal/service/audio2subtitle.go
+++ b/internal/service/audio2subtitle.go
@@ -92,7 +92,7 @@ func splitAudio(stepParam *types.SubtitleTaskStepParam) error {
return nil
}
-func (s Service) transcribeAudio(audioFilePath string, language string, taskBasePath string) (*types.TranscriptionData, error) {
+func (s Service) transcribeAudio(id int, audioFilePath string, language string, taskBasePath string) (*types.TranscriptionData, error) {
defer func() {
if r := recover(); r != nil {
log.GetLogger().Error("audioToSubtitle transcribeAudio panic recovered", zap.Any("panic", r), zap.String("stack", string(debug.Stack())))
@@ -108,6 +108,8 @@ func (s Service) transcribeAudio(audioFilePath string, language string, taskBase
return nil, fmt.Errorf("audioToSubtitle transcribeAudio Transcription err: %w", err)
}
+ _ = util.SaveToDisk(transcriptionData, filepath.Join(taskBasePath, fmt.Sprintf(types.SubtitleTaskAudioTranscriptionDataPersistenceFileNamePattern, id)))
+
if transcriptionData.Text == "" {
log.GetLogger().Info("audioToSubtitle transcribeAudio TranscriptionData.Text is empty", zap.Any("audioFilePath", audioFilePath), zap.Any("taskBasePath", taskBasePath))
}
@@ -225,7 +227,7 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
log.GetLogger().Info("Begin transcribe", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", audioFileItem.Id))
// 语音转文字
for range config.Conf.App.TranscribeMaxAttempts {
- transcriptionData, err = s.transcribeAudio(audioFileItem.Data, string(stepParam.OriginLanguage), stepParam.TaskBasePath)
+ transcriptionData, err = s.transcribeAudio(audioFileItem.Id, audioFileItem.Data, string(stepParam.OriginLanguage), stepParam.TaskBasePath)
if err == nil {
break
}
@@ -654,6 +656,9 @@ func getSentenceTimestamps(words []types.Word, sentence string, lastTs float64,
beginWord := sentenceWords[beginWordIndex]
endWord := sentenceWords[endWordIndex]
+ //sequence := util.FindClosestConsecutiveWords(words, sentence)
+ //beginWord := sequence[0]
+ //endWord := sequence[len(sequence)-1]
srtSt.Start = beginWord.Start
if srtSt.Start < thisLastTs {
@@ -711,6 +716,10 @@ func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int, []types.Word)
return -1, -1, nil
}
+ if len(words) == 1 {
+ return 0, 0, words
+ }
+
// dp[i] 表示以 words[i] 结束的递增子数组的长度
dp := make([]int, len(words))
// prev[i] 用来记录与当前递增子数组相连的前一个元素的索引
@@ -757,9 +766,11 @@ func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int, []types.Word)
}
// 构造结果子数组
- result := []types.Word{}
- for i := startIdx; i != -1; i = prev[i] {
- result = append(result, words[i])
+ result := make([]types.Word, 0, maxLen)
+ current := endIdx
+ for current != -1 {
+ result = append(result, words[current])
+ current = prev[current]
}
// 由于是从后往前构造的子数组,需要反转
diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.go
index 3ffde187..a26c7605 100644
--- a/internal/types/subtitle_task.go
+++ b/internal/types/subtitle_task.go
@@ -130,29 +130,30 @@ const (
)
const (
- SubtitleTaskAudioFileName = "origin_audio.mp3"
- SubtitleTaskVideoFileName = "origin_video.mp4"
- SubtitleTaskSplitAudioFileNamePrefix = "split_audio"
- SubtitleTaskSplitAudioFileNamePattern = SubtitleTaskSplitAudioFileNamePrefix + "_%03d.mp3"
- SubtitleTaskSplitAudioTxtFileNamePattern = "split_audio_txt_%d.txt"
- SubtitleTaskSplitAudioWordsFileNamePattern = "split_audio_words_%d.txt"
- SubtitleTaskSplitSrtNoTimestampFileNamePattern = "srt_no_ts_%d.srt"
- SubtitleTaskSrtNoTimestampFileName = "srt_no_ts.srt"
- SubtitleTaskSplitBilingualSrtFileNamePattern = "split_bilingual_srt_%d.srt"
- SubtitleTaskSplitShortOriginMixedSrtFileNamePattern = "split_short_origin_mixed_srt_%d.srt" //长中文+短英文
- SubtitleTaskSplitShortOriginSrtFileNamePattern = "split_short_origin_srt_%d.srt" //短英文
- SubtitleTaskBilingualSrtFileName = "bilingual_srt.srt"
- SubtitleTaskShortOriginMixedSrtFileName = "short_origin_mixed_srt.srt" //长中文+短英文
- SubtitleTaskShortOriginSrtFileName = "short_origin_srt.srt" //短英文
- SubtitleTaskOriginLanguageSrtFileName = "origin_language_srt.srt"
- SubtitleTaskOriginLanguageTextFileName = "origin_language.txt"
- SubtitleTaskTargetLanguageSrtFileName = "target_language_srt.srt"
- SubtitleTaskTargetLanguageTextFileName = "target_language.txt"
- SubtitleTaskStepParamGobPersistenceFileName = "step_param.gob"
- SubtitleTaskTransferredVerticalVideoFileName = "transferred_vertical_video.mp4"
- SubtitleTaskHorizontalEmbedVideoFileName = "horizontal_embed.mp4"
- SubtitleTaskVerticalEmbedVideoFileName = "vertical_embed.mp4"
- SubtitleTaskVideoWithTtsFileName = "video_with_tts.mp4"
+ SubtitleTaskAudioFileName = "origin_audio.mp3"
+ SubtitleTaskVideoFileName = "origin_video.mp4"
+ SubtitleTaskSplitAudioFileNamePrefix = "split_audio"
+ SubtitleTaskSplitAudioFileNamePattern = SubtitleTaskSplitAudioFileNamePrefix + "_%03d.mp3"
+ SubtitleTaskSplitAudioTxtFileNamePattern = "split_audio_txt_%d.txt"
+ SubtitleTaskSplitAudioWordsFileNamePattern = "split_audio_words_%d.txt"
+ SubtitleTaskSplitSrtNoTimestampFileNamePattern = "srt_no_ts_%d.srt"
+ SubtitleTaskSrtNoTimestampFileName = "srt_no_ts.srt"
+ SubtitleTaskSplitBilingualSrtFileNamePattern = "split_bilingual_srt_%d.srt"
+ SubtitleTaskSplitShortOriginMixedSrtFileNamePattern = "split_short_origin_mixed_srt_%d.srt" //长中文+短英文
+ SubtitleTaskSplitShortOriginSrtFileNamePattern = "split_short_origin_srt_%d.srt" //短英文
+ SubtitleTaskBilingualSrtFileName = "bilingual_srt.srt"
+ SubtitleTaskShortOriginMixedSrtFileName = "short_origin_mixed_srt.srt" //长中文+短英文
+ SubtitleTaskShortOriginSrtFileName = "short_origin_srt.srt" //短英文
+ SubtitleTaskOriginLanguageSrtFileName = "origin_language_srt.srt"
+ SubtitleTaskOriginLanguageTextFileName = "origin_language.txt"
+ SubtitleTaskTargetLanguageSrtFileName = "target_language_srt.srt"
+ SubtitleTaskTargetLanguageTextFileName = "target_language.txt"
+ SubtitleTaskStepParamGobPersistenceFileName = "step_param.gob"
+ SubtitleTaskAudioTranscriptionDataPersistenceFileNamePattern = "audio_transcription_data_%d.json"
+ SubtitleTaskTransferredVerticalVideoFileName = "transferred_vertical_video.mp4"
+ SubtitleTaskHorizontalEmbedVideoFileName = "horizontal_embed.mp4"
+ SubtitleTaskVerticalEmbedVideoFileName = "vertical_embed.mp4"
+ SubtitleTaskVideoWithTtsFileName = "video_with_tts.mp4"
)
const (
diff --git a/pkg/util/base.go b/pkg/util/base.go
index 01245228..c0635551 100644
--- a/pkg/util/base.go
+++ b/pkg/util/base.go
@@ -2,9 +2,12 @@ package util
import (
"archive/zip"
+ "encoding/json"
"fmt"
"github.com/google/uuid"
+ "github.com/texttheater/golang-levenshtein/levenshtein"
"io"
+ "krillin-ai/internal/types"
"math"
"math/rand"
"net/url"
@@ -221,3 +224,82 @@ func SanitizePathName(name string) string {
return sanitized
}
+
+// FindClosestConsecutiveWords 查找 words 中 Num 连续递增的一组词,使得其拼接后的文本与 inputStr 的编辑距离最小。
+func FindClosestConsecutiveWords(words []types.Word, inputStr string) []types.Word {
+ if len(words) == 0 {
+ return nil
+ }
+
+ // 先将输入按 Num 排序(如果你已经保证是有序的可跳过此步骤)
+ // sort.Slice(words, func(i, j int) bool { return words[i].Num < words[j].Num })
+
+ // Step 1: 获取所有 Num 连续递增的 []types.Word 组合
+ var groups [][]types.Word
+ var currentGroup []types.Word
+
+ for i, word := range words {
+ if i == 0 {
+ currentGroup = append(currentGroup, word)
+ continue
+ }
+
+ if word.Num == words[i-1].Num+1 {
+ currentGroup = append(currentGroup, word)
+ } else {
+ if len(currentGroup) > 0 {
+ groups = append(groups, currentGroup)
+ }
+ currentGroup = []types.Word{word}
+ }
+ }
+ if len(currentGroup) > 0 {
+ groups = append(groups, currentGroup)
+ }
+
+ // Step 2: 比较编辑距离,找最接近 inputStr 的那个组
+ minDistance := -1
+ var bestGroup []types.Word
+
+ for _, group := range groups {
+ var sb strings.Builder
+ for _, w := range group {
+ sb.WriteString(w.Text)
+ }
+ groupText := sb.String()
+
+ dist := levenshtein.DistanceForStrings([]rune(groupText), []rune(inputStr), levenshtein.DefaultOptions)
+
+ if minDistance == -1 || dist < minDistance {
+ minDistance = dist
+ bestGroup = group
+ }
+ }
+
+ return bestGroup
+}
+
+func SaveToDisk(data any, filename string) error {
+ file, err := os.Create(filename)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ encoder := json.NewEncoder(file)
+ encoder.SetIndent("", " ") // 美化输出
+ return encoder.Encode(data)
+}
+
+func LoadFromDisk(filename string) (any, error) {
+ var data any
+ file, err := os.Open(filename)
+ if err != nil {
+ return data, err
+ }
+ defer file.Close()
+
+ decoder := json.NewDecoder(file)
+ err = decoder.Decode(&data)
+ return data, err
+}