Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
Add a new timestamp matching algorithm
wulien40d9dde00e9e
diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.goindex e0edb33e..ebde1ff1 100644--- a/internal/service/audio2subtitle.go+++ b/internal/service/audio2subtitle.go@@ -113,13 +113,22 @@ func (s Service) transcribeAudio(id int, audioFilePath string, language string,return transcriptionData, nil}+func (s Service) IsSplitUseSpace(language types.StandardLanguageCode) bool {+ if language == types.LanguageNameSimplifiedChinese || language == types.LanguageNameTraditionalChinese ||+ language == types.LanguageNameJapanese || language == types.LanguageNameKorean || language == types.LanguageNameThai {+ return true+ }++ return false+}+func (s Service) splitTextAndTranslateV2(basePath, inputText string, originLang, targetLang types.StandardLanguageCode, enableModalFilter bool, id int) ([]*TranslatedItem, error) {- sentences := util.SplitTextSentences(inputText)+ sentences := util.SplitTextSentences(inputText, config.Conf.App.MaxSentenceLength)if len(sentences) == 0 {return []*TranslatedItem{}, nil}// 补丁:whisper转录中文的时候很多句子后面不输出符号,导致上面基于符号的切分失效- if originLang == types.LanguageNameSimplifiedChinese || originLang == types.LanguageNameTraditionalChinese {+ if s.IsSplitUseSpace(originLang) {newSentences := make([]string, 0)for _, sentence := range sentences {newSentences = append(newSentences, strings.Split(sentence, " ")...)@@ -128,28 +137,12 @@ func (s Service) splitTextAndTranslateV2(basePath, inputText string, originLang,}shortSentences := make([]string, 0)- lastMerged := false//判断句子如果还是过长,就继续用大模型拆句- for i, sentence := range sentences {- // 如果前一次循环已经处理了当前句子(作为合并的一部分),跳过- if lastMerged {- lastMerged = false+ for _, sentence := range sentences {+ if sentence == "" {continue}-- // 如果当前句子小于10个字符,尝试与下一个句子合并- if len(sentence) < 10 && i < len(sentences)-1 {- // 与下一个句子合并- mergedSentence := sentence + " " + sentences[i+1]- if calcLength(mergedSentence) <= float64(config.Conf.App.MaxSentenceLength) {- shortSentences = append(shortSentences, mergedSentence)- // 标记下一个句子已处理,需要在外层循环中跳过- lastMerged = true- continue- }- }-- if calcLength(sentence) <= float64(config.Conf.App.MaxSentenceLength) {+ if util.CountEffectiveChars(sentence) <= config.Conf.App.MaxSentenceLength {shortSentences = append(shortSentences, sentence)continue}@@ -162,7 +155,7 @@ func (s Service) splitTextAndTranslateV2(basePath, inputText string, originLang,}//拆完之后还长,就再拆一次for _, item := range splitItems {- if calcLength(item) <= float64(config.Conf.App.MaxSentenceLength) {+ if util.CountEffectiveChars(item) <= config.Conf.App.MaxSentenceLength {shortSentences = append(shortSentences, item)continue}@@ -472,12 +465,12 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt}// 保存不带时间戳的原始字幕for i, translatedItem := range translatedItems.Data {- if util.IsAsianLanguage(stepParam.TargetLanguage) {- translatedItem.TranslatedText = util.BeautifyAsianLanguageSentence(translatedItem.TranslatedText)- }- if util.IsAsianLanguage(stepParam.OriginLanguage) {- translatedItem.OriginText = util.BeautifyAsianLanguageSentence(translatedItem.OriginText)- }+ // if util.IsAsianLanguage(stepParam.TargetLanguage) {+ // translatedItem.TranslatedText = util.BeautifyAsianLanguageSentence(translatedItem.TranslatedText)+ // }+ // if util.IsAsianLanguage(stepParam.OriginLanguage) {+ // translatedItem.OriginText = util.BeautifyAsianLanguageSentence(translatedItem.OriginText)+ // }_, _ = originNoTsSrtFile.WriteString(fmt.Sprintf("%d\n", i+1))_, _ = originNoTsSrtFile.WriteString(fmt.Sprintf("%s\n", translatedItem.TranslatedText))_, _ = originNoTsSrtFile.WriteString(fmt.Sprintf("%s\n\n", translatedItem.OriginText))@@ -985,6 +978,12 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, tsOffset float64, wor// 获取每个字幕块的时间戳var lastTs float64shortOriginSrtMap := make(map[int][]util.SrtBlock, 0)+ timeMatcher := NewTimestampGenerator()+ newSrtBlocks, err := timeMatcher.GenerateTimestamps(srtBlocks, words, stepParam.OriginLanguage, tsOffset)+ if err != nil {+ return fmt.Errorf("audioToSubtitle generateTimestamps GenerateTimestamps error: %w", err)+ }+for _, srtBlock := range srtBlocks {if srtBlock.OriginLanguageSentence == "" {continue@@ -1090,7 +1089,7 @@ func generateSrtWithTimestamps(srtBlocks []*util.SrtBlock, tsOffset float64, wordefer finalBilingualSrtFile.Close()// 写入字幕文件- for _, srtBlock := range srtBlocks {+ for _, srtBlock := range newSrtBlocks {_, _ = finalBilingualSrtFile.WriteString(fmt.Sprintf("%d\n", srtBlock.Index))_, _ = finalBilingualSrtFile.WriteString(srtBlock.Timestamp + "\n")if stepParam.SubtitleResultType == types.SubtitleResultTypeBilingualTranslationOnTop {@@ -1257,8 +1256,7 @@ func calcLength(text string) float64 {// splitTranslateItem 根据字符权重和最大长度分割长句func (s Service) splitTranslateItem(items []*TranslatedItem) ([]*TranslatedItem, error) {var result []*TranslatedItem- maxLength := 70 // todo 先写死- //targetMultiplier := config.Conf.Subtitle.TargetMultiplier+ maxLength := config.Conf.App.MaxSentenceLength + 30for _, item := range items {// 计算翻译文本的加权长度@@ -1313,7 +1311,7 @@ func (s Service) splitLongSentence(item *TranslatedItem) ([]*TranslatedItem, err}func (s Service) splitOriginLongSentence(sentence string) ([]string, error) {- prompt := fmt.Sprintf(types.SplitOriginLongSentencePrompt, sentence, config.Conf.App.MaxSentenceLength)+ prompt := fmt.Sprintf(types.SplitOriginLongSentencePrompt, sentence)var response stringvar err errordiff --git a/internal/service/timestamps.go b/internal/service/timestamps.gonew file mode 100644index 00000000..2a33ee23--- /dev/null+++ b/internal/service/timestamps.go@@ -0,0 +1,571 @@+package service++import (+ "fmt"+ "strings"+ "unicode"++ "krillin-ai/internal/types"+ "krillin-ai/log"++ "krillin-ai/pkg/util"++ "go.uber.org/zap"+)++// TimestampMatcher defines the interface for different language timestamp matching algorithms+type TimestampMatcher interface {+ // MatchSentenceTimestamp finds the start and end timestamps for a sentence+ // lastTs is the last known timestamp+ MatchSentenceTimestamp(sentence string, words []types.Word, lastTs float64) (startTime, endTime float64, err error)+ // GetLanguageType returns the language type this matcher supports+ GetLanguageType() types.StandardLanguageCode+}++// TimestampGenerator handles timestamp generation for multiple languages+type TimestampGenerator struct {+ matchers map[types.StandardLanguageCode]TimestampMatcher+}++// NewTimestampGenerator creates a new timestamp generator with default matchers+func NewTimestampGenerator() *TimestampGenerator {+ generator := &TimestampGenerator{+ matchers: make(map[types.StandardLanguageCode]TimestampMatcher),+ }++ // Register Base matcher+ BaseMatcher := &BaseLanguageMatcher{language: types.StandardLanguageCode("base")}+ generator.RegisterMatcher(types.StandardLanguageCode("base"), BaseMatcher)+ generator.RegisterMatcher(types.LanguageNameJapanese, BaseMatcher)+ generator.RegisterMatcher(types.LanguageNameSimplifiedChinese, BaseMatcher)+ generator.RegisterMatcher(types.LanguageNameTraditionalChinese, BaseMatcher)+ generator.RegisterMatcher(types.LanguageNameThai, BaseMatcher)+ generator.RegisterMatcher(types.LanguageNameEnglish, BaseMatcher)+ generator.RegisterMatcher(types.LanguageNameFinnish, BaseMatcher)++ return generator+}++// RegisterMatcher registers a timestamp matcher for a specific language+func (tg *TimestampGenerator) RegisterMatcher(language types.StandardLanguageCode, matcher TimestampMatcher) {+ tg.matchers[language] = matcher+}++// GenerateTimestamps generates timestamps for SRT blocks using the appropriate language matcher+func (tg *TimestampGenerator) GenerateTimestamps(srtBlocks []*util.SrtBlock, words []types.Word, language types.StandardLanguageCode, tsOffset float64) ([]*util.SrtBlock, error) {+ matcher, exists := tg.matchers[language]+ if !exists {+ // Fallback to English matcher for unsupported languages+ matcher = tg.matchers[types.StandardLanguageCode("base")]+ if matcher == nil {+ return nil, fmt.Errorf("no timestamp matcher available for language: %s", language)+ }+ log.GetLogger().Warn("Using fallback base matcher for unsupported language",+ zap.String("language", string(language)))+ }++ var lastEndTime float64+ updatedBlocks := make([]*util.SrtBlock, len(srtBlocks))++ for i, block := range srtBlocks {+ updatedBlocks[i] = block++ if block.OriginLanguageSentence == "" {+ // Skip empty sentences+ continue+ }++ startTime, endTime, err := matcher.MatchSentenceTimestamp(block.OriginLanguageSentence, words, lastEndTime)+ if err != nil {+ log.GetLogger().Warn("Failed to match sentence timestamp",+ zap.String("sentence", block.OriginLanguageSentence),+ zap.Error(err))+ // Use fallback timing+ startTime = lastEndTime+ endTime = lastEndTime+ } else {+ // Ensure timestamps don't overlap with previous block+ if startTime < lastEndTime {+ startTime = lastEndTime+ }+ if endTime <= startTime {+ endTime = startTime + 1.0 // Minimum 1 second duration+ }+ }++ // Generate timestamp string+ updatedBlocks[i].Timestamp = util.ConvertTimes(float32(startTime+tsOffset), float32(endTime+tsOffset))+ if endTime-startTime < 5 {+ lastEndTime = endTime+ }++ log.GetLogger().Debug("Generated timestamp for sentence",+ zap.Int("index", block.Index),+ zap.String("sentence", block.OriginLanguageSentence),+ zap.Any("startTime", startTime),+ zap.Any("endTime", endTime))+ }++ return updatedBlocks, nil+}++// // AlphabeticLanguageMatcher handles timestamp matching for alphabetic languages (English, French, etc.)+// type AlphabeticLanguageMatcher struct {+// language types.StandardLanguageCode+// }++// func (alm *AlphabeticLanguageMatcher) GetLanguageType() types.StandardLanguageCode {+// return alm.language+// }++// func (alm *AlphabeticLanguageMatcher) MatchSentenceTimestamp(sentence string, words []types.Word, lastTs float64) (startTime, endTime float64, err error) {+// if sentence == "" || len(words) == 0 {+// return 0, 0, fmt.Errorf("empty sentence or words")+// }++// // Clean and split sentence into words+// sentenceWords := alm.extractWords(sentence)+// if len(sentenceWords) == 0 {+// return 0, 0, fmt.Errorf("no valid words found in sentence")+// }++// // Find matching words using fuzzy matching, prioritizing unused words+// matchedWords := alm.findMatchingWords(sentenceWords, words, lastTs)+// if len(matchedWords) == 0 {+// return 0, 0, fmt.Errorf("no matching words found")+// }++// // Find the maximum increasing subsequence for better accuracy+// beginWordIndex, endWordIndex := jumpFindMaxIncreasingSubArrayTs(matchedWords)+// if (endWordIndex - beginWordIndex) == 0 {+// log.GetLogger().Warn("getSentenceTimestamps no valid sentence",+// zap.String("sentence", sentence),+// zap.Int("beginWordIndex", beginWordIndex),+// zap.Int("endWordIndex", endWordIndex))+// return 0, 0, errors.New("getSentenceTimestamps no valid sentence")+// }++// // 找到最大连续子数组后,再去找整个句子开始和结束的时间戳+// beginWord := matchedWords[beginWordIndex]+// endWord := matchedWords[endWordIndex-1]+// if endWordIndex-beginWordIndex == len(matchedWords) {+// return beginWord.Start, endWord.End, nil+// }++// if beginWordIndex > 0 {+// for i, j := beginWordIndex-1, beginWord.Num-1; i >= 0 && j >= 0; {+// if words[j].Text == "" {+// j--+// continue+// }+// if strings.EqualFold(words[j].Text, matchedWords[i].Text) {+// beginWord = words[j]+// matchedWords[i] = beginWord+// } else {+// break+// }++// i--+// j--+// }+// }++// if endWordIndex < len(matchedWords) {+// for i, j := endWordIndex, endWord.Num+1; i < len(matchedWords) && j < len(words); {+// if words[j].Text == "" {+// j+++// continue+// }+// if strings.EqualFold(words[j].Text, matchedWords[i].Text) {+// endWord = words[j]+// matchedWords[i] = endWord+// } else {+// break+// }++// i+++// j+++// }+// }++// if beginWord.Num > matchedWords[0].Num && beginWord.Num-matchedWords[0].Num < 10 {+// beginWord = matchedWords[0]+// }++// if matchedWords[len(matchedWords)-1].Num > endWord.Num && matchedWords[len(matchedWords)-1].Num-endWord.Num < 10 {+// endWord = matchedWords[len(matchedWords)-1]+// }++// startTs := beginWord.Start+// endTs := endWord.End+// if startTs < lastTs {+// startTs = lastTs+// }+// if beginWord.Num != endWord.Num && endWord.End > lastTs {+// // Update lastTs for potential future use+// _ = endWord.End+// }++// return startTs, endTs, nil+// }++// func (alm *AlphabeticLanguageMatcher) extractWords(sentence string) []string {+// // Remove punctuation and split by whitespace+// cleaned := regexp.MustCompile(`[^\w\s']+`).ReplaceAllString(sentence, " ")+// words := strings.Fields(strings.ToLower(cleaned))++// var result []string+// for _, word := range words {+// if len(word) > 1 { // Filter out single characters+// result = append(result, word)+// }+// }+// return result+// }++// func (alm *AlphabeticLanguageMatcher) findMatchingWords(sentenceWords []string, words []types.Word, lastTs float64) []types.Word {+// newSentenceWords := make([]types.Word, 0)+// thisLastTs := lastTs+// sentenceWordIndex := 0+// wordNow := words[sentenceWordIndex]+// for _, sentenceWord := range sentenceWords {+// for sentenceWordIndex < len(words) {+// for sentenceWordIndex < len(words) && !strings.EqualFold(words[sentenceWordIndex].Text, sentenceWord) {+// sentenceWordIndex+++// }++// if sentenceWordIndex >= len(words) {+// break+// }++// wordNow = words[sentenceWordIndex]+// if wordNow.Start < thisLastTs {+// sentenceWordIndex+++// continue+// } else {+// break+// }+// }++// if sentenceWordIndex >= len(words) {+// newSentenceWords = append(newSentenceWords, types.Word{+// Text: sentenceWord,+// })+// sentenceWordIndex = 0+// continue+// }++// newSentenceWords = append(newSentenceWords, wordNow)+// sentenceWordIndex = 0+// }++// // for _, sentenceWord := range newSentenceWords {+// // // Find best matching word, prioritizing unused words+// // bestMatch, _, similarity := alm.findBestMatch(sentenceWord, words, lastTs)+// // if similarity > 0.6 { // Similarity threshold+// // matched = append(matched, bestMatch)+// // }+// // }++// return newSentenceWords+// }++// func (alm *AlphabeticLanguageMatcher) findBestMatch(target string, words []types.Word, lastTs float64) (types.Word, int, float64) {+// var bestWord types.Word+// var bestSimilarity float64+// bestIndex := -1++// // First pass: try to find unused words+// for i, word := range words {+// if word.End < lastTs {+// continue // Skip already used words+// }++// cleanWord := strings.ToLower(util.CleanPunctuation(word.Text))+// similarity := alm.calculateSimilarity(target, cleanWord)+// if similarity > bestSimilarity {+// bestSimilarity = similarity+// bestWord = word+// bestIndex = i+// }+// }++// // If no unused word found with good similarity, fallback to used words+// if bestSimilarity < 0.6 {+// for i, word := range words {+// cleanWord := strings.ToLower(util.CleanPunctuation(word.Text))+// similarity := alm.calculateSimilarity(target, cleanWord)+// if similarity > bestSimilarity {+// bestSimilarity = similarity+// bestWord = word+// bestIndex = i+// }+// }+// }++// return bestWord, bestIndex, bestSimilarity+// }++// // calculateSimilarity calculates the similarity between two strings+// // Currently unused but kept for potential future fuzzy matching features+// func (alm *AlphabeticLanguageMatcher) calculateSimilarity(s1, s2 string) float64 {+// if s1 == s2 {+// return 1.0+// }+// if strings.Contains(s2, s1) || strings.Contains(s1, s2) {+// return 0.8+// }+// // Simple edit distance based similarity+// maxLen := len(s1)+// if len(s2) > maxLen {+// maxLen = len(s2)+// }+// return 1.0 - float64(levenshteinDistance(s1, s2))/float64(maxLen)+// }++// BaseLanguageMatcher handles timestamp matching for Base+type BaseLanguageMatcher struct {+ language types.StandardLanguageCode+}++func (jlm *BaseLanguageMatcher) GetLanguageType() types.StandardLanguageCode {+ return jlm.language+}++func (jlm *BaseLanguageMatcher) MatchSentenceTimestamp(sentence string, words []types.Word, lastTs float64) (startTime, endTime float64, err error) {+ if sentence == "" || len(words) == 0 {+ return 0, 0, fmt.Errorf("empty sentence or words")+ }++ // 使用新的字符串匹配方法+ return jlm.matchSentenceByStringAlignment(sentence, words, lastTs)+}++// matchSentenceByStringAlignment 使用字符串对齐的方法匹配句子时间戳+func (jlm *BaseLanguageMatcher) matchSentenceByStringAlignment(sentence string, words []types.Word, lastTs float64) (startTime, endTime float64, err error) {+ // 步骤1: 合并 Whisper 的所有词文本+ whisperFullText := jlm.buildWhisperFullText(words)+ if whisperFullText == "" {+ return 0, 0, fmt.Errorf("no valid text from whisper words")+ }++ // 步骤2: 清理句子,移除多余的空格和标点符号+ cleanSentence := jlm.cleanBaseText(sentence)+ cleanWhisperText := jlm.cleanBaseText(whisperFullText)++ if cleanSentence == "" {+ return 0, 0, fmt.Errorf("empty sentence after cleaning")+ }++ // 步骤3: 在完整文本中查找句子的所有可能位置+ allMatches := jlm.findAllMatches(cleanSentence, cleanWhisperText)+ if len(allMatches) == 0 {+ // 如果直接匹配失败,尝试模糊匹配+ return jlm.fuzzyMatchSentence(cleanSentence, words, lastTs)+ }++ // 步骤4: 对每个可能的匹配位置,尝试计算时间戳+ var bestStartTime, bestEndTime float64+ var bestErr error+ found := false++ for _, startCharIndex := range allMatches {+ endCharIndex := startCharIndex + len([]rune(cleanSentence))++ startTime, endTime, err := jlm.calculateTimestampsByCharIndex(startCharIndex, endCharIndex, words, lastTs)+ if err == nil && startTime >= lastTs {+ // 找到一个有效的匹配,使用它+ bestStartTime = startTime+ bestEndTime = endTime+ found = true+ break+ } else if err == nil {+ // 记录这个匹配,但继续寻找更好的+ if !found || startTime > bestStartTime {+ bestStartTime = startTime+ bestEndTime = endTime+ bestErr = nil+ found = true+ }+ } else {+ // 记录错误,以防没有找到有效匹配+ if bestErr == nil {+ bestErr = err+ }+ }+ }++ if found {+ return bestStartTime, bestEndTime, bestErr+ }++ // 如果所有匹配都失败,回退到模糊匹配+ // return jlm.fuzzyMatchSentence(cleanSentence, words, lastTs)+ return 0, 0, fmt.Errorf("no valid timestamps found for sentence: %s, error: %v", cleanSentence, bestErr)+}++// buildWhisperFullText 合并所有 Whisper 词的文本+func (jlm *BaseLanguageMatcher) buildWhisperFullText(words []types.Word) string {+ var fullText strings.Builder+ for _, word := range words {+ if word.Text != "" {+ fullText.WriteString(word.Text)+ }+ }+ return fullText.String()+}++// cleanBaseText 清理文本,移除空格、标点符号和特殊符号+func (jlm *BaseLanguageMatcher) cleanBaseText(text string) string {+ var cleaned strings.Builder+ for _, r := range text {+ // 移除标点符号、空格和特殊符号,保留字母、数字和其他语言字符+ if !unicode.IsPunct(r) && !unicode.IsSpace(r) && !unicode.IsSymbol(r) {+ cleaned.WriteRune(r)+ }+ }+ return cleaned.String()+}++// findAllMatches 找到所有匹配位置+func (jlm *BaseLanguageMatcher) findAllMatches(sentence, fullText string) []int {+ var matches []int+ sentenceRunes := []rune(sentence)+ fullTextRunes := []rune(fullText)++ if len(sentenceRunes) == 0 || len(fullTextRunes) == 0 {+ return matches+ }++ for i := 0; i <= len(fullTextRunes)-len(sentenceRunes); i++ {+ if jlm.matchesAt(fullTextRunes, sentenceRunes, i) {+ matches = append(matches, i)+ }+ }++ return matches+}++// matchesAt 检查在指定位置是否匹配+func (jlm *BaseLanguageMatcher) matchesAt(fullText, sentence []rune, pos int) bool {+ if pos+len(sentence) > len(fullText) {+ return false+ }++ for i, r := range sentence {+ if fullText[pos+i] != r {+ return false+ }+ }+ return true+}++// calculateTimestampsByCharIndex 根据字符索引计算时间戳+func (jlm *BaseLanguageMatcher) calculateTimestampsByCharIndex(startCharIndex, endCharIndex int, words []types.Word, lastTs float64) (startTime, endTime float64, err error) {+ var resultStartTime, resultEndTime float64+ var startWordFound, endWordFound bool+ currentCharIndex := 0++ for _, word := range words {+ if word.Text == "" {+ continue+ }++ // 计算当前词的清理后文本长度+ cleanWordText := jlm.cleanBaseText(word.Text)+ wordCharLength := len([]rune(cleanWordText))++ wordStartIndex := currentCharIndex+ wordEndIndex := currentCharIndex + wordCharLength++ // 检查是否找到开始时间戳+ if !startWordFound && wordEndIndex > startCharIndex {+ if word.Start >= lastTs {+ resultStartTime = word.Start+ startWordFound = true+ }+ }++ // 检查是否找到结束时间戳+ if wordStartIndex < endCharIndex {+ if word.End >= lastTs {+ resultEndTime = word.End+ endWordFound = true+ }+ }++ // 如果已经超过了结束索引,停止搜索+ if wordStartIndex >= endCharIndex {+ break+ }++ currentCharIndex = wordEndIndex+ }++ if !startWordFound || !endWordFound || resultStartTime < lastTs || resultStartTime >= resultEndTime {+ return 0, 0, fmt.Errorf("could not find valid timestamps for character range [%d, %d)", startCharIndex, endCharIndex)+ }++ return resultStartTime, resultEndTime, nil+}++// fuzzyMatchSentence 当直接匹配失败时的模糊匹配方法+func (jlm *BaseLanguageMatcher) fuzzyMatchSentence(sentence string, words []types.Word, lastTs float64) (startTime, endTime float64, err error) {+ // 将句子拆分成字符,寻找包含这些字符的词+ sentenceRunes := []rune(sentence)+ var matchedWords []types.Word++ for _, word := range words {+ if word.Start < lastTs {+ continue+ }++ // 检查词是否包含句子中的字符+ wordRunes := []rune(jlm.cleanBaseText(word.Text))+ if jlm.containsBaseChars(wordRunes, sentenceRunes) {+ matchedWords = append(matchedWords, word)+ }+ }++ if len(matchedWords) == 0 {+ // 如果在 lastTs 之后没找到,从头开始搜索+ for _, word := range words {+ wordRunes := []rune(jlm.cleanBaseText(word.Text))+ if jlm.containsBaseChars(wordRunes, sentenceRunes) {+ matchedWords = append(matchedWords, word)+ }+ }+ }++ if len(matchedWords) == 0 {+ return 0, 0, fmt.Errorf("no matching words found for fuzzy matching")+ }++ // 使用第一个和最后一个匹配的词来确定时间范围+ resultStartTime := matchedWords[0].Start+ resultEndTime := matchedWords[len(matchedWords)-1].End++ // 应用 lastTs 约束+ if resultStartTime < lastTs {+ resultStartTime = lastTs+ }++ return resultStartTime, resultEndTime, nil+}++// containsBaseChars 检查词中是否包含句子中的字符+func (jlm *BaseLanguageMatcher) containsBaseChars(wordRunes, sentenceRunes []rune) bool {+ for _, sentenceChar := range sentenceRunes {+ found := false+ for _, wordChar := range wordRunes {+ if wordChar == sentenceChar {+ found = true+ break+ }+ }+ if found {+ return true+ }+ }+ return false+}diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.goindex bddfae68..2518bd63 100644--- a/internal/types/subtitle_task.go+++ b/internal/types/subtitle_task.go@@ -129,15 +129,16 @@ var SplitLongSentencePrompt = `请将以下原文和译文分割成多个部分4. 务必返回JSON格式,包含origin_part和translated_part数组,例如:{"align":[{"origin_part":"原文部分1","translated_part":"译文部分1"},{"origin_part":"原文部分2","translated_part":"译文部分2"}]}`-var SplitOriginLongSentencePrompt = `Please split the following text into multiple parts, ensuring each part does not exceed the character limit:+var SplitOriginLongSentencePrompt = `Please split the following text into multiple parts, ensuring it's divided into at most 3 short sentences, preferably 2:Original text: %sRequirements:1. The split sentences must exactly match the original text, absolutely no changes to the original text are allowed-2. Split based on sentence meaning, keeping each part under %d characters-3. Return in JSON format only, no other descriptions or explanations-4. Example format:+2. Split based on sentence meaning, dividing into at most 3 parts, preferably 2 parts+3. Try to make the split as balanced as possible while maintaining sentence integrity+4. Return in JSON format only, no other descriptions or explanations+5. Example format:{"short_sentences":[{"text": "split sentence 1"},{"text": "split sentence 2"}]}`@@ -177,18 +178,19 @@ Requirements:// Please provide only the translation result:`-var SplitTextWithContextPrompt = `You are a professional translation expert.+var SplitTextWithContextPrompt = `You are a professional subtitle translation expert.[STRICT TRANSLATION TASK]**Objective**:-Translate ONLY the "Target Sentence" below into %s.-Must reference but NEVER translate surrounding context.+Translate ONLY the "Target Sentence" below into %s.+Use "Previous Sentences" ONLY to understand the context of referents (e.g. pronouns or ellipses), not to infer meaning.**Critical Rules**:-1. OUTPUT MUST BE A SINGLE LINE containing ONLY the target sentence's translation-2. NEVER continue/complete the target sentence's thought-3. IGNORE all "Next Sentences" completely-4. Maintain contextual flow from "Previous Sentences" implicitly+1. OUTPUT MUST BE A SINGLE LINE: only the translation of the target sentence+2. Do NOT infer or explain the meaning of the target sentence. Do NOT add any logical connections or causal phrases+3. If the sentence is fragmentary or dependent (e.g. starts with "that"), KEEP IT THAT WAY in translation+4. Do NOT complete or rewrite the sentence for fluency+5. IGNORE the "Next Sentences" completely**Context**:[Previous Sentences]@@ -200,7 +202,7 @@ Must reference but NEVER translate surrounding context.[Next Sentences]%s-**Provide only the translation result:**`+**Your output must be literal, minimal, and on a single line. Provide only the translation result:**`type SmallAudio struct {AudioFile stringdiff --git a/pkg/util/subtitle.go b/pkg/util/subtitle.goindex b2e81314..b752de88 100644--- a/pkg/util/subtitle.go+++ b/pkg/util/subtitle.go@@ -29,20 +29,20 @@ func ProcessBlock(block []string, targetLanguageFile, targetLanguageTextFile, orif len(targetLines) == 2 && len(originLines) == 2 { // 刚写完编号和时间戳,到了上方的文字行if isTargetOnTop {targetLines = append(targetLines, line)- targetLanguageTextFile.WriteString(line) // 文稿文件+ targetLanguageTextFile.WriteString(line + " ") // 文稿文件} else {originLines = append(originLines, line)- originLanguageTextFile.WriteString(line)+ originLanguageTextFile.WriteString(line + " ")}continue}// 到了下方的文字行if isTargetOnTop {originLines = append(originLines, line)- originLanguageTextFile.WriteString(line)+ originLanguageTextFile.WriteString(line + " ")} else {targetLines = append(targetLines, line)- targetLanguageTextFile.WriteString(line)+ targetLanguageTextFile.WriteString(line + " ")}}@@ -355,48 +355,272 @@ func BeautifyAsianLanguageSentence(input string) string {}// SplitTextSentences 将文本按常见的半全角分隔符号切分成句子,会考虑一些特殊的不用切分的情况-func SplitTextSentences(text string) []string {- const (- dotPlaceholder = "\u0001"- commaPlaceholder = "\u0002"- timePlaceholder = "\u0003"- )-- // 时间- timeRegex := regexp.MustCompile(`\b\d{1,2}(?::|\.)\d{2}\s+[ap]\.m\.`)- text = timeRegex.ReplaceAllStringFunc(text, func(m string) string {- return strings.ReplaceAll(m, ".", timePlaceholder)- })+// maxChars: 最小字符数,完整句子小于此字符数时不切割,否则连逗号也要切割+// 使用示例:+//+// SplitTextSentences("你好,世界!", 5) // 返回: ["你好,世界!"] (不切割,因为总字符数<5)+// SplitTextSentences("这是一个很长的句子,包含很多内容。", 10) // 返回: ["这是一个很长的句子", "包含很多内容。"] (切割逗号)+func SplitTextSentences(text string, maxChars int) []string {+ if strings.TrimSpace(text) == "" {+ return []string{}+ }- // 千位分隔符- text = regexp.MustCompile(`\b\d{1,3}(?:,\d{3})+\b`).ReplaceAllStringFunc(text, func(m string) string {- return strings.ReplaceAll(m, ",", commaPlaceholder)- })+ // 第一步:保护特殊模式(数字、时间、缩写等)+ text = protectSpecialNumbers(text)- // 小数- text = regexp.MustCompile(`\b\d+\.\d+\b`).ReplaceAllStringFunc(text, func(m string) string {- return strings.ReplaceAll(m, ".", dotPlaceholder)- })+ // 第二步:智能切割 - 首先按完整句子分割+ completeSentences := splitByCompleteSentences(text)++ var result []string+ for _, sentence := range completeSentences {+ sentence = strings.TrimSpace(sentence)+ if sentence == "" {+ continue+ }++ // 统计有效字符数(排除标点和空格)+ effectiveChars := CountEffectiveChars(sentence)++ // 如果完整句子小于最小字符数,不切割+ if effectiveChars < maxChars {+ cleaned := restoreProtectedPatterns(sentence)+ result = append(result, strings.TrimSpace(cleaned))+ } else {+ // 完整句子过长,需要进一步按逗号等标点切割+ subSentences := splitByAllPunctuation(sentence)+ merged := mergeShortSentences(subSentences, 20, maxChars)++ for _, subSentence := range merged {+ cleaned := restoreProtectedPatterns(subSentence)+ cleaned = strings.TrimSpace(cleaned)+ if cleaned != "" {+ result = append(result, cleaned)+ }+ }+ }+ }- // 缩写词- text = regexp.MustCompile(`\b(?:[A-Za-z]\.){2,}[A-Za-z]?\b|\b[A-Z][a-z]*\.(?:[A-Z][a-z]*\.)+`).ReplaceAllStringFunc(text, func(m string) string {- return strings.ReplaceAll(m, ".", dotPlaceholder)+ return result+}++// protectedPatterns 存储被保护的模式+var protectedPatterns map[string]string++// protectSpecialNumbers 保护数字、时间、缩写等不被误切+func protectSpecialNumbers(text string) string {+ protectedPatterns = make(map[string]string)++ // 使用更直接的方法来保护列表编号模式+ // 先处理特定的模式,如 "1.value", "2.be", "3.give" 等+ listNumberPattern := regexp.MustCompile(`\b\d+\.[a-zA-Z]`)+ text = listNumberPattern.ReplaceAllStringFunc(text, func(match string) string {+ placeholder := fmt.Sprintf("\uE000%d\uE000", len(protectedPatterns))+ protectedPatterns[placeholder] = match+ return placeholder})- text = regexp.MustCompile(`([。.!!??;;,,\n]+)`).ReplaceAllString(text, "${1}\u0000")+ patterns := []struct {+ regex *regexp.Regexp+ name string+ }{+ // 保护域名和网址(如 .com, .org, .net 等)+ {regexp.MustCompile(`\b[a-zA-Z0-9-]+\.(?:com|org|net|edu|gov|mil|int|co|io|ai|me|tv|fm|am|pm|uk|cn|jp|de|fr|it|es|ru|in|au|ca|br|mx|ar|cl|pe|ve|ec|py|uy|bo|gf|sr|gy|fk|gs|sh|ac|ad|ae|af|ag|al|am|an|ao|aq|as|at|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|cc|cd|cf|cg|ch|ci|ck|cm|co|cr|cs|cu|cv|cx|cy|cz|dj|dk|dm|do|dz|eg|eh|er|et|eu|fi|fj|fk|fo|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|iq|ir|is|je|jm|jo|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|qa|re|ro|rs|rw|sa|sb|sc|sd|se|sg|si|sj|sk|sl|sm|sn|so|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tz|ua|ug|um|us|uy|uz|va|vc|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw)\b`), "domain"},+ // 保护 a.m., p.m., A.M., P.M. 这类缩写+ {regexp.MustCompile(`(?i)\b[ap]\.m\.`), "ampm"},+ // 时间格式+ {regexp.MustCompile(`\b\d{1,2}[:\.]\d{2}\s*(?:[ap]\.?m\.?|AM|PM)?\b`), "time"},+ // 小数(包括多位小数)+ {regexp.MustCompile(`\b\d+\.\d+\b`), "decimal"},+ // 千位分隔符+ {regexp.MustCompile(`\b\d{1,3}(?:,\d{3})+(?:\.\d+)?\b`), "thousands"},+ // 版本号(如 1.0, 2.5.1 等)+ {regexp.MustCompile(`\b\d+(?:\.\d+)+\b`), "version"},+ // 英文缩写+ {regexp.MustCompile(`\b(?:[A-Z][a-z]*\.){2,}|(?:[A-Z]\.){2,}[A-Z]?\b`), "abbrev"},+ // Mr., Mrs., Dr. 等称谓+ {regexp.MustCompile(`\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr)\.`), "title"},+ // 列表编号(如 1., 2., 3. 等)- 数字+点+空格+ {regexp.MustCompile(`\b\d+\.\s`), "list_number_with_space"},+ // 字母编号(如 a., b., c. 等)+ {regexp.MustCompile(`\b[a-zA-Z]\.\s`), "letter_number_with_space"},+ }++ for _, pattern := range patterns {+ text = pattern.regex.ReplaceAllStringFunc(text, func(match string) string {+ placeholder := fmt.Sprintf("\uE000%d\uE000", len(protectedPatterns))+ protectedPatterns[placeholder] = match+ return placeholder+ })+ }++ return text+}++// splitByCompleteSentences 按完整句子标点分割(句号、感叹号、问号等)+func splitByCompleteSentences(text string) []string {+ // 只按句末标点分割,不包含逗号+ completeSentenceMarkers := []string{+ ".", "!", "?", "。", "!", "?", ";", "\n", "\r\n",+ }++ // 创建正则表达式模式+ var patterns []string+ for _, marker := range completeSentenceMarkers {+ patterns = append(patterns, regexp.QuoteMeta(marker))+ }++ // 匹配连续的句末标点符号+ regexPattern := fmt.Sprintf(`([%s]+)`, strings.Join(patterns, ""))+ regex := regexp.MustCompile(regexPattern)- parts := strings.Split(text, "\u0000")+ // 在标点符号后添加分隔符+ text = regex.ReplaceAllString(text, "${1}\uE001")- var sentences []string+ // 按分隔符分割+ parts := strings.Split(text, "\uE001")++ var segments []stringfor _, part := range parts {- s := strings.TrimSpace(part)- s = strings.ReplaceAll(s, timePlaceholder, ".")- s = strings.ReplaceAll(s, dotPlaceholder, ".")- s = strings.ReplaceAll(s, commaPlaceholder, ",")- if s != "" {- sentences = append(sentences, s)+ trimmed := strings.TrimSpace(part)+ if trimmed != "" {+ segments = append(segments, trimmed)}}- return sentences+ return segments+}++// countEffectiveChars 统计有效字符数(排除标点和空格)+func CountEffectiveChars(text string) int {+ effectiveText := regexp.MustCompile(`[^\p{L}\p{N}]`).ReplaceAllString(text, "")+ return len([]rune(effectiveText))+}++// splitByAllPunctuation 按所有标点符号分割文本+func splitByAllPunctuation(text string) []string {+ // 注意:这里的text已经在SplitTextSentences中被保护过了,不需要再次保护++ // 定义分割标点符号(包括中英文标点)+ punctuationMarkers := []string{+ // 句末标点+ ".", "!", "?", ";", "。", "!", "?", ";",+ // 句内标点(也要分割)+ ",", ",", ";",+ // 换行符+ "\n", "\r\n",+ }++ // 创建正则表达式模式+ var patterns []string+ for _, marker := range punctuationMarkers {+ patterns = append(patterns, regexp.QuoteMeta(marker))+ }++ // 匹配连续的标点符号+ regexPattern := fmt.Sprintf(`([%s]+)`, strings.Join(patterns, ""))+ regex := regexp.MustCompile(regexPattern)++ // 在标点符号后添加分隔符+ text = regex.ReplaceAllString(text, "${1}\uE001")++ // 按分隔符分割+ parts := strings.Split(text, "\uE001")++ var segments []string+ for _, part := range parts {+ trimmed := strings.TrimSpace(part)+ if trimmed != "" {+ segments = append(segments, trimmed)+ }+ }++ return segments+}++// mergeShortSentences 合并过短的句子+// maxChars: 最小字符数,句子小于此值时考虑合并+// maxChars: 最大字符数,合并后的句子不能超过此值+func mergeShortSentences(segments []string, minChars, maxChars int) []string {+ if len(segments) == 0 {+ return segments+ }++ var result []string+ var current strings.Builder++ for i, segment := range segments {+ segment = strings.TrimSpace(segment)+ if segment == "" {+ continue+ }++ // 添加到当前句子+ if current.Len() > 0 {+ current.WriteString(" ")+ }+ current.WriteString(segment)++ currentText := current.String()+ currentEffectiveChars := CountEffectiveChars(currentText)++ // 检查是否应该合并下一个片段+ shouldMerge := false+ if i < len(segments)-1 { // 还有下一个片段+ nextSegment := strings.TrimSpace(segments[i+1])+ if nextSegment != "" {+ // 计算合并后的长度+ potentialMerged := currentText + " " + nextSegment+ mergedEffectiveChars := CountEffectiveChars(potentialMerged)++ // 只有当前句子小于minChars,并且合并后不超过maxChars才合并+ shouldMerge = currentEffectiveChars < minChars && mergedEffectiveChars <= maxChars+ }+ }++ if !shouldMerge {+ // 不合并,输出当前句子并重置+ result = append(result, strings.TrimSpace(currentText))+ current.Reset()+ }+ // 如果shouldMerge为true,继续循环到下一个片段进行合并+ }++ // 处理最后的片段+ if current.Len() > 0 {+ result = append(result, strings.TrimSpace(current.String()))+ }++ return result+}++// isTooShort 判断句子是否过短需要合并+func isTooShort(text string, maxChars int) bool {+ text = strings.TrimSpace(text)++ // 计算有效字符数(排除标点和空格)+ effectiveChars := CountEffectiveChars(text)++ // 如果有效字符少于最小字符数,认为过短+ if effectiveChars < maxChars {+ return true+ }++ // 如果只有一个单词,也认为过短(除非已经达到最小字符数)+ words := strings.Fields(text)+ return len(words) <= 1 && effectiveChars < maxChars+}++// restoreProtectedPatterns 恢复被保护的模式+func restoreProtectedPatterns(text string) string {+ for placeholder, original := range protectedPatterns {+ text = strings.ReplaceAll(text, placeholder, original)+ }+ return text+}++// 将start和end转换为指定格式+func ConvertTimes(start, end float32) string {+ startTime := FormatTime(start)+ endTime := FormatTime(end)+ return fmt.Sprintf("%s --> %s", startTime, endTime)}diff --git a/pkg/util/subtitle_test.go b/pkg/util/subtitle_test.gonew file mode 100644index 00000000..f34469ea--- /dev/null+++ b/pkg/util/subtitle_test.go@@ -0,0 +1,20 @@+package util++import (+ "fmt"+ "testing"+)++func TestSplitTextSentences(t *testing.T) {+ origin_sentence := "Now, I'm Ryan D'Aris, founder and CEO of flowstate.com"+ sentences := SplitTextSentences(origin_sentence, 55)+ fmt.Println("origin sentence:", origin_sentence)+ for i, sentence := range sentences {+ fmt.Printf("sentence %d, got '%s'\n", i, sentence)+ }++ // 期望的结果:应该保持为一个完整句子,因为有效字符数小于70+ if len(sentences) != 1 {+ t.Errorf("Expected 1 sentence, got %d sentences", len(sentences))+ }+}diff --git a/pkg/util/util_test.go b/pkg/util/util_test.godeleted file mode 100644index 49904036..00000000--- a/pkg/util/util_test.go+++ /dev/null@@ -1,16 +0,0 @@-package util--import (- "fmt"- "testing"-)--func TestSplitSentence(t *testing.T) {- testText := `这有一个14,000千克的集装箱。The U.S. sent to me. 现在是 11:23 p.m. 重复,11.23 p.m. 我身高1.8米。But it also makes communications with a submarine almost impossible. The high frequency signals that surface ships use for communications can transmit large amounts of data while being encrypted and resistant to jamming.`-- sentences := SplitTextSentences(testText)-- for i, s := range sentences {- fmt.Printf("句子 %d: %s\n", i+1, s)- }-}