Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

fix(krillinai): repair sentence-level VTT routing and artifact validation

wulien-center4d347c46576e
diff --git a/runtime/krillinai/VTT_SUBTITLES.md b/runtime/krillinai/VTT_SUBTITLES.md
new file mode 100644
index 00000000..c24b8976
--- /dev/null
+++ b/runtime/krillinai/VTT_SUBTITLES.md
@@ -0,0 +1,17 @@
+# YouTube 字幕格式兼容性
+
+HTTP Server 和 CLI 共用字幕处理入口。处理 VTT 前始终识别字幕格式:
+
+- 含行内单词时间戳的字幕沿用逐词处理。
+- 只有字幕块时间戳的整句字幕直接保留每个字幕块的完整正文和时间轴。
+- 格式检测失败时返回错误,不默认猜测为逐词格式。
+
+旧配置字段 enable_block_vtt_batch 保留读取兼容性,但不再决定是否检测字幕格式。关闭该开关也能正确处理整句字幕。
+
+仅原文模式不执行翻译。双语模式生成原文 SRT、译文 SRT、双语 SRT,以及 output/origin_language.txt 和 output/target_language.txt。空字幕、无效时间区间、缺少原文或译文、必需文本产物写入失败时,不报告处理完成。
+
+定向验证命令:
+
+ go test ./internal/service -run 'TestProcessYouTubeSubtitle|TestDetectVttFormat|TestValidateYouTubeSubtitleBlocks' -count=1
+
+设置 KRILLIN_VTT_TASKS_DIR 为保存线上失败字幕的 tasks 目录后,会额外使用三个原始失败输入运行回归测试;测试只读取原始文件,并在临时目录生成产物。
diff --git a/runtime/krillinai/internal/service/subtitle_service.go b/runtime/krillinai/internal/service/subtitle_service.go
index 08cef94f..6be39b26 100644
--- a/runtime/krillinai/internal/service/subtitle_service.go
+++ b/runtime/krillinai/internal/service/subtitle_service.go
@@ -179,29 +179,7 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.Star
}
req.VttFile = vttFile
- // 检测VTT格式类型
- hasWordTimestamps := true // 默认假设有单词级时间戳
- if config.Conf.App.EnableBlockVttBatch {
- // 只有启用了新功能才进行格式检测
- detected, detectErr := s.YouTubeSubtitleSrv.DetectVttFormat(vttFile)
- if detectErr != nil {
- log.GetLogger().Warn("VTT格式检测失败,使用默认处理方式",
- zap.String("taskId", taskId), zap.Error(detectErr))
- } else {
- hasWordTimestamps = detected
- }
- }
-
- var srtFile string
- if hasWordTimestamps {
- // 使用原有的word-level处理流程(完全不变)
- log.GetLogger().Info("使用word-level VTT处理流程", zap.String("taskId", taskId))
- srtFile, err = s.YouTubeSubtitleSrv.processYouTubeSubtitle(ctx, req)
- } else {
- // 使用新的block-level处理流程
- log.GetLogger().Info("使用block-level VTT处理流程", zap.String("taskId", taskId))
- srtFile, err = s.YouTubeSubtitleSrv.ProcessBlockLevelVtt(ctx, req)
- }
+ srtFile, err := s.YouTubeSubtitleSrv.processYouTubeSubtitle(ctx, req)
if err != nil {
// 处理字幕失败,回退到音频转录方式
diff --git a/runtime/krillinai/internal/service/youtube_subtitle.go b/runtime/krillinai/internal/service/youtube_subtitle.go
index 1a588b02..43ace96b 100644
--- a/runtime/krillinai/internal/service/youtube_subtitle.go
+++ b/runtime/krillinai/internal/service/youtube_subtitle.go
@@ -509,6 +509,14 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req
if req.VttFile == "" {
return "", fmt.Errorf("processYouTubeSubtitle: no original subtitle file found")
}
+ hasWordTimestamps, err := s.DetectVttFormat(req.VttFile)
+ if err != nil {
+ return "", fmt.Errorf("字幕格式检测失败: %w", err)
+ }
+ if !hasWordTimestamps {
+ return s.ProcessBlockLevelVtt(ctx, req)
+ }
+ log.GetLogger().Info("使用word-level VTT处理流程", zap.String("taskId", req.TaskId))
log.GetLogger().Info("processYouTubeSubtitle start",
zap.String("taskId", req.TaskId),
@@ -566,6 +574,9 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req
log.GetLogger().Info("生成原始语言SRT完成",
zap.String("文件", originSrtFile),
zap.Int("块数", len(srtBlocks)))
+ if err := s.validateYouTubeSubtitleBlocks(srtBlocks, false); err != nil {
+ return "", err
+ }
// 更新进度:原始SRT生成完成
if req.TaskPtr != nil {
@@ -584,6 +595,9 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req
return "", fmt.Errorf("failed to batch translate: %w", err)
}
log.GetLogger().Info("批量翻译完成", zap.Int("块数", len(srtBlocks)))
+ if err := s.validateYouTubeSubtitleBlocks(srtBlocks, true); err != nil {
+ return "", err
+ }
// 更新进度:翻译完成
if req.TaskPtr != nil {
@@ -632,20 +646,8 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req
}
// 8. 生成纯文本文件到output目录
- outputDir := filepath.Join(req.TaskBasePath, "output")
- err = os.MkdirAll(outputDir, 0755)
- if err != nil {
- log.GetLogger().Warn("创建output目录失败", zap.Error(err))
- }
- originTxtFile := filepath.Join(outputDir, "origin_language.txt")
- targetTxtFile := filepath.Join(outputDir, "target_language.txt")
- err = s.generateTextFiles(srtBlocks, originTxtFile, targetTxtFile, req.TargetLanguage)
- if err != nil {
- log.GetLogger().Warn("生成文本文件失败", zap.Error(err))
- } else {
- log.GetLogger().Info("生成文本文件完成",
- zap.String("原文", originTxtFile),
- zap.String("译文", targetTxtFile))
+ if err := s.writeYouTubeSubtitleTextFiles(srtBlocks, req); err != nil {
+ return "", err
}
// 更新进度:完成
@@ -3454,12 +3456,9 @@ func (s *YouTubeSubtitleService) DetectVttFormat(vttFile string) (bool, error) {
// 匹配单词级行内时间戳的正则表达式
wordTimeRegex := regexp.MustCompile(`<(\d{2}:\d{2}:\d{2}\.\d{3})>`)
- lineCount := 0
- maxLinesToCheck := 100 // 检查前100行即可判断格式
-
- for scanner.Scan() && lineCount < maxLinesToCheck {
+ scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+ for scanner.Scan() {
line := scanner.Text()
- lineCount++
// 如果发现行内时间戳,说明是word-level格式
if wordTimeRegex.MatchString(line) {
@@ -3481,6 +3480,7 @@ func (s *YouTubeSubtitleService) DetectVttFormat(vttFile string) (bool, error) {
// ProcessBlockLevelVtt 处理块级时间戳的VTT文件(无单词级时间戳)
// 流程: VTT → SRT → 批量翻译 → 双语字幕
func (s *YouTubeSubtitleService) ProcessBlockLevelVtt(ctx context.Context, req *YoutubeSubtitleReq) (string, error) {
+ log.GetLogger().Info("使用block-level VTT处理流程", zap.String("taskId", req.TaskId))
log.GetLogger().Info("开始处理block-level VTT字幕",
zap.String("taskId", req.TaskId),
zap.String("vttFile", req.VttFile))
@@ -3526,7 +3526,17 @@ func (s *YouTubeSubtitleService) ProcessBlockLevelVtt(ctx context.Context, req *
if err != nil {
return "", fmt.Errorf("解析SRT文件失败: %w", err)
}
+ if err := s.validateYouTubeSubtitleBlocks(srtBlocks, false); err != nil {
+ return "", err
+ }
log.GetLogger().Info("解析SRT完成", zap.Int("字幕块数", len(srtBlocks)))
+ if req.SourceOnly {
+ if req.TaskPtr != nil {
+ req.TaskPtr.SetProgress(100)
+ }
+ _ = os.Remove(tempSrtFile)
+ return originSrtFile, nil
+ }
// 4. 批量翻译(40%-90%的进度在BatchTranslateSrtBlocks内部更新)
err = s.translator.BatchTranslateSrtBlocks(srtBlocks, req.OriginLanguage, req.TargetLanguage, req.TaskPtr)
@@ -3534,6 +3544,9 @@ func (s *YouTubeSubtitleService) ProcessBlockLevelVtt(ctx context.Context, req *
return "", fmt.Errorf("批量翻译失败: %w", err)
}
log.GetLogger().Info("批量翻译完成", zap.Int("翻译块数", len(srtBlocks)))
+ if err := s.validateYouTubeSubtitleBlocks(srtBlocks, true); err != nil {
+ return "", err
+ }
// 更新进度:翻译完成
if req.TaskPtr != nil {
@@ -3555,11 +3568,14 @@ func (s *YouTubeSubtitleService) ProcessBlockLevelVtt(ctx context.Context, req *
// 6. 生成双语字幕文件(bilingual_srt.srt)
bilingualSrtFile := filepath.Join(req.TaskBasePath, types.SubtitleTaskBilingualSrtFileName)
- err = s.writeSrtBlocksToFile(srtBlocks, bilingualSrtFile)
+ err = s.writeBilingualSrtFile(srtBlocks, bilingualSrtFile, req.TargetLanguageFirst)
if err != nil {
return "", fmt.Errorf("写入双语字幕失败: %w", err)
}
log.GetLogger().Info("生成双语字幕完成", zap.String("bilingualSrtFile", bilingualSrtFile))
+ if err := s.writeYouTubeSubtitleTextFiles(srtBlocks, req); err != nil {
+ return "", err
+ }
// 更新进度:完成
if req.TaskPtr != nil {
@@ -3576,6 +3592,38 @@ func (s *YouTubeSubtitleService) ProcessBlockLevelVtt(ctx context.Context, req *
return bilingualSrtFile, nil
}
+func (s *YouTubeSubtitleService) validateYouTubeSubtitleBlocks(blocks []*util.SrtBlock, translated bool) error {
+ if len(blocks) == 0 {
+ return fmt.Errorf("字幕质量检查失败: 未生成有效字幕")
+ }
+ for index, block := range blocks {
+ if block == nil || strings.TrimSpace(block.OriginLanguageSentence) == "" {
+ return fmt.Errorf("字幕质量检查失败: 第%d条缺少原文", index+1)
+ }
+ start, end, err := s.parseSrtTimestamp(block.Timestamp)
+ if err != nil || start < 0 || end <= start {
+ return fmt.Errorf("字幕质量检查失败: 第%d条时间区间无效", index+1)
+ }
+ if translated && strings.TrimSpace(block.TargetLanguageSentence) == "" {
+ return fmt.Errorf("字幕质量检查失败: 第%d条缺少译文", index+1)
+ }
+ }
+ return nil
+}
+
+func (s *YouTubeSubtitleService) writeYouTubeSubtitleTextFiles(blocks []*util.SrtBlock, req *YoutubeSubtitleReq) error {
+ outputDir := filepath.Join(req.TaskBasePath, "output")
+ if err := os.MkdirAll(outputDir, 0755); err != nil {
+ return fmt.Errorf("创建字幕文本目录失败: %w", err)
+ }
+ originTxtFile := filepath.Join(outputDir, "origin_language.txt")
+ targetTxtFile := filepath.Join(outputDir, "target_language.txt")
+ if err := s.generateTextFiles(blocks, originTxtFile, targetTxtFile, req.TargetLanguage); err != nil {
+ return fmt.Errorf("生成字幕文本文件失败: %w", err)
+ }
+ return nil
+}
+
// groupWordsByCharLength 按字符长度将 VTT 单词分组
// maxChars: 每组最大字符数(包含单词间的空格)
func (s *YouTubeSubtitleService) groupWordsByCharLength(words []VttWord, maxChars int) [][]VttWord {
diff --git a/runtime/krillinai/internal/service/youtube_subtitle_routing_test.go b/runtime/krillinai/internal/service/youtube_subtitle_routing_test.go
new file mode 100644
index 00000000..f8bc5f62
--- /dev/null
+++ b/runtime/krillinai/internal/service/youtube_subtitle_routing_test.go
@@ -0,0 +1,235 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "testing"
+
+ "go.uber.org/zap"
+ "krillin-ai/config"
+ "krillin-ai/internal/types"
+ "krillin-ai/log"
+ "krillin-ai/pkg/util"
+)
+
+type subtitleRoutingCompleter struct{}
+
+func (subtitleRoutingCompleter) ChatCompletion(prompt string) (string, error) {
+ _, input, found := strings.Cut(prompt, "Input subtitles:\n")
+ if !found {
+ return "", nil
+ }
+ input, _, _ = strings.Cut(input, "\nRequired JSON format")
+ matches := regexp.MustCompile("(?m)^(\\d+)\\. (.+)$").FindAllStringSubmatch(input, -1)
+ translations := make([]map[string]any, 0, len(matches))
+ for _, match := range matches {
+ index, err := strconv.Atoi(match[1])
+ if err != nil {
+ return "", err
+ }
+ translations = append(translations, map[string]any{"index": index, "text": "测试译文:" + match[2]})
+ }
+ data, err := json.Marshal(map[string]any{"translations": translations})
+ return string(data), err
+}
+
+func subtitleRoutingTestService(test *testing.T) *YouTubeSubtitleService {
+ test.Helper()
+ previousLogger := log.Logger
+ log.Logger = zap.NewNop()
+ previousApp := config.Conf.App
+ config.Conf.App.EnableBlockVttBatch = false
+ config.Conf.App.VttBatchSize = 10
+ test.Cleanup(func() {
+ log.Logger = previousLogger
+ config.Conf.App = previousApp
+ })
+ return &YouTubeSubtitleService{translator: &Translator{chatCompleter: subtitleRoutingCompleter{}}}
+}
+
+func subtitleRoutingTestRequest(test *testing.T, content string) *YoutubeSubtitleReq {
+ test.Helper()
+ directory := test.TempDir()
+ input := filepath.Join(directory, "source.en.vtt")
+ if err := os.WriteFile(input, []byte(content), 0644); err != nil {
+ test.Fatal(err)
+ }
+ return &YoutubeSubtitleReq{
+ TaskBasePath: directory,
+ TaskId: "routing-test",
+ OriginLanguage: "en",
+ TargetLanguage: "zh_cn",
+ VttFile: input,
+ TaskPtr: &types.SubtitleTask{},
+ }
+}
+
+const blockSubtitleRoutingFixture = "WEBVTT\nKind: captions\nLanguage: en\n\n00:00:00.000 --> 00:00:03.000\nThere's a great expression that says, go together.\n\n00:00:10.000 --> 00:00:14.000\nPeople need leadership.\n\n00:00:21.000 --> 00:00:22.000\nNothing.\n"
+
+func TestProcessYouTubeSubtitleRoutesBlockVttWhenBatchDisabled(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ service.translator = nil
+ request := subtitleRoutingTestRequest(test, blockSubtitleRoutingFixture)
+ request.SourceOnly = true
+ output, err := service.processYouTubeSubtitle(context.Background(), request)
+ if err != nil {
+ test.Fatal(err)
+ }
+ blocks, err := util.ParseSrtFile(output)
+ if err != nil {
+ test.Fatal(err)
+ }
+ if len(blocks) != 3 || blocks[0].OriginLanguageSentence != "There's a great expression that says, go together." {
+ test.Fatalf("整句字幕正文丢失: %+v", blocks)
+ }
+ if request.TaskPtr.ProcessPct != 100 {
+ test.Fatalf("有效原文字幕未完成: %d", request.TaskPtr.ProcessPct)
+ }
+ if _, err := os.Stat(filepath.Join(request.TaskBasePath, types.SubtitleTaskTargetLanguageSrtFileName)); !os.IsNotExist(err) {
+ test.Fatalf("仅原文模式不应生成翻译字幕: %v", err)
+ }
+}
+
+func TestProcessYouTubeSubtitleBlockVttProducesRequiredOutputs(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ for _, targetFirst := range []bool{false, true} {
+ test.Run(strconv.FormatBool(targetFirst), func(test *testing.T) {
+ request := subtitleRoutingTestRequest(test, blockSubtitleRoutingFixture)
+ request.TargetLanguageFirst = targetFirst
+ output, err := service.processYouTubeSubtitle(context.Background(), request)
+ if err != nil {
+ test.Fatal(err)
+ }
+ for _, name := range []string{
+ types.SubtitleTaskOriginLanguageSrtFileName,
+ types.SubtitleTaskTargetLanguageSrtFileName,
+ types.SubtitleTaskBilingualSrtFileName,
+ filepath.Join("output", "origin_language.txt"),
+ filepath.Join("output", "target_language.txt"),
+ } {
+ data, err := os.ReadFile(filepath.Join(request.TaskBasePath, name))
+ if err != nil || len(data) == 0 {
+ test.Fatalf("必需产物缺失或为空 %s: %v", name, err)
+ }
+ }
+ blocks, err := util.ParseSrtFile(output)
+ if err != nil || len(blocks) != 3 {
+ test.Fatalf("双语字幕数量=%d: %v", len(blocks), err)
+ }
+ firstIsTarget := strings.HasPrefix(blocks[0].OriginLanguageSentence, "测试译文:")
+ if firstIsTarget != targetFirst {
+ test.Fatal("双语字幕顺序与请求不一致")
+ }
+ if request.TaskPtr.ProcessPct != 100 {
+ test.Fatalf("完成进度=%d", request.TaskPtr.ProcessPct)
+ }
+ })
+ }
+}
+
+func TestProcessYouTubeSubtitleRejectsInvalidBlockVtt(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ for _, content := range []string{
+ "WEBVTT\n",
+ "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\n\n",
+ "WEBVTT\n\n00:00:02.000 --> 00:00:01.000\nInvalid time interval.\n",
+ } {
+ request := subtitleRoutingTestRequest(test, content)
+ request.SourceOnly = true
+ if _, err := service.processYouTubeSubtitle(context.Background(), request); err == nil {
+ test.Fatal("空字幕或无效时间轴不应成功")
+ }
+ if request.TaskPtr.ProcessPct == 100 {
+ test.Fatal("不合格字幕不应报告完成")
+ }
+ }
+}
+
+func TestProcessYouTubeSubtitleRejectsMissingVtt(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ request := subtitleRoutingTestRequest(test, blockSubtitleRoutingFixture)
+ request.VttFile = filepath.Join(request.TaskBasePath, "missing.vtt")
+ if _, err := service.processYouTubeSubtitle(context.Background(), request); err == nil {
+ test.Fatal("格式检测失败不应默认按逐词字幕处理")
+ }
+}
+
+func TestDetectVttFormatScansBeyondFirstHundredLines(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ content := "WEBVTT\n" + strings.Repeat("\n", 110) + "00:00:00.000 --> 00:00:02.000\nHello<00:00:00.800><c> world.</c>\n"
+ request := subtitleRoutingTestRequest(test, content)
+ hasWordTimestamps, err := service.DetectVttFormat(request.VttFile)
+ if err != nil || !hasWordTimestamps {
+ test.Fatalf("后段逐词时间戳未识别: %v", err)
+ }
+}
+
+func TestProcessYouTubeSubtitleDoesNotCompleteWithoutTextOutputs(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ request := subtitleRoutingTestRequest(test, blockSubtitleRoutingFixture)
+ if err := os.WriteFile(filepath.Join(request.TaskBasePath, "output"), []byte("阻止创建文本目录"), 0644); err != nil {
+ test.Fatal(err)
+ }
+ if _, err := service.processYouTubeSubtitle(context.Background(), request); err == nil {
+ test.Fatal("必需文本产物无法生成时不应成功")
+ }
+ if request.TaskPtr.ProcessPct == 100 {
+ test.Fatal("必需产物缺失时不应报告完成")
+ }
+}
+
+func TestValidateYouTubeSubtitleBlocksRejectsMissingContent(test *testing.T) {
+ service := subtitleRoutingTestService(test)
+ for _, blocks := range [][]*util.SrtBlock{
+ nil,
+ {nil},
+ {{Timestamp: "00:00:00,000 --> 00:00:01,000", OriginLanguageSentence: " "}},
+ {{Timestamp: "00:00:00,000 --> 00:00:01,000", OriginLanguageSentence: "Hello.", TargetLanguageSentence: " "}},
+ } {
+ if err := service.validateYouTubeSubtitleBlocks(blocks, true); err == nil {
+ test.Fatal("字幕缺少原文或译文时不应成功")
+ }
+ }
+}
+
+func TestProcessYouTubeSubtitleProductionVttRegression(test *testing.T) {
+ root := os.Getenv("KRILLIN_VTT_TASKS_DIR")
+ if root == "" {
+ test.Skip("设置KRILLIN_VTT_TASKS_DIR后运行真实失败字幕回归")
+ }
+ service := subtitleRoutingTestService(test)
+ service.translator = nil
+ fixtures := []struct {
+ directory string
+ videoID string
+ wanted int
+ }{
+ {"watch_vwZYnJfL4_2fzY", "wZYnJfL4c2Y", 914},
+ {"watch_vGABhTBPy_oySd", "GABhTBPyg_E", 785},
+ {"watch_vGsP3OeE0_Jffq", "GsP3OeE0eMo", 726},
+ }
+ for _, fixture := range fixtures {
+ test.Run(fixture.videoID, func(test *testing.T) {
+ data, err := os.ReadFile(filepath.Join(root, fixture.directory, fixture.videoID+".en.vtt"))
+ if err != nil {
+ test.Fatal(err)
+ }
+ request := subtitleRoutingTestRequest(test, string(data))
+ request.SourceOnly = true
+ output, err := service.processYouTubeSubtitle(context.Background(), request)
+ if err != nil {
+ test.Fatal(err)
+ }
+ blocks, err := util.ParseSrtFile(output)
+ if err != nil || len(blocks) != fixture.wanted {
+ test.Fatalf("输出字幕=%d,预期=%d: %v", len(blocks), fixture.wanted, err)
+ }
+ test.Logf("修复后保留全部%d条字幕", len(blocks))
+ })
+ }
+}