Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
fix(creator): resolve YouTube origin captions
wulien-center88c4d7a8c9ec
diff --git a/runtime/krillinai/internal/service/youtube_subtitle.go b/runtime/krillinai/internal/service/youtube_subtitle.goindex e2a5e056..dfbfcc6f 100644--- a/runtime/krillinai/internal/service/youtube_subtitle.go+++ b/runtime/krillinai/internal/service/youtube_subtitle.go@@ -3,9 +3,14 @@ package serviceimport ("bufio""context"+ "encoding/json""fmt"+ "net/url""os"+ "os/exec""path/filepath"+ "regexp"+ "sort""strconv""strings""time"@@ -16,8 +21,6 @@ import ("krillin-ai/log""krillin-ai/pkg/util"- "regexp"-"go.uber.org/zap")@@ -47,6 +50,22 @@ type YouTubeSubtitleService struct {timestampGenerator *TimestampGenerator}+type youtubeSubtitleFormat struct {+ URL string `json:"url"`+ Name string `json:"name"`+}++type youtubeSubtitleMetadata struct {+ Language string `json:"language"`+ Subtitles map[string][]youtubeSubtitleFormat `json:"subtitles"`+ AutomaticCaptions map[string][]youtubeSubtitleFormat `json:"automatic_captions"`+}++type youtubeSubtitleTrack struct {+ Language string+ Automatic bool+}+// NewYouTubeSubtitleService creates a new YouTubeSubtitleService.func NewYouTubeSubtitleService() *YouTubeSubtitleService {return &YouTubeSubtitleService{@@ -129,32 +148,33 @@ func (s *YouTubeSubtitleService) downloadYouTubeSubtitle(ctx context.Context, re}// 确定要下载的字幕语言- subtitleLang := util.MapLanguageForYouTube(req.OriginLanguage)+ track, err := s.resolveYouTubeSubtitleTrack(ctx, req)+ if err != nil {+ return "", fmt.Errorf("downloadYouTubeSubtitle: resolve origin subtitle: %w", err)+ }+ if strings.EqualFold(strings.TrimSpace(req.OriginLanguage), "auto") {+ req.OriginLanguage = internalLanguageCodeFromYouTube(track.Language)+ }// 构造yt-dlp命令参数,使用视频ID作为文件名outputPattern := filepath.Join(req.TaskBasePath, videoID+".%(ext)s")- cmdArgs := []string{- "--write-auto-subs",- "--sub-langs", subtitleLang,- "--skip-download",- "-o", outputPattern,- req.URL,- }+ cmdArgs := youtubeSubtitleCommandArgs(track, outputPattern)// 添加代理设置- if config.Conf.App.Proxy != "" {- cmdArgs = append(cmdArgs, "--proxy", config.Conf.App.Proxy)- }+ cmdArgs = appendYouTubeAccessArgs(cmdArgs)// 添加cookies(如果存在且格式有效)- cmdArgs = appendCookiesArgs(cmdArgs, youtubeCookiesPath)// 添加ffmpeg路径if storage.FfmpegPath != "ffmpeg" {cmdArgs = append(cmdArgs, "--ffmpeg-location", storage.FfmpegPath)}+ cmdArgs = append(cmdArgs, req.URL)- log.GetLogger().Info("downloadYouTubeSubtitle starting", zap.Any("taskId", req.TaskId), zap.Any("cmdArgs", cmdArgs))+ log.GetLogger().Info("downloadYouTubeSubtitle starting",+ zap.String("taskId", req.TaskId),+ zap.String("subtitleLanguage", track.Language),+ zap.Bool("automatic", track.Automatic))// 添加重试机制maxAttempts := 3@@ -174,7 +194,7 @@ func (s *YouTubeSubtitleService) downloadYouTubeSubtitle(ctx context.Context, relog.GetLogger().Info("downloadYouTubeSubtitle completed", zap.Any("taskId", req.TaskId), zap.String("output", string(output)))// 查找下载的字幕文件- subtitleFile, err := s.findDownloadedSubtitleFile(req.TaskBasePath, subtitleLang, videoID)+ subtitleFile, err := s.findDownloadedSubtitleFile(req.TaskBasePath, track.Language, videoID)if err != nil {log.GetLogger().Error("downloadYouTubeSubtitle findDownloadedSubtitleFile error", zap.Any("stepParam", req), zap.Error(err))return "", fmt.Errorf("downloadYouTubeSubtitle findDownloadedSubtitleFile error: %w", err)@@ -208,6 +228,237 @@ func (s *YouTubeSubtitleService) downloadYouTubeSubtitle(ctx context.Context, re}// 查找下载的字幕文件+func (s *YouTubeSubtitleService) resolveYouTubeSubtitleTrack(ctx context.Context, req *YoutubeSubtitleReq) (youtubeSubtitleTrack, error) {+ if err := ctx.Err(); err != nil {+ return youtubeSubtitleTrack{}, err+ }++ cmdArgs := []string{"--dump-single-json", "--skip-download", "--no-playlist"}+ cmdArgs = appendYouTubeAccessArgs(cmdArgs)+ cmdArgs = append(cmdArgs, req.URL)++ output, err := storage.YtdlpCommand(cmdArgs...).Output()+ if err != nil {+ details := ""+ if exitErr, ok := err.(*exec.ExitError); ok {+ details = compactCommandOutput(exitErr.Stderr)+ }+ if details != "" {+ return youtubeSubtitleTrack{}, fmt.Errorf("yt-dlp subtitle metadata error: %w: %s", err, details)+ }+ return youtubeSubtitleTrack{}, fmt.Errorf("yt-dlp subtitle metadata error: %w", err)+ }++ var metadata youtubeSubtitleMetadata+ if err := json.Unmarshal(output, &metadata); err != nil {+ return youtubeSubtitleTrack{}, fmt.Errorf("parse yt-dlp subtitle metadata: %w", err)+ }++ manualLanguages := availableLanguageKeys(metadata.Subtitles)+ automaticLanguages := automaticOriginLanguageKeys(metadata.AutomaticCaptions)+ markedOriginLanguages := markedOriginalLanguageKeys(metadata.AutomaticCaptions)+ track, err := selectYouTubeSubtitleTrack(metadata, req.OriginLanguage)+ if err != nil {+ log.GetLogger().Warn("No matching YouTube origin subtitle track",+ zap.String("taskId", req.TaskId),+ zap.String("metadataLanguage", metadata.Language),+ zap.Strings("manualSubtitleLanguages", manualLanguages),+ zap.Strings("automaticSourceLanguages", automaticLanguages),+ zap.Strings("markedOriginLanguages", markedOriginLanguages),+ zap.Error(err))+ return youtubeSubtitleTrack{}, err+ }++ log.GetLogger().Info("Resolved YouTube origin subtitle track",+ zap.String("taskId", req.TaskId),+ zap.String("metadataLanguage", metadata.Language),+ zap.Strings("manualSubtitleLanguages", manualLanguages),+ zap.Strings("automaticSourceLanguages", automaticLanguages),+ zap.Strings("markedOriginLanguages", markedOriginLanguages),+ zap.String("selectedLanguage", track.Language),+ zap.Bool("automatic", track.Automatic))+ return track, nil+}++func selectYouTubeSubtitleTrack(metadata youtubeSubtitleMetadata, requestedLanguage string) (youtubeSubtitleTrack, error) {+ manualLanguages := availableLanguageKeys(metadata.Subtitles)+ automaticLanguages := automaticOriginLanguageKeys(metadata.AutomaticCaptions)+ markedOriginLanguages := markedOriginalLanguageKeys(metadata.AutomaticCaptions)+ requestedLanguage = strings.TrimSpace(requestedLanguage)++ if requestedLanguage != "" && !strings.EqualFold(requestedLanguage, "auto") {+ youtubeLanguage := util.MapLanguageForYouTube(requestedLanguage)+ if language, ok := matchAvailableLanguage(youtubeLanguage, manualLanguages); ok {+ return youtubeSubtitleTrack{Language: language}, nil+ }+ if language, ok := matchAvailableLanguage(youtubeLanguage, automaticLanguages); ok {+ return youtubeSubtitleTrack{Language: language, Automatic: true}, nil+ }+ return youtubeSubtitleTrack{}, fmt.Errorf("origin subtitle language %q is unavailable", youtubeLanguage)+ }++ if metadata.Language != "" {+ if language, ok := matchMarkedOriginLanguage(metadata.Language, markedOriginLanguages); ok {+ return youtubeSubtitleTrack{Language: language, Automatic: true}, nil+ }+ }+ if len(markedOriginLanguages) == 1 {+ return youtubeSubtitleTrack{Language: markedOriginLanguages[0], Automatic: true}, nil+ }++ if metadata.Language != "" {+ if language, ok := matchAvailableLanguage(metadata.Language, manualLanguages); ok {+ return youtubeSubtitleTrack{Language: language}, nil+ }+ if language, ok := matchAvailableLanguage(metadata.Language, automaticLanguages); ok {+ return youtubeSubtitleTrack{Language: language, Automatic: true}, nil+ }+ }++ if len(manualLanguages) == 1 {+ return youtubeSubtitleTrack{Language: manualLanguages[0]}, nil+ }+ if len(automaticLanguages) == 1 {+ return youtubeSubtitleTrack{Language: automaticLanguages[0], Automatic: true}, nil+ }++ return youtubeSubtitleTrack{}, fmt.Errorf("cannot determine the origin subtitle from available tracks")+}++func youtubeSubtitleCommandArgs(track youtubeSubtitleTrack, outputPattern string) []string {+ writeFlag := "--write-subs"+ if track.Automatic {+ writeFlag = "--write-auto-subs"+ }+ return []string{+ writeFlag,+ "--sub-langs", track.Language,+ "--sub-format", "vtt",+ "--skip-download",+ "--no-playlist",+ "-o", outputPattern,+ }+}++func appendYouTubeAccessArgs(cmdArgs []string) []string {+ if config.Conf.App.Proxy != "" {+ cmdArgs = append(cmdArgs, "--proxy", config.Conf.App.Proxy)+ }+ return appendCookiesArgs(cmdArgs, youtubeCookiesPath)+}++func availableLanguageKeys(tracks map[string][]youtubeSubtitleFormat) []string {+ languages := make([]string, 0, len(tracks))+ for language, formats := range tracks {+ if strings.TrimSpace(language) != "" && len(formats) > 0 {+ languages = append(languages, language)+ }+ }+ sort.Strings(languages)+ return languages+}++func automaticOriginLanguageKeys(tracks map[string][]youtubeSubtitleFormat) []string {+ languages := make([]string, 0, len(tracks))+ for language, formats := range tracks {+ if strings.TrimSpace(language) != "" && hasUntranslatedCaptionFormat(formats) {+ languages = append(languages, language)+ }+ }+ sort.Strings(languages)+ return languages+}++func markedOriginalLanguageKeys(tracks map[string][]youtubeSubtitleFormat) []string {+ languages := make([]string, 0, len(tracks))+ for language, formats := range tracks {+ if strings.HasSuffix(canonicalYouTubeLanguage(language), "-orig") || hasOriginalCaptionName(formats) {+ languages = append(languages, language)+ }+ }+ sort.Strings(languages)+ return languages+}++func hasOriginalCaptionName(formats []youtubeSubtitleFormat) bool {+ for _, format := range formats {+ if strings.Contains(strings.ToLower(format.Name), "original") {+ return true+ }+ }+ return false+}++func hasUntranslatedCaptionFormat(formats []youtubeSubtitleFormat) bool {+ for _, format := range formats {+ captionURL, err := url.Parse(format.URL)+ if err == nil && format.URL != "" && !captionURL.Query().Has("tlang") {+ return true+ }+ }+ return false+}++func matchAvailableLanguage(requestedLanguage string, availableLanguages []string) (string, bool) {+ requested := canonicalYouTubeLanguage(requestedLanguage)+ if requested == "" {+ return "", false+ }+ for _, language := range availableLanguages {+ if canonicalYouTubeLanguage(language) == requested {+ return language, true+ }+ }++ requestedBase := strings.Split(requested, "-")[0]+ var baseMatches []string+ for _, language := range availableLanguages {+ if strings.Split(canonicalYouTubeLanguage(language), "-")[0] == requestedBase {+ baseMatches = append(baseMatches, language)+ }+ }+ if len(baseMatches) == 1 {+ return baseMatches[0], true+ }+ return "", false+}++func matchMarkedOriginLanguage(requestedLanguage string, availableLanguages []string) (string, bool) {+ normalized := make([]string, 0, len(availableLanguages))+ for _, language := range availableLanguages {+ normalized = append(normalized, strings.TrimSuffix(canonicalYouTubeLanguage(language), "-orig"))+ }+ matched, ok := matchAvailableLanguage(requestedLanguage, normalized)+ if !ok {+ return "", false+ }+ for index, language := range normalized {+ if language == matched {+ return availableLanguages[index], true+ }+ }+ return "", false+}++func canonicalYouTubeLanguage(language string) string {+ language = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(language), "_", "-"))+ if language == "iw" {+ return "he"+ }+ return language+}++func internalLanguageCodeFromYouTube(language string) string {+ canonical := canonicalYouTubeLanguage(language)+ switch canonical {+ case "zh-hans", "zh-cn", "zh-sg", "zh":+ return "zh_cn"+ case "zh-hant", "zh-tw", "zh-hk", "zh-mo":+ return "zh_tw"+ }+ return strings.Split(canonical, "-")[0]+}+func (s *YouTubeSubtitleService) findDownloadedSubtitleFile(taskBasePath, language, videoID string) (string, error) {// 支持的字幕文件扩展名extensions := []string{".vtt", ".srt"}diff --git a/runtime/krillinai/internal/service/youtube_subtitle_test.go b/runtime/krillinai/internal/service/youtube_subtitle_test.goindex 98892bd1..eca1d109 100644--- a/runtime/krillinai/internal/service/youtube_subtitle_test.go+++ b/runtime/krillinai/internal/service/youtube_subtitle_test.go@@ -4,13 +4,168 @@ import ("context""krillin-ai/config""krillin-ai/internal/deps"+ "krillin-ai/internal/storage""krillin-ai/internal/types""krillin-ai/log""os"+ "os/exec""path/filepath"+ "reflect"+ "strings""testing")+func TestSelectYouTubeSubtitleTrackPrefersManualOriginLanguage(t *testing.T) {+ metadata := youtubeSubtitleMetadata{+ Language: "en",+ Subtitles: map[string][]youtubeSubtitleFormat{+ "en": {{URL: "https://example.test/caption?lang=en"}},+ "es": {{URL: "https://example.test/caption?lang=es"}},+ },+ AutomaticCaptions: map[string][]youtubeSubtitleFormat{+ "en": {{URL: "https://example.test/auto?lang=en"}},+ },+ }++ track, err := selectYouTubeSubtitleTrack(metadata, "auto")+ if err != nil {+ t.Fatal(err)+ }+ if track.Language != "en" || track.Automatic {+ t.Fatalf("track = %+v, want manual en", track)+ }+}++func TestSelectYouTubeSubtitleTrackUsesOriginalAutomaticCaption(t *testing.T) {+ metadata := youtubeSubtitleMetadata{+ Language: "en",+ AutomaticCaptions: map[string][]youtubeSubtitleFormat{+ "en": {{URL: "https://example.test/auto?lang=en"}},+ "es": {{URL: "https://example.test/auto?lang=en&tlang=es"}},+ },+ }++ track, err := selectYouTubeSubtitleTrack(metadata, "auto")+ if err != nil {+ t.Fatal(err)+ }+ if track.Language != "en" || !track.Automatic {+ t.Fatalf("track = %+v, want automatic en", track)+ }+ if got := automaticOriginLanguageKeys(metadata.AutomaticCaptions); !reflect.DeepEqual(got, []string{"en"}) {+ t.Fatalf("automatic origin languages = %v, want [en]", got)+ }+}++func TestSelectYouTubeSubtitleTrackPrefersMarkedOriginalCaptionForAuto(t *testing.T) {+ metadata := youtubeSubtitleMetadata{+ Language: "en",+ Subtitles: map[string][]youtubeSubtitleFormat{+ "en": {{URL: "https://example.test/caption?lang=en"}},+ },+ AutomaticCaptions: map[string][]youtubeSubtitleFormat{+ "en-orig": {{URL: "https://example.test/auto?lang=en", Name: "English (Original)"}},+ },+ }++ track, err := selectYouTubeSubtitleTrack(metadata, "auto")+ if err != nil {+ t.Fatal(err)+ }+ if track.Language != "en-orig" || !track.Automatic {+ t.Fatalf("track = %+v, want automatic en-orig", track)+ }+}++func TestSelectYouTubeSubtitleTrackFallsBackOnlyForUniqueTrack(t *testing.T) {+ metadata := youtubeSubtitleMetadata{+ Subtitles: map[string][]youtubeSubtitleFormat{+ "ja": {{URL: "https://example.test/caption?lang=ja"}},+ },+ }+ track, err := selectYouTubeSubtitleTrack(metadata, "auto")+ if err != nil {+ t.Fatal(err)+ }+ if track.Language != "ja" || track.Automatic {+ t.Fatalf("track = %+v, want manual ja", track)+ }++ metadata.Subtitles["en"] = []youtubeSubtitleFormat{{URL: "https://example.test/caption?lang=en"}}+ if _, err := selectYouTubeSubtitleTrack(metadata, "auto"); err == nil {+ t.Fatal("selectYouTubeSubtitleTrack() error = nil, want ambiguous origin error")+ }+}++func TestSelectYouTubeSubtitleTrackMatchesExplicitLanguage(t *testing.T) {+ metadata := youtubeSubtitleMetadata{+ Subtitles: map[string][]youtubeSubtitleFormat{+ "zh-Hans": {{URL: "https://example.test/caption?lang=zh-Hans"}},+ },+ }+ track, err := selectYouTubeSubtitleTrack(metadata, "zh_cn")+ if err != nil {+ t.Fatal(err)+ }+ if track.Language != "zh-Hans" || track.Automatic {+ t.Fatalf("track = %+v, want manual zh-Hans", track)+ }+}++func TestYouTubeSubtitleCommandArgsUsesResolvedTrack(t *testing.T) {+ manualArgs := youtubeSubtitleCommandArgs(youtubeSubtitleTrack{Language: "en"}, "video.%(ext)s")+ joinedManual := strings.Join(manualArgs, " ")+ if strings.Contains(joinedManual, "auto") || !strings.Contains(joinedManual, "--write-subs --sub-langs en --sub-format vtt") {+ t.Fatalf("manual args = %v", manualArgs)+ }++ automaticArgs := youtubeSubtitleCommandArgs(youtubeSubtitleTrack{Language: "en", Automatic: true}, "video.%(ext)s")+ joinedAutomatic := strings.Join(automaticArgs, " ")+ if strings.Contains(joinedAutomatic, "--sub-langs auto") || !strings.Contains(joinedAutomatic, "--write-auto-subs --sub-langs en --sub-format vtt") {+ t.Fatalf("automatic args = %v", automaticArgs)+ }+}++func TestDownloadYouTubeSubtitleAutoIntegration(t *testing.T) {+ if os.Getenv("KRILLIN_RUN_YOUTUBE_SUBTITLE_AUTO_INTEGRATION") != "1" {+ t.Skip("set KRILLIN_RUN_YOUTUBE_SUBTITLE_AUTO_INTEGRATION=1 to run the YouTube auto-language integration test")+ }+ log.InitLogger()+ ytdlpPath, err := exec.LookPath("yt-dlp")+ if err != nil {+ t.Fatalf("find yt-dlp: %v", err)+ }+ previousPath, previousPrefix := storage.YtdlpPath, storage.YtdlpPrefixArgs+ storage.YtdlpPath, storage.YtdlpPrefixArgs = ytdlpPath, nil+ t.Cleanup(func() {+ storage.YtdlpPath, storage.YtdlpPrefixArgs = previousPath, previousPrefix+ })++ req := &YoutubeSubtitleReq{+ TaskBasePath: t.TempDir(),+ TaskId: "youtube-auto-origin",+ OriginLanguage: "auto",+ URL: "https://www.youtube.com/watch?v=paF--WGA8dU",+ }+ path, err := NewYouTubeSubtitleService().downloadYouTubeSubtitle(context.Background(), req)+ if err != nil {+ t.Fatal(err)+ }+ if req.OriginLanguage != "en" {+ t.Fatalf("origin language = %q, want en", req.OriginLanguage)+ }+ info, err := os.Stat(path)+ if err != nil {+ t.Fatalf("stat downloaded subtitle: %v", err)+ }+ if info.Size() == 0 {+ t.Fatal("downloaded subtitle is empty")+ }+ if !strings.Contains(filepath.Base(path), ".en-orig.") {+ t.Fatalf("downloaded subtitle = %q, want the en-orig track", path)+ }+}+func Test_YoutubeSubtitle(t *testing.T) {skipYouTubeSubtitleIntegrationTest(t)