Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
fix: stabilize stickman source transcription
wulien-center6ae78372f288
diff --git a/apps/daemon/src/creator/krillin/cli-runner.ts b/apps/daemon/src/creator/krillin/cli-runner.tsindex 6bc171a3..8831bda5 100644--- a/apps/daemon/src/creator/krillin/cli-runner.ts+++ b/apps/daemon/src/creator/krillin/cli-runner.ts@@ -218,14 +218,15 @@ export function buildKrillinCliCommandArguments(if (command === 'subtitle') {const source = resolveKrillinCliSource(artifacts, options);if (!source) throw new CreatorExecutorError('creator_stage_input_missing', 'Subtitle input video or URL is required');+ const sourceOnly = booleanOption(options, 'sourceOnly', false);return ['subtitle',source,- '--prepare-video',+ ...(sourceOnly ? [] : ['--prepare-video']),'--origin-lang', requiredOption(options, 'originLanguage'),'--target-lang', requiredOption(options, 'targetLanguage'),'--caption-source', stringOption(options, 'captionSource') ?? 'any',- ...(booleanOption(options, 'sourceOnly', false) ? ['--source-only'] : []),+ ...(sourceOnly ? ['--source-only'] : []),`--bilingual-top=${booleanOption(options, 'bilingualTop', true)}`,...common,...styleArgument(stylePath)diff --git a/apps/daemon/test/unit/creator-stickman-krillin.test.ts b/apps/daemon/test/unit/creator-stickman-krillin.test.tsindex 70497b57..5bd2231b 100644--- a/apps/daemon/test/unit/creator-stickman-krillin.test.ts+++ b/apps/daemon/test/unit/creator-stickman-krillin.test.ts@@ -68,12 +68,14 @@ describe('stickman KrillinAI mapping', () => {};const stickmanOptions = buildKrillinStageOptions(sourceTranscript as never);expect(stickmanOptions.sourceOnly).toBe(true);- expect(buildKrillinCliCommandArguments(+ const stickmanArgs = buildKrillinCliCommandArguments(sourceTranscript as never,[],stickmanOptions,undefined- )).toContain('--source-only');+ );+ expect(stickmanArgs).toContain('--source-only');+ expect(stickmanArgs).not.toContain('--prepare-video');const translationStage = {...sourceTranscript,@@ -82,12 +84,14 @@ describe('stickman KrillinAI mapping', () => {};const translationOptions = buildKrillinStageOptions(translationStage as never);expect(translationOptions.sourceOnly).toBe(false);- expect(buildKrillinCliCommandArguments(+ const translationArgs = buildKrillinCliCommandArguments(translationStage as never,[],translationOptions,undefined- )).not.toContain('--source-only');+ );+ expect(translationArgs).not.toContain('--source-only');+ expect(translationArgs).toContain('--prepare-video');});it('rejects undeclared or missing stickman outputs and aliases valid subtitles', async () => {diff --git a/runtime/krillinai/internal/cli/commands.go b/runtime/krillinai/internal/cli/commands.goindex 38436c7f..70a7d0a1 100644--- a/runtime/krillinai/internal/cli/commands.go+++ b/runtime/krillinai/internal/cli/commands.go@@ -121,7 +121,8 @@ Flags:--workdir <dir> Task working directory--task-id <id> Optional task id--caption-source <source> any, platform, manual, auto, or whisper- --prepare-video download original video for later rendering+ --prepare-video download original video for later rendering+ --source-only generate source-language subtitles without translation--bilingual-top Put target subtitle on top (default true)--max-word-one-line <n> Max words per subtitle line--subtitle-style-file <file> JSON subtitle style override file@@ -489,6 +490,7 @@ func parseSubtitle(name string, args []string) (Command, error) {taskID := fs.String("task-id", "", "task id")captionSource := fs.String("caption-source", string(pipeline.CaptionSourceAny), "caption source")prepareVideo := fs.Bool("prepare-video", false, "prepare original video for later rendering")+ sourceOnly := fs.Bool("source-only", false, "generate source-language subtitles without translation")bilingualTop := fs.Bool("bilingual-top", true, "put target subtitle on top")maxWordOneLine := fs.Int("max-word-one-line", 0, "max words per line")subtitleStyleFile := fs.String("subtitle-style-file", "", "subtitle style JSON file")@@ -521,6 +523,7 @@ func parseSubtitle(name string, args []string) (Command, error) {UserLang: *userLang,CaptionSource: pipeline.CaptionSource(*captionSource),PrepareVideo: *prepareVideo,+ SourceOnly: *sourceOnly,BilingualTop: *bilingualTop,MaxWordOneLine: *maxWordOneLine,},diff --git a/runtime/krillinai/internal/cli/commands_test.go b/runtime/krillinai/internal/cli/commands_test.goindex 8648ef0a..1a929f7b 100644--- a/runtime/krillinai/internal/cli/commands_test.go+++ b/runtime/krillinai/internal/cli/commands_test.go@@ -53,6 +53,23 @@ func TestParseSubtitlePrepareVideo(t *testing.T) {}}+func TestParseSubtitleSourceOnly(t *testing.T) {+ cmd, err := Parse([]string{+ "subtitle",+ "https://www.youtube.com/watch?v=abc",+ "--source-only",+ })+ if err != nil {+ t.Fatal(err)+ }+ if !cmd.Subtitle.SourceOnly {+ t.Fatal("SourceOnly = false, want true")+ }+ if !strings.Contains(Help(Command{Name: "subtitle"}), "--source-only") {+ t.Fatal("subtitle help does not advertise --source-only")+ }+}+func TestParseSubtitleCommandCanPutTargetLanguageOnBottom(t *testing.T) {cmd, err := Parse([]string{"subtitle",diff --git a/runtime/krillinai/internal/deps/checker.go b/runtime/krillinai/internal/deps/checker.goindex af85d4dd..32feec67 100644--- a/runtime/krillinai/internal/deps/checker.go+++ b/runtime/krillinai/internal/deps/checker.go@@ -157,11 +157,11 @@ func configurePackagedTranscriptionDependency() error {return err}case "whispercpp":- name := "whispercpp"+ name := "whisper-cli"if runtime.GOOS == "windows" {name += ".exe"}- if storage.WhispercppPath, err = resourcepath.RequireFile("bin", name); err != nil {+ if storage.WhispercppPath, err = resourcepath.RequireFile("bin", "whispercpp", name); err != nil {return err}if _, err = resourcepath.RequireFile("models", "whispercpp", fmt.Sprintf("ggml-%s.bin", config.Conf.Transcribe.Whispercpp.Model)); err != nil {diff --git a/runtime/krillinai/internal/deps/checker_test.go b/runtime/krillinai/internal/deps/checker_test.goindex dd8b8c83..94a6520b 100644--- a/runtime/krillinai/internal/deps/checker_test.go+++ b/runtime/krillinai/internal/deps/checker_test.go@@ -129,6 +129,42 @@ func TestConfigurePackagedTranscriptionDependencyUsesWhisperKitArchiveDirectory(}}+func TestConfigurePackagedTranscriptionDependencyUsesWhisperCppInstallerLayout(t *testing.T) {+ root := t.TempDir()+ t.Setenv(resourcepath.RootEnv, root)++ executableName := "whisper-cli"+ if runtime.GOOS == "windows" {+ executableName += ".exe"+ }+ executablePath := filepath.Join(root, "bin", "whispercpp", executableName)+ modelPath := filepath.Join(root, "models", "whispercpp", "ggml-tiny.bin")+ for _, path := range []string{executablePath, modelPath} {+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(path, []byte("test"), 0755); err != nil {+ t.Fatal(err)+ }+ }++ previousTranscribe := config.Conf.Transcribe+ previousWhisperCppPath := storage.WhispercppPath+ t.Cleanup(func() {+ config.Conf.Transcribe = previousTranscribe+ storage.WhispercppPath = previousWhisperCppPath+ })+ config.Conf.Transcribe.Provider = "whispercpp"+ config.Conf.Transcribe.Whispercpp.Model = "tiny"++ if err := configurePackagedTranscriptionDependency(); err != nil {+ t.Fatalf("configurePackagedTranscriptionDependency() error = %v", err)+ }+ if storage.WhispercppPath != executablePath {+ t.Fatalf("WhispercppPath = %q, want %q", storage.WhispercppPath, executablePath)+ }+}+func TestResolveYtDlpUpdatesExistingBundledBinaryToStableRelease(t *testing.T) {env := testYtDlpEnv(t, "darwin")env.stat = func(path string) (os.FileInfo, error) {diff --git a/runtime/krillinai/internal/pipeline/subtitle.go b/runtime/krillinai/internal/pipeline/subtitle.goindex 1ae26d01..a7ca4375 100644--- a/runtime/krillinai/internal/pipeline/subtitle.go+++ b/runtime/krillinai/internal/pipeline/subtitle.go@@ -26,6 +26,7 @@ type SubtitleRequest struct {MaxWordOneLine intSubtitleStyle *subtitlestyle.StyleSetPrepareVideo bool+ SourceOnly boolReportProgress func(phase string, percent int, message string)}@@ -74,15 +75,17 @@ func GenerateSubtitles(ctx context.Context, svc StageService, req SubtitleReques}if err == nil {manifest.CaptionSource = "youtube_vtt"- reportSubtitleProgress(req, "preparing_original_media", 76, "正在补齐原始视频")- stepParam.TaskPtr.SetProgressReporter(func(percent uint8) {- overall := 76 + minInt(int(percent), 10)*14/10- reportSubtitleProgress(req, "preparing_original_media", overall, "正在补齐原始视频")- })- if err := prepareOriginalMediaForRendering(ctx, svc, stepParam); err != nil {- return failSubtitleStage(req, manifest, ErrorKindRetryable, "prepare_media_for_render_failed", err)+ if !req.SourceOnly || req.PrepareVideo {+ reportSubtitleProgress(req, "preparing_original_media", 76, "正在补齐原始视频")+ stepParam.TaskPtr.SetProgressReporter(func(percent uint8) {+ overall := 76 + minInt(int(percent), 10)*14/10+ reportSubtitleProgress(req, "preparing_original_media", overall, "正在补齐原始视频")+ })+ if err := prepareOriginalMediaForRendering(ctx, svc, stepParam); err != nil {+ return failSubtitleStage(req, manifest, ErrorKindRetryable, "prepare_media_for_render_failed", err)+ }+ syncPreparedMediaOutputs(manifest, stepParam)}- syncPreparedMediaOutputs(manifest, stepParam)reportSubtitleProgress(req, "collecting_outputs", 95, "正在整理字幕和视频产物")return saveSubtitleSuccess(manifest, req, CaptionSource("youtube_vtt"))}@@ -141,6 +144,18 @@ func audioSubtitleProgressReporter(req SubtitleRequest) func(uint8) {overall := 25 + value*65/100phase := "transcribing_audio"message := "正在转录并翻译音频字幕"+ if req.SourceOnly {+ message = "正在转录音频字幕"+ if value <= 10 {+ phase = "preparing_audio"+ message = "正在准备音频转录"+ } else if value >= 90 {+ phase = "collecting_subtitles"+ message = "正在生成原文字幕"+ }+ reportSubtitleProgress(req, phase, overall, message)+ return+ }if value <= 10 {phase = "preparing_audio"message = "正在准备音频转录"@@ -188,7 +203,9 @@ func subtitleStepParam(req SubtitleRequest) *types.SubtitleTaskStepParam {userLang = string(types.LanguageNameSimplifiedChinese)}resultType := types.SubtitleResultTypeBilingualTranslationOnBottom- if req.BilingualTop {+ if req.SourceOnly {+ resultType = types.SubtitleResultTypeOriginOnly+ } else if req.BilingualTop {resultType = types.SubtitleResultTypeBilingualTranslationOnTop}maxWordOneLine := req.MaxWordOneLine@@ -236,6 +253,7 @@ func subtitleYouTubeReq(req SubtitleRequest, taskPtr *types.SubtitleTask) *serviOriginLanguage: req.OriginLang,TargetLanguage: req.TargetLang,TaskPtr: taskPtr,+ SourceOnly: req.SourceOnly,TargetLanguageFirst: req.BilingualTop,}}@@ -248,6 +266,13 @@ func saveSubtitleSuccess(manifest *Manifest, req SubtitleRequest, captionSourcereturn failSubtitleStage(req, manifest, ErrorKindRetryable, "source_video_missing", errors.New("original video was requested but was not produced"))}}+ if req.SourceOnly {+ manifest.Outputs.TargetSRT = ""+ manifest.Outputs.BilingualSRT = ""+ manifest.Outputs.ShortOriginSRT = ""+ manifest.Outputs.ShortOriginMixedSRT = ""+ manifest.Outputs.TargetText = ""+ }if err := validateExistingSubtitleOutputs(manifest.Outputs); err != nil {manifest.MarkStage(StageSubtitle, false, err.Error())_ = manifest.Save()diff --git a/runtime/krillinai/internal/pipeline/subtitle_test.go b/runtime/krillinai/internal/pipeline/subtitle_test.goindex 935a2872..1f83d855 100644--- a/runtime/krillinai/internal/pipeline/subtitle_test.go+++ b/runtime/krillinai/internal/pipeline/subtitle_test.go@@ -19,6 +19,7 @@ type fakeStageService struct {prepareVTT []boolprepareEmbedTypes []stringlastPrepare *types.SubtitleTaskStepParam+ lastYouTube *service.YoutubeSubtitleReqlastSpeech *types.SubtitleTaskStepParamlastCoverPrompt stringlastCoverSize string@@ -76,6 +77,13 @@ func (f *fakeStageService) DownloadYouTubeSubtitle(context.Context, *service.Youfunc (f *fakeStageService) ProcessYouTubeSubtitle(_ context.Context, req *service.YoutubeSubtitleReq) (string, error) {f.calls = append(f.calls, "process-youtube")+ f.lastYouTube = req+ if req.SourceOnly {+ path := filepath.Join(req.TaskBasePath, types.SubtitleTaskOriginLanguageSrtFileName)+ if err := os.WriteFile(path, []byte("1\n00:00:00,000 --> 00:00:01,000\nsource\n\n"), 0600); err != nil {+ return "", err+ }+ }if req.TaskPtr != nil {req.TaskPtr.SetProgress(40)req.TaskPtr.SetProgress(65)@@ -269,6 +277,40 @@ func TestGenerateSubtitlesYouTubeCaptionsDoNotUseAudio(t *testing.T) {}}+func TestGenerateSubtitlesSourceOnlySkipsTranslationAndVideoPreparation(t *testing.T) {+ dir := t.TempDir()+ fake := &fakeStageService{}+ req := SubtitleRequest{+ Input: "https://www.youtube.com/watch?v=abc",+ Workdir: dir,+ TaskID: "source-only",+ OriginLang: "en",+ TargetLang: "zh_cn",+ CaptionSource: CaptionSourceAny,+ SourceOnly: true,+ }++ resp, err := GenerateSubtitles(context.Background(), fake, req)+ if err != nil || !resp.OK {+ t.Fatalf("response = %+v, error = %v", resp, err)+ }+ if got := fake.calls; len(got) != 3 || got[0] != "prepare" || got[1] != "download-youtube" || got[2] != "process-youtube" {+ t.Fatalf("calls = %v, want source preparation and platform captions only", got)+ }+ if fake.lastPrepare == nil || fake.lastPrepare.SubtitleResultType != types.SubtitleResultTypeOriginOnly {+ t.Fatalf("SubtitleResultType = %v, want origin only", fake.lastPrepare.SubtitleResultType)+ }+ if fake.lastYouTube == nil || !fake.lastYouTube.SourceOnly {+ t.Fatal("YouTube request did not preserve SourceOnly")+ }+ if resp.Outputs.OriginSRT == "" {+ t.Fatal("OriginSRT is empty")+ }+ if resp.Outputs.TargetSRT != "" || resp.Outputs.BilingualSRT != "" || resp.Outputs.ShortOriginMixedSRT != "" {+ t.Fatalf("source-only response advertised translated outputs: %+v", resp.Outputs)+ }+}+func TestGenerateSubtitlesYouTubeCaptionsPrepareOriginalMediaForRendering(t *testing.T) {dir := t.TempDir()fake := &fakeStageService{}diff --git a/runtime/krillinai/internal/service/audio2subtitle.go b/runtime/krillinai/internal/service/audio2subtitle.goindex ae258035..9fd61909 100644--- a/runtime/krillinai/internal/service/audio2subtitle.go+++ b/runtime/krillinai/internal/service/audio2subtitle.go@@ -16,6 +16,7 @@ import ("runtime/debug""strconv""strings"+ "unicode""go.uber.org/zap""golang.org/x/sync/errgroup"@@ -59,6 +60,13 @@ type contextChatCompleter interface {}func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {+ if stepParam.SubtitleResultType == types.SubtitleResultTypeOriginOnly {+ if err := s.audioToOriginSubtitle(ctx, stepParam); err != nil {+ return fmt.Errorf("audioToSubtitle audioToOriginSubtitle error: %w", err)+ }+ stepParam.TaskPtr.SetProgress(95)+ return nil+ }var err errorerr = s.audioToSrt(ctx, stepParam) // 这里进度更新到90%了if err != nil {@@ -73,6 +81,236 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTreturn nil}+type originSubtitleCue struct {+ Start float64+ End float64+ Text string+}++func (s Service) audioToOriginSubtitle(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {+ timePoints, err := s.getSplitPointsForAudio(stepParam)+ if err != nil {+ return err+ }+ cues := make([]originSubtitleCue, 0)+ segmentCount := len(timePoints) - 1+ for segmentIndex := 0; segmentIndex < segmentCount; segmentIndex++ {+ select {+ case <-ctx.Done():+ return ctx.Err()+ default:+ }++ audioPath := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitAudioFileNamePattern, segmentIndex))+ if err := ClipAudio(stepParam.AudioFilePath, audioPath, timePoints[segmentIndex], timePoints[segmentIndex+1]); err != nil {+ return err+ }+ var transcription *types.TranscriptionData+ attempts := config.Conf.App.TranscribeMaxAttempts+ if attempts < 1 {+ attempts = 1+ }+ for range attempts {+ transcription, err = s.transcribeAudio(+ segmentIndex,+ audioPath,+ string(stepParam.OriginLanguage),+ stepParam.TaskBasePath,+ func(percent int) {+ completed := float64(segmentIndex) + clampFraction(float64(percent)/100)+ stepParam.TaskPtr.SetProgress(uint8(15 + completed/float64(segmentCount)*75))+ },+ )+ if err == nil {+ break+ }+ }+ if err != nil {+ return fmt.Errorf("transcribe audio segment %d: %w", segmentIndex, err)+ }+ cues = append(cues, originCuesFromTranscription(+ transcription,+ timePoints[segmentIndex],+ timePoints[segmentIndex+1],+ stepParam.MaxWordOneLine,+ stepParam.OriginLanguage,+ )...)+ stepParam.TaskPtr.SetProgress(uint8(15 + float64(segmentIndex+1)/float64(segmentCount)*75))+ }+ if len(cues) == 0 {+ return errors.New("transcription produced no subtitle cues")+ }+ return writeOriginSubtitleFile(+ filepath.Join(stepParam.TaskBasePath, types.SubtitleTaskOriginLanguageSrtFileName),+ cues,+ )+}++func originCuesFromTranscription(+ transcription *types.TranscriptionData,+ offset float64,+ segmentEnd float64,+ maxWords int,+ language types.StandardLanguageCode,+) []originSubtitleCue {+ if transcription == nil {+ return nil+ }+ if maxWords <= 0 {+ maxWords = 12+ }+ words := make([]types.Word, 0, len(transcription.Words))+ for _, word := range transcription.Words {+ if strings.TrimSpace(word.Text) != "" {+ words = append(words, word)+ }+ }+ if len(words) == 0 {+ text := strings.TrimSpace(transcription.Text)+ if text == "" {+ return nil+ }+ return []originSubtitleCue{{Start: offset, End: segmentEnd, Text: text}}+ }+ if !util.IsAsianLanguage(language) {+ if alignedWords, ok := alignTranscriptionWords(transcription.Text, words); ok {+ words = alignedWords+ }+ }++ cues := make([]originSubtitleCue, 0, (len(words)+maxWords-1)/maxWords)+ for start := 0; start < len(words); start += maxWords {+ end := min(start+maxWords, len(words))+ parts := make([]string, 0, end-start)+ for _, word := range words[start:end] {+ parts = append(parts, strings.TrimSpace(word.Text))+ }+ separator := " "+ if util.IsAsianLanguage(language) {+ separator = ""+ }+ cueStart := offset + words[start].Start+ cueEnd := offset + words[end-1].End+ if cueEnd <= cueStart {+ cueEnd = cueStart + 0.5+ }+ cues = append(cues, originSubtitleCue{+ Start: cueStart,+ End: cueEnd,+ Text: strings.Join(parts, separator),+ })+ }+ return cues+}++func alignTranscriptionWords(text string, words []types.Word) ([]types.Word, bool) {+ textTokens := strings.Fields(text)+ if len(textTokens) == 0 {+ return nil, false+ }++ timedWords := make([]types.Word, 0, len(words))+ for _, word := range words {+ if normalizeTranscriptionToken(word.Text) != "" {+ timedWords = append(timedWords, word)+ }+ }+ if len(timedWords) == 0 {+ return nil, false+ }++ aligned := make([]types.Word, 0, len(textTokens))+ wordIndex := 0+ prefix := ""+ for _, textToken := range textTokens {+ target := normalizeTranscriptionToken(textToken)+ if target == "" {+ if len(aligned) == 0 {+ prefix += textToken + " "+ } else {+ aligned[len(aligned)-1].Text += " " + textToken+ }+ continue+ }++ start := wordIndex+ matched := ""+ for wordIndex < len(timedWords) && len(matched) < len(target) {+ matched += normalizeTranscriptionToken(timedWords[wordIndex].Text)+ wordIndex+++ if !strings.HasPrefix(target, matched) {+ return nil, false+ }+ }+ if matched != target {+ return nil, false+ }+ aligned = append(aligned, types.Word{+ Text: prefix + textToken,+ Start: timedWords[start].Start,+ End: timedWords[wordIndex-1].End,+ })+ prefix = ""+ }+ if wordIndex != len(timedWords) {+ return nil, false+ }+ if prefix != "" && len(aligned) > 0 {+ aligned[len(aligned)-1].Text += " " + strings.TrimSpace(prefix)+ }+ return aligned, len(aligned) > 0+}++func normalizeTranscriptionToken(text string) string {+ return strings.Map(func(r rune) rune {+ if unicode.IsLetter(r) || unicode.IsNumber(r) {+ return unicode.ToLower(r)+ }+ return -1+ }, text)+}++func writeOriginSubtitleFile(path string, cues []originSubtitleCue) error {+ file, err := os.Create(path)+ if err != nil {+ return err+ }+ defer file.Close()++ previousEnd := 0.0+ index := 0+ for _, cue := range cues {+ text := strings.TrimSpace(cue.Text)+ if text == "" {+ continue+ }+ start := cue.Start+ if start < previousEnd {+ start = previousEnd+ }+ end := cue.End+ if end <= start {+ end = start + 0.5+ }+ index+++ if _, err := fmt.Fprintf(+ file,+ "%d\n%s --> %s\n%s\n\n",+ index,+ util.FormatTime(float32(start)),+ util.FormatTime(float32(end)),+ text,+ ); err != nil {+ return err+ }+ previousEnd = end+ }+ if index == 0 {+ return errors.New("transcription produced no subtitle cues")+ }+ return file.Sync()+}+//func splitAudio(stepParam *types.SubtitleTaskStepParam) error {// log.GetLogger().Info("audioToSubtitle.splitAudio start", zap.String("task id", stepParam.TaskId))// var err errordiff --git a/runtime/krillinai/internal/service/audio2subtitle_test.go b/runtime/krillinai/internal/service/audio2subtitle_test.goindex 9ee637c4..230d9b42 100644--- a/runtime/krillinai/internal/service/audio2subtitle_test.go+++ b/runtime/krillinai/internal/service/audio2subtitle_test.go@@ -184,6 +184,80 @@ func TestAudioProcessPercentUsesDetailedTranscriptionAndTranslationProgress(t *t}}+func TestOriginCuesFromTranscriptionUseWordTimingsWithoutTranslation(t *testing.T) {+ transcription := &types.TranscriptionData{+ Text: "one two three four",+ Words: []types.Word{+ {Text: "one", Start: 0.1, End: 0.4},+ {Text: "two", Start: 0.5, End: 0.9},+ {Text: "three", Start: 1.0, End: 1.4},+ {Text: "four", Start: 1.5, End: 2.0},+ },+ }+ cues := originCuesFromTranscription(transcription, 10, 13, 2, types.LanguageNameEnglish)+ if len(cues) != 2 {+ t.Fatalf("cue count = %d, want 2", len(cues))+ }+ if cues[0].Text != "one two" || cues[0].Start != 10.1 || cues[0].End != 10.9 {+ t.Fatalf("first cue = %+v", cues[0])+ }+ if cues[1].Text != "three four" || cues[1].Start != 11 || cues[1].End != 12 {+ t.Fatalf("second cue = %+v", cues[1])+ }++ path := filepath.Join(t.TempDir(), types.SubtitleTaskOriginLanguageSrtFileName)+ if err := writeOriginSubtitleFile(path, cues); err != nil {+ t.Fatal(err)+ }+ if err := NormalizeSRTFile(path, false); err != nil {+ t.Fatalf("generated source subtitle is invalid: %v", err)+ }+}++func TestOriginCuesFromTranscriptionAlignsSubwordTokensWithTranscript(t *testing.T) {+ transcription := &types.TranscriptionData{+ Text: `"Saying no" isn't social conformity in the 1950s.`,+ Words: []types.Word{+ {Text: "S", Start: 0.1, End: 0.2},+ {Text: "aying", Start: 0.2, End: 0.6},+ {Text: "no", Start: 0.7, End: 0.9},+ {Text: "isn", Start: 1.0, End: 1.2},+ {Text: "t", Start: 1.2, End: 1.3},+ {Text: "social", Start: 1.4, End: 1.8},+ {Text: "conform", Start: 1.9, End: 2.4},+ {Text: "ity", Start: 2.4, End: 2.6},+ {Text: "in", Start: 2.7, End: 2.8},+ {Text: "the", Start: 2.9, End: 3.0},+ {Text: "1950", Start: 3.1, End: 3.5},+ {Text: "s", Start: 3.5, End: 3.6},+ },+ }++ cues := originCuesFromTranscription(transcription, 10, 14, 4, types.LanguageNameEnglish)+ if len(cues) != 2 {+ t.Fatalf("cue count = %d, want 2", len(cues))+ }+ if cues[0].Text != `"Saying no" isn't social` || cues[0].Start != 10.1 || cues[0].End != 11.8 {+ t.Fatalf("first cue = %+v", cues[0])+ }+ if cues[1].Text != "conformity in the 1950s." || cues[1].Start != 11.9 || cues[1].End != 13.6 {+ t.Fatalf("second cue = %+v", cues[1])+ }+}++func TestOriginCuesFromTranscriptionFallBackToTranscriptText(t *testing.T) {+ cues := originCuesFromTranscription(+ &types.TranscriptionData{Text: "fallback transcript"},+ 20,+ 24,+ 12,+ types.LanguageNameEnglish,+ )+ if len(cues) != 1 || cues[0].Text != "fallback transcript" || cues[0].Start != 20 || cues[0].End != 24 {+ t.Fatalf("cues = %+v", cues)+ }+}+type echoBatchCompleter struct {batchSizes []int}diff --git a/runtime/krillinai/internal/service/youtube_subtitle.go b/runtime/krillinai/internal/service/youtube_subtitle.goindex 855ddfad..e2a5e056 100644--- a/runtime/krillinai/internal/service/youtube_subtitle.go+++ b/runtime/krillinai/internal/service/youtube_subtitle.go@@ -37,6 +37,7 @@ type YoutubeSubtitleReq struct {TargetLanguage stringVttFile stringTaskPtr *types.SubtitleTask+ SourceOnly boolTargetLanguageFirst bool // 是否将目标语言放在上面(双语字幕)}@@ -286,7 +287,12 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req}// 2. 组织成句子- sentences := s.groupWordsIntoSentences(vttWords)+ var sentences []Sentence+ if req.SourceOnly {+ sentences = s.groupWordsIntoSentencesWithoutLLM(vttWords)+ } else {+ sentences = s.groupWordsIntoSentences(vttWords)+ }if len(sentences) == 0 {return "", fmt.Errorf("no sentences formed from VTT words")}@@ -311,6 +317,12 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, reqif req.TaskPtr != nil {req.TaskPtr.SetProgress(40)}+ if req.SourceOnly {+ if req.TaskPtr != nil {+ req.TaskPtr.SetProgress(100)+ }+ return originSrtFile, nil+ }// 4. 批量翻译生成目标语言SRT(40%-90%进度)err = s.translator.BatchTranslateSrtBlocks(srtBlocks, req.OriginLanguage, req.TargetLanguage, req.TaskPtr)@@ -1030,6 +1042,25 @@ type Sentence struct {EndTime string // 句子结束时间}+func (s *YouTubeSubtitleService) groupWordsIntoSentencesWithoutLLM(words []VttWord) []Sentence {+ primarySentences := s.splitByPrimarySentencePunctuation(words)+ deterministicSentences := make([]Sentence, 0, len(primarySentences))+ for _, sentence := range primarySentences {+ if util.CountEffectiveChars(sentence.Text) <= config.Conf.App.MaxSentenceLength {+ deterministicSentences = append(deterministicSentences, sentence)+ continue+ }+ for _, candidate := range s.splitByCommasPunctuation(sentence.Words) {+ if util.CountEffectiveChars(candidate.Text) > config.Conf.App.MaxSentenceLength {+ deterministicSentences = append(deterministicSentences, s.splitByFixedLength(candidate.Words)...)+ } else {+ deterministicSentences = append(deterministicSentences, candidate)+ }+ }+ }+ return s.cleanupPunctuationOnlySentences(deterministicSentences)+}+// groupWordsIntoSentences 根据标点符号将单词分组成完整的句子func (s *YouTubeSubtitleService) groupWordsIntoSentences(words []VttWord) []Sentence {if len(words) == 0 {diff --git a/runtime/krillinai/internal/service/youtube_subtitle_test.go b/runtime/krillinai/internal/service/youtube_subtitle_test.goindex 7b23bc93..98892bd1 100644--- a/runtime/krillinai/internal/service/youtube_subtitle_test.go+++ b/runtime/krillinai/internal/service/youtube_subtitle_test.go@@ -4,6 +4,7 @@ import ("context""krillin-ai/config""krillin-ai/internal/deps"+ "krillin-ai/internal/types""krillin-ai/log""os""path/filepath"@@ -86,6 +87,39 @@ func Test_processYouTubeSubtitle(t *testing.T) {}}+func TestProcessYouTubeSubtitleSourceOnlySkipsTranslator(t *testing.T) {+ log.InitLogger()+ service := NewYouTubeSubtitleService()+ service.translator = nil+ workdir := t.TempDir()+ vttFile := copyTestVtt(t, workdir, "source-only.en.vtt")+ task := &types.SubtitleTask{}+ req := &YoutubeSubtitleReq{+ TaskBasePath: workdir,+ TaskId: "source-only",+ OriginLanguage: "en",+ TargetLanguage: "zh_cn",+ VttFile: vttFile,+ TaskPtr: task,+ SourceOnly: true,+ }++ output, err := service.processYouTubeSubtitle(context.Background(), req)+ if err != nil {+ t.Fatal(err)+ }+ want := filepath.Join(workdir, types.SubtitleTaskOriginLanguageSrtFileName)+ if output != want {+ t.Fatalf("output = %q, want %q", output, want)+ }+ if task.ProcessPct != 100 {+ t.Fatalf("progress = %d, want 100", task.ProcessPct)+ }+ if _, err := os.Stat(filepath.Join(workdir, types.SubtitleTaskTargetLanguageSrtFileName)); !os.IsNotExist(err) {+ t.Fatalf("target subtitle should not be generated, stat error = %v", err)+ }+}+func skipYouTubeSubtitleIntegrationTest(t *testing.T) {t.Helper()if os.Getenv("KRILLIN_RUN_YOUTUBE_SUBTITLE_INTEGRATION") != "1" {