Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

fix: stabilize stickman source transcription

wulien-center6ae78372f288
diff --git a/apps/daemon/src/creator/krillin/cli-runner.ts b/apps/daemon/src/creator/krillin/cli-runner.ts
index 6bc171a3..8831bda5 100644
--- a/apps/daemon/src/creator/krillin/cli-runner.ts
+++ b/apps/daemon/src/creator/krillin/cli-runner.ts
@@ -218,14 +218,15 @@ export function buildKrillinCliCommandArguments(
if (command === 'subtitle') {
const source = resolveKrillinCliSource(artifacts, options);
if (!source) throw new CreatorExecutorError('creator_stage_input_missing', 'Subtitle input video or URL is required');
+ const sourceOnly = booleanOption(options, 'sourceOnly', false);
return [
'subtitle',
source,
- '--prepare-video',
+ ...(sourceOnly ? [] : ['--prepare-video']),
'--origin-lang', requiredOption(options, 'originLanguage'),
'--target-lang', requiredOption(options, 'targetLanguage'),
'--caption-source', stringOption(options, 'captionSource') ?? 'any',
- ...(booleanOption(options, 'sourceOnly', false) ? ['--source-only'] : []),
+ ...(sourceOnly ? ['--source-only'] : []),
`--bilingual-top=${booleanOption(options, 'bilingualTop', true)}`,
...common,
...styleArgument(stylePath)
diff --git a/apps/daemon/test/unit/creator-stickman-krillin.test.ts b/apps/daemon/test/unit/creator-stickman-krillin.test.ts
index 70497b57..5bd2231b 100644
--- a/apps/daemon/test/unit/creator-stickman-krillin.test.ts
+++ b/apps/daemon/test/unit/creator-stickman-krillin.test.ts
@@ -68,12 +68,14 @@ describe('stickman KrillinAI mapping', () => {
};
const stickmanOptions = buildKrillinStageOptions(sourceTranscript as never);
expect(stickmanOptions.sourceOnly).toBe(true);
- expect(buildKrillinCliCommandArguments(
+ const stickmanArgs = buildKrillinCliCommandArguments(
sourceTranscript as never,
[],
stickmanOptions,
undefined
- )).toContain('--source-only');
+ );
+ expect(stickmanArgs).toContain('--source-only');
+ expect(stickmanArgs).not.toContain('--prepare-video');
const translationStage = {
...sourceTranscript,
@@ -82,12 +84,14 @@ describe('stickman KrillinAI mapping', () => {
};
const translationOptions = buildKrillinStageOptions(translationStage as never);
expect(translationOptions.sourceOnly).toBe(false);
- expect(buildKrillinCliCommandArguments(
+ const translationArgs = buildKrillinCliCommandArguments(
translationStage as never,
[],
translationOptions,
undefined
- )).not.toContain('--source-only');
+ );
+ expect(translationArgs).not.toContain('--source-only');
+ expect(translationArgs).toContain('--prepare-video');
});
it('rejects undeclared or missing stickman outputs and aliases valid subtitles', async () => {
diff --git a/runtime/krillinai/internal/cli/commands.go b/runtime/krillinai/internal/cli/commands.go
index 38436c7f..70a7d0a1 100644
--- a/runtime/krillinai/internal/cli/commands.go
+++ b/runtime/krillinai/internal/cli/commands.go
@@ -121,7 +121,8 @@ Flags:
--workdir <dir> Task working directory
--task-id <id> Optional task id
--caption-source <source> any, platform, manual, auto, or whisper
- --prepare-video download original video for later rendering
+ --prepare-video download original video for later rendering
+ --source-only generate source-language subtitles without translation
--bilingual-top Put target subtitle on top (default true)
--max-word-one-line <n> Max words per subtitle line
--subtitle-style-file <file> JSON subtitle style override file
@@ -489,6 +490,7 @@ func parseSubtitle(name string, args []string) (Command, error) {
taskID := fs.String("task-id", "", "task id")
captionSource := fs.String("caption-source", string(pipeline.CaptionSourceAny), "caption source")
prepareVideo := fs.Bool("prepare-video", false, "prepare original video for later rendering")
+ sourceOnly := fs.Bool("source-only", false, "generate source-language subtitles without translation")
bilingualTop := fs.Bool("bilingual-top", true, "put target subtitle on top")
maxWordOneLine := fs.Int("max-word-one-line", 0, "max words per line")
subtitleStyleFile := fs.String("subtitle-style-file", "", "subtitle style JSON file")
@@ -521,6 +523,7 @@ func parseSubtitle(name string, args []string) (Command, error) {
UserLang: *userLang,
CaptionSource: pipeline.CaptionSource(*captionSource),
PrepareVideo: *prepareVideo,
+ SourceOnly: *sourceOnly,
BilingualTop: *bilingualTop,
MaxWordOneLine: *maxWordOneLine,
},
diff --git a/runtime/krillinai/internal/cli/commands_test.go b/runtime/krillinai/internal/cli/commands_test.go
index 8648ef0a..1a929f7b 100644
--- a/runtime/krillinai/internal/cli/commands_test.go
+++ b/runtime/krillinai/internal/cli/commands_test.go
@@ -53,6 +53,23 @@ func TestParseSubtitlePrepareVideo(t *testing.T) {
}
}
+func TestParseSubtitleSourceOnly(t *testing.T) {
+ cmd, err := Parse([]string{
+ "subtitle",
+ "https://www.youtube.com/watch?v=abc",
+ "--source-only",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !cmd.Subtitle.SourceOnly {
+ t.Fatal("SourceOnly = false, want true")
+ }
+ if !strings.Contains(Help(Command{Name: "subtitle"}), "--source-only") {
+ t.Fatal("subtitle help does not advertise --source-only")
+ }
+}
+
func TestParseSubtitleCommandCanPutTargetLanguageOnBottom(t *testing.T) {
cmd, err := Parse([]string{
"subtitle",
diff --git a/runtime/krillinai/internal/deps/checker.go b/runtime/krillinai/internal/deps/checker.go
index af85d4dd..32feec67 100644
--- a/runtime/krillinai/internal/deps/checker.go
+++ b/runtime/krillinai/internal/deps/checker.go
@@ -157,11 +157,11 @@ func configurePackagedTranscriptionDependency() error {
return err
}
case "whispercpp":
- name := "whispercpp"
+ name := "whisper-cli"
if runtime.GOOS == "windows" {
name += ".exe"
}
- if storage.WhispercppPath, err = resourcepath.RequireFile("bin", name); err != nil {
+ if storage.WhispercppPath, err = resourcepath.RequireFile("bin", "whispercpp", name); err != nil {
return err
}
if _, err = resourcepath.RequireFile("models", "whispercpp", fmt.Sprintf("ggml-%s.bin", config.Conf.Transcribe.Whispercpp.Model)); err != nil {
diff --git a/runtime/krillinai/internal/deps/checker_test.go b/runtime/krillinai/internal/deps/checker_test.go
index dd8b8c83..94a6520b 100644
--- a/runtime/krillinai/internal/deps/checker_test.go
+++ b/runtime/krillinai/internal/deps/checker_test.go
@@ -129,6 +129,42 @@ func TestConfigurePackagedTranscriptionDependencyUsesWhisperKitArchiveDirectory(
}
}
+func TestConfigurePackagedTranscriptionDependencyUsesWhisperCppInstallerLayout(t *testing.T) {
+ root := t.TempDir()
+ t.Setenv(resourcepath.RootEnv, root)
+
+ executableName := "whisper-cli"
+ if runtime.GOOS == "windows" {
+ executableName += ".exe"
+ }
+ executablePath := filepath.Join(root, "bin", "whispercpp", executableName)
+ modelPath := filepath.Join(root, "models", "whispercpp", "ggml-tiny.bin")
+ for _, path := range []string{executablePath, modelPath} {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte("test"), 0755); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ previousTranscribe := config.Conf.Transcribe
+ previousWhisperCppPath := storage.WhispercppPath
+ t.Cleanup(func() {
+ config.Conf.Transcribe = previousTranscribe
+ storage.WhispercppPath = previousWhisperCppPath
+ })
+ config.Conf.Transcribe.Provider = "whispercpp"
+ config.Conf.Transcribe.Whispercpp.Model = "tiny"
+
+ if err := configurePackagedTranscriptionDependency(); err != nil {
+ t.Fatalf("configurePackagedTranscriptionDependency() error = %v", err)
+ }
+ if storage.WhispercppPath != executablePath {
+ t.Fatalf("WhispercppPath = %q, want %q", storage.WhispercppPath, executablePath)
+ }
+}
+
func TestResolveYtDlpUpdatesExistingBundledBinaryToStableRelease(t *testing.T) {
env := testYtDlpEnv(t, "darwin")
env.stat = func(path string) (os.FileInfo, error) {
diff --git a/runtime/krillinai/internal/pipeline/subtitle.go b/runtime/krillinai/internal/pipeline/subtitle.go
index 1ae26d01..a7ca4375 100644
--- a/runtime/krillinai/internal/pipeline/subtitle.go
+++ b/runtime/krillinai/internal/pipeline/subtitle.go
@@ -26,6 +26,7 @@ type SubtitleRequest struct {
MaxWordOneLine int
SubtitleStyle *subtitlestyle.StyleSet
PrepareVideo bool
+ SourceOnly bool
ReportProgress func(phase string, percent int, message string)
}
@@ -74,15 +75,17 @@ func GenerateSubtitles(ctx context.Context, svc StageService, req SubtitleReques
}
if err == nil {
manifest.CaptionSource = "youtube_vtt"
- reportSubtitleProgress(req, "preparing_original_media", 76, "正在补齐原始视频")
- stepParam.TaskPtr.SetProgressReporter(func(percent uint8) {
- overall := 76 + minInt(int(percent), 10)*14/10
- reportSubtitleProgress(req, "preparing_original_media", overall, "正在补齐原始视频")
- })
- if err := prepareOriginalMediaForRendering(ctx, svc, stepParam); err != nil {
- return failSubtitleStage(req, manifest, ErrorKindRetryable, "prepare_media_for_render_failed", err)
+ if !req.SourceOnly || req.PrepareVideo {
+ reportSubtitleProgress(req, "preparing_original_media", 76, "正在补齐原始视频")
+ stepParam.TaskPtr.SetProgressReporter(func(percent uint8) {
+ overall := 76 + minInt(int(percent), 10)*14/10
+ reportSubtitleProgress(req, "preparing_original_media", overall, "正在补齐原始视频")
+ })
+ if err := prepareOriginalMediaForRendering(ctx, svc, stepParam); err != nil {
+ return failSubtitleStage(req, manifest, ErrorKindRetryable, "prepare_media_for_render_failed", err)
+ }
+ syncPreparedMediaOutputs(manifest, stepParam)
}
- syncPreparedMediaOutputs(manifest, stepParam)
reportSubtitleProgress(req, "collecting_outputs", 95, "正在整理字幕和视频产物")
return saveSubtitleSuccess(manifest, req, CaptionSource("youtube_vtt"))
}
@@ -141,6 +144,18 @@ func audioSubtitleProgressReporter(req SubtitleRequest) func(uint8) {
overall := 25 + value*65/100
phase := "transcribing_audio"
message := "正在转录并翻译音频字幕"
+ if req.SourceOnly {
+ message = "正在转录音频字幕"
+ if value <= 10 {
+ phase = "preparing_audio"
+ message = "正在准备音频转录"
+ } else if value >= 90 {
+ phase = "collecting_subtitles"
+ message = "正在生成原文字幕"
+ }
+ reportSubtitleProgress(req, phase, overall, message)
+ return
+ }
if value <= 10 {
phase = "preparing_audio"
message = "正在准备音频转录"
@@ -188,7 +203,9 @@ func subtitleStepParam(req SubtitleRequest) *types.SubtitleTaskStepParam {
userLang = string(types.LanguageNameSimplifiedChinese)
}
resultType := types.SubtitleResultTypeBilingualTranslationOnBottom
- if req.BilingualTop {
+ if req.SourceOnly {
+ resultType = types.SubtitleResultTypeOriginOnly
+ } else if req.BilingualTop {
resultType = types.SubtitleResultTypeBilingualTranslationOnTop
}
maxWordOneLine := req.MaxWordOneLine
@@ -236,6 +253,7 @@ func subtitleYouTubeReq(req SubtitleRequest, taskPtr *types.SubtitleTask) *servi
OriginLanguage: req.OriginLang,
TargetLanguage: req.TargetLang,
TaskPtr: taskPtr,
+ SourceOnly: req.SourceOnly,
TargetLanguageFirst: req.BilingualTop,
}
}
@@ -248,6 +266,13 @@ func saveSubtitleSuccess(manifest *Manifest, req SubtitleRequest, captionSource
return failSubtitleStage(req, manifest, ErrorKindRetryable, "source_video_missing", errors.New("original video was requested but was not produced"))
}
}
+ if req.SourceOnly {
+ manifest.Outputs.TargetSRT = ""
+ manifest.Outputs.BilingualSRT = ""
+ manifest.Outputs.ShortOriginSRT = ""
+ manifest.Outputs.ShortOriginMixedSRT = ""
+ manifest.Outputs.TargetText = ""
+ }
if err := validateExistingSubtitleOutputs(manifest.Outputs); err != nil {
manifest.MarkStage(StageSubtitle, false, err.Error())
_ = manifest.Save()
diff --git a/runtime/krillinai/internal/pipeline/subtitle_test.go b/runtime/krillinai/internal/pipeline/subtitle_test.go
index 935a2872..1f83d855 100644
--- a/runtime/krillinai/internal/pipeline/subtitle_test.go
+++ b/runtime/krillinai/internal/pipeline/subtitle_test.go
@@ -19,6 +19,7 @@ type fakeStageService struct {
prepareVTT []bool
prepareEmbedTypes []string
lastPrepare *types.SubtitleTaskStepParam
+ lastYouTube *service.YoutubeSubtitleReq
lastSpeech *types.SubtitleTaskStepParam
lastCoverPrompt string
lastCoverSize string
@@ -76,6 +77,13 @@ func (f *fakeStageService) DownloadYouTubeSubtitle(context.Context, *service.You
func (f *fakeStageService) ProcessYouTubeSubtitle(_ context.Context, req *service.YoutubeSubtitleReq) (string, error) {
f.calls = append(f.calls, "process-youtube")
+ f.lastYouTube = req
+ if req.SourceOnly {
+ path := filepath.Join(req.TaskBasePath, types.SubtitleTaskOriginLanguageSrtFileName)
+ if err := os.WriteFile(path, []byte("1\n00:00:00,000 --> 00:00:01,000\nsource\n\n"), 0600); err != nil {
+ return "", err
+ }
+ }
if req.TaskPtr != nil {
req.TaskPtr.SetProgress(40)
req.TaskPtr.SetProgress(65)
@@ -269,6 +277,40 @@ func TestGenerateSubtitlesYouTubeCaptionsDoNotUseAudio(t *testing.T) {
}
}
+func TestGenerateSubtitlesSourceOnlySkipsTranslationAndVideoPreparation(t *testing.T) {
+ dir := t.TempDir()
+ fake := &fakeStageService{}
+ req := SubtitleRequest{
+ Input: "https://www.youtube.com/watch?v=abc",
+ Workdir: dir,
+ TaskID: "source-only",
+ OriginLang: "en",
+ TargetLang: "zh_cn",
+ CaptionSource: CaptionSourceAny,
+ SourceOnly: true,
+ }
+
+ resp, err := GenerateSubtitles(context.Background(), fake, req)
+ if err != nil || !resp.OK {
+ t.Fatalf("response = %+v, error = %v", resp, err)
+ }
+ if got := fake.calls; len(got) != 3 || got[0] != "prepare" || got[1] != "download-youtube" || got[2] != "process-youtube" {
+ t.Fatalf("calls = %v, want source preparation and platform captions only", got)
+ }
+ if fake.lastPrepare == nil || fake.lastPrepare.SubtitleResultType != types.SubtitleResultTypeOriginOnly {
+ t.Fatalf("SubtitleResultType = %v, want origin only", fake.lastPrepare.SubtitleResultType)
+ }
+ if fake.lastYouTube == nil || !fake.lastYouTube.SourceOnly {
+ t.Fatal("YouTube request did not preserve SourceOnly")
+ }
+ if resp.Outputs.OriginSRT == "" {
+ t.Fatal("OriginSRT is empty")
+ }
+ if resp.Outputs.TargetSRT != "" || resp.Outputs.BilingualSRT != "" || resp.Outputs.ShortOriginMixedSRT != "" {
+ t.Fatalf("source-only response advertised translated outputs: %+v", resp.Outputs)
+ }
+}
+
func TestGenerateSubtitlesYouTubeCaptionsPrepareOriginalMediaForRendering(t *testing.T) {
dir := t.TempDir()
fake := &fakeStageService{}
diff --git a/runtime/krillinai/internal/service/audio2subtitle.go b/runtime/krillinai/internal/service/audio2subtitle.go
index ae258035..9fd61909 100644
--- a/runtime/krillinai/internal/service/audio2subtitle.go
+++ b/runtime/krillinai/internal/service/audio2subtitle.go
@@ -16,6 +16,7 @@ import (
"runtime/debug"
"strconv"
"strings"
+ "unicode"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
@@ -59,6 +60,13 @@ type contextChatCompleter interface {
}
func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {
+ if stepParam.SubtitleResultType == types.SubtitleResultTypeOriginOnly {
+ if err := s.audioToOriginSubtitle(ctx, stepParam); err != nil {
+ return fmt.Errorf("audioToSubtitle audioToOriginSubtitle error: %w", err)
+ }
+ stepParam.TaskPtr.SetProgress(95)
+ return nil
+ }
var err error
err = s.audioToSrt(ctx, stepParam) // 这里进度更新到90%了
if err != nil {
@@ -73,6 +81,236 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleT
return nil
}
+type originSubtitleCue struct {
+ Start float64
+ End float64
+ Text string
+}
+
+func (s Service) audioToOriginSubtitle(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {
+ timePoints, err := s.getSplitPointsForAudio(stepParam)
+ if err != nil {
+ return err
+ }
+ cues := make([]originSubtitleCue, 0)
+ segmentCount := len(timePoints) - 1
+ for segmentIndex := 0; segmentIndex < segmentCount; segmentIndex++ {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
+
+ audioPath := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskSplitAudioFileNamePattern, segmentIndex))
+ if err := ClipAudio(stepParam.AudioFilePath, audioPath, timePoints[segmentIndex], timePoints[segmentIndex+1]); err != nil {
+ return err
+ }
+ var transcription *types.TranscriptionData
+ attempts := config.Conf.App.TranscribeMaxAttempts
+ if attempts < 1 {
+ attempts = 1
+ }
+ for range attempts {
+ transcription, err = s.transcribeAudio(
+ segmentIndex,
+ audioPath,
+ string(stepParam.OriginLanguage),
+ stepParam.TaskBasePath,
+ func(percent int) {
+ completed := float64(segmentIndex) + clampFraction(float64(percent)/100)
+ stepParam.TaskPtr.SetProgress(uint8(15 + completed/float64(segmentCount)*75))
+ },
+ )
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ return fmt.Errorf("transcribe audio segment %d: %w", segmentIndex, err)
+ }
+ cues = append(cues, originCuesFromTranscription(
+ transcription,
+ timePoints[segmentIndex],
+ timePoints[segmentIndex+1],
+ stepParam.MaxWordOneLine,
+ stepParam.OriginLanguage,
+ )...)
+ stepParam.TaskPtr.SetProgress(uint8(15 + float64(segmentIndex+1)/float64(segmentCount)*75))
+ }
+ if len(cues) == 0 {
+ return errors.New("transcription produced no subtitle cues")
+ }
+ return writeOriginSubtitleFile(
+ filepath.Join(stepParam.TaskBasePath, types.SubtitleTaskOriginLanguageSrtFileName),
+ cues,
+ )
+}
+
+func originCuesFromTranscription(
+ transcription *types.TranscriptionData,
+ offset float64,
+ segmentEnd float64,
+ maxWords int,
+ language types.StandardLanguageCode,
+) []originSubtitleCue {
+ if transcription == nil {
+ return nil
+ }
+ if maxWords <= 0 {
+ maxWords = 12
+ }
+ words := make([]types.Word, 0, len(transcription.Words))
+ for _, word := range transcription.Words {
+ if strings.TrimSpace(word.Text) != "" {
+ words = append(words, word)
+ }
+ }
+ if len(words) == 0 {
+ text := strings.TrimSpace(transcription.Text)
+ if text == "" {
+ return nil
+ }
+ return []originSubtitleCue{{Start: offset, End: segmentEnd, Text: text}}
+ }
+ if !util.IsAsianLanguage(language) {
+ if alignedWords, ok := alignTranscriptionWords(transcription.Text, words); ok {
+ words = alignedWords
+ }
+ }
+
+ cues := make([]originSubtitleCue, 0, (len(words)+maxWords-1)/maxWords)
+ for start := 0; start < len(words); start += maxWords {
+ end := min(start+maxWords, len(words))
+ parts := make([]string, 0, end-start)
+ for _, word := range words[start:end] {
+ parts = append(parts, strings.TrimSpace(word.Text))
+ }
+ separator := " "
+ if util.IsAsianLanguage(language) {
+ separator = ""
+ }
+ cueStart := offset + words[start].Start
+ cueEnd := offset + words[end-1].End
+ if cueEnd <= cueStart {
+ cueEnd = cueStart + 0.5
+ }
+ cues = append(cues, originSubtitleCue{
+ Start: cueStart,
+ End: cueEnd,
+ Text: strings.Join(parts, separator),
+ })
+ }
+ return cues
+}
+
+func alignTranscriptionWords(text string, words []types.Word) ([]types.Word, bool) {
+ textTokens := strings.Fields(text)
+ if len(textTokens) == 0 {
+ return nil, false
+ }
+
+ timedWords := make([]types.Word, 0, len(words))
+ for _, word := range words {
+ if normalizeTranscriptionToken(word.Text) != "" {
+ timedWords = append(timedWords, word)
+ }
+ }
+ if len(timedWords) == 0 {
+ return nil, false
+ }
+
+ aligned := make([]types.Word, 0, len(textTokens))
+ wordIndex := 0
+ prefix := ""
+ for _, textToken := range textTokens {
+ target := normalizeTranscriptionToken(textToken)
+ if target == "" {
+ if len(aligned) == 0 {
+ prefix += textToken + " "
+ } else {
+ aligned[len(aligned)-1].Text += " " + textToken
+ }
+ continue
+ }
+
+ start := wordIndex
+ matched := ""
+ for wordIndex < len(timedWords) && len(matched) < len(target) {
+ matched += normalizeTranscriptionToken(timedWords[wordIndex].Text)
+ wordIndex++
+ if !strings.HasPrefix(target, matched) {
+ return nil, false
+ }
+ }
+ if matched != target {
+ return nil, false
+ }
+ aligned = append(aligned, types.Word{
+ Text: prefix + textToken,
+ Start: timedWords[start].Start,
+ End: timedWords[wordIndex-1].End,
+ })
+ prefix = ""
+ }
+ if wordIndex != len(timedWords) {
+ return nil, false
+ }
+ if prefix != "" && len(aligned) > 0 {
+ aligned[len(aligned)-1].Text += " " + strings.TrimSpace(prefix)
+ }
+ return aligned, len(aligned) > 0
+}
+
+func normalizeTranscriptionToken(text string) string {
+ return strings.Map(func(r rune) rune {
+ if unicode.IsLetter(r) || unicode.IsNumber(r) {
+ return unicode.ToLower(r)
+ }
+ return -1
+ }, text)
+}
+
+func writeOriginSubtitleFile(path string, cues []originSubtitleCue) error {
+ file, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ previousEnd := 0.0
+ index := 0
+ for _, cue := range cues {
+ text := strings.TrimSpace(cue.Text)
+ if text == "" {
+ continue
+ }
+ start := cue.Start
+ if start < previousEnd {
+ start = previousEnd
+ }
+ end := cue.End
+ if end <= start {
+ end = start + 0.5
+ }
+ index++
+ if _, err := fmt.Fprintf(
+ file,
+ "%d\n%s --> %s\n%s\n\n",
+ index,
+ util.FormatTime(float32(start)),
+ util.FormatTime(float32(end)),
+ text,
+ ); err != nil {
+ return err
+ }
+ previousEnd = end
+ }
+ if index == 0 {
+ return errors.New("transcription produced no subtitle cues")
+ }
+ return file.Sync()
+}
+
//func splitAudio(stepParam *types.SubtitleTaskStepParam) error {
// log.GetLogger().Info("audioToSubtitle.splitAudio start", zap.String("task id", stepParam.TaskId))
// var err error
diff --git a/runtime/krillinai/internal/service/audio2subtitle_test.go b/runtime/krillinai/internal/service/audio2subtitle_test.go
index 9ee637c4..230d9b42 100644
--- a/runtime/krillinai/internal/service/audio2subtitle_test.go
+++ b/runtime/krillinai/internal/service/audio2subtitle_test.go
@@ -184,6 +184,80 @@ func TestAudioProcessPercentUsesDetailedTranscriptionAndTranslationProgress(t *t
}
}
+func TestOriginCuesFromTranscriptionUseWordTimingsWithoutTranslation(t *testing.T) {
+ transcription := &types.TranscriptionData{
+ Text: "one two three four",
+ Words: []types.Word{
+ {Text: "one", Start: 0.1, End: 0.4},
+ {Text: "two", Start: 0.5, End: 0.9},
+ {Text: "three", Start: 1.0, End: 1.4},
+ {Text: "four", Start: 1.5, End: 2.0},
+ },
+ }
+ cues := originCuesFromTranscription(transcription, 10, 13, 2, types.LanguageNameEnglish)
+ if len(cues) != 2 {
+ t.Fatalf("cue count = %d, want 2", len(cues))
+ }
+ if cues[0].Text != "one two" || cues[0].Start != 10.1 || cues[0].End != 10.9 {
+ t.Fatalf("first cue = %+v", cues[0])
+ }
+ if cues[1].Text != "three four" || cues[1].Start != 11 || cues[1].End != 12 {
+ t.Fatalf("second cue = %+v", cues[1])
+ }
+
+ path := filepath.Join(t.TempDir(), types.SubtitleTaskOriginLanguageSrtFileName)
+ if err := writeOriginSubtitleFile(path, cues); err != nil {
+ t.Fatal(err)
+ }
+ if err := NormalizeSRTFile(path, false); err != nil {
+ t.Fatalf("generated source subtitle is invalid: %v", err)
+ }
+}
+
+func TestOriginCuesFromTranscriptionAlignsSubwordTokensWithTranscript(t *testing.T) {
+ transcription := &types.TranscriptionData{
+ Text: `"Saying no" isn't social conformity in the 1950s.`,
+ Words: []types.Word{
+ {Text: "S", Start: 0.1, End: 0.2},
+ {Text: "aying", Start: 0.2, End: 0.6},
+ {Text: "no", Start: 0.7, End: 0.9},
+ {Text: "isn", Start: 1.0, End: 1.2},
+ {Text: "t", Start: 1.2, End: 1.3},
+ {Text: "social", Start: 1.4, End: 1.8},
+ {Text: "conform", Start: 1.9, End: 2.4},
+ {Text: "ity", Start: 2.4, End: 2.6},
+ {Text: "in", Start: 2.7, End: 2.8},
+ {Text: "the", Start: 2.9, End: 3.0},
+ {Text: "1950", Start: 3.1, End: 3.5},
+ {Text: "s", Start: 3.5, End: 3.6},
+ },
+ }
+
+ cues := originCuesFromTranscription(transcription, 10, 14, 4, types.LanguageNameEnglish)
+ if len(cues) != 2 {
+ t.Fatalf("cue count = %d, want 2", len(cues))
+ }
+ if cues[0].Text != `"Saying no" isn't social` || cues[0].Start != 10.1 || cues[0].End != 11.8 {
+ t.Fatalf("first cue = %+v", cues[0])
+ }
+ if cues[1].Text != "conformity in the 1950s." || cues[1].Start != 11.9 || cues[1].End != 13.6 {
+ t.Fatalf("second cue = %+v", cues[1])
+ }
+}
+
+func TestOriginCuesFromTranscriptionFallBackToTranscriptText(t *testing.T) {
+ cues := originCuesFromTranscription(
+ &types.TranscriptionData{Text: "fallback transcript"},
+ 20,
+ 24,
+ 12,
+ types.LanguageNameEnglish,
+ )
+ if len(cues) != 1 || cues[0].Text != "fallback transcript" || cues[0].Start != 20 || cues[0].End != 24 {
+ t.Fatalf("cues = %+v", cues)
+ }
+}
+
type echoBatchCompleter struct {
batchSizes []int
}
diff --git a/runtime/krillinai/internal/service/youtube_subtitle.go b/runtime/krillinai/internal/service/youtube_subtitle.go
index 855ddfad..e2a5e056 100644
--- a/runtime/krillinai/internal/service/youtube_subtitle.go
+++ b/runtime/krillinai/internal/service/youtube_subtitle.go
@@ -37,6 +37,7 @@ type YoutubeSubtitleReq struct {
TargetLanguage string
VttFile string
TaskPtr *types.SubtitleTask
+ SourceOnly bool
TargetLanguageFirst bool // 是否将目标语言放在上面(双语字幕)
}
@@ -286,7 +287,12 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req
}
// 2. 组织成句子
- sentences := s.groupWordsIntoSentences(vttWords)
+ var sentences []Sentence
+ if req.SourceOnly {
+ sentences = s.groupWordsIntoSentencesWithoutLLM(vttWords)
+ } else {
+ sentences = s.groupWordsIntoSentences(vttWords)
+ }
if len(sentences) == 0 {
return "", fmt.Errorf("no sentences formed from VTT words")
}
@@ -311,6 +317,12 @@ func (s *YouTubeSubtitleService) processYouTubeSubtitle(ctx context.Context, req
if req.TaskPtr != nil {
req.TaskPtr.SetProgress(40)
}
+ if req.SourceOnly {
+ if req.TaskPtr != nil {
+ req.TaskPtr.SetProgress(100)
+ }
+ return originSrtFile, nil
+ }
// 4. 批量翻译生成目标语言SRT(40%-90%进度)
err = s.translator.BatchTranslateSrtBlocks(srtBlocks, req.OriginLanguage, req.TargetLanguage, req.TaskPtr)
@@ -1030,6 +1042,25 @@ type Sentence struct {
EndTime string // 句子结束时间
}
+func (s *YouTubeSubtitleService) groupWordsIntoSentencesWithoutLLM(words []VttWord) []Sentence {
+ primarySentences := s.splitByPrimarySentencePunctuation(words)
+ deterministicSentences := make([]Sentence, 0, len(primarySentences))
+ for _, sentence := range primarySentences {
+ if util.CountEffectiveChars(sentence.Text) <= config.Conf.App.MaxSentenceLength {
+ deterministicSentences = append(deterministicSentences, sentence)
+ continue
+ }
+ for _, candidate := range s.splitByCommasPunctuation(sentence.Words) {
+ if util.CountEffectiveChars(candidate.Text) > config.Conf.App.MaxSentenceLength {
+ deterministicSentences = append(deterministicSentences, s.splitByFixedLength(candidate.Words)...)
+ } else {
+ deterministicSentences = append(deterministicSentences, candidate)
+ }
+ }
+ }
+ return s.cleanupPunctuationOnlySentences(deterministicSentences)
+}
+
// groupWordsIntoSentences 根据标点符号将单词分组成完整的句子
func (s *YouTubeSubtitleService) groupWordsIntoSentences(words []VttWord) []Sentence {
if len(words) == 0 {
diff --git a/runtime/krillinai/internal/service/youtube_subtitle_test.go b/runtime/krillinai/internal/service/youtube_subtitle_test.go
index 7b23bc93..98892bd1 100644
--- a/runtime/krillinai/internal/service/youtube_subtitle_test.go
+++ b/runtime/krillinai/internal/service/youtube_subtitle_test.go
@@ -4,6 +4,7 @@ import (
"context"
"krillin-ai/config"
"krillin-ai/internal/deps"
+ "krillin-ai/internal/types"
"krillin-ai/log"
"os"
"path/filepath"
@@ -86,6 +87,39 @@ func Test_processYouTubeSubtitle(t *testing.T) {
}
}
+func TestProcessYouTubeSubtitleSourceOnlySkipsTranslator(t *testing.T) {
+ log.InitLogger()
+ service := NewYouTubeSubtitleService()
+ service.translator = nil
+ workdir := t.TempDir()
+ vttFile := copyTestVtt(t, workdir, "source-only.en.vtt")
+ task := &types.SubtitleTask{}
+ req := &YoutubeSubtitleReq{
+ TaskBasePath: workdir,
+ TaskId: "source-only",
+ OriginLanguage: "en",
+ TargetLanguage: "zh_cn",
+ VttFile: vttFile,
+ TaskPtr: task,
+ SourceOnly: true,
+ }
+
+ output, err := service.processYouTubeSubtitle(context.Background(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := filepath.Join(workdir, types.SubtitleTaskOriginLanguageSrtFileName)
+ if output != want {
+ t.Fatalf("output = %q, want %q", output, want)
+ }
+ if task.ProcessPct != 100 {
+ t.Fatalf("progress = %d, want 100", task.ProcessPct)
+ }
+ if _, err := os.Stat(filepath.Join(workdir, types.SubtitleTaskTargetLanguageSrtFileName)); !os.IsNotExist(err) {
+ t.Fatalf("target subtitle should not be generated, stat error = %v", err)
+ }
+}
+
func skipYouTubeSubtitleIntegrationTest(t *testing.T) {
t.Helper()
if os.Getenv("KRILLIN_RUN_YOUTUBE_SUBTITLE_INTEGRATION") != "1" {