Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: add tts pipeline stage

wulien2c8739e19240
diff --git a/internal/pipeline/subtitle_test.go b/internal/pipeline/subtitle_test.go
index ce7c5777..fde5b363 100644
--- a/internal/pipeline/subtitle_test.go
+++ b/internal/pipeline/subtitle_test.go
@@ -13,6 +13,7 @@ type fakeStageService struct {
processErr error
calls []string
prepareVTT []bool
+ lastSpeech *types.SubtitleTaskStepParam
}
func (f *fakeStageService) PrepareMedia(_ context.Context, p *types.SubtitleTaskStepParam) error {
@@ -26,7 +27,9 @@ func (f *fakeStageService) GenerateSubtitlesFromAudio(context.Context, *types.Su
return nil
}
-func (f *fakeStageService) GenerateSpeechFromSRT(context.Context, *types.SubtitleTaskStepParam) error {
+func (f *fakeStageService) GenerateSpeechFromSRT(_ context.Context, p *types.SubtitleTaskStepParam) error {
+ f.calls = append(f.calls, "speech")
+ f.lastSpeech = p
return nil
}
diff --git a/internal/pipeline/tts.go b/internal/pipeline/tts.go
new file mode 100644
index 00000000..1d636fbe
--- /dev/null
+++ b/internal/pipeline/tts.go
@@ -0,0 +1,125 @@
+package pipeline
+
+import (
+ "context"
+ "errors"
+ "krillin-ai/internal/types"
+ "os"
+ "path/filepath"
+)
+
+const ttsInputFileName = "tts_input.srt"
+
+type TTSRequest struct {
+ Workdir string
+ TaskID string
+ InputSRT string
+ LineMode LineMode
+ Video string
+ Voice string
+ VoiceCloneSource string
+}
+
+func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Response, error) {
+ if req.LineMode == "" {
+ req.LineMode = LineModeTargetOnly
+ }
+ manifest, err := ttsManifest(req)
+ if err != nil {
+ return ttsFailureResponse(req, nil, ErrorKindInternal, "load_manifest_failed", err), err
+ }
+ manifest.TaskID = req.TaskID
+ manifest.Workdir = req.Workdir
+ if err := manifest.ApplyDefaultOutputs(); err != nil {
+ return ttsFailureResponse(req, manifest, ErrorKindInternal, "apply_outputs_failed", err), err
+ }
+
+ inputSRT := req.InputSRT
+ if inputSRT == "" {
+ inputSRT = manifest.Outputs.TargetSRT
+ }
+ ttsSource := inputSRT
+ if req.LineMode != LineModeTargetOnly {
+ ttsSource = filepath.Join(req.Workdir, ttsInputFileName)
+ if err := ExtractTargetSRT(inputSRT, ttsSource, req.LineMode); err != nil {
+ return failTTSStage(req, manifest, "extract_tts_input_failed", err)
+ }
+ }
+
+ inputVideo := req.Video
+ if inputVideo == "" {
+ inputVideo = manifest.Outputs.OriginVideo
+ }
+ stepParam := &types.SubtitleTaskStepParam{
+ TaskId: req.TaskID,
+ TaskPtr: &types.SubtitleTask{TaskId: req.TaskID, Status: types.SubtitleTaskStatusProcessing},
+ TaskBasePath: req.Workdir,
+ EnableTts: true,
+ TtsSourceFilePath: ttsSource,
+ TtsResultFilePath: manifest.Outputs.TTSAudio,
+ InputVideoPath: inputVideo,
+ TtsVoiceCode: req.Voice,
+ VoiceCloneAudioUrl: req.VoiceCloneSource,
+ VideoWithTtsFilePath: manifest.Outputs.VideoWithTTS,
+ }
+ if err := svc.GenerateSpeechFromSRT(ctx, stepParam); err != nil {
+ return failTTSStage(req, manifest, "generate_speech_failed", err)
+ }
+ if stepParam.TtsResultFilePath != "" {
+ manifest.Outputs.TTSAudio = stepParam.TtsResultFilePath
+ }
+ if stepParam.VideoWithTtsFilePath != "" {
+ manifest.Outputs.VideoWithTTS = stepParam.VideoWithTtsFilePath
+ }
+ manifest.MarkStage(StageTTS, true, "")
+ if err := manifest.Save(); err != nil {
+ return ttsFailureResponse(req, manifest, ErrorKindInternal, "save_manifest_failed", err), err
+ }
+ return ttsResponse(true, req, manifest, nil), nil
+}
+
+func ttsManifest(req TTSRequest) (*Manifest, error) {
+ manifest, err := LoadManifest(req.Workdir)
+ if err == nil {
+ return manifest, nil
+ }
+ if errors.Is(err, os.ErrNotExist) {
+ return NewManifest(req.TaskID, req.Workdir), nil
+ }
+ return nil, err
+}
+
+func failTTSStage(req TTSRequest, manifest *Manifest, code string, err error) (Response, error) {
+ if manifest != nil {
+ manifest.MarkStage(StageTTS, false, err.Error())
+ _ = manifest.Save()
+ }
+ return ttsFailureResponse(req, manifest, ErrorKindRetryable, code, err), err
+}
+
+func ttsFailureResponse(req TTSRequest, manifest *Manifest, kind ErrorKind, code string, err error) Response {
+ return ttsResponse(false, req, manifest, &Error{
+ Kind: kind,
+ Code: code,
+ Message: err.Error(),
+ Retryable: kind == ErrorKindRetryable,
+ })
+}
+
+func ttsResponse(ok bool, req TTSRequest, manifest *Manifest, pipelineErr *Error) Response {
+ resp := Response{
+ OK: ok,
+ Stage: StageTTS,
+ Workdir: req.Workdir,
+ TaskID: req.TaskID,
+ Error: pipelineErr,
+ }
+ if manifest != nil {
+ resp.Workdir = manifest.Workdir
+ resp.TaskID = manifest.TaskID
+ resp.Outputs = manifest.Outputs
+ resp.Warnings = manifest.Warnings
+ resp.FailedIndexes = manifest.FailedIndexes
+ }
+ return resp
+}
diff --git a/internal/pipeline/tts_test.go b/internal/pipeline/tts_test.go
new file mode 100644
index 00000000..226e8139
--- /dev/null
+++ b/internal/pipeline/tts_test.go
@@ -0,0 +1,44 @@
+package pipeline
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestGenerateTTSExtractsBilingualTargetBeforeSpeech(t *testing.T) {
+ dir := t.TempDir()
+ input := filepath.Join(dir, "bilingual.srt")
+ if err := os.WriteFile(input, []byte("1\n00:00:00,000 --> 00:00:01,000\nhello\n你好\n\n"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ fake := &fakeStageService{}
+ req := TTSRequest{
+ Workdir: dir,
+ TaskID: "demo",
+ InputSRT: input,
+ LineMode: LineModeBilingualTargetBottom,
+ }
+ resp, err := GenerateTTS(context.Background(), fake, req)
+ if err != nil {
+ t.Fatalf("GenerateTTS() error = %v", err)
+ }
+ if !resp.OK {
+ t.Fatalf("OK = false, want true")
+ }
+ extracted := filepath.Join(dir, "tts_input.srt")
+ data, err := os.ReadFile(extracted)
+ if err != nil {
+ t.Fatalf("tts input not written: %v", err)
+ }
+ if string(data) != "1\n00:00:00,000 --> 00:00:01,000\n你好\n\n" {
+ t.Fatalf("tts input = %q", string(data))
+ }
+ if fake.lastSpeech == nil {
+ t.Fatalf("GenerateSpeechFromSRT was not called")
+ }
+ if fake.lastSpeech.TtsSourceFilePath != extracted {
+ t.Fatalf("TtsSourceFilePath = %q, want %q", fake.lastSpeech.TtsSourceFilePath, extracted)
+ }
+}