Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: add tts pipeline stage
wulien2c8739e19240
diff --git a/internal/pipeline/subtitle_test.go b/internal/pipeline/subtitle_test.goindex ce7c5777..fde5b363 100644--- a/internal/pipeline/subtitle_test.go+++ b/internal/pipeline/subtitle_test.go@@ -13,6 +13,7 @@ type fakeStageService struct {processErr errorcalls []stringprepareVTT []bool+ lastSpeech *types.SubtitleTaskStepParam}func (f *fakeStageService) PrepareMedia(_ context.Context, p *types.SubtitleTaskStepParam) error {@@ -26,7 +27,9 @@ func (f *fakeStageService) GenerateSubtitlesFromAudio(context.Context, *types.Sureturn nil}-func (f *fakeStageService) GenerateSpeechFromSRT(context.Context, *types.SubtitleTaskStepParam) error {+func (f *fakeStageService) GenerateSpeechFromSRT(_ context.Context, p *types.SubtitleTaskStepParam) error {+ f.calls = append(f.calls, "speech")+ f.lastSpeech = preturn nil}diff --git a/internal/pipeline/tts.go b/internal/pipeline/tts.gonew file mode 100644index 00000000..1d636fbe--- /dev/null+++ b/internal/pipeline/tts.go@@ -0,0 +1,125 @@+package pipeline++import (+ "context"+ "errors"+ "krillin-ai/internal/types"+ "os"+ "path/filepath"+)++const ttsInputFileName = "tts_input.srt"++type TTSRequest struct {+ Workdir string+ TaskID string+ InputSRT string+ LineMode LineMode+ Video string+ Voice string+ VoiceCloneSource string+}++func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Response, error) {+ if req.LineMode == "" {+ req.LineMode = LineModeTargetOnly+ }+ manifest, err := ttsManifest(req)+ if err != nil {+ return ttsFailureResponse(req, nil, ErrorKindInternal, "load_manifest_failed", err), err+ }+ manifest.TaskID = req.TaskID+ manifest.Workdir = req.Workdir+ if err := manifest.ApplyDefaultOutputs(); err != nil {+ return ttsFailureResponse(req, manifest, ErrorKindInternal, "apply_outputs_failed", err), err+ }++ inputSRT := req.InputSRT+ if inputSRT == "" {+ inputSRT = manifest.Outputs.TargetSRT+ }+ ttsSource := inputSRT+ if req.LineMode != LineModeTargetOnly {+ ttsSource = filepath.Join(req.Workdir, ttsInputFileName)+ if err := ExtractTargetSRT(inputSRT, ttsSource, req.LineMode); err != nil {+ return failTTSStage(req, manifest, "extract_tts_input_failed", err)+ }+ }++ inputVideo := req.Video+ if inputVideo == "" {+ inputVideo = manifest.Outputs.OriginVideo+ }+ stepParam := &types.SubtitleTaskStepParam{+ TaskId: req.TaskID,+ TaskPtr: &types.SubtitleTask{TaskId: req.TaskID, Status: types.SubtitleTaskStatusProcessing},+ TaskBasePath: req.Workdir,+ EnableTts: true,+ TtsSourceFilePath: ttsSource,+ TtsResultFilePath: manifest.Outputs.TTSAudio,+ InputVideoPath: inputVideo,+ TtsVoiceCode: req.Voice,+ VoiceCloneAudioUrl: req.VoiceCloneSource,+ VideoWithTtsFilePath: manifest.Outputs.VideoWithTTS,+ }+ if err := svc.GenerateSpeechFromSRT(ctx, stepParam); err != nil {+ return failTTSStage(req, manifest, "generate_speech_failed", err)+ }+ if stepParam.TtsResultFilePath != "" {+ manifest.Outputs.TTSAudio = stepParam.TtsResultFilePath+ }+ if stepParam.VideoWithTtsFilePath != "" {+ manifest.Outputs.VideoWithTTS = stepParam.VideoWithTtsFilePath+ }+ manifest.MarkStage(StageTTS, true, "")+ if err := manifest.Save(); err != nil {+ return ttsFailureResponse(req, manifest, ErrorKindInternal, "save_manifest_failed", err), err+ }+ return ttsResponse(true, req, manifest, nil), nil+}++func ttsManifest(req TTSRequest) (*Manifest, error) {+ manifest, err := LoadManifest(req.Workdir)+ if err == nil {+ return manifest, nil+ }+ if errors.Is(err, os.ErrNotExist) {+ return NewManifest(req.TaskID, req.Workdir), nil+ }+ return nil, err+}++func failTTSStage(req TTSRequest, manifest *Manifest, code string, err error) (Response, error) {+ if manifest != nil {+ manifest.MarkStage(StageTTS, false, err.Error())+ _ = manifest.Save()+ }+ return ttsFailureResponse(req, manifest, ErrorKindRetryable, code, err), err+}++func ttsFailureResponse(req TTSRequest, manifest *Manifest, kind ErrorKind, code string, err error) Response {+ return ttsResponse(false, req, manifest, &Error{+ Kind: kind,+ Code: code,+ Message: err.Error(),+ Retryable: kind == ErrorKindRetryable,+ })+}++func ttsResponse(ok bool, req TTSRequest, manifest *Manifest, pipelineErr *Error) Response {+ resp := Response{+ OK: ok,+ Stage: StageTTS,+ Workdir: req.Workdir,+ TaskID: req.TaskID,+ Error: pipelineErr,+ }+ if manifest != nil {+ resp.Workdir = manifest.Workdir+ resp.TaskID = manifest.TaskID+ resp.Outputs = manifest.Outputs+ resp.Warnings = manifest.Warnings+ resp.FailedIndexes = manifest.FailedIndexes+ }+ return resp+}diff --git a/internal/pipeline/tts_test.go b/internal/pipeline/tts_test.gonew file mode 100644index 00000000..226e8139--- /dev/null+++ b/internal/pipeline/tts_test.go@@ -0,0 +1,44 @@+package pipeline++import (+ "context"+ "os"+ "path/filepath"+ "testing"+)++func TestGenerateTTSExtractsBilingualTargetBeforeSpeech(t *testing.T) {+ dir := t.TempDir()+ input := filepath.Join(dir, "bilingual.srt")+ if err := os.WriteFile(input, []byte("1\n00:00:00,000 --> 00:00:01,000\nhello\n你好\n\n"), 0644); err != nil {+ t.Fatal(err)+ }+ fake := &fakeStageService{}+ req := TTSRequest{+ Workdir: dir,+ TaskID: "demo",+ InputSRT: input,+ LineMode: LineModeBilingualTargetBottom,+ }+ resp, err := GenerateTTS(context.Background(), fake, req)+ if err != nil {+ t.Fatalf("GenerateTTS() error = %v", err)+ }+ if !resp.OK {+ t.Fatalf("OK = false, want true")+ }+ extracted := filepath.Join(dir, "tts_input.srt")+ data, err := os.ReadFile(extracted)+ if err != nil {+ t.Fatalf("tts input not written: %v", err)+ }+ if string(data) != "1\n00:00:00,000 --> 00:00:01,000\n你好\n\n" {+ t.Fatalf("tts input = %q", string(data))+ }+ if fake.lastSpeech == nil {+ t.Fatalf("GenerateSpeechFromSRT was not called")+ }+ if fake.lastSpeech.TtsSourceFilePath != extracted {+ t.Fatalf("TtsSourceFilePath = %q, want %q", fake.lastSpeech.TtsSourceFilePath, extracted)+ }+}