Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: add dubbing tts and audio helpers
wulien82bb76922e94
diff --git a/internal/service/dubbing/audio.go b/internal/service/dubbing/audio.gonew file mode 100644index 00000000..37b00879--- /dev/null+++ b/internal/service/dubbing/audio.go@@ -0,0 +1,65 @@+package dubbing++import (+ "fmt"+ "krillin-ai/internal/storage"+ "os/exec"+ "strings"+)++func defaultFFmpegRunner(args []string) error {+ cmd := exec.Command(storage.FfmpegPath, args...)+ output, err := cmd.CombinedOutput()+ if err != nil {+ return fmt.Errorf("ffmpeg error: %w, output: %s", err, string(output))+ }+ return nil+}++func WriteTinySilence(output string, run CommandRunner) error {+ if run == nil {+ run = defaultFFmpegRunner+ }+ return run([]string{+ "-y",+ "-f", "lavfi",+ "-i", "anullsrc=channel_layout=mono:sample_rate=44100",+ "-t", "0.100",+ "-ar", "44100",+ "-ac", "1",+ "-c:a", "pcm_s16le",+ output,+ })+}++func buildAtempoFilter(speed float64) string {+ if speed <= 0 {+ speed = 1+ }++ parts := []string{}+ for speed > 2.0 {+ parts = append(parts, "atempo=2.000")+ speed /= 2.0+ }+ for speed < 0.5 {+ parts = append(parts, "atempo=0.500")+ speed /= 0.5+ }+ parts = append(parts, fmt.Sprintf("atempo=%.3f", speed))+ return strings.Join(parts, ",")+}++func buildMuxArgs(inputVideo, inputAudio, outputVideo string) []string {+ return []string{+ "-y",+ "-i", inputVideo,+ "-i", inputAudio,+ "-c:v", "copy",+ "-map", "0:v:0",+ "-map", "1:a:0",+ "-c:a", "aac",+ "-b:a", "192k",+ outputVideo,+ }+}diff --git a/internal/service/dubbing/audio_test.go b/internal/service/dubbing/audio_test.gonew file mode 100644index 00000000..e8f6445e--- /dev/null+++ b/internal/service/dubbing/audio_test.go@@ -0,0 +1,21 @@+package dubbing++import (+ "strings"+ "testing"+)++func TestBuildAtempoFilterChainsLargeSpeed(t *testing.T) {+ got := buildAtempoFilter(3.0)+ if got != "atempo=2.000,atempo=1.500" {+ t.Fatalf("buildAtempoFilter(3) = %q", got)+ }+}++func TestBuildMuxArgsMapsVideoAndDubAudio(t *testing.T) {+ args := buildMuxArgs("input.mp4", "dub.wav", "out.mp4")+ joined := strings.Join(args, " ")+ if !strings.Contains(joined, "-map 0:v:0") || !strings.Contains(joined, "-map 1:a:0") {+ t.Fatalf("args should map original video and dub audio: %v", args)+ }+}diff --git a/internal/service/dubbing/tts.go b/internal/service/dubbing/tts.gonew file mode 100644index 00000000..f91336a6--- /dev/null+++ b/internal/service/dubbing/tts.go@@ -0,0 +1,69 @@+package dubbing++import (+ "context"+ "errors"+ "fmt"+ "krillin-ai/internal/types"+ "os"+ "path/filepath"+)++func GenerateRawSegments(ctx context.Context, tts types.Ttser, plan []PlanItem, voice, dir string, run CommandRunner, duration DurationProbe) ([]PlanItem, error) {+ if ctx == nil {+ ctx = context.Background()+ }+ if run == nil {+ run = defaultFFmpegRunner+ }+ if duration == nil {+ return nil, errors.New("duration probe is required")+ }++ rawDir := filepath.Join(dir, "raw")+ if err := os.MkdirAll(rawDir, 0755); err != nil {+ return nil, err+ }++ for i := range plan {+ if err := ctx.Err(); err != nil {+ return nil, err+ }++ output := filepath.Join(rawDir, fmt.Sprintf("%d.wav", plan[i].Index))+ if IsSilenceOnlyText(plan[i].SpokenText) {+ if err := WriteTinySilence(output, run); err != nil {+ return nil, err+ }+ } else {+ if tts == nil {+ return nil, errors.New("tts is required for non-silence text")+ }+ if err := retryTTS(tts, plan[i].SpokenText, voice, output, 3); err != nil {+ return nil, fmt.Errorf("tts segment %d failed: %w", plan[i].Index, err)+ }+ }++ dur, err := duration(output)+ if err != nil {+ return nil, err+ }+ plan[i].ActualDuration = dur+ }++ return plan, nil+}++func retryTTS(tts types.Ttser, text, voice, output string, attempts int) error {+ var last error+ for i := 0; i < attempts; i++ {+ last = tts.Text2Speech(text, voice, output)+ if last == nil {+ if _, err := os.Stat(output); err == nil {+ return nil+ }+ last = fmt.Errorf("output file missing: %s", output)+ }+ }+ return last+}diff --git a/internal/service/dubbing/tts_test.go b/internal/service/dubbing/tts_test.gonew file mode 100644index 00000000..7f7cd961--- /dev/null+++ b/internal/service/dubbing/tts_test.go@@ -0,0 +1,43 @@+package dubbing++import (+ "context"+ "errors"+ "os"+ "path/filepath"+ "testing"+)++type fakeTTS struct {+ failures int+ calls int+}++func (f *fakeTTS) Text2Speech(text, voice, outputFile string) error {+ f.calls+++ if f.calls <= f.failures {+ return errors.New("tts failed")+ }+ return os.WriteFile(outputFile, []byte("wav"), 0644)+}++func TestGenerateRawSegmentsRetriesAndWritesFiles(t *testing.T) {+ dir := t.TempDir()+ tts := &fakeTTS{failures: 1}+ plan := []PlanItem{{Index: 1, SpokenText: "你好"}}+ got, err := GenerateRawSegments(context.Background(), tts, plan, "voice", dir, nil, func(string) (float64, error) {+ return 1.2, nil+ })+ if err != nil {+ t.Fatalf("GenerateRawSegments() error = %v", err)+ }+ if tts.calls != 2 {+ t.Fatalf("calls = %d, want 2", tts.calls)+ }+ if got[0].ActualDuration != 1.2 {+ t.Fatalf("ActualDuration = %v", got[0].ActualDuration)+ }+ if _, err := os.Stat(filepath.Join(dir, "raw", "1.wav")); err != nil {+ t.Fatalf("raw file missing: %v", err)+ }+}