Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: add dubbing tts and audio helpers

wulien82bb76922e94
diff --git a/internal/service/dubbing/audio.go b/internal/service/dubbing/audio.go
new file mode 100644
index 00000000..37b00879
--- /dev/null
+++ b/internal/service/dubbing/audio.go
@@ -0,0 +1,65 @@
+package dubbing
+
+import (
+ "fmt"
+ "krillin-ai/internal/storage"
+ "os/exec"
+ "strings"
+)
+
+func defaultFFmpegRunner(args []string) error {
+ cmd := exec.Command(storage.FfmpegPath, args...)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("ffmpeg error: %w, output: %s", err, string(output))
+ }
+ return nil
+}
+
+func WriteTinySilence(output string, run CommandRunner) error {
+ if run == nil {
+ run = defaultFFmpegRunner
+ }
+ return run([]string{
+ "-y",
+ "-f", "lavfi",
+ "-i", "anullsrc=channel_layout=mono:sample_rate=44100",
+ "-t", "0.100",
+ "-ar", "44100",
+ "-ac", "1",
+ "-c:a", "pcm_s16le",
+ output,
+ })
+}
+
+func buildAtempoFilter(speed float64) string {
+ if speed <= 0 {
+ speed = 1
+ }
+
+ parts := []string{}
+ for speed > 2.0 {
+ parts = append(parts, "atempo=2.000")
+ speed /= 2.0
+ }
+ for speed < 0.5 {
+ parts = append(parts, "atempo=0.500")
+ speed /= 0.5
+ }
+ parts = append(parts, fmt.Sprintf("atempo=%.3f", speed))
+ return strings.Join(parts, ",")
+}
+
+func buildMuxArgs(inputVideo, inputAudio, outputVideo string) []string {
+ return []string{
+ "-y",
+ "-i", inputVideo,
+ "-i", inputAudio,
+ "-c:v", "copy",
+ "-map", "0:v:0",
+ "-map", "1:a:0",
+ "-c:a", "aac",
+ "-b:a", "192k",
+ outputVideo,
+ }
+}
diff --git a/internal/service/dubbing/audio_test.go b/internal/service/dubbing/audio_test.go
new file mode 100644
index 00000000..e8f6445e
--- /dev/null
+++ b/internal/service/dubbing/audio_test.go
@@ -0,0 +1,21 @@
+package dubbing
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestBuildAtempoFilterChainsLargeSpeed(t *testing.T) {
+ got := buildAtempoFilter(3.0)
+ if got != "atempo=2.000,atempo=1.500" {
+ t.Fatalf("buildAtempoFilter(3) = %q", got)
+ }
+}
+
+func TestBuildMuxArgsMapsVideoAndDubAudio(t *testing.T) {
+ args := buildMuxArgs("input.mp4", "dub.wav", "out.mp4")
+ joined := strings.Join(args, " ")
+ if !strings.Contains(joined, "-map 0:v:0") || !strings.Contains(joined, "-map 1:a:0") {
+ t.Fatalf("args should map original video and dub audio: %v", args)
+ }
+}
diff --git a/internal/service/dubbing/tts.go b/internal/service/dubbing/tts.go
new file mode 100644
index 00000000..f91336a6
--- /dev/null
+++ b/internal/service/dubbing/tts.go
@@ -0,0 +1,69 @@
+package dubbing
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "krillin-ai/internal/types"
+ "os"
+ "path/filepath"
+)
+
+func GenerateRawSegments(ctx context.Context, tts types.Ttser, plan []PlanItem, voice, dir string, run CommandRunner, duration DurationProbe) ([]PlanItem, error) {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ if run == nil {
+ run = defaultFFmpegRunner
+ }
+ if duration == nil {
+ return nil, errors.New("duration probe is required")
+ }
+
+ rawDir := filepath.Join(dir, "raw")
+ if err := os.MkdirAll(rawDir, 0755); err != nil {
+ return nil, err
+ }
+
+ for i := range plan {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+
+ output := filepath.Join(rawDir, fmt.Sprintf("%d.wav", plan[i].Index))
+ if IsSilenceOnlyText(plan[i].SpokenText) {
+ if err := WriteTinySilence(output, run); err != nil {
+ return nil, err
+ }
+ } else {
+ if tts == nil {
+ return nil, errors.New("tts is required for non-silence text")
+ }
+ if err := retryTTS(tts, plan[i].SpokenText, voice, output, 3); err != nil {
+ return nil, fmt.Errorf("tts segment %d failed: %w", plan[i].Index, err)
+ }
+ }
+
+ dur, err := duration(output)
+ if err != nil {
+ return nil, err
+ }
+ plan[i].ActualDuration = dur
+ }
+
+ return plan, nil
+}
+
+func retryTTS(tts types.Ttser, text, voice, output string, attempts int) error {
+ var last error
+ for i := 0; i < attempts; i++ {
+ last = tts.Text2Speech(text, voice, output)
+ if last == nil {
+ if _, err := os.Stat(output); err == nil {
+ return nil
+ }
+ last = fmt.Errorf("output file missing: %s", output)
+ }
+ }
+ return last
+}
diff --git a/internal/service/dubbing/tts_test.go b/internal/service/dubbing/tts_test.go
new file mode 100644
index 00000000..7f7cd961
--- /dev/null
+++ b/internal/service/dubbing/tts_test.go
@@ -0,0 +1,43 @@
+package dubbing
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+type fakeTTS struct {
+ failures int
+ calls int
+}
+
+func (f *fakeTTS) Text2Speech(text, voice, outputFile string) error {
+ f.calls++
+ if f.calls <= f.failures {
+ return errors.New("tts failed")
+ }
+ return os.WriteFile(outputFile, []byte("wav"), 0644)
+}
+
+func TestGenerateRawSegmentsRetriesAndWritesFiles(t *testing.T) {
+ dir := t.TempDir()
+ tts := &fakeTTS{failures: 1}
+ plan := []PlanItem{{Index: 1, SpokenText: "你好"}}
+ got, err := GenerateRawSegments(context.Background(), tts, plan, "voice", dir, nil, func(string) (float64, error) {
+ return 1.2, nil
+ })
+ if err != nil {
+ t.Fatalf("GenerateRawSegments() error = %v", err)
+ }
+ if tts.calls != 2 {
+ t.Fatalf("calls = %d, want 2", tts.calls)
+ }
+ if got[0].ActualDuration != 1.2 {
+ t.Fatalf("ActualDuration = %v", got[0].ActualDuration)
+ }
+ if _, err := os.Stat(filepath.Join(dir, "raw", "1.wav")); err != nil {
+ t.Fatalf("raw file missing: %v", err)
+ }
+}