Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: assemble dubbing outputs
wuliena558d964fd73
diff --git a/internal/service/dubbing/audio.go b/internal/service/dubbing/audio.goindex 0069dc10..f751140f 100644--- a/internal/service/dubbing/audio.go+++ b/internal/service/dubbing/audio.go@@ -4,7 +4,9 @@ import ("fmt""krillin-ai/internal/storage""math"+ "os""os/exec"+ "path/filepath""strings")@@ -65,3 +67,98 @@ func buildMuxArgs(inputVideo, inputAudio, outputVideo string) []string {outputVideo,}}++func BuildDubCues(plan []PlanItem) []Cue {+ cues := make([]Cue, len(plan))+ for i, item := range plan {+ cues[i] = Cue{+ Index: i + 1,+ Start: item.NewStart,+ End: item.NewEnd,+ Text: item.SpokenText,+ }+ }+ return cues+}++func fittedSegmentPath(segmentsDir string, index int) string {+ return filepath.Join(segmentsDir, "fitted", fmt.Sprintf("%d.wav", index))+}++func AssembleAudio(plan []PlanItem, segmentsDir, outputAudio string, run CommandRunner) error {+ if run == nil {+ run = defaultFFmpegRunner+ }++ fittedDir := filepath.Join(segmentsDir, "fitted")+ if err := os.MkdirAll(fittedDir, 0755); err != nil {+ return err+ }++ concatLines := make([]string, 0, len(plan)*2)+ lastEnd := 0.0+ for _, item := range plan {+ raw := filepath.Join(segmentsDir, "raw", fmt.Sprintf("%d.wav", item.Index))+ if err := ensureNonEmptyFile(raw, "raw segment"); err != nil {+ return err+ }++ fitted := fittedSegmentPath(segmentsDir, item.Index)+ filter, err := buildAtempoFilter(item.SpeedFactor)+ if err != nil {+ return err+ }+ if err := run([]string{+ "-y",+ "-i", raw,+ "-filter:a", filter,+ "-ar", "44100",+ "-ac", "1",+ "-c:a", "pcm_s16le",+ fitted,+ }); err != nil {+ return fmt.Errorf("fit segment %d: %w", item.Index, err)+ }++ if item.NewStart < lastEnd {+ return fmt.Errorf("plan item %d starts before previous end: start %.3f lastEnd %.3f", item.Index, item.NewStart, lastEnd)+ }+ if item.NewStart > lastEnd {+ silence := filepath.Join(fittedDir, fmt.Sprintf("silence_%d.wav", item.Index))+ if err := run([]string{+ "-y",+ "-f", "lavfi",+ "-i", "anullsrc=channel_layout=mono:sample_rate=44100",+ "-t", fmt.Sprintf("%.3f", item.NewStart-lastEnd),+ "-ar", "44100",+ "-ac", "1",+ "-c:a", "pcm_s16le",+ silence,+ }); err != nil {+ return fmt.Errorf("write silence before segment %d: %w", item.Index, err)+ }+ concatLines = append(concatLines, fmt.Sprintf("file '%s'", filepath.Base(silence)))+ }++ concatLines = append(concatLines, fmt.Sprintf("file '%s'", filepath.Base(fitted)))+ lastEnd = item.NewEnd+ }++ concatPath := filepath.Join(fittedDir, "concat.txt")+ if err := os.WriteFile(concatPath, []byte(strings.Join(concatLines, "\n")+"\n"), 0644); err != nil {+ return err+ }++ if err := run([]string{+ "-y",+ "-f", "concat",+ "-safe", "0",+ "-i", concatPath,+ "-c", "copy",+ outputAudio,+ }); err != nil {+ return fmt.Errorf("concat fitted audio: %w", err)+ }++ return nil+}diff --git a/internal/service/dubbing/audio_assemble_test.go b/internal/service/dubbing/audio_assemble_test.gonew file mode 100644index 00000000..787675a2--- /dev/null+++ b/internal/service/dubbing/audio_assemble_test.go@@ -0,0 +1,33 @@+package dubbing++import (+ "os"+ "path/filepath"+ "strings"+ "testing"+)++func TestAssembleAudioWritesConcatListInFittedDir(t *testing.T) {+ dir := t.TempDir()+ rawDir := filepath.Join(dir, "raw")+ if err := os.MkdirAll(rawDir, 0755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(rawDir, "1.wav"), []byte("raw"), 0644); err != nil {+ t.Fatal(err)+ }+ plan := []PlanItem{{Index: 1, NewStart: 0.5, NewEnd: 1.3, SpeedFactor: 1.0}}+ err := AssembleAudio(plan, dir, filepath.Join(dir, "out.wav"), func(args []string) error {+ return os.WriteFile(args[len(args)-1], []byte("media"), 0644)+ })+ if err != nil {+ t.Fatalf("AssembleAudio() error = %v", err)+ }+ data, err := os.ReadFile(filepath.Join(dir, "fitted", "concat.txt"))+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), "silence_1.wav") || !strings.Contains(string(data), "1.wav") {+ t.Fatalf("concat list = %q", string(data))+ }+}diff --git a/internal/service/dubbing/runner.go b/internal/service/dubbing/runner.gonew file mode 100644index 00000000..3cf2a749--- /dev/null+++ b/internal/service/dubbing/runner.go@@ -0,0 +1,159 @@+package dubbing++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "krillin-ai/internal/types"+ "krillin-ai/pkg/util"+ "os"+ "path/filepath"+)++type Result struct {+ Plan []PlanItem+ Chunks []Chunk+ Report Report+ DubSRT string+ Audio string+ Video string+}++type Runner struct {+ deps Dependencies+}++func NewRunner(deps Dependencies) *Runner {+ if deps.Config.MaxChunkSize <= 0 {+ deps.Config = DefaultConfig()+ }+ if deps.FFmpeg == nil {+ deps.FFmpeg = defaultFFmpegRunner+ }+ if deps.Duration == nil {+ deps.Duration = util.GetAudioDuration+ }+ if deps.OutputAudio == "" && deps.Workdir != "" {+ deps.OutputAudio = filepath.Join(deps.Workdir, types.TtsResultAudioFileName)+ }+ if deps.OutputVideo == "" && deps.Workdir != "" {+ deps.OutputVideo = filepath.Join(deps.Workdir, types.SubtitleTaskVideoWithTtsFileName)+ }+ return &Runner{deps: deps}+}++func (r *Runner) Run(ctx context.Context) (Result, error) {+ if ctx == nil {+ ctx = context.Background()+ }+ if err := r.validate(); err != nil {+ return Result{}, err+ }++ dubbingDir := filepath.Join(r.deps.Workdir, DubbingDirName)+ segmentsDir := filepath.Join(dubbingDir, "segments")+ if err := os.MkdirAll(segmentsDir, 0755); err != nil {+ return Result{}, err+ }++ cues, err := ParseSRTFile(r.deps.InputSRT)+ if err != nil {+ return Result{}, err+ }+ if err := WriteSRTFile(filepath.Join(dubbingDir, DubbingInputFileName), cues); err != nil {+ return Result{}, err+ }++ planner := NewPlanner(r.deps.Config, NewStatisticalEstimator(), NewLLMOptimizer(r.deps.Chat))+ plan, chunks, err := planner.Plan(cues, r.deps.Language)+ if err != nil {+ return Result{}, err+ }++ plan, err = GenerateRawSegments(ctx, r.deps.TTS, plan, r.deps.Voice, segmentsDir, r.deps.FFmpeg, r.deps.Duration)+ if err != nil {+ return Result{}, err+ }++ fitted, report, err := FitTimeline(plan, chunks, r.deps.Config)+ if err != nil {+ return Result{}, err+ }++ dubSRT := filepath.Join(dubbingDir, DubSubtitleFileName)+ if err := WriteSRTFile(dubSRT, BuildDubCues(fitted)); err != nil {+ return Result{}, err+ }+ if err := writeJSON(filepath.Join(dubbingDir, DubbingPlanFileName), fitted); err != nil {+ return Result{}, err+ }+ if err := writeJSON(filepath.Join(dubbingDir, DubbingReportName), report); err != nil {+ return Result{}, err+ }++ if err := AssembleAudio(fitted, segmentsDir, r.deps.OutputAudio, r.deps.FFmpeg); err != nil {+ return Result{}, err+ }+ if err := ensureNonEmptyFile(r.deps.OutputAudio, "output audio"); err != nil {+ return Result{}, err+ }++ if err := r.deps.FFmpeg(buildMuxArgs(r.deps.InputVideo, r.deps.OutputAudio, r.deps.OutputVideo)); err != nil {+ return Result{}, err+ }+ if err := ensureNonEmptyFile(r.deps.OutputVideo, "output video"); err != nil {+ return Result{}, err+ }++ return Result{+ Plan: fitted,+ Chunks: chunks,+ Report: report,+ DubSRT: dubSRT,+ Audio: r.deps.OutputAudio,+ Video: r.deps.OutputVideo,+ }, nil+}++func (r *Runner) validate() error {+ if r.deps.Workdir == "" {+ return errors.New("workdir is required")+ }+ if r.deps.InputSRT == "" {+ return errors.New("input srt is required")+ }+ if r.deps.TTS == nil {+ return errors.New("tts is required")+ }+ if r.deps.InputVideo == "" {+ return errors.New("input video is required")+ }+ if err := ensureNonEmptyFile(r.deps.InputVideo, "input video"); err != nil {+ return err+ }+ return nil+}++func ensureNonEmptyFile(path, label string) error {+ info, err := os.Stat(path)+ if err != nil {+ return fmt.Errorf("%s %s: %w", label, path, err)+ }+ if info.IsDir() {+ return fmt.Errorf("%s %s is a directory", label, path)+ }+ if info.Size() == 0 {+ return fmt.Errorf("%s %s is empty", label, path)+ }+ return nil+}++func writeJSON(path string, value any) error {+ data, err := json.MarshalIndent(value, "", " ")+ if err != nil {+ return err+ }+ data = append(data, '\n')+ return os.WriteFile(path, data, 0644)+}diff --git a/internal/service/dubbing/runner_test.go b/internal/service/dubbing/runner_test.gonew file mode 100644index 00000000..e4a9423d--- /dev/null+++ b/internal/service/dubbing/runner_test.go@@ -0,0 +1,88 @@+package dubbing++import (+ "context"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++func fakeRunnerWritingOutputs(dir string) CommandRunner {+ return func(args []string) error {+ out := args[len(args)-1]+ if strings.HasSuffix(out, ".wav") || strings.HasSuffix(out, ".mp4") {+ return os.WriteFile(out, []byte("media"), 0644)+ }+ return nil+ }+}++func TestRunWritesDubbingArtifactsWithFakeTTS(t *testing.T) {+ dir := t.TempDir()+ input := filepath.Join(dir, "input.srt")+ video := filepath.Join(dir, "origin.mp4")+ if err := os.WriteFile(input, []byte("1\n00:00:00,000 --> 00:00:01,000\n你好\n\n"), 0644); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(video, []byte("video"), 0644); err != nil {+ t.Fatal(err)+ }+ deps := Dependencies{+ TTS: &fakeTTS{writeOnReturn: true},+ Language: "zh_cn",+ Voice: "voice",+ Workdir: dir,+ InputSRT: input,+ InputVideo: video,+ OutputAudio: filepath.Join(dir, "tts_final_audio.wav"),+ OutputVideo: filepath.Join(dir, "video_with_tts.mp4"),+ Config: DefaultConfig(),+ FFmpeg: fakeRunnerWritingOutputs(dir),+ Duration: func(string) (float64, error) {+ return 0.8, nil+ },+ }+ result, err := NewRunner(deps).Run(context.Background())+ if err != nil {+ t.Fatalf("Run() error = %v", err)+ }+ for _, path := range []string{+ filepath.Join(dir, DubbingDirName, DubbingInputFileName),+ filepath.Join(dir, DubbingDirName, DubbingPlanFileName),+ filepath.Join(dir, DubbingDirName, DubbingReportName),+ filepath.Join(dir, DubbingDirName, DubSubtitleFileName),+ result.Audio,+ result.Video,+ } {+ if info, err := os.Stat(path); err != nil || info.Size() == 0 {+ t.Fatalf("missing output %s: info=%v err=%v", path, info, err)+ }+ }+}++func TestRunnerRequiresInputVideoForMux(t *testing.T) {+ dir := t.TempDir()+ input := filepath.Join(dir, "input.srt")+ if err := os.WriteFile(input, []byte("1\n00:00:00,000 --> 00:00:01,000\n你好\n\n"), 0644); err != nil {+ t.Fatal(err)+ }+ deps := Dependencies{+ TTS: &fakeTTS{writeOnReturn: true},+ Language: "zh_cn",+ Voice: "voice",+ Workdir: dir,+ InputSRT: input,+ OutputAudio: filepath.Join(dir, "tts_final_audio.wav"),+ OutputVideo: filepath.Join(dir, "video_with_tts.mp4"),+ Config: DefaultConfig(),+ FFmpeg: fakeRunnerWritingOutputs(dir),+ Duration: func(string) (float64, error) {+ return 0.8, nil+ },+ }+ _, err := NewRunner(deps).Run(context.Background())+ if err == nil {+ t.Fatalf("Run() error = nil, want missing input video error")+ }+}