Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: plan and fit dubbing timeline

wulienc1eac36a09ad
diff --git a/internal/service/dubbing/fit.go b/internal/service/dubbing/fit.go
new file mode 100644
index 00000000..9de6dad4
--- /dev/null
+++ b/internal/service/dubbing/fit.go
@@ -0,0 +1,61 @@
+package dubbing
+
+import (
+ "fmt"
+)
+
+func FitTimeline(plan []PlanItem, chunks []Chunk, cfg Config) ([]PlanItem, Report, error) {
+ fitted := append([]PlanItem(nil), plan...)
+ report := Report{}
+
+ for _, chunk := range chunks {
+ available := chunk.End - chunk.Start
+ if available <= 0 {
+ return nil, report, fmt.Errorf("chunk %d has non-positive duration: %.3f", chunk.ID, available)
+ }
+
+ actual := 0.0
+ for _, idx := range chunk.Items {
+ if idx < 0 || idx >= len(fitted) {
+ return nil, report, fmt.Errorf("chunk %d references plan item %d out of range", chunk.ID, idx)
+ }
+ actual += fitted[idx].ActualDuration
+ }
+
+ speed := 1.0
+ if actual > available {
+ speed = actual / available
+ }
+ if speed > report.MaxSpeedFactor {
+ report.MaxSpeedFactor = speed
+ }
+ if speed > cfg.SpeedAccept {
+ report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d speed %.2f exceeds acceptable %.2f", chunk.ID, speed, cfg.SpeedAccept))
+ }
+ if speed > cfg.SpeedMax {
+ report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d speed %.2f exceeds max %.2f", chunk.ID, speed, cfg.SpeedMax))
+ }
+ appliedSpeed := speed
+ if appliedSpeed < cfg.SpeedMin {
+ appliedSpeed = cfg.SpeedMin
+ }
+
+ cursor := chunk.Start
+ for _, idx := range chunk.Items {
+ duration := fitted[idx].ActualDuration
+ if appliedSpeed > 0 {
+ duration = duration / appliedSpeed
+ }
+ fitted[idx].NewStart = cursor
+ fitted[idx].NewEnd = cursor + duration
+ fitted[idx].SpeedFactor = appliedSpeed
+ cursor = fitted[idx].NewEnd
+ }
+
+ if cursor > chunk.End+0.6 {
+ report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d overflows by %.2fs", chunk.ID, cursor-chunk.End))
+ }
+ }
+
+ return fitted, report, nil
+}
diff --git a/internal/service/dubbing/fit_test.go b/internal/service/dubbing/fit_test.go
new file mode 100644
index 00000000..a9755e11
--- /dev/null
+++ b/internal/service/dubbing/fit_test.go
@@ -0,0 +1,22 @@
+package dubbing
+
+import "testing"
+
+func TestFitTimelineProducesMonotonicTimesAndChunkSpeed(t *testing.T) {
+ cfg := DefaultConfig()
+ plan := []PlanItem{
+ {Index: 1, OriginalStart: 0, OriginalEnd: 1, SpokenText: "一", ActualDuration: 0.8, ChunkID: 1},
+ {Index: 2, OriginalStart: 1.1, OriginalEnd: 2, SpokenText: "二", ActualDuration: 0.8, ChunkID: 1},
+ }
+ chunks := []Chunk{{ID: 1, Items: []int{0, 1}, Start: 0, End: 2.5}}
+ got, report, err := FitTimeline(plan, chunks, cfg)
+ if err != nil {
+ t.Fatalf("FitTimeline() error = %v", err)
+ }
+ if got[0].NewStart != 0 || got[1].NewStart < got[0].NewEnd {
+ t.Fatalf("timeline overlaps: %+v", got)
+ }
+ if report.MaxSpeedFactor <= 0 {
+ t.Fatalf("MaxSpeedFactor not set: %+v", report)
+ }
+}
diff --git a/internal/service/dubbing/planner.go b/internal/service/dubbing/planner.go
new file mode 100644
index 00000000..7357c37e
--- /dev/null
+++ b/internal/service/dubbing/planner.go
@@ -0,0 +1,107 @@
+package dubbing
+
+import (
+ "context"
+
+ "krillin-ai/internal/types"
+)
+
+type Planner struct {
+ cfg Config
+ estimator DurationEstimator
+ optimizer TextOptimizer
+}
+
+func NewPlanner(cfg Config, estimator DurationEstimator, optimizer TextOptimizer) *Planner {
+ if cfg.MaxChunkSize <= 0 {
+ cfg = DefaultConfig()
+ }
+ if estimator == nil {
+ estimator = NewStatisticalEstimator()
+ }
+ return &Planner{
+ cfg: cfg,
+ estimator: estimator,
+ optimizer: optimizer,
+ }
+}
+
+func (p *Planner) Plan(cues []Cue, language types.StandardLanguageCode) ([]PlanItem, []Chunk, error) {
+ if len(cues) == 0 {
+ return nil, nil, nil
+ }
+
+ plan := make([]PlanItem, len(cues))
+ for i, cue := range cues {
+ clean := CleanTextForSpeech(cue.Text)
+ estimate, confidence, err := p.estimator.Estimate(clean, language)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ spoken := clean
+ rewriteAttempts := 0
+ available := cue.Duration() + p.cfg.GapTolerance
+ if p.cfg.EnableTextRewrite && estimate > available && p.optimizer != nil {
+ optimized, err := p.optimizer.Optimize(context.Background(), clean, available, "estimated_too_long")
+ if err == nil && optimized != "" {
+ spoken = optimized
+ rewriteAttempts = 1
+ }
+ }
+
+ plan[i] = PlanItem{
+ Index: cue.Index,
+ OriginalStart: cue.Start,
+ OriginalEnd: cue.End,
+ OriginalText: cue.Text,
+ CleanText: clean,
+ SpokenText: spoken,
+ EstimatedDuration: estimate,
+ EstimateConfidence: confidence,
+ RewriteAttempts: rewriteAttempts,
+ }
+ }
+
+ chunks := p.makeChunks(cues, plan)
+ for _, chunk := range chunks {
+ for _, item := range chunk.Items {
+ plan[item].ChunkID = chunk.ID
+ }
+ }
+
+ return plan, chunks, nil
+}
+
+func (p *Planner) makeChunks(cues []Cue, plan []PlanItem) []Chunk {
+ if len(cues) == 0 || len(plan) == 0 {
+ return nil
+ }
+
+ chunks := make([]Chunk, 0, len(cues))
+ current := Chunk{ID: 1, Start: cues[0].Start}
+
+ for i, cue := range cues {
+ if len(current.Items) == 0 {
+ current.Start = cue.Start
+ current.End = cue.End
+ current.Items = append(current.Items, i)
+ continue
+ }
+
+ prev := cues[i-1]
+ gap := cue.Start - prev.End
+ mustSplit := len(current.Items) >= p.cfg.MaxChunkSize || gap > p.cfg.GapTolerance
+ if mustSplit {
+ chunks = append(chunks, current)
+ current = Chunk{ID: len(chunks) + 1, Start: cue.Start, End: cue.End, Items: []int{i}}
+ continue
+ }
+
+ current.Items = append(current.Items, i)
+ current.End = cue.End
+ }
+
+ chunks = append(chunks, current)
+ return chunks
+}
diff --git a/internal/service/dubbing/planner_test.go b/internal/service/dubbing/planner_test.go
new file mode 100644
index 00000000..31414806
--- /dev/null
+++ b/internal/service/dubbing/planner_test.go
@@ -0,0 +1,29 @@
+package dubbing
+
+import (
+ "krillin-ai/internal/types"
+ "testing"
+)
+
+func TestPlannerMergesShortAdjacentCues(t *testing.T) {
+ cfg := DefaultConfig()
+ cues := []Cue{
+ {Index: 1, Start: 0, End: 0.8, Text: "你好"},
+ {Index: 2, Start: 1.0, End: 2.2, Text: "我们开始吧"},
+ {Index: 3, Start: 5.0, End: 6.0, Text: "下一段"},
+ }
+ planner := NewPlanner(cfg, NewStatisticalEstimator(), nil)
+ plan, chunks, err := planner.Plan(cues, types.LanguageNameSimplifiedChinese)
+ if err != nil {
+ t.Fatalf("Plan() error = %v", err)
+ }
+ if len(plan) != 3 || len(chunks) != 2 {
+ t.Fatalf("plan=%+v chunks=%+v", plan, chunks)
+ }
+ if plan[0].ChunkID != plan[1].ChunkID {
+ t.Fatalf("first short cue should merge with second: %+v", plan)
+ }
+ if plan[2].ChunkID == plan[1].ChunkID {
+ t.Fatalf("large gap should start a new chunk: %+v", plan)
+ }
+}