Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: plan and fit dubbing timeline
wulienc1eac36a09ad
diff --git a/internal/service/dubbing/fit.go b/internal/service/dubbing/fit.gonew file mode 100644index 00000000..9de6dad4--- /dev/null+++ b/internal/service/dubbing/fit.go@@ -0,0 +1,61 @@+package dubbing++import (+ "fmt"+)++func FitTimeline(plan []PlanItem, chunks []Chunk, cfg Config) ([]PlanItem, Report, error) {+ fitted := append([]PlanItem(nil), plan...)+ report := Report{}++ for _, chunk := range chunks {+ available := chunk.End - chunk.Start+ if available <= 0 {+ return nil, report, fmt.Errorf("chunk %d has non-positive duration: %.3f", chunk.ID, available)+ }++ actual := 0.0+ for _, idx := range chunk.Items {+ if idx < 0 || idx >= len(fitted) {+ return nil, report, fmt.Errorf("chunk %d references plan item %d out of range", chunk.ID, idx)+ }+ actual += fitted[idx].ActualDuration+ }++ speed := 1.0+ if actual > available {+ speed = actual / available+ }+ if speed > report.MaxSpeedFactor {+ report.MaxSpeedFactor = speed+ }+ if speed > cfg.SpeedAccept {+ report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d speed %.2f exceeds acceptable %.2f", chunk.ID, speed, cfg.SpeedAccept))+ }+ if speed > cfg.SpeedMax {+ report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d speed %.2f exceeds max %.2f", chunk.ID, speed, cfg.SpeedMax))+ }+ appliedSpeed := speed+ if appliedSpeed < cfg.SpeedMin {+ appliedSpeed = cfg.SpeedMin+ }++ cursor := chunk.Start+ for _, idx := range chunk.Items {+ duration := fitted[idx].ActualDuration+ if appliedSpeed > 0 {+ duration = duration / appliedSpeed+ }+ fitted[idx].NewStart = cursor+ fitted[idx].NewEnd = cursor + duration+ fitted[idx].SpeedFactor = appliedSpeed+ cursor = fitted[idx].NewEnd+ }++ if cursor > chunk.End+0.6 {+ report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d overflows by %.2fs", chunk.ID, cursor-chunk.End))+ }+ }++ return fitted, report, nil+}diff --git a/internal/service/dubbing/fit_test.go b/internal/service/dubbing/fit_test.gonew file mode 100644index 00000000..a9755e11--- /dev/null+++ b/internal/service/dubbing/fit_test.go@@ -0,0 +1,22 @@+package dubbing++import "testing"++func TestFitTimelineProducesMonotonicTimesAndChunkSpeed(t *testing.T) {+ cfg := DefaultConfig()+ plan := []PlanItem{+ {Index: 1, OriginalStart: 0, OriginalEnd: 1, SpokenText: "一", ActualDuration: 0.8, ChunkID: 1},+ {Index: 2, OriginalStart: 1.1, OriginalEnd: 2, SpokenText: "二", ActualDuration: 0.8, ChunkID: 1},+ }+ chunks := []Chunk{{ID: 1, Items: []int{0, 1}, Start: 0, End: 2.5}}+ got, report, err := FitTimeline(plan, chunks, cfg)+ if err != nil {+ t.Fatalf("FitTimeline() error = %v", err)+ }+ if got[0].NewStart != 0 || got[1].NewStart < got[0].NewEnd {+ t.Fatalf("timeline overlaps: %+v", got)+ }+ if report.MaxSpeedFactor <= 0 {+ t.Fatalf("MaxSpeedFactor not set: %+v", report)+ }+}diff --git a/internal/service/dubbing/planner.go b/internal/service/dubbing/planner.gonew file mode 100644index 00000000..7357c37e--- /dev/null+++ b/internal/service/dubbing/planner.go@@ -0,0 +1,107 @@+package dubbing++import (+ "context"++ "krillin-ai/internal/types"+)++type Planner struct {+ cfg Config+ estimator DurationEstimator+ optimizer TextOptimizer+}++func NewPlanner(cfg Config, estimator DurationEstimator, optimizer TextOptimizer) *Planner {+ if cfg.MaxChunkSize <= 0 {+ cfg = DefaultConfig()+ }+ if estimator == nil {+ estimator = NewStatisticalEstimator()+ }+ return &Planner{+ cfg: cfg,+ estimator: estimator,+ optimizer: optimizer,+ }+}++func (p *Planner) Plan(cues []Cue, language types.StandardLanguageCode) ([]PlanItem, []Chunk, error) {+ if len(cues) == 0 {+ return nil, nil, nil+ }++ plan := make([]PlanItem, len(cues))+ for i, cue := range cues {+ clean := CleanTextForSpeech(cue.Text)+ estimate, confidence, err := p.estimator.Estimate(clean, language)+ if err != nil {+ return nil, nil, err+ }++ spoken := clean+ rewriteAttempts := 0+ available := cue.Duration() + p.cfg.GapTolerance+ if p.cfg.EnableTextRewrite && estimate > available && p.optimizer != nil {+ optimized, err := p.optimizer.Optimize(context.Background(), clean, available, "estimated_too_long")+ if err == nil && optimized != "" {+ spoken = optimized+ rewriteAttempts = 1+ }+ }++ plan[i] = PlanItem{+ Index: cue.Index,+ OriginalStart: cue.Start,+ OriginalEnd: cue.End,+ OriginalText: cue.Text,+ CleanText: clean,+ SpokenText: spoken,+ EstimatedDuration: estimate,+ EstimateConfidence: confidence,+ RewriteAttempts: rewriteAttempts,+ }+ }++ chunks := p.makeChunks(cues, plan)+ for _, chunk := range chunks {+ for _, item := range chunk.Items {+ plan[item].ChunkID = chunk.ID+ }+ }++ return plan, chunks, nil+}++func (p *Planner) makeChunks(cues []Cue, plan []PlanItem) []Chunk {+ if len(cues) == 0 || len(plan) == 0 {+ return nil+ }++ chunks := make([]Chunk, 0, len(cues))+ current := Chunk{ID: 1, Start: cues[0].Start}++ for i, cue := range cues {+ if len(current.Items) == 0 {+ current.Start = cue.Start+ current.End = cue.End+ current.Items = append(current.Items, i)+ continue+ }++ prev := cues[i-1]+ gap := cue.Start - prev.End+ mustSplit := len(current.Items) >= p.cfg.MaxChunkSize || gap > p.cfg.GapTolerance+ if mustSplit {+ chunks = append(chunks, current)+ current = Chunk{ID: len(chunks) + 1, Start: cue.Start, End: cue.End, Items: []int{i}}+ continue+ }++ current.Items = append(current.Items, i)+ current.End = cue.End+ }++ chunks = append(chunks, current)+ return chunks+}diff --git a/internal/service/dubbing/planner_test.go b/internal/service/dubbing/planner_test.gonew file mode 100644index 00000000..31414806--- /dev/null+++ b/internal/service/dubbing/planner_test.go@@ -0,0 +1,29 @@+package dubbing++import (+ "krillin-ai/internal/types"+ "testing"+)++func TestPlannerMergesShortAdjacentCues(t *testing.T) {+ cfg := DefaultConfig()+ cues := []Cue{+ {Index: 1, Start: 0, End: 0.8, Text: "你好"},+ {Index: 2, Start: 1.0, End: 2.2, Text: "我们开始吧"},+ {Index: 3, Start: 5.0, End: 6.0, Text: "下一段"},+ }+ planner := NewPlanner(cfg, NewStatisticalEstimator(), nil)+ plan, chunks, err := planner.Plan(cues, types.LanguageNameSimplifiedChinese)+ if err != nil {+ t.Fatalf("Plan() error = %v", err)+ }+ if len(plan) != 3 || len(chunks) != 2 {+ t.Fatalf("plan=%+v chunks=%+v", plan, chunks)+ }+ if plan[0].ChunkID != plan[1].ChunkID {+ t.Fatalf("first short cue should merge with second: %+v", plan)+ }+ if plan[2].ChunkID == plan[1].ChunkID {+ t.Fatalf("large gap should start a new chunk: %+v", plan)+ }+}