Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat: add dubbing config and model
wulien926f2c74de05
diff --git a/config/config-example.toml b/config/config-example.tomlindex 8a1bde11..7394f6b0 100644--- a/config/config-example.toml+++ b/config/config-example.toml@@ -62,6 +62,17 @@access_key_secret = ""app_key= ""+[dubbing]+ min_subtitle_duration = 2.5 # 最短配音字幕时长,短句会优先合并+ max_chunk_size = 5 # 单个配音 chunk 最多合并字幕条数+ gap_tolerance = 1.5 # 可吸收的相邻字幕空隙,单位秒+ speed_min = 0.95 # 允许的最慢调速倍率+ speed_accept = 1.15 # 推荐的最大自然调速倍率+ speed_max = 1.30 # 硬上限,超过后优先改写文本+ enable_text_rewrite = true # 是否允许 LLM 改写为自然口播+ rewrite_max_attempts = 2 # 单条字幕最多改写次数+ estimator = "statistical" # 估时器,当前支持 statistical+[image]provider = "openai-compatible"[image.openai]diff --git a/config/config.go b/config/config.goindex af99ec22..9bce3604 100644--- a/config/config.go+++ b/config/config.go@@ -83,6 +83,18 @@ type Tts struct {Aliyun AliyunTtsConfig `toml:"aliyun"`}+type Dubbing struct {+ MinSubtitleDuration float64 `toml:"min_subtitle_duration"`+ MaxChunkSize int `toml:"max_chunk_size"`+ GapTolerance float64 `toml:"gap_tolerance"`+ SpeedMin float64 `toml:"speed_min"`+ SpeedAccept float64 `toml:"speed_accept"`+ SpeedMax float64 `toml:"speed_max"`+ EnableTextRewrite bool `toml:"enable_text_rewrite"`+ RewriteMaxAttempts int `toml:"rewrite_max_attempts"`+ Estimator string `toml:"estimator"`+}+type Image struct {Provider string `toml:"provider"`Openai OpenaiCompatibleConfig `toml:"openai"`@@ -99,6 +111,7 @@ type Config struct {Llm OpenaiCompatibleConfig `toml:"llm"`Transcribe Transcribe `toml:"transcribe"`Tts Tts `toml:"tts"`+ Dubbing Dubbing `toml:"dubbing"`Image Image `toml:"image"`}@@ -142,6 +155,17 @@ var Conf = Config{Model: "gpt-4o-mini-tts",},},+ Dubbing: Dubbing{+ MinSubtitleDuration: 2.5,+ MaxChunkSize: 5,+ GapTolerance: 1.5,+ SpeedMin: 0.95,+ SpeedAccept: 1.15,+ SpeedMax: 1.30,+ EnableTextRewrite: true,+ RewriteMaxAttempts: 2,+ Estimator: "statistical",+ },Image: Image{Provider: "openai-compatible",Openai: OpenaiCompatibleConfig{diff --git a/config/config_test.go b/config/config_test.goindex 8fbd9f1d..40e1f59f 100644--- a/config/config_test.go+++ b/config/config_test.go@@ -10,3 +10,27 @@ func TestDefaultImageConfig(t *testing.T) {t.Fatalf("Image.Openai.Model is empty")}}++func TestDefaultDubbingConfig(t *testing.T) {+ if Conf.Dubbing.MinSubtitleDuration != 2.5 {+ t.Fatalf("MinSubtitleDuration = %v, want 2.5", Conf.Dubbing.MinSubtitleDuration)+ }+ if Conf.Dubbing.MaxChunkSize != 5 {+ t.Fatalf("MaxChunkSize = %d, want 5", Conf.Dubbing.MaxChunkSize)+ }+ if Conf.Dubbing.GapTolerance != 1.5 {+ t.Fatalf("GapTolerance = %v, want 1.5", Conf.Dubbing.GapTolerance)+ }+ if Conf.Dubbing.SpeedMin != 0.95 || Conf.Dubbing.SpeedAccept != 1.15 || Conf.Dubbing.SpeedMax != 1.30 {+ t.Fatalf("speed config = %+v", Conf.Dubbing)+ }+ if !Conf.Dubbing.EnableTextRewrite {+ t.Fatalf("EnableTextRewrite = false, want true")+ }+ if Conf.Dubbing.RewriteMaxAttempts != 2 {+ t.Fatalf("RewriteMaxAttempts = %d, want 2", Conf.Dubbing.RewriteMaxAttempts)+ }+ if Conf.Dubbing.Estimator != "statistical" {+ t.Fatalf("Estimator = %q, want statistical", Conf.Dubbing.Estimator)+ }+}diff --git a/internal/service/dubbing/types.go b/internal/service/dubbing/types.gonew file mode 100644index 00000000..7ad45a24--- /dev/null+++ b/internal/service/dubbing/types.go@@ -0,0 +1,105 @@+package dubbing++import (+ "context"+ "krillin-ai/internal/types"+)++const (+ DubbingDirName = "dubbing"+ DubbingInputFileName = "dubbing_input.srt"+ DubbingPlanFileName = "dubbing_plan.json"+ DubbingReportName = "dubbing_report.json"+ DubSubtitleFileName = "dub.srt"+)++type Config struct {+ MinSubtitleDuration float64+ MaxChunkSize int+ GapTolerance float64+ SpeedMin float64+ SpeedAccept float64+ SpeedMax float64+ EnableTextRewrite bool+ RewriteMaxAttempts int+ Estimator string+}++func DefaultConfig() Config {+ return Config{+ MinSubtitleDuration: 2.5,+ MaxChunkSize: 5,+ GapTolerance: 1.5,+ SpeedMin: 0.95,+ SpeedAccept: 1.15,+ SpeedMax: 1.30,+ EnableTextRewrite: true,+ RewriteMaxAttempts: 2,+ Estimator: "statistical",+ }+}++type Cue struct {+ Index int+ Start float64+ End float64+ Text string+}++func (c Cue) Duration() float64 {+ return c.End - c.Start+}++type PlanItem struct {+ Index int `json:"index"`+ OriginalStart float64 `json:"original_start"`+ OriginalEnd float64 `json:"original_end"`+ NewStart float64 `json:"new_start"`+ NewEnd float64 `json:"new_end"`+ OriginalText string `json:"original_text"`+ CleanText string `json:"clean_text"`+ SpokenText string `json:"spoken_text"`+ EstimatedDuration float64 `json:"estimated_duration"`+ EstimateConfidence float64 `json:"estimate_confidence"`+ ActualDuration float64 `json:"actual_duration"`+ SpeedFactor float64 `json:"speed_factor"`+ ChunkID int `json:"chunk_id"`+ RewriteAttempts int `json:"rewrite_attempts"`+ Warning string `json:"warning,omitempty"`+}++type Chunk struct {+ ID int+ Items []int+ Start float64+ End float64+}++type Report struct {+ Warnings []string `json:"warnings"`+ FailedIndexes []int `json:"failed_indexes"`+ MaxSpeedFactor float64 `json:"max_speed_factor"`+ RewriteCount int `json:"rewrite_count"`+}++type CommandRunner func(args []string) error+type DurationProbe func(path string) (float64, error)++type Dependencies struct {+ TTS types.Ttser+ Chat types.ChatCompleter+ Language types.StandardLanguageCode+ Voice string+ Workdir string+ InputSRT string+ InputVideo string+ OutputAudio string+ OutputVideo string+ Config Config+ FFmpeg CommandRunner+ Duration DurationProbe+}++type TextOptimizer interface {+ Optimize(ctx context.Context, text string, availableSeconds float64, reason string) (string, error)+}diff --git a/internal/service/dubbing/types_test.go b/internal/service/dubbing/types_test.gonew file mode 100644index 00000000..12258844--- /dev/null+++ b/internal/service/dubbing/types_test.go@@ -0,0 +1,23 @@+package dubbing++import "testing"++func TestDefaultConfigValues(t *testing.T) {+ cfg := DefaultConfig()+ if cfg.MinSubtitleDuration != 2.5 || cfg.MaxChunkSize != 5 || cfg.GapTolerance != 1.5 {+ t.Fatalf("DefaultConfig timing = %+v", cfg)+ }+ if cfg.SpeedMin != 0.95 || cfg.SpeedAccept != 1.15 || cfg.SpeedMax != 1.30 {+ t.Fatalf("DefaultConfig speed = %+v", cfg)+ }+ if !cfg.EnableTextRewrite || cfg.RewriteMaxAttempts != 2 || cfg.Estimator != "statistical" {+ t.Fatalf("DefaultConfig rewrite = %+v", cfg)+ }+}++func TestCueDuration(t *testing.T) {+ cue := Cue{Start: 1.25, End: 3.75}+ if cue.Duration() != 2.5 {+ t.Fatalf("Duration() = %v, want 2.5", cue.Duration())+ }+}