Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat: add dubbing estimator and optimizer

wulienbb29d779cdc3
diff --git a/internal/service/dubbing/estimator.go b/internal/service/dubbing/estimator.go
new file mode 100644
index 00000000..b3748d71
--- /dev/null
+++ b/internal/service/dubbing/estimator.go
@@ -0,0 +1,201 @@
+package dubbing
+
+import (
+ "math"
+ "strings"
+ "sync"
+ "unicode"
+
+ "krillin-ai/internal/types"
+)
+
+type DurationEstimator interface {
+ Estimate(text string, language types.StandardLanguageCode) (float64, float64, error)
+}
+
+type CalibratingEstimator interface {
+ DurationEstimator
+ Calibrate(language types.StandardLanguageCode, estimatedSeconds, actualSeconds float64)
+}
+
+type speechRateProfile struct {
+ runePerSecond float64
+ confidence float64
+ spaceDiscount float64
+ pauseWeight float64
+ numberWeight float64
+ acronymWeight float64
+}
+
+var speechProfiles = map[types.StandardLanguageCode]speechRateProfile{
+ types.LanguageNameSimplifiedChinese: {runePerSecond: 4.2, confidence: 0.95, spaceDiscount: 0.6, pauseWeight: 0.30, numberWeight: 0.22, acronymWeight: 0.12},
+ types.LanguageNameTraditionalChinese: {runePerSecond: 4.1, confidence: 0.95, spaceDiscount: 0.6, pauseWeight: 0.30, numberWeight: 0.22, acronymWeight: 0.12},
+ types.LanguageNameJapanese: {runePerSecond: 4.0, confidence: 0.94, spaceDiscount: 0.6, pauseWeight: 0.28, numberWeight: 0.20, acronymWeight: 0.12},
+ types.LanguageNameKorean: {runePerSecond: 4.3, confidence: 0.93, spaceDiscount: 0.6, pauseWeight: 0.28, numberWeight: 0.20, acronymWeight: 0.12},
+ types.LanguageNameEnglish: {runePerSecond: 13.5, confidence: 0.92, spaceDiscount: 1.0, pauseWeight: 0.24, numberWeight: 0.26, acronymWeight: 0.32},
+ types.LanguageNameGerman: {runePerSecond: 11.8, confidence: 0.91, spaceDiscount: 1.0, pauseWeight: 0.24, numberWeight: 0.25, acronymWeight: 0.28},
+ types.LanguageNameRussian: {runePerSecond: 10.8, confidence: 0.90, spaceDiscount: 0.95, pauseWeight: 0.24, numberWeight: 0.24, acronymWeight: 0.24},
+ types.LanguageNameTurkish: {runePerSecond: 12.0, confidence: 0.91, spaceDiscount: 0.98, pauseWeight: 0.24, numberWeight: 0.24, acronymWeight: 0.26},
+}
+
+type StatisticalEstimator struct {
+ mu sync.RWMutex
+ calibration map[types.StandardLanguageCode]float64
+}
+
+func NewStatisticalEstimator() *StatisticalEstimator {
+ return &StatisticalEstimator{
+ calibration: map[types.StandardLanguageCode]float64{},
+ }
+}
+
+func (e *StatisticalEstimator) Estimate(text string, language types.StandardLanguageCode) (float64, float64, error) {
+ profile, ok := speechProfiles[language]
+ if !ok {
+ return NewHeuristicEstimator().Estimate(text, language)
+ }
+
+ runeCount := nonSpaceRuneCount(text)
+ if runeCount == 0 {
+ return 0, profile.confidence, nil
+ }
+
+ baseDuration := float64(runeCount) / profile.runePerSecond
+ duration := baseDuration + punctuationPause(text, profile) + numberPenalty(text, profile) + acronymPenalty(text, profile)
+ duration *= e.calibrationFactor(language)
+ if duration < 0 {
+ duration = 0
+ }
+ return duration, profile.confidence, nil
+}
+
+func (e *StatisticalEstimator) Calibrate(language types.StandardLanguageCode, estimatedSeconds, actualSeconds float64) {
+ if estimatedSeconds <= 0 || actualSeconds <= 0 {
+ return
+ }
+
+ target := actualSeconds / estimatedSeconds
+ if target <= 0 || math.IsNaN(target) || math.IsInf(target, 0) {
+ return
+ }
+
+ e.mu.Lock()
+ defer e.mu.Unlock()
+
+ current := e.calibration[language]
+ if current == 0 {
+ current = 1.0
+ }
+ e.calibration[language] = current*0.7 + target*0.3
+}
+
+func (e *StatisticalEstimator) calibrationFactor(language types.StandardLanguageCode) float64 {
+ e.mu.RLock()
+ defer e.mu.RUnlock()
+
+ if factor, ok := e.calibration[language]; ok && factor > 0 {
+ return factor
+ }
+ return 1.0
+}
+
+type HeuristicEstimator struct{}
+
+func NewHeuristicEstimator() *HeuristicEstimator {
+ return &HeuristicEstimator{}
+}
+
+func (e *HeuristicEstimator) Estimate(text string, language types.StandardLanguageCode) (float64, float64, error) {
+ runeCount := nonSpaceRuneCount(text)
+ if runeCount == 0 {
+ return 0, 0.5, nil
+ }
+
+ wordCount := len(strings.Fields(text))
+ letterCount := 0
+ digitCount := 0
+ for _, r := range text {
+ switch {
+ case unicode.IsDigit(r):
+ digitCount++
+ case unicode.IsLetter(r):
+ letterCount++
+ }
+ }
+
+ baseDuration := float64(runeCount) / 8.5
+ if wordCount > 0 {
+ baseDuration += float64(wordCount) * 0.12
+ }
+ baseDuration += float64(digitCount) * 0.08
+ baseDuration += float64(letterCount) * 0.01
+ baseDuration += punctuationPause(text, speechRateProfile{pauseWeight: 0.18, numberWeight: 0.10, acronymWeight: 0.10})
+ baseDuration += numberPenalty(text, speechRateProfile{pauseWeight: 0.18, numberWeight: 0.10, acronymWeight: 0.10})
+ baseDuration += acronymPenalty(text, speechRateProfile{pauseWeight: 0.18, numberWeight: 0.10, acronymWeight: 0.10})
+ if baseDuration < 0.2 {
+ baseDuration = 0.2
+ }
+ return baseDuration, 0.5, nil
+}
+
+func nonSpaceRuneCount(text string) int {
+ count := 0
+ for _, r := range text {
+ if !unicode.IsSpace(r) {
+ count++
+ }
+ }
+ return count
+}
+
+func baseDuration(text string, profile speechRateProfile) float64 {
+ runeCount := nonSpaceRuneCount(text)
+ if runeCount == 0 {
+ return 0
+ }
+ return float64(runeCount) / profile.runePerSecond
+}
+
+func punctuationPause(text string, profile speechRateProfile) float64 {
+ var pauses float64
+ for _, r := range text {
+ switch r {
+ case ',', ',', '、', ';', ';', ':', ':':
+ pauses += 0.22 * profile.pauseWeight
+ case '.', '。', '!', '!', '?', '?':
+ pauses += 0.28 * profile.pauseWeight
+ case '…', '—', '~':
+ pauses += 0.34 * profile.pauseWeight
+ }
+ }
+ return pauses
+}
+
+func numberPenalty(text string, profile speechRateProfile) float64 {
+ var count int
+ for _, r := range text {
+ if unicode.IsDigit(r) {
+ count++
+ }
+ }
+ return float64(count) * 0.12 * profile.numberWeight
+}
+
+func acronymPenalty(text string, profile speechRateProfile) float64 {
+ var penalty float64
+ var run int
+ for _, r := range text {
+ if unicode.IsUpper(r) {
+ run++
+ continue
+ }
+ if run >= 2 {
+ penalty += float64(run) * 0.18 * profile.acronymWeight
+ }
+ run = 0
+ }
+ if run >= 2 {
+ penalty += float64(run) * 0.18 * profile.acronymWeight
+ }
+ return penalty
+}
diff --git a/internal/service/dubbing/estimator_test.go b/internal/service/dubbing/estimator_test.go
new file mode 100644
index 00000000..be04ede3
--- /dev/null
+++ b/internal/service/dubbing/estimator_test.go
@@ -0,0 +1,42 @@
+package dubbing
+
+import (
+ "krillin-ai/internal/types"
+ "testing"
+)
+
+func TestStatisticalEstimatorAddsPunctuationAndNumberPenalty(t *testing.T) {
+ est := NewStatisticalEstimator()
+ plain, _, err := est.Estimate("你好世界", types.LanguageNameSimplifiedChinese)
+ if err != nil {
+ t.Fatal(err)
+ }
+ withPause, _, err := est.Estimate("你好,世界。2026", types.LanguageNameSimplifiedChinese)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if withPause <= plain {
+ t.Fatalf("withPause = %v, plain = %v; want punctuation and number to add duration", withPause, plain)
+ }
+}
+
+func TestEstimatorCalibrationAdjustsFutureEstimates(t *testing.T) {
+ est := NewStatisticalEstimator()
+ before, _, _ := est.Estimate("这是一个测试句子", types.LanguageNameSimplifiedChinese)
+ est.Calibrate(types.LanguageNameSimplifiedChinese, before, before*1.5)
+ after, _, _ := est.Estimate("这是一个测试句子", types.LanguageNameSimplifiedChinese)
+ if after <= before {
+ t.Fatalf("after = %v, before = %v; want calibration to increase estimate", after, before)
+ }
+}
+
+func TestHeuristicFallbackReturnsLowConfidence(t *testing.T) {
+ est := NewHeuristicEstimator()
+ seconds, confidence, err := est.Estimate("hello world", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if seconds <= 0 || confidence >= 0.7 {
+ t.Fatalf("seconds=%v confidence=%v", seconds, confidence)
+ }
+}
diff --git a/internal/service/dubbing/optimizer.go b/internal/service/dubbing/optimizer.go
new file mode 100644
index 00000000..1bbfe4b0
--- /dev/null
+++ b/internal/service/dubbing/optimizer.go
@@ -0,0 +1,47 @@
+package dubbing
+
+import (
+ "context"
+ "fmt"
+ "krillin-ai/internal/types"
+ "strings"
+)
+
+type LLMOptimizer struct {
+ chat types.ChatCompleter
+}
+
+func NewLLMOptimizer(chat types.ChatCompleter) *LLMOptimizer {
+ return &LLMOptimizer{chat: chat}
+}
+
+func (o *LLMOptimizer) Optimize(ctx context.Context, text string, availableSeconds float64, reason string) (string, error) {
+ if o == nil || o.chat == nil {
+ return text, nil
+ }
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ prompt := fmt.Sprintf(`请把下面字幕改写成更自然、更短、 更适合口播的一句话。
+要求:
+1. 保留核心含义,不添加新事实。
+2. 输出目标语言文本,不要解释。
+3. 输出单行纯文本。
+4. 尽量适合 %.1f 秒内自然朗读。
+触发原因:%s
+
+字幕:
+%s`, availableSeconds, reason, text)
+ resp, err := o.chat.ChatCompletion(prompt)
+ if err != nil {
+ return "", err
+ }
+ resp = strings.TrimSpace(resp)
+ resp = strings.ReplaceAll(resp, "\r", " ")
+ resp = strings.ReplaceAll(resp, "\n", " ")
+ resp = strings.Join(strings.Fields(resp), " ")
+ if resp == "" {
+ return text, nil
+ }
+ return resp, nil
+}
diff --git a/internal/service/dubbing/optimizer_test.go b/internal/service/dubbing/optimizer_test.go
new file mode 100644
index 00000000..501cbd80
--- /dev/null
+++ b/internal/service/dubbing/optimizer_test.go
@@ -0,0 +1,32 @@
+package dubbing
+
+import (
+ "context"
+ "testing"
+)
+
+type fakeChat struct {
+ response string
+ query string
+ err error
+}
+
+func (f *fakeChat) ChatCompletion(query string) (string, error) {
+ f.query = query
+ return f.response, f.err
+}
+
+func TestLLMOptimizerReturnsSingleLineTrimmedText(t *testing.T) {
+ chat := &fakeChat{response: " 更自然的说法\n"}
+ opt := NewLLMOptimizer(chat)
+ got, err := opt.Optimize(context.Background(), "字幕腔文本", 2.5, "estimated_too_long")
+ if err != nil {
+ t.Fatalf("Optimize() error = %v", err)
+ }
+ if got != "更自然的说法" {
+ t.Fatalf("Optimize() = %q", got)
+ }
+ if chat.query == "" {
+ t.Fatalf("expected optimizer to call chat")
+ }
+}