Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

fix: harden dubbing estimator and optimizer

wulienc1f6b235fa2f
diff --git a/internal/service/dubbing/estimator.go b/internal/service/dubbing/estimator.go
index b3748d71..e5a915f6 100644
--- a/internal/service/dubbing/estimator.go
+++ b/internal/service/dubbing/estimator.go
@@ -21,21 +21,20 @@ type CalibratingEstimator interface {
type speechRateProfile struct {
runePerSecond float64
confidence float64
- spaceDiscount float64
pauseWeight float64
numberWeight float64
acronymWeight float64
}
var speechProfiles = map[types.StandardLanguageCode]speechRateProfile{
- types.LanguageNameSimplifiedChinese: {runePerSecond: 4.2, confidence: 0.95, spaceDiscount: 0.6, pauseWeight: 0.30, numberWeight: 0.22, acronymWeight: 0.12},
- types.LanguageNameTraditionalChinese: {runePerSecond: 4.1, confidence: 0.95, spaceDiscount: 0.6, pauseWeight: 0.30, numberWeight: 0.22, acronymWeight: 0.12},
- types.LanguageNameJapanese: {runePerSecond: 4.0, confidence: 0.94, spaceDiscount: 0.6, pauseWeight: 0.28, numberWeight: 0.20, acronymWeight: 0.12},
- types.LanguageNameKorean: {runePerSecond: 4.3, confidence: 0.93, spaceDiscount: 0.6, pauseWeight: 0.28, numberWeight: 0.20, acronymWeight: 0.12},
- types.LanguageNameEnglish: {runePerSecond: 13.5, confidence: 0.92, spaceDiscount: 1.0, pauseWeight: 0.24, numberWeight: 0.26, acronymWeight: 0.32},
- types.LanguageNameGerman: {runePerSecond: 11.8, confidence: 0.91, spaceDiscount: 1.0, pauseWeight: 0.24, numberWeight: 0.25, acronymWeight: 0.28},
- types.LanguageNameRussian: {runePerSecond: 10.8, confidence: 0.90, spaceDiscount: 0.95, pauseWeight: 0.24, numberWeight: 0.24, acronymWeight: 0.24},
- types.LanguageNameTurkish: {runePerSecond: 12.0, confidence: 0.91, spaceDiscount: 0.98, pauseWeight: 0.24, numberWeight: 0.24, acronymWeight: 0.26},
+ types.LanguageNameSimplifiedChinese: {runePerSecond: 4.2, confidence: 0.95, pauseWeight: 0.30, numberWeight: 0.22, acronymWeight: 0.12},
+ types.LanguageNameTraditionalChinese: {runePerSecond: 4.1, confidence: 0.95, pauseWeight: 0.30, numberWeight: 0.22, acronymWeight: 0.12},
+ types.LanguageNameJapanese: {runePerSecond: 4.0, confidence: 0.94, pauseWeight: 0.28, numberWeight: 0.20, acronymWeight: 0.12},
+ types.LanguageNameKorean: {runePerSecond: 4.3, confidence: 0.93, pauseWeight: 0.28, numberWeight: 0.20, acronymWeight: 0.12},
+ types.LanguageNameEnglish: {runePerSecond: 13.5, confidence: 0.92, pauseWeight: 0.24, numberWeight: 0.26, acronymWeight: 0.32},
+ types.LanguageNameGerman: {runePerSecond: 11.8, confidence: 0.91, pauseWeight: 0.24, numberWeight: 0.25, acronymWeight: 0.28},
+ types.LanguageNameRussian: {runePerSecond: 10.8, confidence: 0.90, pauseWeight: 0.24, numberWeight: 0.24, acronymWeight: 0.24},
+ types.LanguageNameTurkish: {runePerSecond: 12.0, confidence: 0.91, pauseWeight: 0.24, numberWeight: 0.24, acronymWeight: 0.26},
}
type StatisticalEstimator struct {
@@ -78,6 +77,7 @@ func (e *StatisticalEstimator) Calibrate(language types.StandardLanguageCode, es
if target <= 0 || math.IsNaN(target) || math.IsInf(target, 0) {
return
}
+ target = math.Max(0.5, math.Min(1.5, target))
e.mu.Lock()
defer e.mu.Unlock()
@@ -148,14 +148,6 @@ func nonSpaceRuneCount(text string) int {
return count
}
-func baseDuration(text string, profile speechRateProfile) float64 {
- runeCount := nonSpaceRuneCount(text)
- if runeCount == 0 {
- return 0
- }
- return float64(runeCount) / profile.runePerSecond
-}
-
func punctuationPause(text string, profile speechRateProfile) float64 {
var pauses float64
for _, r := range text {
diff --git a/internal/service/dubbing/estimator_test.go b/internal/service/dubbing/estimator_test.go
index be04ede3..713f0b11 100644
--- a/internal/service/dubbing/estimator_test.go
+++ b/internal/service/dubbing/estimator_test.go
@@ -30,6 +30,21 @@ func TestEstimatorCalibrationAdjustsFutureEstimates(t *testing.T) {
}
}
+func TestEstimatorCalibrationClampsOutliers(t *testing.T) {
+ est := NewStatisticalEstimator()
+ before, _, _ := est.Estimate("这是一个测试句子", types.LanguageNameSimplifiedChinese)
+ est.Calibrate(types.LanguageNameSimplifiedChinese, before, before*100)
+ afterHigh, _, _ := est.Estimate("这是一个测试句子", types.LanguageNameSimplifiedChinese)
+ if afterHigh > before*1.6 {
+ t.Fatalf("outlier high calibration after = %v, before = %v", afterHigh, before)
+ }
+ est.Calibrate(types.LanguageNameSimplifiedChinese, before, before*0.01)
+ afterLow, _, _ := est.Estimate("这是一个测试句子", types.LanguageNameSimplifiedChinese)
+ if afterLow < before*0.6 {
+ t.Fatalf("outlier low calibration after = %v, before = %v", afterLow, before)
+ }
+}
+
func TestHeuristicFallbackReturnsLowConfidence(t *testing.T) {
est := NewHeuristicEstimator()
seconds, confidence, err := est.Estimate("hello world", "")
diff --git a/internal/service/dubbing/optimizer.go b/internal/service/dubbing/optimizer.go
index 1bbfe4b0..b5d47fc3 100644
--- a/internal/service/dubbing/optimizer.go
+++ b/internal/service/dubbing/optimizer.go
@@ -19,8 +19,10 @@ func (o *LLMOptimizer) Optimize(ctx context.Context, text string, availableSecon
if o == nil || o.chat == nil {
return text, nil
}
- if err := ctx.Err(); err != nil {
- return "", err
+ if ctx != nil {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
}
prompt := fmt.Sprintf(`请把下面字幕改写成更自然、更短、 更适合口播的一句话。
要求:
diff --git a/internal/service/dubbing/optimizer_test.go b/internal/service/dubbing/optimizer_test.go
index 501cbd80..0232c939 100644
--- a/internal/service/dubbing/optimizer_test.go
+++ b/internal/service/dubbing/optimizer_test.go
@@ -30,3 +30,27 @@ func TestLLMOptimizerReturnsSingleLineTrimmedText(t *testing.T) {
t.Fatalf("expected optimizer to call chat")
}
}
+
+func TestLLMOptimizerHandlesNilChatAndEmptyResponse(t *testing.T) {
+ if got, err := NewLLMOptimizer(nil).Optimize(context.Background(), "原文", 1.2, "test"); err != nil || got != "原文" {
+ t.Fatalf("nil chat Optimize() = %q, %v", got, err)
+ }
+ chat := &fakeChat{response: " \n"}
+ got, err := NewLLMOptimizer(chat).Optimize(context.Background(), "原文", 1.2, "test")
+ if err != nil || got != "原文" {
+ t.Fatalf("empty response Optimize() = %q, %v", got, err)
+ }
+}
+
+func TestLLMOptimizerHandlesNilAndCancelledContext(t *testing.T) {
+ chat := &fakeChat{response: "改写"}
+ got, err := NewLLMOptimizer(chat).Optimize(nil, "原文", 1.2, "test")
+ if err != nil || got != "改写" {
+ t.Fatalf("nil context Optimize() = %q, %v", got, err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if _, err := NewLLMOptimizer(chat).Optimize(ctx, "原文", 1.2, "test"); err == nil {
+ t.Fatalf("cancelled context should return error")
+ }
+}