Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

fix: avoid unnecessary CJK translation hard splits

Leggett508c22c4bbad
diff --git a/runtime/krillinai/internal/service/paired_translation.go b/runtime/krillinai/internal/service/paired_translation.go
index bbed90ad..8bf2c2ab 100644
--- a/runtime/krillinai/internal/service/paired_translation.go
+++ b/runtime/krillinai/internal/service/paired_translation.go
@@ -103,7 +103,10 @@ type budgetedChatCompleter interface {
}
func pairedTranslationResponseBudget(text string, limit int) int {
- return 4*len(text) + (3*subtitleTargetLength(text)/max(40, limit)+1)*48
+ // UTF-8 byte length overestimates CJK input by roughly 3x and can trigger
+ // an unnecessary 800-byte semantic seam before the model sees the text.
+ sourceUnits := utf8.RuneCountInString(text)
+ return 4*sourceUnits + (3*subtitleTargetLength(text)/max(40, limit)+1)*48
}
func buildPairedTranslationBatches(texts []string, originLang, targetLang types.StandardLanguageCode, limit, batchSize int, terms ...translationTerm) ([]pairedTranslationInput, error) {
diff --git a/runtime/krillinai/internal/service/translation_input_budget_test.go b/runtime/krillinai/internal/service/translation_input_budget_test.go
index 1695cedc..8606701b 100644
--- a/runtime/krillinai/internal/service/translation_input_budget_test.go
+++ b/runtime/krillinai/internal/service/translation_input_budget_test.go
@@ -17,6 +17,16 @@ func TestTranslationInputUsesAvailableResponseBudget(t *testing.T) {
}
}
+func TestTranslationInputKeepsUnpunctuatedCJKSentenceTogether(t *testing.T) {
+ // This is approximately the shape of the reported transcript: it fits the
+ // 1600-byte input limit, but the old byte-based estimate forced an 800-byte cut.
+ text := strings.Repeat("中文无标点", 98)
+ parts, err := splitTranslationSentenceForBudget(text, translationSentenceBytes, 60)
+ if err != nil || len(parts) != 1 || parts[0] != text {
+ t.Fatalf("unpunctuated CJK sentence acquired a hard seam: parts=%d err=%v", len(parts), err)
+ }
+}
+
func TestTranslationInputCutsStillRespectAllBudgets(t *testing.T) {
for _, text := range []string{
strings.Repeat("Please check No. 35 before sending it to J. R. Smith. ", 90),