Skip to content

World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.

fix: keep non-ASCII words in transcript and scoring tokenizers (#776)

Eliran Geffen90409ef21bb6
diff --git a/lib/scoring.py b/lib/scoring.py
index ef3642d4..ab7344a3 100644
--- a/lib/scoring.py
+++ b/lib/scoring.py
@@ -150,7 +150,10 @@ _SYNONYM_CLUSTERS: list[set[str]] = [
{"music", "soundtrack", "background-music", "score", "ambient"},
]
-_TOKEN_RE = re.compile(r"[a-z0-9](?:[a-z0-9+._-]*[a-z0-9])?")
+# Letters/digits of any script (e.g. Hebrew prompts), not just ASCII.
+_TOKEN_RE = re.compile(r"[^\W_](?:[\w+.-]*[^\W_])?")
+# Hebrew niqqud/cantillation marks; stripped so pointed and unpointed words match.
+_HEBREW_MARKS_RE = re.compile(r"[\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7]")
_GENERATED_VISUAL_TERMS = {
"animated",
"animation",
@@ -194,7 +197,7 @@ _IMAGE_EDIT_TERMS = {
def _tokenize_text(value: str) -> list[str]:
- return _TOKEN_RE.findall((value or "").lower())
+ return _TOKEN_RE.findall(_HEBREW_MARKS_RE.sub("", value or "").lower())
def _expand_synonyms(words: set[str]) -> set[str]:
diff --git a/tests/tools/test_unicode_tokenizers.py b/tests/tools/test_unicode_tokenizers.py
new file mode 100644
index 00000000..6d289add
--- /dev/null
+++ b/tests/tools/test_unicode_tokenizers.py
@@ -0,0 +1,78 @@
+"""Tokenizers must keep non-ASCII (e.g. Hebrew) words and leave ASCII behaviour unchanged."""
+
+from __future__ import annotations
+
+import random
+import re
+import string
+
+from lib.scoring import _tokenize_text
+from tools.video.video_compose import VideoCompose
+
+_OLD_COMPOSE_RE = re.compile(r"[^A-Za-z0-9\-' ]+")
+_OLD_SCORING_RE = re.compile(r"[a-z0-9](?:[a-z0-9+._-]*[a-z0-9])?")
+
+
+def _old_compose_tokenize(text: str) -> list[str]:
+ cleaned = _OLD_COMPOSE_RE.sub(" ", text.lower())
+ return [t for t in cleaned.split() if t and t != "-"]
+
+
+def _random_ascii(rng: random.Random) -> str:
+ return "".join(rng.choice(string.printable) for _ in range(rng.randint(0, 60)))
+
+
+def test_compose_tokenize_keeps_hebrew():
+ assert VideoCompose._tokenize("שלום עולם, זה מבחן!") == ["שלום", "עולם", "זה", "מבחן"]
+
+
+def test_compose_tokenize_pointed_matches_unpointed():
+ pointed = "בְּרֵאשִׁית בָּרָא אֱלֹהִים"
+ assert VideoCompose._tokenize(pointed) == VideoCompose._tokenize("בראשית ברא אלהים")
+
+
+def test_compose_tokenize_hebrew_punctuation():
+ # Maqaf acts as a hyphen, geresh as an apostrophe, underscore splits.
+ assert VideoCompose._tokenize("בית־ספר") == ["בית-ספר"]
+ assert VideoCompose._tokenize("ג׳ירפה") == VideoCompose._tokenize("ג'ירפה")
+ assert VideoCompose._tokenize("foo_bar") == ["foo", "bar"]
+
+
+def test_compose_tokenize_ascii_unchanged():
+ assert VideoCompose._tokenize("The many-worlds view... isn't - new!") == [
+ "the", "many-worlds", "view", "isn't", "new",
+ ]
+ rng = random.Random(0)
+ for _ in range(5000):
+ text = _random_ascii(rng)
+ assert VideoCompose._tokenize(text) == _old_compose_tokenize(text), text
+
+
+def test_compose_transcript_check_runs_on_hebrew(tmp_path):
+ transcript = tmp_path / "t.json"
+ words = ["שלום", "לכולם", "וברוכים", "הבאים"]
+ transcript.write_text(
+ '{"word_timestamps": [' + ",".join(f'{{"word": "{w}"}}' for w in words) + "]}",
+ encoding="utf-8",
+ )
+ result = VideoCompose._compare_transcript_to_script(
+ transcript, "שָׁלוֹם לְכוּלָּם, וּבְרוּכִים הַבָּאִים."
+ )
+ assert result["issues"] == []
+ assert result["word_accuracy"] == 1.0
+ assert result["transcript_matches_script"] is True
+
+
+def test_scoring_tokenize_keeps_hebrew():
+ assert _tokenize_text("סרטון קולנועי של נוף") == ["סרטון", "קולנועי", "של", "נוף"]
+ assert _tokenize_text("קוֹלְנוֹעִי") == ["קולנועי"]
+
+
+def test_scoring_tokenize_ascii_unchanged():
+ assert _tokenize_text("Cinematic 4K b-roll, v2.1 + Text_To_Video") == [
+ "cinematic", "4k", "b-roll", "v2.1", "text_to_video",
+ ]
+ rng = random.Random(1)
+ for _ in range(5000):
+ text = _random_ascii(rng)
+ assert _tokenize_text(text) == _OLD_SCORING_RE.findall(text.lower()), text
diff --git a/tools/video/video_compose.py b/tools/video/video_compose.py
index 4ca31d00..358b276b 100644
--- a/tools/video/video_compose.py
+++ b/tools/video/video_compose.py
@@ -2169,9 +2169,13 @@ class VideoCompose(BaseTool):
stripped, numeric-word-aware). Empty tokens dropped."""
import re
+ # Strip Hebrew niqqud/cantillation so pointed and unpointed text match;
+ # maqaf acts as a hyphen and geresh as an apostrophe.
+ text = re.sub(r"[\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7]", "", text)
+ text = text.replace("\u05BE", "-").replace("\u05F3", "'")
# Preserve hyphenated words as single tokens ("many-worlds" -> "many-worlds").
- # Drop everything except letters, digits, hyphens, apostrophes.
- cleaned = re.sub(r"[^A-Za-z0-9\-' ]+", " ", text.lower())
+ # Drop everything except letters/digits of any script, hyphens, apostrophes.
+ cleaned = re.sub(r"[^\w\-' ]+|_", " ", text.lower())
return [t for t in cleaned.split() if t and t != "-"]
@classmethod