World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.
Python agentagentic-aiaiclaudecopilotcursorelevenlabsffmpegfluximage-generationopen-sourceopenaipythonremotionstable-diffusiontext-to-speechtext-to-videovideo-generationvideo-production
fix: keep non-ASCII words in transcript and scoring tokenizers (#776)
Eliran Geffen90409ef21bb6
diff --git a/lib/scoring.py b/lib/scoring.pyindex ef3642d4..ab7344a3 100644--- a/lib/scoring.py+++ b/lib/scoring.py@@ -150,7 +150,10 @@ _SYNONYM_CLUSTERS: list[set[str]] = [{"music", "soundtrack", "background-music", "score", "ambient"},]-_TOKEN_RE = re.compile(r"[a-z0-9](?:[a-z0-9+._-]*[a-z0-9])?")+# Letters/digits of any script (e.g. Hebrew prompts), not just ASCII.+_TOKEN_RE = re.compile(r"[^\W_](?:[\w+.-]*[^\W_])?")+# Hebrew niqqud/cantillation marks; stripped so pointed and unpointed words match.+_HEBREW_MARKS_RE = re.compile(r"[\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7]")_GENERATED_VISUAL_TERMS = {"animated","animation",@@ -194,7 +197,7 @@ _IMAGE_EDIT_TERMS = {def _tokenize_text(value: str) -> list[str]:- return _TOKEN_RE.findall((value or "").lower())+ return _TOKEN_RE.findall(_HEBREW_MARKS_RE.sub("", value or "").lower())def _expand_synonyms(words: set[str]) -> set[str]:diff --git a/tests/tools/test_unicode_tokenizers.py b/tests/tools/test_unicode_tokenizers.pynew file mode 100644index 00000000..6d289add--- /dev/null+++ b/tests/tools/test_unicode_tokenizers.py@@ -0,0 +1,78 @@+"""Tokenizers must keep non-ASCII (e.g. Hebrew) words and leave ASCII behaviour unchanged."""++from __future__ import annotations++import random+import re+import string++from lib.scoring import _tokenize_text+from tools.video.video_compose import VideoCompose++_OLD_COMPOSE_RE = re.compile(r"[^A-Za-z0-9\-' ]+")+_OLD_SCORING_RE = re.compile(r"[a-z0-9](?:[a-z0-9+._-]*[a-z0-9])?")+++def _old_compose_tokenize(text: str) -> list[str]:+ cleaned = _OLD_COMPOSE_RE.sub(" ", text.lower())+ return [t for t in cleaned.split() if t and t != "-"]+++def _random_ascii(rng: random.Random) -> str:+ return "".join(rng.choice(string.printable) for _ in range(rng.randint(0, 60)))+++def test_compose_tokenize_keeps_hebrew():+ assert VideoCompose._tokenize("שלום עולם, זה מבחן!") == ["שלום", "עולם", "זה", "מבחן"]+++def test_compose_tokenize_pointed_matches_unpointed():+ pointed = "בְּרֵאשִׁית בָּרָא אֱלֹהִים"+ assert VideoCompose._tokenize(pointed) == VideoCompose._tokenize("בראשית ברא אלהים")+++def test_compose_tokenize_hebrew_punctuation():+ # Maqaf acts as a hyphen, geresh as an apostrophe, underscore splits.+ assert VideoCompose._tokenize("בית־ספר") == ["בית-ספר"]+ assert VideoCompose._tokenize("ג׳ירפה") == VideoCompose._tokenize("ג'ירפה")+ assert VideoCompose._tokenize("foo_bar") == ["foo", "bar"]+++def test_compose_tokenize_ascii_unchanged():+ assert VideoCompose._tokenize("The many-worlds view... isn't - new!") == [+ "the", "many-worlds", "view", "isn't", "new",+ ]+ rng = random.Random(0)+ for _ in range(5000):+ text = _random_ascii(rng)+ assert VideoCompose._tokenize(text) == _old_compose_tokenize(text), text+++def test_compose_transcript_check_runs_on_hebrew(tmp_path):+ transcript = tmp_path / "t.json"+ words = ["שלום", "לכולם", "וברוכים", "הבאים"]+ transcript.write_text(+ '{"word_timestamps": [' + ",".join(f'{{"word": "{w}"}}' for w in words) + "]}",+ encoding="utf-8",+ )+ result = VideoCompose._compare_transcript_to_script(+ transcript, "שָׁלוֹם לְכוּלָּם, וּבְרוּכִים הַבָּאִים."+ )+ assert result["issues"] == []+ assert result["word_accuracy"] == 1.0+ assert result["transcript_matches_script"] is True+++def test_scoring_tokenize_keeps_hebrew():+ assert _tokenize_text("סרטון קולנועי של נוף") == ["סרטון", "קולנועי", "של", "נוף"]+ assert _tokenize_text("קוֹלְנוֹעִי") == ["קולנועי"]+++def test_scoring_tokenize_ascii_unchanged():+ assert _tokenize_text("Cinematic 4K b-roll, v2.1 + Text_To_Video") == [+ "cinematic", "4k", "b-roll", "v2.1", "text_to_video",+ ]+ rng = random.Random(1)+ for _ in range(5000):+ text = _random_ascii(rng)+ assert _tokenize_text(text) == _OLD_SCORING_RE.findall(text.lower()), textdiff --git a/tools/video/video_compose.py b/tools/video/video_compose.pyindex 4ca31d00..358b276b 100644--- a/tools/video/video_compose.py+++ b/tools/video/video_compose.py@@ -2169,9 +2169,13 @@ class VideoCompose(BaseTool):stripped, numeric-word-aware). Empty tokens dropped."""import re+ # Strip Hebrew niqqud/cantillation so pointed and unpointed text match;+ # maqaf acts as a hyphen and geresh as an apostrophe.+ text = re.sub(r"[\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7]", "", text)+ text = text.replace("\u05BE", "-").replace("\u05F3", "'")# Preserve hyphenated words as single tokens ("many-worlds" -> "many-worlds").- # Drop everything except letters, digits, hyphens, apostrophes.- cleaned = re.sub(r"[^A-Za-z0-9\-' ]+", " ", text.lower())+ # Drop everything except letters/digits of any script, hyphens, apostrophes.+ cleaned = re.sub(r"[^\w\-' ]+|_", " ", text.lower())return [t for t in cleaned.split() if t and t != "-"]@classmethod