Skip to content

World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.

fix(scoring): strip trailing punctuation from tokens

Your Name32d34a7ca894
diff --git a/lib/scoring.py b/lib/scoring.py
index 618c6bb2..e1a8759b 100644
--- a/lib/scoring.py
+++ b/lib/scoring.py
@@ -147,7 +147,7 @@ _SYNONYM_CLUSTERS: list[set[str]] = [
{"music", "soundtrack", "background-music", "score", "ambient"},
]
-_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9+._-]*")
+_TOKEN_RE = re.compile(r"[a-z0-9](?:[a-z0-9+._-]*[a-z0-9])?")
_GENERATED_VISUAL_TERMS = {
"animated",
"animation",
@@ -498,7 +498,7 @@ def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore:
# lip-sync from quoted dialogue. This is what makes Seedance 2.0 (and
# peer premium APIs) meaningfully better than generic clip providers.
if asset_type == "video":
- intent_words = _expand_synonyms(set(intent.lower().split())) | set(style_keywords)
+ intent_words = _expand_synonyms(set(_tokenize_text(intent))) | set(style_keywords)
cinematic_signal = bool(
intent_words & {"cinematic", "film", "movie", "trailer", "teaser", "dramatic", "epic", "premium"}
)
diff --git a/tests/tools/test_scoring.py b/tests/tools/test_scoring.py
new file mode 100644
index 00000000..0b2072a3
--- /dev/null
+++ b/tests/tools/test_scoring.py
@@ -0,0 +1,47 @@
+"""Regression tests for provider scoring tokenization."""
+
+from __future__ import annotations
+
+from lib.scoring import _tokenize_text, score_provider
+from tools.base_tool import ToolStatus
+
+
+class _FakeVideoTool:
+ name = "fake-video"
+
+ def get_info(self) -> dict[str, object]:
+ return {
+ "name": "fake-video",
+ "provider": "fake",
+ "best_for": ["cinematic video"],
+ "supports": {
+ "native_audio": True,
+ "multi_shot": True,
+ "camera_direction": True,
+ "lip_sync": True,
+ "cinematic_quality": True,
+ },
+ "stability": "production",
+ "runtime": "api",
+ }
+
+ def get_status(self) -> ToolStatus:
+ return ToolStatus.AVAILABLE
+
+ def estimate_cost(self, inputs: dict[str, object]) -> float:
+ return 0.0
+
+
+def test_tokenize_text_strips_trailing_punctuation() -> None:
+ assert _tokenize_text("cinematic.") == ["cinematic"]
+ assert _tokenize_text("v1.5.") == ["v1.5"]
+ assert _tokenize_text("gpt-4.1") == ["gpt-4.1"]
+
+
+def test_cinematic_bonus_ignores_adjacent_punctuation() -> None:
+ tool = _FakeVideoTool()
+ plain = score_provider(tool, {"asset_type": "video", "intent": "make it cinematic and fast"})
+ punctuated = score_provider(tool, {"asset_type": "video", "intent": "make it cinematic, and fast"})
+
+ assert punctuated.task_fit == plain.task_fit
+ assert punctuated.output_quality == plain.output_quality