Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(embedding): speed up token guards without changing estimates (#5997)

bot-of-qin-ctxb218cd8058ec
diff --git a/openviking/utils/embedding_input.py b/openviking/utils/embedding_input.py
index 07e5bddcd..eb3be56cd 100644
--- a/openviking/utils/embedding_input.py
+++ b/openviking/utils/embedding_input.py
@@ -4,24 +4,16 @@
from __future__ import annotations
-import math
+import re
EMBEDDING_TRUNCATION_SUFFIX = "\n...(truncated for embedding)"
+_CJK_RUN = re.compile("[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]+")
def estimate_embedding_input_tokens(text: str) -> int:
- """Estimate tokens for the raw text embedding input guard."""
- if not text:
- return 0
- cjk_chars = sum(
- 1
- for char in text
- if "\u4e00" <= char <= "\u9fff"
- or "\u3040" <= char <= "\u30ff"
- or "\uac00" <= char <= "\ud7af"
- )
- other_chars = len(text) - cjk_chars
- return max(1, cjk_chars + math.ceil(other_chars / 4))
+ """Estimate CJK=1 and all other characters=1/4 tokens, rounding up."""
+ other_chars = len(text) if text.isascii() else len(_CJK_RUN.sub("", text))
+ return len(text) - other_chars + (other_chars + 3) // 4
def truncate_embedding_input(
@@ -29,23 +21,38 @@ def truncate_embedding_input(
max_tokens: int,
suffix: str = EMBEDDING_TRUNCATION_SUFFIX,
) -> str:
- """Trim raw text before embedding using the local estimate above."""
+ """Keep the original estimated-token prefix budget, then append the marker."""
if not text:
return text
if max_tokens <= 0:
return suffix.lstrip()
- if estimate_embedding_input_tokens(text) <= max_tokens:
+ if len(text) <= max_tokens:
return text
- low = 0
- high = len(text)
- while low < high:
- mid = (low + high + 1) // 2
- if estimate_embedding_input_tokens(text[:mid]) <= max_tokens:
- low = mid
- else:
- high = mid - 1
- return text[:low].rstrip() + suffix
+ # Quarter-token units preserve the original estimate without rounding each
+ # run. No prefix longer than 4 * max_tokens can fit, even with only ASCII.
+ max_units = max_tokens * 4
+ if text.isascii():
+ return text if len(text) <= max_units else text[:max_units].rstrip() + suffix
+ scan_end = min(len(text), max_units + 1)
+ units = position = 0
+ for match in _CJK_RUN.finditer(text, 0, scan_end):
+ start, end = match.span()
+ # Ordinary characters preceding the CJK run cost one unit each.
+ if units + start - position > max_units:
+ prefix_end = position + max_units - units
+ return text[:prefix_end].rstrip() + suffix
+ units += start - position
+ if units + (end - start) * 4 > max_units:
+ prefix_end = start + (max_units - units) // 4
+ return text[:prefix_end].rstrip() + suffix
+ units += (end - start) * 4
+ position = end
+
+ if units + scan_end - position <= max_units:
+ return text
+ prefix_end = position + max_units - units
+ return text[:prefix_end].rstrip() + suffix
def resolve_embedding_max_input_tokens(
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 4af2395f0..059439063 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -12,6 +12,10 @@ from openviking.parse.parsers.media.utils import (
)
from openviking.storage.index_action import FieldPatch
from openviking.utils import embedding_utils
+from openviking.utils.embedding_input import (
+ EMBEDDING_TRUNCATION_SUFFIX,
+ estimate_embedding_input_tokens,
+)
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.utils.config.parser_config import ImageConfig
@@ -1069,9 +1073,19 @@ async def test_vectorize_content_only_reads_embedding_input(monkeypatch):
@pytest.mark.asyncio
-async def test_vectorize_file_bounds_content_before_enqueue(monkeypatch):
+ "content, expected, expected_tokens",
+ [
+ ("a1👍" * 100, "a1👍" * 26 + "a1" + EMBEDDING_TRUNCATION_SUFFIX, 28),
+ ("中文あ가L、" * 100, "中文あ가L、" * 4 + "中文" + EMBEDDING_TRUNCATION_SUFFIX, 28),
+ ("a1中👍" * 5, "a1中👍" * 5, 9),
+ ],
+ ids=["ordinary-characters", "wide-text", "within-budget"],
+)
+async def test_vectorize_file_bounds_content_before_enqueue(
+ monkeypatch, content, expected, expected_tokens
+):
queue = DummyQueue()
- content = " ".join(f"token-{i}" for i in range(200))
fs = DummyFS(content)
monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)
@@ -1090,8 +1104,9 @@ async def test_vectorize_file_bounds_content_before_enqueue(monkeypatch):
)
assert len(queue.items) == 1
- assert queue.items[0].message != content
- assert queue.items[0].message.endswith("...(truncated for embedding)")
+ message = queue.items[0].message
+ assert message == expected
+ assert estimate_embedding_input_tokens(message) == expected_tokens
assert fs.read_file_calls == 1