Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(embedding): speed up token guards without changing estimates (#5997)
bot-of-qin-ctxb218cd8058ec
diff --git a/openviking/utils/embedding_input.py b/openviking/utils/embedding_input.pyindex 07e5bddcd..eb3be56cd 100644--- a/openviking/utils/embedding_input.py+++ b/openviking/utils/embedding_input.py@@ -4,24 +4,16 @@from __future__ import annotations-import math+import reEMBEDDING_TRUNCATION_SUFFIX = "\n...(truncated for embedding)"+_CJK_RUN = re.compile("[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]+")def estimate_embedding_input_tokens(text: str) -> int:- """Estimate tokens for the raw text embedding input guard."""- if not text:- return 0- cjk_chars = sum(- 1- for char in text- if "\u4e00" <= char <= "\u9fff"- or "\u3040" <= char <= "\u30ff"- or "\uac00" <= char <= "\ud7af"- )- other_chars = len(text) - cjk_chars- return max(1, cjk_chars + math.ceil(other_chars / 4))+ """Estimate CJK=1 and all other characters=1/4 tokens, rounding up."""+ other_chars = len(text) if text.isascii() else len(_CJK_RUN.sub("", text))+ return len(text) - other_chars + (other_chars + 3) // 4def truncate_embedding_input(@@ -29,23 +21,38 @@ def truncate_embedding_input(max_tokens: int,suffix: str = EMBEDDING_TRUNCATION_SUFFIX,) -> str:- """Trim raw text before embedding using the local estimate above."""+ """Keep the original estimated-token prefix budget, then append the marker."""if not text:return textif max_tokens <= 0:return suffix.lstrip()- if estimate_embedding_input_tokens(text) <= max_tokens:+ if len(text) <= max_tokens:return text- low = 0- high = len(text)- while low < high:- mid = (low + high + 1) // 2- if estimate_embedding_input_tokens(text[:mid]) <= max_tokens:- low = mid- else:- high = mid - 1- return text[:low].rstrip() + suffix+ # Quarter-token units preserve the original estimate without rounding each+ # run. No prefix longer than 4 * max_tokens can fit, even with only ASCII.+ max_units = max_tokens * 4+ if text.isascii():+ return text if len(text) <= max_units else text[:max_units].rstrip() + suffix+ scan_end = min(len(text), max_units + 1)+ units = position = 0+ for match in _CJK_RUN.finditer(text, 0, scan_end):+ start, end = match.span()+ # Ordinary characters preceding the CJK run cost one unit each.+ if units + start - position > max_units:+ prefix_end = position + max_units - units+ return text[:prefix_end].rstrip() + suffix+ units += start - position+ if units + (end - start) * 4 > max_units:+ prefix_end = start + (max_units - units) // 4+ return text[:prefix_end].rstrip() + suffix+ units += (end - start) * 4+ position = end++ if units + scan_end - position <= max_units:+ return text+ prefix_end = position + max_units - units+ return text[:prefix_end].rstrip() + suffixdef resolve_embedding_max_input_tokens(diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex 4af2395f0..059439063 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -12,6 +12,10 @@ from openviking.parse.parsers.media.utils import ()from openviking.storage.index_action import FieldPatchfrom openviking.utils import embedding_utils+from openviking.utils.embedding_input import (+ EMBEDDING_TRUNCATION_SUFFIX,+ estimate_embedding_input_tokens,+)from openviking.utils.ingest_options import IngestOptionsfrom openviking_cli.utils.config.parser_config import ImageConfig@@ -1069,9 +1073,19 @@ async def test_vectorize_content_only_reads_embedding_input(monkeypatch):@pytest.mark.asyncio-async def test_vectorize_file_bounds_content_before_enqueue(monkeypatch):+ "content, expected, expected_tokens",+ [+ ("a1👍" * 100, "a1👍" * 26 + "a1" + EMBEDDING_TRUNCATION_SUFFIX, 28),+ ("中文あ가L、" * 100, "中文あ가L、" * 4 + "中文" + EMBEDDING_TRUNCATION_SUFFIX, 28),+ ("a1中👍" * 5, "a1中👍" * 5, 9),+ ],+ ids=["ordinary-characters", "wide-text", "within-budget"],+)+async def test_vectorize_file_bounds_content_before_enqueue(+ monkeypatch, content, expected, expected_tokens+):queue = DummyQueue()- content = " ".join(f"token-{i}" for i in range(200))fs = DummyFS(content)monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)@@ -1090,8 +1104,9 @@ async def test_vectorize_file_bounds_content_before_enqueue(monkeypatch):)assert len(queue.items) == 1- assert queue.items[0].message != content- assert queue.items[0].message.endswith("...(truncated for embedding)")+ message = queue.items[0].message+ assert message == expected+ assert estimate_embedding_input_tokens(message) == expected_tokensassert fs.read_file_calls == 1