RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(token_utils): build the tiktoken encoder on first use, not at import (#18986)
MarMar Labs2f13f103a0a4
diff --git a/common/token_utils.py b/common/token_utils.pyindex b98288a1af..391f0f3b89 100644--- a/common/token_utils.py+++ b/common/token_utils.py@@ -41,8 +41,24 @@ def _ensure_tiktoken_cache() -> str:tiktoken_cache_dir = _ensure_tiktoken_cache()os.environ["TIKTOKEN_CACHE_DIR"] = tiktoken_cache_dir-# encoder = tiktoken.encoding_for_model("gpt-3.5-turbo")-encoder = tiktoken.get_encoding("cl100k_base")++# Built on first use, not at import. `tiktoken.get_encoding` reads the BPE table from+# TIKTOKEN_CACHE_DIR and downloads it from openaipublic.blob.core.windows.net when that+# cache is cold, so building it here made importing this module a network operation.+# Nothing in `rag.nlp` wants the encoder itself; it imports `num_tokens_from_string`.+# The build ran regardless, so an unreachable blob host failed any import that reached+# this file, including unit tests that never tokenize anything.+# tiktoken caches each built encoding in `tiktoken.registry.ENCODINGS`, so this+# module keeps no cache of its own.+++def get_encoder():+ """Return the cl100k_base encoder, building it on first use.++ Failures propagate: a missing BPE table is a real problem and callers must not+ silently treat it as an empty tokenization.+ """+ return tiktoken.get_encoding("cl100k_base")# Per-run token usage sink. An agent run (Canvas.run) installs a mutable dict here@@ -125,8 +141,11 @@ def usage_from_response(resp) -> dict:def num_tokens_from_string(string: str) -> int:"""Returns the number of tokens in a text string."""+ # Resolve the encoder outside the try: an unavailable BPE table must fail loudly+ # rather than be reported as a zero-token string by the guard below.+ enc = get_encoder()try:- code_list = encoder.encode(string, disallowed_special=())+ code_list = enc.encode(string, disallowed_special=())return len(code_list)except Exception:return 0@@ -182,4 +201,5 @@ def total_token_count_from_response(resp):def truncate(string: str, max_len: int) -> str:"""Returns truncated text if the length of text exceed max_len."""- return encoder.decode(encoder.encode(string, disallowed_special=())[:max_len])+ enc = get_encoder()+ return enc.decode(enc.encode(string, disallowed_special=())[:max_len])diff --git a/internal/ingestion/component/chunker/tool-py/capture_golden.py b/internal/ingestion/component/chunker/tool-py/capture_golden.pyindex 8d4a0e9c79..aedb2e4a1b 100644--- a/internal/ingestion/component/chunker/tool-py/capture_golden.py+++ b/internal/ingestion/component/chunker/tool-py/capture_golden.py@@ -37,11 +37,12 @@ that file replaces ``num_tokens_from_string`` with ``lambda text: 1`` and``naive_merge`` with a function returning ``[]``. Golden values captured underthose stubs would be systematically wrong.-``num_tokens_from_string`` swallows every exception and returns 0-(``common/token_utils.py``), so an unavailable tiktoken encoding silently turns-every token budget into "never exceeded" and collapses each case into a single-chunk. That failure looks like a legitimate baseline, so ``_assert_tokenizer_alive``-runs first and aborts the capture rather than writing a poisoned golden.+``num_tokens_from_string`` (``common/token_utils.py``) builds the tiktoken encoder+on first use and lets an unavailable encoding raise, but a failed encode of a+valid string is swallowed and reported as 0, which silently turns every token+budget into "never exceeded" and collapses each case into a single chunk. That+failure looks like a legitimate baseline, so ``_assert_tokenizer_alive`` runs+first and aborts the capture on either shape rather than writing a poisoned golden."""from __future__ import annotations@@ -125,23 +126,26 @@ def _assert_tokenizer_alive() -> None:"""Abort unless the real tiktoken encoder is loaded and counting.Every token-budget decision in ``naive_merge`` reads- ``num_tokens_from_string``. When that returns 0 the budget is never- exceeded and every case degenerates to one chunk -- a plausible-looking- baseline that is entirely an artefact of the capture environment.+ ``num_tokens_from_string``, which fails in two shapes: an unavailable BPE+ table raises out of the lazy encoder build, and a failed encode of a valid+ string is reported as 0. The raise would only crash the capture; the 0 means+ the budget is never exceeded and every case degenerates to one chunk -- a+ plausible-looking baseline that is entirely an artefact of the capture+ environment. Both abort here before any golden is written."""from rag.nlp import num_tokens_from_stringprobe = "RAGFlow chunker parity tokenizer liveness probe."- got = num_tokens_from_string(probe)+ hint = "Fix the encoding first: uv run python3 ragflow_deps/download_deps.py (provides ragflow_deps/cl100k_base.tiktoken)."+ # Two shapes to catch. common/token_utils.py builds the encoder on first use and+ # lets that failure propagate, so an unavailable BPE table raises here; a failure+ # to encode an otherwise valid string is still reported as 0.+ try:+ got = num_tokens_from_string(probe)+ except Exception as exc:+ sys.exit(f"tokenizer is dead: num_tokens_from_string() raised {exc!r}.\n{hint}")if got <= 0:- sys.exit(- "tokenizer is dead: num_tokens_from_string() returned "- f"{got} for a {len(probe)}-character probe.\n"- "common/token_utils.py returns 0 on any encoder failure, so capturing now "- "would write goldens where no token budget is ever exceeded.\n"- "Fix the encoding first: uv run python3 ragflow_deps/download_deps.py "- "(provides ragflow_deps/cl100k_base.tiktoken)."- )+ sys.exit(f"tokenizer is dead: num_tokens_from_string() returned {got} for a {len(probe)}-character probe, so capturing now would write goldens where no token budget is ever exceeded.\n{hint}")def run_case(case: dict) -> dict:diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.pyindex c212e1461d..fa3a81b3de 100644--- a/rag/flow/chunker/title_chunker/common.py+++ b/rag/flow/chunker/title_chunker/common.py@@ -120,8 +120,8 @@ class BaseTitleChunker(ABC):"""Hard-split a boundary-less run into <= cap-token pieces.Mirrors common.token_utils.truncate semantics (prefix bounded by- ``cap`` tokens). Falls back to a character prefix if the tokenizer is- unavailable so the ceiling still holds.+ ``cap`` tokens). Falls back to a character prefix if the tokenizer+ rejects the input so the ceiling still holds."""out = []rest = text or ""@@ -129,8 +129,9 @@ class BaseTitleChunker(ABC):try:head = truncate(rest, cap)except (ValueError, TypeError, UnicodeError):- # Tokenizer unavailable/failed; fall back to a character prefix- # so the ceiling still holds instead of raising.+ # Tokenizer rejected the input; fall back to a character prefix+ # so the ceiling still holds. An unavailable BPE table raises+ # from truncate and is not caught here; see _token_count.head = rest[:cap]# truncate decodes the first `cap` tokens; when that lands mid# multibyte character it emits a U+FFFD, which is not a true prefix@@ -198,9 +199,10 @@ class BaseTitleChunker(ABC):def _token_count(self, text):"""Count tokens for ``text``.- ``num_tokens_from_string`` returns 0 when the encoder is unavailable;- in that case fall back to the character count so the cap is still- enforced rather than silently skipped.+ ``num_tokens_from_string`` returns 0 when it cannot encode the input;+ fall back to the character count so the cap is still enforced. An+ unavailable BPE table raises from it, as it does from ``truncate``+ above, so the chunking task fails instead of counting characters."""n = num_tokens_from_string(text or "")if n == 0 and text:diff --git a/rag/flow/tests/test_token_chunker_delimiter.py b/rag/flow/tests/test_token_chunker_delimiter.pyindex af5d78010a..9980478532 100644--- a/rag/flow/tests/test_token_chunker_delimiter.py+++ b/rag/flow/tests/test_token_chunker_delimiter.py@@ -18,10 +18,13 @@ because of the ``""`` passed at token_chunker.py:326; forwarding``"".join(self._param.delimiters)`` to ``naive_merge`` (or at least``"\\n。;!?"``) makes it pass and aligns Python with the Go side.-Note: the test needs a live tiktoken encoder. ``common.token_utils`` returns 0 on-any encoder failure, which would make every token budget "never exceeded" and-hide the bug behind a single-chunk baseline. We skip rather than pass under a-dead tokenizer, mirroring ``capture_golden.py::_assert_tokenizer_alive``.+Note: the test needs a live tiktoken encoder. ``common.token_utils`` builds it on+first use, so an unavailable BPE table raises out of ``num_tokens_from_string``,+while a failure to encode a valid string is reported as 0. Only the zero produces+the false "budget never exceeded" baseline that would hide the bug behind a single+chunk; the raise just makes the test unable to run at all. Neither leaves a usable+token budget, so we treat both as a dead tokenizer and skip rather than pass,+mirroring ``capture_golden.py::_assert_tokenizer_alive``."""import asyncio@@ -64,13 +67,25 @@ def _make_sentences(n: int) -> list[str]:return [f"Sentence number {i}. alpha beta gamma delta epsilon zeta eta theta iota kappa" for i in range(n)]+def _tokenizer_is_alive() -> bool:+ """True only when the real encoder counts tokens.++ An unavailable BPE table raises out of ``num_tokens_from_string``; a failed+ encode of a valid string returns 0. Both mean the token budget is useless here.+ """+ try:+ return num_tokens_from_string("alive tokenizer probe sentence") > 0+ except Exception:+ return False++def test_token_chunker_token_size_mode_does_not_split_sentences():# Guard: a dead tokenizer would collapse everything to one chunk and hide# the bug. Skip instead of recording a poisoned pass.- if num_tokens_from_string("alive tokenizer probe sentence") <= 0:+ if not _tokenizer_is_alive():import pytest- pytest.skip("tiktoken encoder unavailable; num_tokens_from_string returned 0")+ pytest.skip("tiktoken encoder unavailable")sentences = _make_sentences(30)payload = "\n".join(sentences)@@ -99,10 +114,10 @@ def test_naive_merge_empty_delimiter_keeps_unit_whole():Documents the new contract: without a delimiter there is nothing to spliton, so the unit is kept whole and the model layer truncates it."""- if num_tokens_from_string("alive tokenizer probe sentence") <= 0:+ if not _tokenizer_is_alive():import pytest- pytest.skip("tiktoken encoder unavailable; num_tokens_from_string returned 0")+ pytest.skip("tiktoken encoder unavailable")sentences = _make_sentences(30)payload = "\n".join(sentences)diff --git a/rag/prompts/generator.py b/rag/prompts/generator.pyindex 450dfeb74a..383e379b49 100644--- a/rag/prompts/generator.py+++ b/rag/prompts/generator.py@@ -27,7 +27,7 @@ from common.misc_utils import hash_str2intfrom rag.nlp import rag_tokenizerfrom rag.prompts.template import load_promptfrom common.constants import TAG_FLD-from common.token_utils import encoder, num_tokens_from_string+from common.token_utils import get_encoder, num_tokens_from_stringSTOP_TOKEN = "<|STOP|>"COMPLETE_TASK = "complete_task"@@ -83,7 +83,8 @@ def message_fit_in(msg, max_length=4000):def trim_content(content, limit):limit = max(0, limit)- return encoder.decode(encoder.encode(content)[:limit])+ enc = get_encoder()+ return enc.decode(enc.encode(content)[:limit])c = count()if c < max_length:diff --git a/test/unit_test/common/test_token_utils.py b/test/unit_test/common/test_token_utils.pyindex bcfc84fe37..4f6bac9b84 100644--- a/test/unit_test/common/test_token_utils.py+++ b/test/unit_test/common/test_token_utils.py@@ -14,11 +14,7 @@# limitations under the License.#-import hashlib-import os--from common.file_utils import get_project_base_directory-from common.token_utils import num_tokens_from_string, total_token_count_from_response, truncate, encoder+from common.token_utils import get_encoder, num_tokens_from_string, total_token_count_from_response, truncateimport pytest@@ -118,12 +114,6 @@ def test_consistency():assert first_result > 0-def test_bundled_cl100k_cache_file_exists():- encoding_url = "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"- cache_path = get_project_base_directory(hashlib.sha1(encoding_url.encode()).hexdigest())- assert os.path.exists(cache_path)--class TestTotalTokenCountFromResponse:"""Test cases for total_token_count_from_response function"""@@ -229,18 +219,18 @@ class TestTruncate:original_string = "hello"result = truncate(original_string, 10)assert result == original_string- assert len(encoder.encode(result)) <= 10+ assert len(get_encoder().encode(result)) <= 10def test_string_equal_to_max_len(self):"""Test string that exactly equals max_len in tokens"""- # Create a string that encodes to exactly 5 tokens+ # max_len is the string's own token length, so truncate must return it whole.test_string = "hello world test"- encoded = encoder.encode(test_string)+ encoded = get_encoder().encode(test_string)exact_length = len(encoded)result = truncate(test_string, exact_length)assert result == test_string- assert len(encoder.encode(result)) == exact_length+ assert len(get_encoder().encode(result)) == exact_lengthdef test_string_longer_than_max_len(self):"""Test string that is longer than max_len"""@@ -248,7 +238,7 @@ class TestTruncate:max_len = 5result = truncate(long_string, max_len)- assert len(encoder.encode(result)) == max_len+ assert len(get_encoder().encode(result)) == max_lenassert result != long_stringdef test_truncation_preserves_beginning(self):@@ -257,10 +247,10 @@ class TestTruncate:max_len = 3result = truncate(test_string, max_len)- encoded_result = encoder.encode(result)+ encoded_result = get_encoder().encode(result)# The truncated result should match the beginning of the original encoding- original_encoded = encoder.encode(test_string)+ original_encoded = get_encoder().encode(test_string)assert encoded_result == original_encoded[:max_len]def test_unicode_characters(self):@@ -269,7 +259,7 @@ class TestTruncate:max_len = 4result = truncate(unicode_string, max_len)- assert len(encoder.encode(result)) == max_len+ assert len(get_encoder().encode(result)) == max_len# Should be a valid stringassert isinstance(result, str)@@ -279,7 +269,7 @@ class TestTruncate:max_len = 3result = truncate(special_string, max_len)- assert len(encoder.encode(result)) == max_len+ assert len(get_encoder().encode(result)) == max_lendef test_whitespace_string(self):"""Test truncation of whitespace-only string"""@@ -287,7 +277,7 @@ class TestTruncate:max_len = 2result = truncate(whitespace_string, max_len)- assert len(encoder.encode(result)) <= max_len+ assert len(get_encoder().encode(result)) <= max_lenassert isinstance(result, str)def test_max_len_zero(self):@@ -295,13 +285,13 @@ class TestTruncate:test_string = "hello world"result = truncate(test_string, 0)assert result == ""- assert len(encoder.encode(result)) == 0+ assert len(get_encoder().encode(result)) == 0def test_max_len_one(self):"""Test truncation with max_len = 1"""test_string = "hello world"result = truncate(test_string, 1)- assert len(encoder.encode(result)) == 1+ assert len(get_encoder().encode(result)) == 1def test_preserves_decoding_encoding_consistency(self):"""Test that truncation preserves encoding-decoding consistency"""@@ -310,7 +300,7 @@ class TestTruncate:result = truncate(test_string, max_len)# Re-encoding the result should give the same token count- re_encoded = encoder.encode(result)+ re_encoded = get_encoder().encode(result)assert len(re_encoded) == max_lendef test_multibyte_characters_truncation(self):@@ -320,7 +310,7 @@ class TestTruncate:max_len = 3result = truncate(multibyte_string, max_len)- assert len(encoder.encode(result)) == max_len+ assert len(get_encoder().encode(result)) == max_lendef test_mixed_english_chinese_text(self):"""Test truncation with mixed English and Chinese text"""@@ -328,7 +318,7 @@ class TestTruncate:max_len = 5result = truncate(mixed_string, max_len)- assert len(encoder.encode(result)) == max_len+ assert len(get_encoder().encode(result)) == max_lendef test_numbers_and_symbols(self):"""Test truncation with numbers and symbols"""@@ -336,4 +326,4 @@ class TestTruncate:max_len = 4result = truncate(number_string, max_len)- assert len(encoder.encode(result)) == max_len+ assert len(get_encoder().encode(result)) == max_lendiff --git a/test/unit_test/common/test_token_utils_lazy_encoder.py b/test/unit_test/common/test_token_utils_lazy_encoder.pynew file mode 100644index 0000000000..e776e193aa--- /dev/null+++ b/test/unit_test/common/test_token_utils_lazy_encoder.py@@ -0,0 +1,171 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import hashlib+import importlib+import json+import os+import subprocess+import sys+from pathlib import Path++import pytest+import tiktoken++import common+++def _reimport_token_utils(monkeypatch, get_encoding):+ """Import common.token_utils fresh with tiktoken.get_encoding replaced.++ Re-importing also rebinds `token_utils` as an attribute of the `common`+ package, and restoring the sys.modules entry does not undo that. Registering+ the attribute with monkeypatch restores it at teardown, so later tests that+ read `common.token_utils` do not get the module built against a fake encoder.+ """+ monkeypatch.setattr(common, "token_utils", getattr(common, "token_utils", None), raising=False)+ # Importing re-runs _ensure_tiktoken_cache, which rewrites TIKTOKEN_CACHE_DIR.+ # Register it first so teardown puts back whatever the suite was using.+ monkeypatch.setenv("TIKTOKEN_CACHE_DIR", os.environ.get("TIKTOKEN_CACHE_DIR", ""))+ monkeypatch.setattr(tiktoken, "get_encoding", get_encoding)+ monkeypatch.delitem(sys.modules, "common.token_utils", raising=False)+ return importlib.import_module("common.token_utils")+++class TestLazyEncoder:+ def test_import_does_not_build_the_encoder(self, monkeypatch):+ """Importing the module must not read or download the BPE table.++ `rag.nlp` imports `num_tokens_from_string` from here, so an import-time+ build made every module in that chain depend on reaching+ openaipublic.blob.core.windows.net, including tests that never tokenize.+ """+ calls = []+ module = _reimport_token_utils(monkeypatch, lambda name: calls.append(name))++ assert calls == []+ assert module.num_tokens_from_string is not None++ def test_get_encoder_delegates_to_tiktoken(self, monkeypatch):+ """`get_encoder()` asks tiktoken for cl100k_base on every call.++ Sharing is tiktoken's registry cache (`tiktoken.registry.ENCODINGS`), not+ this module's. Recording every requested name pins both halves: each call+ reaches tiktoken, and nothing here short-circuits the second one.+ """+ shared = tiktoken.Encoding(+ name="test",+ pat_str=r".",+ mergeable_ranks={b"a": 0},+ special_tokens={},+ )+ requested = []++ def fake_get_encoding(name):+ requested.append(name)+ return shared++ module = _reimport_token_utils(monkeypatch, fake_get_encoding)+ assert requested == []++ assert module.get_encoder() is shared+ assert module.get_encoder() is shared+ assert requested == ["cl100k_base", "cl100k_base"]++ def test_unavailable_table_is_not_reported_as_zero_tokens(self, monkeypatch):+ """A missing BPE table must fail loudly, not count as an empty string.++ `num_tokens_from_string` guards `encode` with `except Exception: return 0`.+ Building the encoder inside that guard would turn an unreachable table into+ a silent zero-token result for every caller.+ """++ def unavailable(name):+ raise RuntimeError("BPE table unavailable")++ module = _reimport_token_utils(monkeypatch, unavailable)++ with pytest.raises(RuntimeError, match="BPE table unavailable"):+ module.num_tokens_from_string("hello")+++class TestImportChain:+ def test_rag_nlp_and_rag_prompts_import_without_the_bpe_table(self):+ """Importing `rag.nlp` and `rag.prompts` must not load the BPE table.++ The tests above re-import `common.token_utils` on its own, so they would+ still pass if an import-time build came back higher up the chain, for+ example in `rag/prompts/generator.py`, which imports `get_encoder` and is+ pulled in by `rag/prompts/__init__.py`. A fresh interpreter covers the+ whole chain. The loader is the patch point: every way to build an+ encoding (`tiktoken.get_encoding`, `tiktoken.encoding_for_model`,+ `tiktoken.registry.get_encoding`) ends in+ `tiktoken.load.load_tiktoken_bpe`, which `tiktoken_ext.openai_public`+ binds on first build, so patching one entry point would let the others+ through.+ """+ code = """+import json+import tiktoken.load++requested = []+++def unreachable(tiktoken_bpe_file, expected_hash=None):+ requested.append(tiktoken_bpe_file)+ raise OSError("BPE table unreachable")+++tiktoken.load.load_tiktoken_bpe = unreachable++import rag.nlp+import rag.prompts++print(json.dumps(requested))+"""+ result = subprocess.run(+ [sys.executable, "-c", code],+ cwd=Path(__file__).resolve().parents[3],+ capture_output=True,+ text=True,+ timeout=120,+ )++ assert result.returncode == 0, result.stderr+ assert json.loads(result.stdout.strip().splitlines()[-1]) == []+++class TestTiktokenCacheOnImport:+ """The helper is only useful if importing the module runs it."""++ def test_import_seeds_the_cache_from_the_bundled_table(self, tmp_path, monkeypatch):+ """Importing must copy the bundled table and point tiktoken at it.++ Testing _ensure_tiktoken_cache alone passes even if nothing calls it,+ so drive it through a real import.+ """+ bundled = tmp_path / "ragflow_deps" / "cl100k_base.tiktoken"+ bundled.parent.mkdir()+ bundled.write_bytes(b"bundled cl100k table")+ monkeypatch.setenv("RAG_PROJECT_BASE", str(tmp_path))++ module = _reimport_token_utils(monkeypatch, lambda name: object())++ encoding_url = "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"+ cache_path = tmp_path / hashlib.sha1(encoding_url.encode()).hexdigest()+ assert cache_path.read_bytes() == b"bundled cl100k table"+ assert os.environ["TIKTOKEN_CACHE_DIR"] == str(tmp_path)+ assert module.tiktoken_cache_dir == str(tmp_path)diff --git a/test/unit_test/rag/prompts/test_generator_message_fit_in.py b/test/unit_test/rag/prompts/test_generator_message_fit_in.pyindex 01b6a35c50..7b788c2a80 100644--- a/test/unit_test/rag/prompts/test_generator_message_fit_in.py+++ b/test/unit_test/rag/prompts/test_generator_message_fit_in.py@@ -52,6 +52,7 @@ def _load_generator_module(monkeypatch):token_utils = ModuleType("common.token_utils")token_utils.encoder = _CharEncoder()+ token_utils.get_encoder = lambda: token_utils.encodertoken_utils.num_tokens_from_string = lambda text: len(text)monkeypatch.setitem(sys.modules, "common.token_utils", token_utils)@@ -82,7 +83,7 @@ def _load_generator_module(monkeypatch):def test_message_fit_in_truncates_user_message_by_system_token_budget(monkeypatch):generator = _load_generator_module(monkeypatch)monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))- monkeypatch.setattr(generator, "encoder", _CharEncoder())+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())messages = [{"role": "system", "content": "1234"},@@ -100,7 +101,7 @@ def test_message_fit_in_truncates_user_message_by_system_token_budget(monkeypatcdef test_message_fit_in_handles_zero_token_messages(monkeypatch):generator = _load_generator_module(monkeypatch)monkeypatch.setattr(generator, "num_tokens_from_string", lambda _text: 0)- monkeypatch.setattr(generator, "encoder", _CharEncoder())+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())messages = [{"role": "system", "content": ""},@@ -117,7 +118,7 @@ def test_message_fit_in_handles_zero_token_messages(monkeypatch):def test_message_fit_in_clamps_negative_slice_lengths(monkeypatch):generator = _load_generator_module(monkeypatch)monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))- monkeypatch.setattr(generator, "encoder", _CharEncoder())+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())messages = [{"role": "system", "content": "1234"},@@ -135,7 +136,7 @@ def test_message_fit_in_clamps_negative_slice_lengths(monkeypatch):def test_message_fit_in_clamps_dominant_last_message_to_budget(monkeypatch):generator = _load_generator_module(monkeypatch)monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))- monkeypatch.setattr(generator, "encoder", _CharEncoder())+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())messages = [{"role": "system", "content": "s" * 41},@@ -153,7 +154,7 @@ def test_message_fit_in_clamps_dominant_last_message_to_budget(monkeypatch):def test_message_fit_in_zero_budget_preserves_non_empty_messages(monkeypatch):generator = _load_generator_module(monkeypatch)monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))- monkeypatch.setattr(generator, "encoder", _CharEncoder())+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())system_len = 8100user_content = "User query: test"diff --git a/test/unit_test/rag/prompts/test_vision_figure_prompt.py b/test/unit_test/rag/prompts/test_vision_figure_prompt.pyindex ca544d2636..e3a7023f38 100644--- a/test/unit_test/rag/prompts/test_vision_figure_prompt.py+++ b/test/unit_test/rag/prompts/test_vision_figure_prompt.py@@ -42,6 +42,7 @@ def _load_generator(monkeypatch):token_utils = ModuleType("common.token_utils")token_utils.encoder = SimpleNamespace()+ token_utils.get_encoder = lambda: token_utils.encodertoken_utils.num_tokens_from_string = lenmonkeypatch.setitem(sys.modules, "common.token_utils", token_utils)