Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(token_utils): build the tiktoken encoder on first use, not at import (#18986)

MarMar Labs2f13f103a0a4
diff --git a/common/token_utils.py b/common/token_utils.py
index b98288a1af..391f0f3b89 100644
--- a/common/token_utils.py
+++ b/common/token_utils.py
@@ -41,8 +41,24 @@ def _ensure_tiktoken_cache() -> str:
tiktoken_cache_dir = _ensure_tiktoken_cache()
os.environ["TIKTOKEN_CACHE_DIR"] = tiktoken_cache_dir
-# encoder = tiktoken.encoding_for_model("gpt-3.5-turbo")
-encoder = tiktoken.get_encoding("cl100k_base")
+
+# Built on first use, not at import. `tiktoken.get_encoding` reads the BPE table from
+# TIKTOKEN_CACHE_DIR and downloads it from openaipublic.blob.core.windows.net when that
+# cache is cold, so building it here made importing this module a network operation.
+# Nothing in `rag.nlp` wants the encoder itself; it imports `num_tokens_from_string`.
+# The build ran regardless, so an unreachable blob host failed any import that reached
+# this file, including unit tests that never tokenize anything.
+# tiktoken caches each built encoding in `tiktoken.registry.ENCODINGS`, so this
+# module keeps no cache of its own.
+
+
+def get_encoder():
+ """Return the cl100k_base encoder, building it on first use.
+
+ Failures propagate: a missing BPE table is a real problem and callers must not
+ silently treat it as an empty tokenization.
+ """
+ return tiktoken.get_encoding("cl100k_base")
# Per-run token usage sink. An agent run (Canvas.run) installs a mutable dict here
@@ -125,8 +141,11 @@ def usage_from_response(resp) -> dict:
def num_tokens_from_string(string: str) -> int:
"""Returns the number of tokens in a text string."""
+ # Resolve the encoder outside the try: an unavailable BPE table must fail loudly
+ # rather than be reported as a zero-token string by the guard below.
+ enc = get_encoder()
try:
- code_list = encoder.encode(string, disallowed_special=())
+ code_list = enc.encode(string, disallowed_special=())
return len(code_list)
except Exception:
return 0
@@ -182,4 +201,5 @@ def total_token_count_from_response(resp):
def truncate(string: str, max_len: int) -> str:
"""Returns truncated text if the length of text exceed max_len."""
- return encoder.decode(encoder.encode(string, disallowed_special=())[:max_len])
+ enc = get_encoder()
+ return enc.decode(enc.encode(string, disallowed_special=())[:max_len])
diff --git a/internal/ingestion/component/chunker/tool-py/capture_golden.py b/internal/ingestion/component/chunker/tool-py/capture_golden.py
index 8d4a0e9c79..aedb2e4a1b 100644
--- a/internal/ingestion/component/chunker/tool-py/capture_golden.py
+++ b/internal/ingestion/component/chunker/tool-py/capture_golden.py
@@ -37,11 +37,12 @@ that file replaces ``num_tokens_from_string`` with ``lambda text: 1`` and
``naive_merge`` with a function returning ``[]``. Golden values captured under
those stubs would be systematically wrong.
-``num_tokens_from_string`` swallows every exception and returns 0
-(``common/token_utils.py``), so an unavailable tiktoken encoding silently turns
-every token budget into "never exceeded" and collapses each case into a single
-chunk. That failure looks like a legitimate baseline, so ``_assert_tokenizer_alive``
-runs first and aborts the capture rather than writing a poisoned golden.
+``num_tokens_from_string`` (``common/token_utils.py``) builds the tiktoken encoder
+on first use and lets an unavailable encoding raise, but a failed encode of a
+valid string is swallowed and reported as 0, which silently turns every token
+budget into "never exceeded" and collapses each case into a single chunk. That
+failure looks like a legitimate baseline, so ``_assert_tokenizer_alive`` runs
+first and aborts the capture on either shape rather than writing a poisoned golden.
"""
from __future__ import annotations
@@ -125,23 +126,26 @@ def _assert_tokenizer_alive() -> None:
"""Abort unless the real tiktoken encoder is loaded and counting.
Every token-budget decision in ``naive_merge`` reads
- ``num_tokens_from_string``. When that returns 0 the budget is never
- exceeded and every case degenerates to one chunk -- a plausible-looking
- baseline that is entirely an artefact of the capture environment.
+ ``num_tokens_from_string``, which fails in two shapes: an unavailable BPE
+ table raises out of the lazy encoder build, and a failed encode of a valid
+ string is reported as 0. The raise would only crash the capture; the 0 means
+ the budget is never exceeded and every case degenerates to one chunk -- a
+ plausible-looking baseline that is entirely an artefact of the capture
+ environment. Both abort here before any golden is written.
"""
from rag.nlp import num_tokens_from_string
probe = "RAGFlow chunker parity tokenizer liveness probe."
- got = num_tokens_from_string(probe)
+ hint = "Fix the encoding first: uv run python3 ragflow_deps/download_deps.py (provides ragflow_deps/cl100k_base.tiktoken)."
+ # Two shapes to catch. common/token_utils.py builds the encoder on first use and
+ # lets that failure propagate, so an unavailable BPE table raises here; a failure
+ # to encode an otherwise valid string is still reported as 0.
+ try:
+ got = num_tokens_from_string(probe)
+ except Exception as exc:
+ sys.exit(f"tokenizer is dead: num_tokens_from_string() raised {exc!r}.\n{hint}")
if got <= 0:
- sys.exit(
- "tokenizer is dead: num_tokens_from_string() returned "
- f"{got} for a {len(probe)}-character probe.\n"
- "common/token_utils.py returns 0 on any encoder failure, so capturing now "
- "would write goldens where no token budget is ever exceeded.\n"
- "Fix the encoding first: uv run python3 ragflow_deps/download_deps.py "
- "(provides ragflow_deps/cl100k_base.tiktoken)."
- )
+ sys.exit(f"tokenizer is dead: num_tokens_from_string() returned {got} for a {len(probe)}-character probe, so capturing now would write goldens where no token budget is ever exceeded.\n{hint}")
def run_case(case: dict) -> dict:
diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.py
index c212e1461d..fa3a81b3de 100644
--- a/rag/flow/chunker/title_chunker/common.py
+++ b/rag/flow/chunker/title_chunker/common.py
@@ -120,8 +120,8 @@ class BaseTitleChunker(ABC):
"""Hard-split a boundary-less run into <= cap-token pieces.
Mirrors common.token_utils.truncate semantics (prefix bounded by
- ``cap`` tokens). Falls back to a character prefix if the tokenizer is
- unavailable so the ceiling still holds.
+ ``cap`` tokens). Falls back to a character prefix if the tokenizer
+ rejects the input so the ceiling still holds.
"""
out = []
rest = text or ""
@@ -129,8 +129,9 @@ class BaseTitleChunker(ABC):
try:
head = truncate(rest, cap)
except (ValueError, TypeError, UnicodeError):
- # Tokenizer unavailable/failed; fall back to a character prefix
- # so the ceiling still holds instead of raising.
+ # Tokenizer rejected the input; fall back to a character prefix
+ # so the ceiling still holds. An unavailable BPE table raises
+ # from truncate and is not caught here; see _token_count.
head = rest[:cap]
# truncate decodes the first `cap` tokens; when that lands mid
# multibyte character it emits a U+FFFD, which is not a true prefix
@@ -198,9 +199,10 @@ class BaseTitleChunker(ABC):
def _token_count(self, text):
"""Count tokens for ``text``.
- ``num_tokens_from_string`` returns 0 when the encoder is unavailable;
- in that case fall back to the character count so the cap is still
- enforced rather than silently skipped.
+ ``num_tokens_from_string`` returns 0 when it cannot encode the input;
+ fall back to the character count so the cap is still enforced. An
+ unavailable BPE table raises from it, as it does from ``truncate``
+ above, so the chunking task fails instead of counting characters.
"""
n = num_tokens_from_string(text or "")
if n == 0 and text:
diff --git a/rag/flow/tests/test_token_chunker_delimiter.py b/rag/flow/tests/test_token_chunker_delimiter.py
index af5d78010a..9980478532 100644
--- a/rag/flow/tests/test_token_chunker_delimiter.py
+++ b/rag/flow/tests/test_token_chunker_delimiter.py
@@ -18,10 +18,13 @@ because of the ``""`` passed at token_chunker.py:326; forwarding
``"".join(self._param.delimiters)`` to ``naive_merge`` (or at least
``"\\n。;!?"``) makes it pass and aligns Python with the Go side.
-Note: the test needs a live tiktoken encoder. ``common.token_utils`` returns 0 on
-any encoder failure, which would make every token budget "never exceeded" and
-hide the bug behind a single-chunk baseline. We skip rather than pass under a
-dead tokenizer, mirroring ``capture_golden.py::_assert_tokenizer_alive``.
+Note: the test needs a live tiktoken encoder. ``common.token_utils`` builds it on
+first use, so an unavailable BPE table raises out of ``num_tokens_from_string``,
+while a failure to encode a valid string is reported as 0. Only the zero produces
+the false "budget never exceeded" baseline that would hide the bug behind a single
+chunk; the raise just makes the test unable to run at all. Neither leaves a usable
+token budget, so we treat both as a dead tokenizer and skip rather than pass,
+mirroring ``capture_golden.py::_assert_tokenizer_alive``.
"""
import asyncio
@@ -64,13 +67,25 @@ def _make_sentences(n: int) -> list[str]:
return [f"Sentence number {i}. alpha beta gamma delta epsilon zeta eta theta iota kappa" for i in range(n)]
+def _tokenizer_is_alive() -> bool:
+ """True only when the real encoder counts tokens.
+
+ An unavailable BPE table raises out of ``num_tokens_from_string``; a failed
+ encode of a valid string returns 0. Both mean the token budget is useless here.
+ """
+ try:
+ return num_tokens_from_string("alive tokenizer probe sentence") > 0
+ except Exception:
+ return False
+
+
def test_token_chunker_token_size_mode_does_not_split_sentences():
# Guard: a dead tokenizer would collapse everything to one chunk and hide
# the bug. Skip instead of recording a poisoned pass.
- if num_tokens_from_string("alive tokenizer probe sentence") <= 0:
+ if not _tokenizer_is_alive():
import pytest
- pytest.skip("tiktoken encoder unavailable; num_tokens_from_string returned 0")
+ pytest.skip("tiktoken encoder unavailable")
sentences = _make_sentences(30)
payload = "\n".join(sentences)
@@ -99,10 +114,10 @@ def test_naive_merge_empty_delimiter_keeps_unit_whole():
Documents the new contract: without a delimiter there is nothing to split
on, so the unit is kept whole and the model layer truncates it.
"""
- if num_tokens_from_string("alive tokenizer probe sentence") <= 0:
+ if not _tokenizer_is_alive():
import pytest
- pytest.skip("tiktoken encoder unavailable; num_tokens_from_string returned 0")
+ pytest.skip("tiktoken encoder unavailable")
sentences = _make_sentences(30)
payload = "\n".join(sentences)
diff --git a/rag/prompts/generator.py b/rag/prompts/generator.py
index 450dfeb74a..383e379b49 100644
--- a/rag/prompts/generator.py
+++ b/rag/prompts/generator.py
@@ -27,7 +27,7 @@ from common.misc_utils import hash_str2int
from rag.nlp import rag_tokenizer
from rag.prompts.template import load_prompt
from common.constants import TAG_FLD
-from common.token_utils import encoder, num_tokens_from_string
+from common.token_utils import get_encoder, num_tokens_from_string
STOP_TOKEN = "<|STOP|>"
COMPLETE_TASK = "complete_task"
@@ -83,7 +83,8 @@ def message_fit_in(msg, max_length=4000):
def trim_content(content, limit):
limit = max(0, limit)
- return encoder.decode(encoder.encode(content)[:limit])
+ enc = get_encoder()
+ return enc.decode(enc.encode(content)[:limit])
c = count()
if c < max_length:
diff --git a/test/unit_test/common/test_token_utils.py b/test/unit_test/common/test_token_utils.py
index bcfc84fe37..4f6bac9b84 100644
--- a/test/unit_test/common/test_token_utils.py
+++ b/test/unit_test/common/test_token_utils.py
@@ -14,11 +14,7 @@
# limitations under the License.
#
-import hashlib
-import os
-
-from common.file_utils import get_project_base_directory
-from common.token_utils import num_tokens_from_string, total_token_count_from_response, truncate, encoder
+from common.token_utils import get_encoder, num_tokens_from_string, total_token_count_from_response, truncate
import pytest
@@ -118,12 +114,6 @@ def test_consistency():
assert first_result > 0
-def test_bundled_cl100k_cache_file_exists():
- encoding_url = "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"
- cache_path = get_project_base_directory(hashlib.sha1(encoding_url.encode()).hexdigest())
- assert os.path.exists(cache_path)
-
-
class TestTotalTokenCountFromResponse:
"""Test cases for total_token_count_from_response function"""
@@ -229,18 +219,18 @@ class TestTruncate:
original_string = "hello"
result = truncate(original_string, 10)
assert result == original_string
- assert len(encoder.encode(result)) <= 10
+ assert len(get_encoder().encode(result)) <= 10
def test_string_equal_to_max_len(self):
"""Test string that exactly equals max_len in tokens"""
- # Create a string that encodes to exactly 5 tokens
+ # max_len is the string's own token length, so truncate must return it whole.
test_string = "hello world test"
- encoded = encoder.encode(test_string)
+ encoded = get_encoder().encode(test_string)
exact_length = len(encoded)
result = truncate(test_string, exact_length)
assert result == test_string
- assert len(encoder.encode(result)) == exact_length
+ assert len(get_encoder().encode(result)) == exact_length
def test_string_longer_than_max_len(self):
"""Test string that is longer than max_len"""
@@ -248,7 +238,7 @@ class TestTruncate:
max_len = 5
result = truncate(long_string, max_len)
- assert len(encoder.encode(result)) == max_len
+ assert len(get_encoder().encode(result)) == max_len
assert result != long_string
def test_truncation_preserves_beginning(self):
@@ -257,10 +247,10 @@ class TestTruncate:
max_len = 3
result = truncate(test_string, max_len)
- encoded_result = encoder.encode(result)
+ encoded_result = get_encoder().encode(result)
# The truncated result should match the beginning of the original encoding
- original_encoded = encoder.encode(test_string)
+ original_encoded = get_encoder().encode(test_string)
assert encoded_result == original_encoded[:max_len]
def test_unicode_characters(self):
@@ -269,7 +259,7 @@ class TestTruncate:
max_len = 4
result = truncate(unicode_string, max_len)
- assert len(encoder.encode(result)) == max_len
+ assert len(get_encoder().encode(result)) == max_len
# Should be a valid string
assert isinstance(result, str)
@@ -279,7 +269,7 @@ class TestTruncate:
max_len = 3
result = truncate(special_string, max_len)
- assert len(encoder.encode(result)) == max_len
+ assert len(get_encoder().encode(result)) == max_len
def test_whitespace_string(self):
"""Test truncation of whitespace-only string"""
@@ -287,7 +277,7 @@ class TestTruncate:
max_len = 2
result = truncate(whitespace_string, max_len)
- assert len(encoder.encode(result)) <= max_len
+ assert len(get_encoder().encode(result)) <= max_len
assert isinstance(result, str)
def test_max_len_zero(self):
@@ -295,13 +285,13 @@ class TestTruncate:
test_string = "hello world"
result = truncate(test_string, 0)
assert result == ""
- assert len(encoder.encode(result)) == 0
+ assert len(get_encoder().encode(result)) == 0
def test_max_len_one(self):
"""Test truncation with max_len = 1"""
test_string = "hello world"
result = truncate(test_string, 1)
- assert len(encoder.encode(result)) == 1
+ assert len(get_encoder().encode(result)) == 1
def test_preserves_decoding_encoding_consistency(self):
"""Test that truncation preserves encoding-decoding consistency"""
@@ -310,7 +300,7 @@ class TestTruncate:
result = truncate(test_string, max_len)
# Re-encoding the result should give the same token count
- re_encoded = encoder.encode(result)
+ re_encoded = get_encoder().encode(result)
assert len(re_encoded) == max_len
def test_multibyte_characters_truncation(self):
@@ -320,7 +310,7 @@ class TestTruncate:
max_len = 3
result = truncate(multibyte_string, max_len)
- assert len(encoder.encode(result)) == max_len
+ assert len(get_encoder().encode(result)) == max_len
def test_mixed_english_chinese_text(self):
"""Test truncation with mixed English and Chinese text"""
@@ -328,7 +318,7 @@ class TestTruncate:
max_len = 5
result = truncate(mixed_string, max_len)
- assert len(encoder.encode(result)) == max_len
+ assert len(get_encoder().encode(result)) == max_len
def test_numbers_and_symbols(self):
"""Test truncation with numbers and symbols"""
@@ -336,4 +326,4 @@ class TestTruncate:
max_len = 4
result = truncate(number_string, max_len)
- assert len(encoder.encode(result)) == max_len
+ assert len(get_encoder().encode(result)) == max_len
diff --git a/test/unit_test/common/test_token_utils_lazy_encoder.py b/test/unit_test/common/test_token_utils_lazy_encoder.py
new file mode 100644
index 0000000000..e776e193aa
--- /dev/null
+++ b/test/unit_test/common/test_token_utils_lazy_encoder.py
@@ -0,0 +1,171 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import hashlib
+import importlib
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import tiktoken
+
+import common
+
+
+def _reimport_token_utils(monkeypatch, get_encoding):
+ """Import common.token_utils fresh with tiktoken.get_encoding replaced.
+
+ Re-importing also rebinds `token_utils` as an attribute of the `common`
+ package, and restoring the sys.modules entry does not undo that. Registering
+ the attribute with monkeypatch restores it at teardown, so later tests that
+ read `common.token_utils` do not get the module built against a fake encoder.
+ """
+ monkeypatch.setattr(common, "token_utils", getattr(common, "token_utils", None), raising=False)
+ # Importing re-runs _ensure_tiktoken_cache, which rewrites TIKTOKEN_CACHE_DIR.
+ # Register it first so teardown puts back whatever the suite was using.
+ monkeypatch.setenv("TIKTOKEN_CACHE_DIR", os.environ.get("TIKTOKEN_CACHE_DIR", ""))
+ monkeypatch.setattr(tiktoken, "get_encoding", get_encoding)
+ monkeypatch.delitem(sys.modules, "common.token_utils", raising=False)
+ return importlib.import_module("common.token_utils")
+
+
+class TestLazyEncoder:
+ def test_import_does_not_build_the_encoder(self, monkeypatch):
+ """Importing the module must not read or download the BPE table.
+
+ `rag.nlp` imports `num_tokens_from_string` from here, so an import-time
+ build made every module in that chain depend on reaching
+ openaipublic.blob.core.windows.net, including tests that never tokenize.
+ """
+ calls = []
+ module = _reimport_token_utils(monkeypatch, lambda name: calls.append(name))
+
+ assert calls == []
+ assert module.num_tokens_from_string is not None
+
+ def test_get_encoder_delegates_to_tiktoken(self, monkeypatch):
+ """`get_encoder()` asks tiktoken for cl100k_base on every call.
+
+ Sharing is tiktoken's registry cache (`tiktoken.registry.ENCODINGS`), not
+ this module's. Recording every requested name pins both halves: each call
+ reaches tiktoken, and nothing here short-circuits the second one.
+ """
+ shared = tiktoken.Encoding(
+ name="test",
+ pat_str=r".",
+ mergeable_ranks={b"a": 0},
+ special_tokens={},
+ )
+ requested = []
+
+ def fake_get_encoding(name):
+ requested.append(name)
+ return shared
+
+ module = _reimport_token_utils(monkeypatch, fake_get_encoding)
+ assert requested == []
+
+ assert module.get_encoder() is shared
+ assert module.get_encoder() is shared
+ assert requested == ["cl100k_base", "cl100k_base"]
+
+ def test_unavailable_table_is_not_reported_as_zero_tokens(self, monkeypatch):
+ """A missing BPE table must fail loudly, not count as an empty string.
+
+ `num_tokens_from_string` guards `encode` with `except Exception: return 0`.
+ Building the encoder inside that guard would turn an unreachable table into
+ a silent zero-token result for every caller.
+ """
+
+ def unavailable(name):
+ raise RuntimeError("BPE table unavailable")
+
+ module = _reimport_token_utils(monkeypatch, unavailable)
+
+ with pytest.raises(RuntimeError, match="BPE table unavailable"):
+ module.num_tokens_from_string("hello")
+
+
+class TestImportChain:
+ def test_rag_nlp_and_rag_prompts_import_without_the_bpe_table(self):
+ """Importing `rag.nlp` and `rag.prompts` must not load the BPE table.
+
+ The tests above re-import `common.token_utils` on its own, so they would
+ still pass if an import-time build came back higher up the chain, for
+ example in `rag/prompts/generator.py`, which imports `get_encoder` and is
+ pulled in by `rag/prompts/__init__.py`. A fresh interpreter covers the
+ whole chain. The loader is the patch point: every way to build an
+ encoding (`tiktoken.get_encoding`, `tiktoken.encoding_for_model`,
+ `tiktoken.registry.get_encoding`) ends in
+ `tiktoken.load.load_tiktoken_bpe`, which `tiktoken_ext.openai_public`
+ binds on first build, so patching one entry point would let the others
+ through.
+ """
+ code = """
+import json
+import tiktoken.load
+
+requested = []
+
+
+def unreachable(tiktoken_bpe_file, expected_hash=None):
+ requested.append(tiktoken_bpe_file)
+ raise OSError("BPE table unreachable")
+
+
+tiktoken.load.load_tiktoken_bpe = unreachable
+
+import rag.nlp
+import rag.prompts
+
+print(json.dumps(requested))
+"""
+ result = subprocess.run(
+ [sys.executable, "-c", code],
+ cwd=Path(__file__).resolve().parents[3],
+ capture_output=True,
+ text=True,
+ timeout=120,
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert json.loads(result.stdout.strip().splitlines()[-1]) == []
+
+
+class TestTiktokenCacheOnImport:
+ """The helper is only useful if importing the module runs it."""
+
+ def test_import_seeds_the_cache_from_the_bundled_table(self, tmp_path, monkeypatch):
+ """Importing must copy the bundled table and point tiktoken at it.
+
+ Testing _ensure_tiktoken_cache alone passes even if nothing calls it,
+ so drive it through a real import.
+ """
+ bundled = tmp_path / "ragflow_deps" / "cl100k_base.tiktoken"
+ bundled.parent.mkdir()
+ bundled.write_bytes(b"bundled cl100k table")
+ monkeypatch.setenv("RAG_PROJECT_BASE", str(tmp_path))
+
+ module = _reimport_token_utils(monkeypatch, lambda name: object())
+
+ encoding_url = "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"
+ cache_path = tmp_path / hashlib.sha1(encoding_url.encode()).hexdigest()
+ assert cache_path.read_bytes() == b"bundled cl100k table"
+ assert os.environ["TIKTOKEN_CACHE_DIR"] == str(tmp_path)
+ assert module.tiktoken_cache_dir == str(tmp_path)
diff --git a/test/unit_test/rag/prompts/test_generator_message_fit_in.py b/test/unit_test/rag/prompts/test_generator_message_fit_in.py
index 01b6a35c50..7b788c2a80 100644
--- a/test/unit_test/rag/prompts/test_generator_message_fit_in.py
+++ b/test/unit_test/rag/prompts/test_generator_message_fit_in.py
@@ -52,6 +52,7 @@ def _load_generator_module(monkeypatch):
token_utils = ModuleType("common.token_utils")
token_utils.encoder = _CharEncoder()
+ token_utils.get_encoder = lambda: token_utils.encoder
token_utils.num_tokens_from_string = lambda text: len(text)
monkeypatch.setitem(sys.modules, "common.token_utils", token_utils)
@@ -82,7 +83,7 @@ def _load_generator_module(monkeypatch):
def test_message_fit_in_truncates_user_message_by_system_token_budget(monkeypatch):
generator = _load_generator_module(monkeypatch)
monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))
- monkeypatch.setattr(generator, "encoder", _CharEncoder())
+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())
messages = [
{"role": "system", "content": "1234"},
@@ -100,7 +101,7 @@ def test_message_fit_in_truncates_user_message_by_system_token_budget(monkeypatc
def test_message_fit_in_handles_zero_token_messages(monkeypatch):
generator = _load_generator_module(monkeypatch)
monkeypatch.setattr(generator, "num_tokens_from_string", lambda _text: 0)
- monkeypatch.setattr(generator, "encoder", _CharEncoder())
+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())
messages = [
{"role": "system", "content": ""},
@@ -117,7 +118,7 @@ def test_message_fit_in_handles_zero_token_messages(monkeypatch):
def test_message_fit_in_clamps_negative_slice_lengths(monkeypatch):
generator = _load_generator_module(monkeypatch)
monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))
- monkeypatch.setattr(generator, "encoder", _CharEncoder())
+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())
messages = [
{"role": "system", "content": "1234"},
@@ -135,7 +136,7 @@ def test_message_fit_in_clamps_negative_slice_lengths(monkeypatch):
def test_message_fit_in_clamps_dominant_last_message_to_budget(monkeypatch):
generator = _load_generator_module(monkeypatch)
monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))
- monkeypatch.setattr(generator, "encoder", _CharEncoder())
+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())
messages = [
{"role": "system", "content": "s" * 41},
@@ -153,7 +154,7 @@ def test_message_fit_in_clamps_dominant_last_message_to_budget(monkeypatch):
def test_message_fit_in_zero_budget_preserves_non_empty_messages(monkeypatch):
generator = _load_generator_module(monkeypatch)
monkeypatch.setattr(generator, "num_tokens_from_string", lambda text: len(text))
- monkeypatch.setattr(generator, "encoder", _CharEncoder())
+ monkeypatch.setattr(generator, "get_encoder", lambda: _CharEncoder())
system_len = 8100
user_content = "User query: test"
diff --git a/test/unit_test/rag/prompts/test_vision_figure_prompt.py b/test/unit_test/rag/prompts/test_vision_figure_prompt.py
index ca544d2636..e3a7023f38 100644
--- a/test/unit_test/rag/prompts/test_vision_figure_prompt.py
+++ b/test/unit_test/rag/prompts/test_vision_figure_prompt.py
@@ -42,6 +42,7 @@ def _load_generator(monkeypatch):
token_utils = ModuleType("common.token_utils")
token_utils.encoder = SimpleNamespace()
+ token_utils.get_encoder = lambda: token_utils.encoder
token_utils.num_tokens_from_string = len
monkeypatch.setitem(sys.modules, "common.token_utils", token_utils)