Skip to content
Python

fix(tokens): count literal special-token spellings as ordinary text

u7-u7e9698a293ced
diff --git a/hy_memory/core/embed_service.py b/hy_memory/core/embed_service.py
index efe7955..0f9de32 100644
--- a/hy_memory/core/embed_service.py
+++ b/hy_memory/core/embed_service.py
@@ -80,7 +80,7 @@ def _count_tokens(text: str) -> int:
try:
import tiktoken
- return len(tiktoken.get_encoding("cl100k_base").encode(text))
+ return len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=()))
except Exception:
return len(text) // _CHARS_PER_TOKEN_FALLBACK + 1
@@ -98,7 +98,8 @@ def _truncate_for_embedding(text: str, limit: int = _EMBED_MAX_TOKENS) -> str:
import tiktoken
enc = tiktoken.get_encoding("cl100k_base")
- ids = enc.encode(text)
+ # Input is ordinary user text, not a tokenizer control-token stream.
+ ids = enc.encode(text, disallowed_special=())
if len(ids) <= limit:
return text
dropped = len(ids) - limit
diff --git a/hy_memory/utils/token_count.py b/hy_memory/utils/token_count.py
index 10e53ad..afbf09d 100644
--- a/hy_memory/utils/token_count.py
+++ b/hy_memory/utils/token_count.py
@@ -36,7 +36,8 @@ def count_tokens(text: str) -> int:
enc = _get_encoder()
if enc:
try:
- return len(enc.encode(text))
+ # Quoted control-token spellings are ordinary input text.
+ return len(enc.encode(text, disallowed_special=()))
except Exception as e:
logger.debug(f"[token_count] encode failed, char fallback: {e}")
return max(1, len(text) // 4)
diff --git a/tests/test_literal_special_token_counting.py b/tests/test_literal_special_token_counting.py
new file mode 100644
index 0000000..a2eb461
--- /dev/null
+++ b/tests/test_literal_special_token_counting.py
@@ -0,0 +1,29 @@
+"""User text may quote tokenizer control spellings without using control tokens."""
+
+import pytest
+import tiktoken
+
+from hy_memory.core.embed_service import _count_tokens, _truncate_for_embedding
+from hy_memory.utils.token_count import count_tokens
+
+
[email protected]('counter', [count_tokens, _count_tokens])
[email protected]('marker', ['<|endoftext|>', '<|fim_prefix|>'])
+def test_literal_special_tokens_are_counted_as_ordinary_text(marker, counter):
+ text = marker + '你好世界' * 20
+ expected = len(tiktoken.get_encoding('cl100k_base').encode_ordinary(text))
+ assert counter(text) == expected
+
+
+def test_literal_special_token_does_not_bypass_embedding_limit():
+ text = '<|endoftext|>' + '你好世界' * 3000
+ encoder = tiktoken.get_encoding('cl100k_base')
+ truncated = _truncate_for_embedding(text)
+ assert truncated != text
+ assert len(encoder.encode_ordinary(truncated)) <= 8192 - 64
+ assert truncated.startswith('<|endoftext|>')
+
+
+def test_short_literal_special_token_text_is_preserved():
+ text = 'Explain <|endoftext|> as a literal string.'
+ assert _truncate_for_embedding(text) == text