Python
fix(tokens): count literal special-token spellings as ordinary text
u7-u7e9698a293ced
diff --git a/hy_memory/core/embed_service.py b/hy_memory/core/embed_service.pyindex efe7955..0f9de32 100644--- a/hy_memory/core/embed_service.py+++ b/hy_memory/core/embed_service.py@@ -80,7 +80,7 @@ def _count_tokens(text: str) -> int:try:import tiktoken- return len(tiktoken.get_encoding("cl100k_base").encode(text))+ return len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=()))except Exception:return len(text) // _CHARS_PER_TOKEN_FALLBACK + 1@@ -98,7 +98,8 @@ def _truncate_for_embedding(text: str, limit: int = _EMBED_MAX_TOKENS) -> str:import tiktokenenc = tiktoken.get_encoding("cl100k_base")- ids = enc.encode(text)+ # Input is ordinary user text, not a tokenizer control-token stream.+ ids = enc.encode(text, disallowed_special=())if len(ids) <= limit:return textdropped = len(ids) - limitdiff --git a/hy_memory/utils/token_count.py b/hy_memory/utils/token_count.pyindex 10e53ad..afbf09d 100644--- a/hy_memory/utils/token_count.py+++ b/hy_memory/utils/token_count.py@@ -36,7 +36,8 @@ def count_tokens(text: str) -> int:enc = _get_encoder()if enc:try:- return len(enc.encode(text))+ # Quoted control-token spellings are ordinary input text.+ return len(enc.encode(text, disallowed_special=()))except Exception as e:logger.debug(f"[token_count] encode failed, char fallback: {e}")return max(1, len(text) // 4)diff --git a/tests/test_literal_special_token_counting.py b/tests/test_literal_special_token_counting.pynew file mode 100644index 0000000..a2eb461--- /dev/null+++ b/tests/test_literal_special_token_counting.py@@ -0,0 +1,29 @@+"""User text may quote tokenizer control spellings without using control tokens."""++import pytest+import tiktoken++from hy_memory.core.embed_service import _count_tokens, _truncate_for_embedding+from hy_memory.utils.token_count import count_tokens++[email protected]('counter', [count_tokens, _count_tokens])[email protected]('marker', ['<|endoftext|>', '<|fim_prefix|>'])+def test_literal_special_tokens_are_counted_as_ordinary_text(marker, counter):+ text = marker + '你好世界' * 20+ expected = len(tiktoken.get_encoding('cl100k_base').encode_ordinary(text))+ assert counter(text) == expected+++def test_literal_special_token_does_not_bypass_embedding_limit():+ text = '<|endoftext|>' + '你好世界' * 3000+ encoder = tiktoken.get_encoding('cl100k_base')+ truncated = _truncate_for_embedding(text)+ assert truncated != text+ assert len(encoder.encode_ordinary(truncated)) <= 8192 - 64+ assert truncated.startswith('<|endoftext|>')+++def test_short_literal_special_token_text_is_preserved():+ text = 'Explain <|endoftext|> as a literal string.'+ assert _truncate_for_embedding(text) == text