Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

fix: handle disallowed special tokens in tiktoken encoder (#382)

Nox1e97dff21437
diff --git a/core/anthropic/tokens.py b/core/anthropic/tokens.py
index 081d28b5..3adaa340 100644
--- a/core/anthropic/tokens.py
+++ b/core/anthropic/tokens.py
@@ -9,6 +9,12 @@ from .content import get_block_attr
ENCODER = tiktoken.get_encoding("cl100k_base")
+_DISALLOWED_SPECIAL: tuple[str, ...] = ()
+
+
+def _count_text_tokens(text: str) -> int:
+ return len(ENCODER.encode(text, disallowed_special=_DISALLOWED_SPECIAL))
+
def get_token_count(
messages: list,
@@ -20,34 +26,34 @@ def get_token_count(
if system:
if isinstance(system, str):
- total_tokens += len(ENCODER.encode(system))
+ total_tokens += _count_text_tokens(system)
elif isinstance(system, list):
for block in system:
text = get_block_attr(block, "text", "")
if text:
- total_tokens += len(ENCODER.encode(str(text)))
+ total_tokens += _count_text_tokens(str(text))
total_tokens += 4
for msg in messages:
if isinstance(msg.content, str):
- total_tokens += len(ENCODER.encode(msg.content))
+ total_tokens += _count_text_tokens(msg.content)
elif isinstance(msg.content, list):
for block in msg.content:
b_type = get_block_attr(block, "type") or None
if b_type == "text":
text = get_block_attr(block, "text", "")
- total_tokens += len(ENCODER.encode(str(text)))
+ total_tokens += _count_text_tokens(str(text))
elif b_type == "thinking":
thinking = get_block_attr(block, "thinking", "")
- total_tokens += len(ENCODER.encode(str(thinking)))
+ total_tokens += _count_text_tokens(str(thinking))
elif b_type == "tool_use":
name = get_block_attr(block, "name", "")
inp = get_block_attr(block, "input", {})
block_id = get_block_attr(block, "id", "")
- total_tokens += len(ENCODER.encode(str(name)))
- total_tokens += len(ENCODER.encode(json.dumps(inp)))
- total_tokens += len(ENCODER.encode(str(block_id)))
+ total_tokens += _count_text_tokens(str(name))
+ total_tokens += _count_text_tokens(json.dumps(inp))
+ total_tokens += _count_text_tokens(str(block_id))
total_tokens += 15
elif b_type == "image":
source = get_block_attr(block, "source")
@@ -63,10 +69,10 @@ def get_token_count(
content = get_block_attr(block, "content", "")
tool_use_id = get_block_attr(block, "tool_use_id", "")
if isinstance(content, str):
- total_tokens += len(ENCODER.encode(content))
+ total_tokens += _count_text_tokens(content)
else:
- total_tokens += len(ENCODER.encode(json.dumps(content)))
- total_tokens += len(ENCODER.encode(str(tool_use_id)))
+ total_tokens += _count_text_tokens(json.dumps(content))
+ total_tokens += _count_text_tokens(str(tool_use_id))
total_tokens += 8
elif b_type in (
"server_tool_use",
@@ -78,16 +84,14 @@ def get_token_count(
else:
blob = block
try:
- total_tokens += len(
- ENCODER.encode(
- json.dumps(blob, default=str, ensure_ascii=False)
- )
+ total_tokens += _count_text_tokens(
+ json.dumps(blob, default=str, ensure_ascii=False)
)
except (TypeError, ValueError, OverflowError) as e:
logger.debug(
"Block encode fallback b_type={} err={}", b_type, e
)
- total_tokens += len(ENCODER.encode(str(blob)))
+ total_tokens += _count_text_tokens(str(blob))
total_tokens += 12
else:
logger.debug(
@@ -95,16 +99,16 @@ def get_token_count(
b_type,
)
try:
- total_tokens += len(ENCODER.encode(json.dumps(block)))
+ total_tokens += _count_text_tokens(json.dumps(block))
except TypeError, ValueError:
- total_tokens += len(ENCODER.encode(str(block)))
+ total_tokens += _count_text_tokens(str(block))
if tools:
for tool in tools:
tool_str = (
tool.name + (tool.description or "") + json.dumps(tool.input_schema)
)
- total_tokens += len(ENCODER.encode(tool_str))
+ total_tokens += _count_text_tokens(tool_str)
total_tokens += len(messages) * 4
if tools:
diff --git a/tests/api/test_request_utils.py b/tests/api/test_request_utils.py
index 6048c9d9..e6a73b97 100644
--- a/tests/api/test_request_utils.py
+++ b/tests/api/test_request_utils.py
@@ -335,6 +335,14 @@ class TestGetTokenCount:
# "Hello world" is ~2-3 tokens plus overhead
assert count >= 3
+ def test_special_token_text_is_counted_as_plain_text(self):
+ """Tiktoken special-token strings should not break token estimates."""
+ msg = MagicMock()
+ msg.content = "<|endoftext|>"
+
+ count = get_token_count([msg], system="<|endoftext|>")
+ assert count > 0
+
def test_message_with_system_prompt(self):
"""Test token count includes system prompt."""
msg = MagicMock()