Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
fix: handle disallowed special tokens in tiktoken encoder (#382)
Nox1e97dff21437
diff --git a/core/anthropic/tokens.py b/core/anthropic/tokens.pyindex 081d28b5..3adaa340 100644--- a/core/anthropic/tokens.py+++ b/core/anthropic/tokens.py@@ -9,6 +9,12 @@ from .content import get_block_attrENCODER = tiktoken.get_encoding("cl100k_base")+_DISALLOWED_SPECIAL: tuple[str, ...] = ()+++def _count_text_tokens(text: str) -> int:+ return len(ENCODER.encode(text, disallowed_special=_DISALLOWED_SPECIAL))+def get_token_count(messages: list,@@ -20,34 +26,34 @@ def get_token_count(if system:if isinstance(system, str):- total_tokens += len(ENCODER.encode(system))+ total_tokens += _count_text_tokens(system)elif isinstance(system, list):for block in system:text = get_block_attr(block, "text", "")if text:- total_tokens += len(ENCODER.encode(str(text)))+ total_tokens += _count_text_tokens(str(text))total_tokens += 4for msg in messages:if isinstance(msg.content, str):- total_tokens += len(ENCODER.encode(msg.content))+ total_tokens += _count_text_tokens(msg.content)elif isinstance(msg.content, list):for block in msg.content:b_type = get_block_attr(block, "type") or Noneif b_type == "text":text = get_block_attr(block, "text", "")- total_tokens += len(ENCODER.encode(str(text)))+ total_tokens += _count_text_tokens(str(text))elif b_type == "thinking":thinking = get_block_attr(block, "thinking", "")- total_tokens += len(ENCODER.encode(str(thinking)))+ total_tokens += _count_text_tokens(str(thinking))elif b_type == "tool_use":name = get_block_attr(block, "name", "")inp = get_block_attr(block, "input", {})block_id = get_block_attr(block, "id", "")- total_tokens += len(ENCODER.encode(str(name)))- total_tokens += len(ENCODER.encode(json.dumps(inp)))- total_tokens += len(ENCODER.encode(str(block_id)))+ total_tokens += _count_text_tokens(str(name))+ total_tokens += _count_text_tokens(json.dumps(inp))+ total_tokens += _count_text_tokens(str(block_id))total_tokens += 15elif b_type == "image":source = get_block_attr(block, "source")@@ -63,10 +69,10 @@ def get_token_count(content = get_block_attr(block, "content", "")tool_use_id = get_block_attr(block, "tool_use_id", "")if isinstance(content, str):- total_tokens += len(ENCODER.encode(content))+ total_tokens += _count_text_tokens(content)else:- total_tokens += len(ENCODER.encode(json.dumps(content)))- total_tokens += len(ENCODER.encode(str(tool_use_id)))+ total_tokens += _count_text_tokens(json.dumps(content))+ total_tokens += _count_text_tokens(str(tool_use_id))total_tokens += 8elif b_type in ("server_tool_use",@@ -78,16 +84,14 @@ def get_token_count(else:blob = blocktry:- total_tokens += len(- ENCODER.encode(- json.dumps(blob, default=str, ensure_ascii=False)- )+ total_tokens += _count_text_tokens(+ json.dumps(blob, default=str, ensure_ascii=False))except (TypeError, ValueError, OverflowError) as e:logger.debug("Block encode fallback b_type={} err={}", b_type, e)- total_tokens += len(ENCODER.encode(str(blob)))+ total_tokens += _count_text_tokens(str(blob))total_tokens += 12else:logger.debug(@@ -95,16 +99,16 @@ def get_token_count(b_type,)try:- total_tokens += len(ENCODER.encode(json.dumps(block)))+ total_tokens += _count_text_tokens(json.dumps(block))except TypeError, ValueError:- total_tokens += len(ENCODER.encode(str(block)))+ total_tokens += _count_text_tokens(str(block))if tools:for tool in tools:tool_str = (tool.name + (tool.description or "") + json.dumps(tool.input_schema))- total_tokens += len(ENCODER.encode(tool_str))+ total_tokens += _count_text_tokens(tool_str)total_tokens += len(messages) * 4if tools:diff --git a/tests/api/test_request_utils.py b/tests/api/test_request_utils.pyindex 6048c9d9..e6a73b97 100644--- a/tests/api/test_request_utils.py+++ b/tests/api/test_request_utils.py@@ -335,6 +335,14 @@ class TestGetTokenCount:# "Hello world" is ~2-3 tokens plus overheadassert count >= 3+ def test_special_token_text_is_counted_as_plain_text(self):+ """Tiktoken special-token strings should not break token estimates."""+ msg = MagicMock()+ msg.content = "<|endoftext|>"++ count = get_token_count([msg], system="<|endoftext|>")+ assert count > 0+def test_message_with_system_prompt(self):"""Test token count includes system prompt."""msg = MagicMock()