Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Make token estimation resilient to unavailable encoder data (#1427)

Ali Khokhare44e48785044
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index ed6f63dd..cd4390ac 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -428,6 +428,14 @@ not mistaken for stalled provider work and cannot receive cancellation from the
generator's timer. Expiry becomes a protocol-neutral, non-retryable 504
`ExecutionFailure`; cancellation and provider-originated timeouts retain their
existing meanings.
+
+[core/token_estimation.py](src/free_claude_code/core/token_estimation.py) owns
+process-wide best-effort plain-text token estimation. It acquires the shared
+encoder once and degrades to a deterministic character estimate when encoder
+data is unavailable, so a cold cache or blocked network cannot prevent FCC from
+starting. Anthropic request counting and the Anthropic/Responses stream ledgers
+retain ownership of their protocol-specific block and structural overhead.
+
[api/response_streams.py](src/free_claude_code/api/response_streams.py) owns public streaming egress
commit timing. It waits for the first protocol chunk before returning a
successful FCC-owned `StreamingResponse`. Its explicit replay iterator owns the
@@ -910,7 +918,8 @@ disjoint input categories into its single `input_tokens` total.
- thinking block handling;
- stream lifecycle through `src/free_claude_code/core/anthropic/streaming`, including the neutral
stream ledger, Anthropic SSE emitter, continuation-body construction, and tool repair;
-- token counting and Anthropic-owned failure-kind-to-wire mapping.
+- Anthropic structural token counting and Anthropic-owned failure-kind-to-wire
+ mapping.
`MessagesRequest` is an ingress model; no current provider sends Anthropic wire
requests downstream. Anthropic request models validate transcript data without
diff --git a/pyproject.toml b/pyproject.toml
index 22619df5..25b57986 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "free-claude-code"
-version = "5.3.0"
+version = "5.3.1"
description = "Local proxy connecting coding agents to OpenAI-compatible AI providers"
readme = "README.md"
requires-python = ">=3.14.0"
diff --git a/src/free_claude_code/core/anthropic/streaming/ledger.py b/src/free_claude_code/core/anthropic/streaming/ledger.py
index 5625945a..90c39e85 100644
--- a/src/free_claude_code/core/anthropic/streaming/ledger.py
+++ b/src/free_claude_code/core/anthropic/streaming/ledger.py
@@ -10,19 +10,14 @@ from typing import Any
from loguru import logger
+from free_claude_code.core.token_estimation import estimate_text_tokens
+
from .emitter import AnthropicSseEmitter
from .recovery import (
ToolSchema,
parse_complete_tool_input,
)
-try:
- import tiktoken
-
- ENCODER = tiktoken.get_encoding("cl100k_base")
-except Exception:
- ENCODER = None
-
def _safe_usage_int(value: object) -> int:
return value if isinstance(value, int) else 0
@@ -446,28 +441,22 @@ class AnthropicStreamLedger:
return "".join(self._thinking_parts)
def estimate_output_tokens(self) -> int:
- if ENCODER:
- text_tokens = len(ENCODER.encode(self.accumulated_text))
- reasoning_tokens = len(ENCODER.encode(self.accumulated_reasoning))
- tool_tokens = 0
- tool_count = 0
- for name, content in self._iter_tool_token_payloads():
- tool_tokens += len(ENCODER.encode(name))
- tool_tokens += len(ENCODER.encode(content))
- tool_tokens += 15
- tool_count += 1
-
- block_count = (
- (1 if self.accumulated_reasoning else 0)
- + (1 if self.accumulated_text else 0)
- + tool_count
- )
- return text_tokens + reasoning_tokens + tool_tokens + (block_count * 4)
-
- text_tokens = len(self.accumulated_text) // 4
- reasoning_tokens = len(self.accumulated_reasoning) // 4
- tool_tokens = sum(1 for _ in self._iter_tool_token_payloads()) * 50
- return text_tokens + reasoning_tokens + tool_tokens
+ text_tokens = estimate_text_tokens(self.accumulated_text)
+ reasoning_tokens = estimate_text_tokens(self.accumulated_reasoning)
+ tool_tokens = 0
+ tool_count = 0
+ for name, content in self._iter_tool_token_payloads():
+ tool_tokens += estimate_text_tokens(name)
+ tool_tokens += estimate_text_tokens(content)
+ tool_tokens += 15
+ tool_count += 1
+
+ block_count = (
+ (1 if self.accumulated_reasoning else 0)
+ + (1 if self.accumulated_text else 0)
+ + tool_count
+ )
+ return text_tokens + reasoning_tokens + tool_tokens + (block_count * 4)
def _iter_tool_token_payloads(self) -> Iterator[tuple[str, str]]:
for block in self.tool_blocks():
diff --git a/src/free_claude_code/core/anthropic/tokens.py b/src/free_claude_code/core/anthropic/tokens.py
index 5fc1750b..72ec3443 100644
--- a/src/free_claude_code/core/anthropic/tokens.py
+++ b/src/free_claude_code/core/anthropic/tokens.py
@@ -2,20 +2,13 @@
import json
-import tiktoken
from loguru import logger
+from free_claude_code.core.token_estimation import estimate_text_tokens
+
from .content import get_block_attr
from .models import Message, SystemContent, Tool
-ENCODER = tiktoken.get_encoding("cl100k_base")
-
-_DISALLOWED_SPECIAL: tuple[str, ...] = ()
-
-
-def _count_text_tokens(text: str) -> int:
- return len(ENCODER.encode(text, disallowed_special=_DISALLOWED_SPECIAL))
-
def get_token_count(
messages: list[Message],
@@ -27,34 +20,34 @@ def get_token_count(
if system:
if isinstance(system, str):
- total_tokens += _count_text_tokens(system)
+ total_tokens += estimate_text_tokens(system)
elif isinstance(system, list):
for block in system:
text = get_block_attr(block, "text", "")
if text:
- total_tokens += _count_text_tokens(str(text))
+ total_tokens += estimate_text_tokens(str(text))
total_tokens += 4
for msg in messages:
if isinstance(msg.content, str):
- total_tokens += _count_text_tokens(msg.content)
+ total_tokens += estimate_text_tokens(msg.content)
elif isinstance(msg.content, list):
for block in msg.content:
b_type = get_block_attr(block, "type") or None
if b_type == "text":
text = get_block_attr(block, "text", "")
- total_tokens += _count_text_tokens(str(text))
+ total_tokens += estimate_text_tokens(str(text))
elif b_type == "thinking":
thinking = get_block_attr(block, "thinking", "")
- total_tokens += _count_text_tokens(str(thinking))
+ total_tokens += estimate_text_tokens(str(thinking))
elif b_type == "tool_use":
name = get_block_attr(block, "name", "")
inp = get_block_attr(block, "input", {})
block_id = get_block_attr(block, "id", "")
- total_tokens += _count_text_tokens(str(name))
- total_tokens += _count_text_tokens(json.dumps(inp))
- total_tokens += _count_text_tokens(str(block_id))
+ total_tokens += estimate_text_tokens(str(name))
+ total_tokens += estimate_text_tokens(json.dumps(inp))
+ total_tokens += estimate_text_tokens(str(block_id))
total_tokens += 15
elif b_type == "image":
source = get_block_attr(block, "source")
@@ -70,10 +63,10 @@ def get_token_count(
content = get_block_attr(block, "content", "")
tool_use_id = get_block_attr(block, "tool_use_id", "")
if isinstance(content, str):
- total_tokens += _count_text_tokens(content)
+ total_tokens += estimate_text_tokens(content)
else:
- total_tokens += _count_text_tokens(json.dumps(content))
- total_tokens += _count_text_tokens(str(tool_use_id))
+ total_tokens += estimate_text_tokens(json.dumps(content))
+ total_tokens += estimate_text_tokens(str(tool_use_id))
total_tokens += 8
elif b_type in (
"server_tool_use",
@@ -85,14 +78,14 @@ def get_token_count(
else:
blob = block
try:
- total_tokens += _count_text_tokens(
+ total_tokens += estimate_text_tokens(
json.dumps(blob, default=str, ensure_ascii=False)
)
except (TypeError, ValueError, OverflowError) as e:
logger.debug(
"Block encode fallback b_type={} err={}", b_type, e
)
- total_tokens += _count_text_tokens(str(blob))
+ total_tokens += estimate_text_tokens(str(blob))
total_tokens += 12
else:
logger.debug(
@@ -100,16 +93,16 @@ def get_token_count(
b_type,
)
try:
- total_tokens += _count_text_tokens(json.dumps(block))
+ total_tokens += estimate_text_tokens(json.dumps(block))
except TypeError, ValueError:
- total_tokens += _count_text_tokens(str(block))
+ total_tokens += estimate_text_tokens(str(block))
if tools:
for tool in tools:
tool_str = (
tool.name + (tool.description or "") + json.dumps(tool.input_schema)
)
- total_tokens += _count_text_tokens(tool_str)
+ total_tokens += estimate_text_tokens(tool_str)
total_tokens += len(messages) * 4
if tools:
diff --git a/src/free_claude_code/core/openai_responses/streaming/ledger.py b/src/free_claude_code/core/openai_responses/streaming/ledger.py
index b312f172..cab4a8d4 100644
--- a/src/free_claude_code/core/openai_responses/streaming/ledger.py
+++ b/src/free_claude_code/core/openai_responses/streaming/ledger.py
@@ -3,7 +3,8 @@
from collections.abc import Mapping
from typing import Any
-from ..usage import estimate_text_tokens
+from free_claude_code.core.token_estimation import estimate_text_tokens
+
from .blocks import BlockState
_ANTHROPIC_INPUT_TOKEN_FIELDS = (
diff --git a/src/free_claude_code/core/openai_responses/usage.py b/src/free_claude_code/core/token_estimation.py
similarity index 61%
rename from src/free_claude_code/core/openai_responses/usage.py
rename to src/free_claude_code/core/token_estimation.py
index a6a2e28c..c3bb8756 100644
--- a/src/free_claude_code/core/openai_responses/usage.py
+++ b/src/free_claude_code/core/token_estimation.py
@@ -1,7 +1,10 @@
-"""Usage helpers for OpenAI Responses payloads."""
+"""Process-wide best-effort plain-text token estimation."""
from typing import Protocol
+import tiktoken
+from loguru import logger
+
_DISALLOWED_SPECIAL: tuple[str, ...] = ()
@@ -12,14 +15,13 @@ class _TokenEncoder(Protocol):
def _load_encoder() -> _TokenEncoder | None:
- try:
- import tiktoken
- except ImportError:
- return None
-
try:
return tiktoken.get_encoding("cl100k_base")
- except ValueError:
+ except Exception as exc:
+ logger.warning(
+ "cl100k_base token encoder unavailable ({}); using approximate token estimates",
+ type(exc).__name__,
+ )
return None
@@ -27,7 +29,7 @@ _ENCODER = _load_encoder()
def estimate_text_tokens(text: str) -> int:
- """Return a best-effort token estimate for Responses usage details."""
+ """Estimate tokens for plain text using the shared process-wide encoder."""
if not text:
return 0
if _ENCODER is not None:
diff --git a/tests/api/test_request_utils.py b/tests/api/test_request_utils.py
index d0126094..9312238e 100644
--- a/tests/api/test_request_utils.py
+++ b/tests/api/test_request_utils.py
@@ -1,6 +1,6 @@
"""Tests for API request detection and token counting helpers."""
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
import pytest
@@ -549,21 +549,19 @@ class TestGetTokenCount:
count = get_token_count([msg])
assert count >= 85
- def test_known_payload_estimate_range(self):
- """Known payload produces estimate within expected range (validation harness)."""
- import tiktoken
-
- enc = tiktoken.get_encoding("cl100k_base")
+ def test_known_payload_structural_overhead(self):
+ """Protocol overhead remains separate from plain-text estimation."""
system_text = "You are a helpful assistant."
user_text = "Hello, how are you?"
- sys_tokens = len(enc.encode(system_text))
- user_tokens = len(enc.encode(user_text))
- # Min: content tokens + system overhead (4) + per-msg overhead (4)
- expected_min = sys_tokens + user_tokens + 4 + 4
msg = MagicMock()
msg.content = user_text
- count = get_token_count([msg], system=system_text)
- assert count >= expected_min, f"count={count} < expected_min={expected_min}"
+ with patch(
+ "free_claude_code.core.anthropic.tokens.estimate_text_tokens",
+ side_effect=len,
+ ):
+ count = get_token_count([msg], system=system_text)
+
+ assert count == len(system_text) + len(user_text) + 4 + 4
# --- Parametrized Edge Case Tests ---
diff --git a/tests/contracts/test_offline_startup.py b/tests/contracts/test_offline_startup.py
new file mode 100644
index 00000000..f60fed4f
--- /dev/null
+++ b/tests/contracts/test_offline_startup.py
@@ -0,0 +1,41 @@
+"""Contracts for starting FCC without token-encoder network access."""
+
+import subprocess
+import sys
+
+
+def test_api_import_survives_unavailable_tiktoken_encoding() -> None:
+ script = """
+import os
+import tempfile
+
+import requests
+
+calls = []
+
+
+def fail(url, *args, **kwargs):
+ del args, kwargs
+ calls.append(url)
+ raise requests.exceptions.ProxyError("Bearer secret")
+
+
+with tempfile.TemporaryDirectory() as cache_dir:
+ os.environ["TIKTOKEN_CACHE_DIR"] = cache_dir
+ requests.get = fail
+ import free_claude_code.api.app
+
+assert len(calls) == 1, calls
+"""
+
+ completed = subprocess.run(
+ [sys.executable, "-c", script],
+ capture_output=True,
+ check=False,
+ text=True,
+ timeout=15,
+ )
+
+ assert completed.returncode == 0, completed.stderr or completed.stdout
+ assert "ProxyError" in completed.stderr
+ assert "Bearer secret" not in completed.stderr
diff --git a/tests/core/anthropic/test_stream_ledger.py b/tests/core/anthropic/test_stream_ledger.py
index 2ebdb9b2..3a20fb5f 100644
--- a/tests/core/anthropic/test_stream_ledger.py
+++ b/tests/core/anthropic/test_stream_ledger.py
@@ -113,14 +113,19 @@ def test_close_unclosed_blocks_closes_each_block_once() -> None:
assert list(ledger.close_unclosed_blocks()) == []
-def test_output_token_estimate_uses_encoder_when_available() -> None:
+def test_output_token_estimate_combines_shared_estimates_and_block_overhead() -> None:
ledger = AnthropicStreamLedger("msg_1", "model")
+ ledger.start_thinking_block()
+ ledger.emit_thinking_delta("why")
+ ledger.stop_thinking_block()
ledger.start_text_block()
ledger.emit_text_delta("abcd")
+ ledger.stop_text_block()
+ ledger.start_tool_block(0, "toolu_1", "Read")
+ ledger.emit_tool_delta(0, "{}")
- class Encoder:
- def encode(self, text: str) -> list[int]:
- return list(range(len(text)))
-
- with patch("free_claude_code.core.anthropic.streaming.ledger.ENCODER", Encoder()):
- assert ledger.estimate_output_tokens() == 8
+ with patch(
+ "free_claude_code.core.anthropic.streaming.ledger.estimate_text_tokens",
+ side_effect=len,
+ ):
+ assert ledger.estimate_output_tokens() == 40
diff --git a/tests/core/test_token_estimation.py b/tests/core/test_token_estimation.py
new file mode 100644
index 00000000..89ba2f46
--- /dev/null
+++ b/tests/core/test_token_estimation.py
@@ -0,0 +1,93 @@
+"""Tests for process-wide plain-text token estimation."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from free_claude_code.core import token_estimation
+
+
+class _RecordingEncoder:
+ def __init__(self, tokens: list[int]) -> None:
+ self.tokens = tokens
+ self.calls: list[tuple[str, tuple[str, ...]]] = []
+
+ def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> list[int]:
+ self.calls.append((text, disallowed_special))
+ return self.tokens
+
+
+def test_encoder_is_loaded_by_canonical_name() -> None:
+ encoder = MagicMock()
+
+ with patch.object(
+ token_estimation.tiktoken,
+ "get_encoding",
+ return_value=encoder,
+ ) as get_encoding:
+ assert token_estimation._load_encoder() is encoder
+
+ get_encoding.assert_called_once_with("cl100k_base")
+
+
+def test_encoder_acquisition_failure_uses_safe_fallback_warning() -> None:
+ with (
+ patch.object(
+ token_estimation.tiktoken,
+ "get_encoding",
+ side_effect=RuntimeError("Bearer secret"),
+ ),
+ patch.object(token_estimation.logger, "warning") as warning,
+ ):
+ assert token_estimation._load_encoder() is None
+
+ warning.assert_called_once_with(
+ "cl100k_base token encoder unavailable ({}); using approximate token estimates",
+ "RuntimeError",
+ )
+ assert "Bearer secret" not in str(warning.call_args)
+
+
+def test_estimate_text_tokens_uses_cached_encoder() -> None:
+ encoder = _RecordingEncoder([1, 2, 3])
+
+ with patch.object(token_estimation, "_ENCODER", encoder):
+ assert token_estimation.estimate_text_tokens("hello") == 3
+
+ assert encoder.calls == [("hello", ())]
+
+
+def test_empty_text_skips_encoder() -> None:
+ encoder = _RecordingEncoder([1])
+
+ with patch.object(token_estimation, "_ENCODER", encoder):
+ assert token_estimation.estimate_text_tokens("") == 0
+
+ assert encoder.calls == []
+
+
+ ("text", "expected"),
+ [
+ ("a", 1),
+ ("abcdefg", 1),
+ ("abcdefgh", 2),
+ ],
+)
+def test_estimate_text_tokens_falls_back_to_character_ratio(
+ text: str,
+ expected: int,
+) -> None:
+ with patch.object(token_estimation, "_ENCODER", None):
+ assert token_estimation.estimate_text_tokens(text) == expected
+
+
+def test_encoder_execution_errors_are_not_hidden() -> None:
+ encoder = MagicMock()
+ encoder.encode.side_effect = RuntimeError("encode failed")
+
+ with (
+ patch.object(token_estimation, "_ENCODER", encoder),
+ pytest.raises(RuntimeError, match="encode failed"),
+ ):
+ token_estimation.estimate_text_tokens("hello")
diff --git a/uv.lock b/uv.lock
index 1ef36fd4..50459d90 100644
--- a/uv.lock
+++ b/uv.lock
@@ -607,7 +607,7 @@ wheels = [
[[package]]
name = "free-claude-code"
-version = "5.3.0"
+version = "5.3.1"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },