Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Make token estimation resilient to unavailable encoder data (#1427)
Ali Khokhare44e48785044
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.mdindex ed6f63dd..cd4390ac 100644--- a/ARCHITECTURE.md+++ b/ARCHITECTURE.md@@ -428,6 +428,14 @@ not mistaken for stalled provider work and cannot receive cancellation from thegenerator's timer. Expiry becomes a protocol-neutral, non-retryable 504`ExecutionFailure`; cancellation and provider-originated timeouts retain theirexisting meanings.++[core/token_estimation.py](src/free_claude_code/core/token_estimation.py) owns+process-wide best-effort plain-text token estimation. It acquires the shared+encoder once and degrades to a deterministic character estimate when encoder+data is unavailable, so a cold cache or blocked network cannot prevent FCC from+starting. Anthropic request counting and the Anthropic/Responses stream ledgers+retain ownership of their protocol-specific block and structural overhead.+[api/response_streams.py](src/free_claude_code/api/response_streams.py) owns public streaming egresscommit timing. It waits for the first protocol chunk before returning asuccessful FCC-owned `StreamingResponse`. Its explicit replay iterator owns the@@ -910,7 +918,8 @@ disjoint input categories into its single `input_tokens` total.- thinking block handling;- stream lifecycle through `src/free_claude_code/core/anthropic/streaming`, including the neutralstream ledger, Anthropic SSE emitter, continuation-body construction, and tool repair;-- token counting and Anthropic-owned failure-kind-to-wire mapping.+- Anthropic structural token counting and Anthropic-owned failure-kind-to-wire+ mapping.`MessagesRequest` is an ingress model; no current provider sends Anthropic wirerequests downstream. Anthropic request models validate transcript data withoutdiff --git a/pyproject.toml b/pyproject.tomlindex 22619df5..25b57986 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "free-claude-code"-version = "5.3.0"+version = "5.3.1"description = "Local proxy connecting coding agents to OpenAI-compatible AI providers"readme = "README.md"requires-python = ">=3.14.0"diff --git a/src/free_claude_code/core/anthropic/streaming/ledger.py b/src/free_claude_code/core/anthropic/streaming/ledger.pyindex 5625945a..90c39e85 100644--- a/src/free_claude_code/core/anthropic/streaming/ledger.py+++ b/src/free_claude_code/core/anthropic/streaming/ledger.py@@ -10,19 +10,14 @@ from typing import Anyfrom loguru import logger+from free_claude_code.core.token_estimation import estimate_text_tokens+from .emitter import AnthropicSseEmitterfrom .recovery import (ToolSchema,parse_complete_tool_input,)-try:- import tiktoken-- ENCODER = tiktoken.get_encoding("cl100k_base")-except Exception:- ENCODER = None-def _safe_usage_int(value: object) -> int:return value if isinstance(value, int) else 0@@ -446,28 +441,22 @@ class AnthropicStreamLedger:return "".join(self._thinking_parts)def estimate_output_tokens(self) -> int:- if ENCODER:- text_tokens = len(ENCODER.encode(self.accumulated_text))- reasoning_tokens = len(ENCODER.encode(self.accumulated_reasoning))- tool_tokens = 0- tool_count = 0- for name, content in self._iter_tool_token_payloads():- tool_tokens += len(ENCODER.encode(name))- tool_tokens += len(ENCODER.encode(content))- tool_tokens += 15- tool_count += 1-- block_count = (- (1 if self.accumulated_reasoning else 0)- + (1 if self.accumulated_text else 0)- + tool_count- )- return text_tokens + reasoning_tokens + tool_tokens + (block_count * 4)-- text_tokens = len(self.accumulated_text) // 4- reasoning_tokens = len(self.accumulated_reasoning) // 4- tool_tokens = sum(1 for _ in self._iter_tool_token_payloads()) * 50- return text_tokens + reasoning_tokens + tool_tokens+ text_tokens = estimate_text_tokens(self.accumulated_text)+ reasoning_tokens = estimate_text_tokens(self.accumulated_reasoning)+ tool_tokens = 0+ tool_count = 0+ for name, content in self._iter_tool_token_payloads():+ tool_tokens += estimate_text_tokens(name)+ tool_tokens += estimate_text_tokens(content)+ tool_tokens += 15+ tool_count += 1++ block_count = (+ (1 if self.accumulated_reasoning else 0)+ + (1 if self.accumulated_text else 0)+ + tool_count+ )+ return text_tokens + reasoning_tokens + tool_tokens + (block_count * 4)def _iter_tool_token_payloads(self) -> Iterator[tuple[str, str]]:for block in self.tool_blocks():diff --git a/src/free_claude_code/core/anthropic/tokens.py b/src/free_claude_code/core/anthropic/tokens.pyindex 5fc1750b..72ec3443 100644--- a/src/free_claude_code/core/anthropic/tokens.py+++ b/src/free_claude_code/core/anthropic/tokens.py@@ -2,20 +2,13 @@import json-import tiktokenfrom loguru import logger+from free_claude_code.core.token_estimation import estimate_text_tokens+from .content import get_block_attrfrom .models import Message, SystemContent, Tool-ENCODER = tiktoken.get_encoding("cl100k_base")--_DISALLOWED_SPECIAL: tuple[str, ...] = ()---def _count_text_tokens(text: str) -> int:- return len(ENCODER.encode(text, disallowed_special=_DISALLOWED_SPECIAL))-def get_token_count(messages: list[Message],@@ -27,34 +20,34 @@ def get_token_count(if system:if isinstance(system, str):- total_tokens += _count_text_tokens(system)+ total_tokens += estimate_text_tokens(system)elif isinstance(system, list):for block in system:text = get_block_attr(block, "text", "")if text:- total_tokens += _count_text_tokens(str(text))+ total_tokens += estimate_text_tokens(str(text))total_tokens += 4for msg in messages:if isinstance(msg.content, str):- total_tokens += _count_text_tokens(msg.content)+ total_tokens += estimate_text_tokens(msg.content)elif isinstance(msg.content, list):for block in msg.content:b_type = get_block_attr(block, "type") or Noneif b_type == "text":text = get_block_attr(block, "text", "")- total_tokens += _count_text_tokens(str(text))+ total_tokens += estimate_text_tokens(str(text))elif b_type == "thinking":thinking = get_block_attr(block, "thinking", "")- total_tokens += _count_text_tokens(str(thinking))+ total_tokens += estimate_text_tokens(str(thinking))elif b_type == "tool_use":name = get_block_attr(block, "name", "")inp = get_block_attr(block, "input", {})block_id = get_block_attr(block, "id", "")- total_tokens += _count_text_tokens(str(name))- total_tokens += _count_text_tokens(json.dumps(inp))- total_tokens += _count_text_tokens(str(block_id))+ total_tokens += estimate_text_tokens(str(name))+ total_tokens += estimate_text_tokens(json.dumps(inp))+ total_tokens += estimate_text_tokens(str(block_id))total_tokens += 15elif b_type == "image":source = get_block_attr(block, "source")@@ -70,10 +63,10 @@ def get_token_count(content = get_block_attr(block, "content", "")tool_use_id = get_block_attr(block, "tool_use_id", "")if isinstance(content, str):- total_tokens += _count_text_tokens(content)+ total_tokens += estimate_text_tokens(content)else:- total_tokens += _count_text_tokens(json.dumps(content))- total_tokens += _count_text_tokens(str(tool_use_id))+ total_tokens += estimate_text_tokens(json.dumps(content))+ total_tokens += estimate_text_tokens(str(tool_use_id))total_tokens += 8elif b_type in ("server_tool_use",@@ -85,14 +78,14 @@ def get_token_count(else:blob = blocktry:- total_tokens += _count_text_tokens(+ total_tokens += estimate_text_tokens(json.dumps(blob, default=str, ensure_ascii=False))except (TypeError, ValueError, OverflowError) as e:logger.debug("Block encode fallback b_type={} err={}", b_type, e)- total_tokens += _count_text_tokens(str(blob))+ total_tokens += estimate_text_tokens(str(blob))total_tokens += 12else:logger.debug(@@ -100,16 +93,16 @@ def get_token_count(b_type,)try:- total_tokens += _count_text_tokens(json.dumps(block))+ total_tokens += estimate_text_tokens(json.dumps(block))except TypeError, ValueError:- total_tokens += _count_text_tokens(str(block))+ total_tokens += estimate_text_tokens(str(block))if tools:for tool in tools:tool_str = (tool.name + (tool.description or "") + json.dumps(tool.input_schema))- total_tokens += _count_text_tokens(tool_str)+ total_tokens += estimate_text_tokens(tool_str)total_tokens += len(messages) * 4if tools:diff --git a/src/free_claude_code/core/openai_responses/streaming/ledger.py b/src/free_claude_code/core/openai_responses/streaming/ledger.pyindex b312f172..cab4a8d4 100644--- a/src/free_claude_code/core/openai_responses/streaming/ledger.py+++ b/src/free_claude_code/core/openai_responses/streaming/ledger.py@@ -3,7 +3,8 @@from collections.abc import Mappingfrom typing import Any-from ..usage import estimate_text_tokens+from free_claude_code.core.token_estimation import estimate_text_tokens+from .blocks import BlockState_ANTHROPIC_INPUT_TOKEN_FIELDS = (diff --git a/src/free_claude_code/core/openai_responses/usage.py b/src/free_claude_code/core/token_estimation.pysimilarity index 61%rename from src/free_claude_code/core/openai_responses/usage.pyrename to src/free_claude_code/core/token_estimation.pyindex a6a2e28c..c3bb8756 100644--- a/src/free_claude_code/core/openai_responses/usage.py+++ b/src/free_claude_code/core/token_estimation.py@@ -1,7 +1,10 @@-"""Usage helpers for OpenAI Responses payloads."""+"""Process-wide best-effort plain-text token estimation."""from typing import Protocol+import tiktoken+from loguru import logger+_DISALLOWED_SPECIAL: tuple[str, ...] = ()@@ -12,14 +15,13 @@ class _TokenEncoder(Protocol):def _load_encoder() -> _TokenEncoder | None:- try:- import tiktoken- except ImportError:- return None-try:return tiktoken.get_encoding("cl100k_base")- except ValueError:+ except Exception as exc:+ logger.warning(+ "cl100k_base token encoder unavailable ({}); using approximate token estimates",+ type(exc).__name__,+ )return None@@ -27,7 +29,7 @@ _ENCODER = _load_encoder()def estimate_text_tokens(text: str) -> int:- """Return a best-effort token estimate for Responses usage details."""+ """Estimate tokens for plain text using the shared process-wide encoder."""if not text:return 0if _ENCODER is not None:diff --git a/tests/api/test_request_utils.py b/tests/api/test_request_utils.pyindex d0126094..9312238e 100644--- a/tests/api/test_request_utils.py+++ b/tests/api/test_request_utils.py@@ -1,6 +1,6 @@"""Tests for API request detection and token counting helpers."""-from unittest.mock import MagicMock+from unittest.mock import MagicMock, patchimport pytest@@ -549,21 +549,19 @@ class TestGetTokenCount:count = get_token_count([msg])assert count >= 85- def test_known_payload_estimate_range(self):- """Known payload produces estimate within expected range (validation harness)."""- import tiktoken-- enc = tiktoken.get_encoding("cl100k_base")+ def test_known_payload_structural_overhead(self):+ """Protocol overhead remains separate from plain-text estimation."""system_text = "You are a helpful assistant."user_text = "Hello, how are you?"- sys_tokens = len(enc.encode(system_text))- user_tokens = len(enc.encode(user_text))- # Min: content tokens + system overhead (4) + per-msg overhead (4)- expected_min = sys_tokens + user_tokens + 4 + 4msg = MagicMock()msg.content = user_text- count = get_token_count([msg], system=system_text)- assert count >= expected_min, f"count={count} < expected_min={expected_min}"+ with patch(+ "free_claude_code.core.anthropic.tokens.estimate_text_tokens",+ side_effect=len,+ ):+ count = get_token_count([msg], system=system_text)++ assert count == len(system_text) + len(user_text) + 4 + 4# --- Parametrized Edge Case Tests ---diff --git a/tests/contracts/test_offline_startup.py b/tests/contracts/test_offline_startup.pynew file mode 100644index 00000000..f60fed4f--- /dev/null+++ b/tests/contracts/test_offline_startup.py@@ -0,0 +1,41 @@+"""Contracts for starting FCC without token-encoder network access."""++import subprocess+import sys+++def test_api_import_survives_unavailable_tiktoken_encoding() -> None:+ script = """+import os+import tempfile++import requests++calls = []+++def fail(url, *args, **kwargs):+ del args, kwargs+ calls.append(url)+ raise requests.exceptions.ProxyError("Bearer secret")+++with tempfile.TemporaryDirectory() as cache_dir:+ os.environ["TIKTOKEN_CACHE_DIR"] = cache_dir+ requests.get = fail+ import free_claude_code.api.app++assert len(calls) == 1, calls+"""++ completed = subprocess.run(+ [sys.executable, "-c", script],+ capture_output=True,+ check=False,+ text=True,+ timeout=15,+ )++ assert completed.returncode == 0, completed.stderr or completed.stdout+ assert "ProxyError" in completed.stderr+ assert "Bearer secret" not in completed.stderrdiff --git a/tests/core/anthropic/test_stream_ledger.py b/tests/core/anthropic/test_stream_ledger.pyindex 2ebdb9b2..3a20fb5f 100644--- a/tests/core/anthropic/test_stream_ledger.py+++ b/tests/core/anthropic/test_stream_ledger.py@@ -113,14 +113,19 @@ def test_close_unclosed_blocks_closes_each_block_once() -> None:assert list(ledger.close_unclosed_blocks()) == []-def test_output_token_estimate_uses_encoder_when_available() -> None:+def test_output_token_estimate_combines_shared_estimates_and_block_overhead() -> None:ledger = AnthropicStreamLedger("msg_1", "model")+ ledger.start_thinking_block()+ ledger.emit_thinking_delta("why")+ ledger.stop_thinking_block()ledger.start_text_block()ledger.emit_text_delta("abcd")+ ledger.stop_text_block()+ ledger.start_tool_block(0, "toolu_1", "Read")+ ledger.emit_tool_delta(0, "{}")- class Encoder:- def encode(self, text: str) -> list[int]:- return list(range(len(text)))-- with patch("free_claude_code.core.anthropic.streaming.ledger.ENCODER", Encoder()):- assert ledger.estimate_output_tokens() == 8+ with patch(+ "free_claude_code.core.anthropic.streaming.ledger.estimate_text_tokens",+ side_effect=len,+ ):+ assert ledger.estimate_output_tokens() == 40diff --git a/tests/core/test_token_estimation.py b/tests/core/test_token_estimation.pynew file mode 100644index 00000000..89ba2f46--- /dev/null+++ b/tests/core/test_token_estimation.py@@ -0,0 +1,93 @@+"""Tests for process-wide plain-text token estimation."""++from unittest.mock import MagicMock, patch++import pytest++from free_claude_code.core import token_estimation+++class _RecordingEncoder:+ def __init__(self, tokens: list[int]) -> None:+ self.tokens = tokens+ self.calls: list[tuple[str, tuple[str, ...]]] = []++ def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> list[int]:+ self.calls.append((text, disallowed_special))+ return self.tokens+++def test_encoder_is_loaded_by_canonical_name() -> None:+ encoder = MagicMock()++ with patch.object(+ token_estimation.tiktoken,+ "get_encoding",+ return_value=encoder,+ ) as get_encoding:+ assert token_estimation._load_encoder() is encoder++ get_encoding.assert_called_once_with("cl100k_base")+++def test_encoder_acquisition_failure_uses_safe_fallback_warning() -> None:+ with (+ patch.object(+ token_estimation.tiktoken,+ "get_encoding",+ side_effect=RuntimeError("Bearer secret"),+ ),+ patch.object(token_estimation.logger, "warning") as warning,+ ):+ assert token_estimation._load_encoder() is None++ warning.assert_called_once_with(+ "cl100k_base token encoder unavailable ({}); using approximate token estimates",+ "RuntimeError",+ )+ assert "Bearer secret" not in str(warning.call_args)+++def test_estimate_text_tokens_uses_cached_encoder() -> None:+ encoder = _RecordingEncoder([1, 2, 3])++ with patch.object(token_estimation, "_ENCODER", encoder):+ assert token_estimation.estimate_text_tokens("hello") == 3++ assert encoder.calls == [("hello", ())]+++def test_empty_text_skips_encoder() -> None:+ encoder = _RecordingEncoder([1])++ with patch.object(token_estimation, "_ENCODER", encoder):+ assert token_estimation.estimate_text_tokens("") == 0++ assert encoder.calls == []+++ ("text", "expected"),+ [+ ("a", 1),+ ("abcdefg", 1),+ ("abcdefgh", 2),+ ],+)+def test_estimate_text_tokens_falls_back_to_character_ratio(+ text: str,+ expected: int,+) -> None:+ with patch.object(token_estimation, "_ENCODER", None):+ assert token_estimation.estimate_text_tokens(text) == expected+++def test_encoder_execution_errors_are_not_hidden() -> None:+ encoder = MagicMock()+ encoder.encode.side_effect = RuntimeError("encode failed")++ with (+ patch.object(token_estimation, "_ENCODER", encoder),+ pytest.raises(RuntimeError, match="encode failed"),+ ):+ token_estimation.estimate_text_tokens("hello")diff --git a/uv.lock b/uv.lockindex 1ef36fd4..50459d90 100644--- a/uv.lock+++ b/uv.lock@@ -607,7 +607,7 @@ wheels = [[[package]]name = "free-claude-code"-version = "5.3.0"+version = "5.3.1"source = { editable = "." }dependencies = [{ name = "aiohttp" },