Skip to content

Building AI agents, atomically

fix: make token counting accurate with proper error handling

KennyVaneetveldea978204f72f3
diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py
index a5c2687..f32acb2 100644
--- a/atomic-agents/atomic_agents/agents/atomic_agent.py
+++ b/atomic-agents/atomic_agents/agents/atomic_agent.py
@@ -10,7 +10,7 @@ from atomic_agents.context.system_prompt_generator import (
SystemPromptGenerator,
)
from atomic_agents.base.base_io_schema import BaseIOSchema
-from atomic_agents.utils.token_counter import TokenCounter, TokenCountResult
+from atomic_agents.utils.token_counter import get_token_counter, TokenCountResult, TokenCountError
import json
from instructor.dsl.partial import PartialBase
@@ -77,6 +77,7 @@ class AgentConfig(BaseModel):
description="The role of the assistant in the conversation. Use 'model' for Gemini, 'assistant' for OpenAI/Anthropic.",
)
model_config = {"arbitrary_types_allowed": True}
+ mode: Mode = Field(default=Mode.TOOLS, description="The Instructor mode used for structured outputs (TOOLS, JSON, etc.).")
model_api_parameters: Optional[dict] = Field(None, description="Additional parameters passed to the API provider.")
@@ -171,6 +172,7 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
self.assistant_role = config.assistant_role
self.initial_history = self.history.copy()
self.current_user_input = None
+ self.mode = config.mode
self.model_api_parameters = config.model_api_parameters or {}
# Hook management attributes
@@ -247,20 +249,50 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
self.messages = self._build_system_messages()
self.messages += self.history.get_history()
- def _build_schema_context(self) -> str:
+ def _build_tools_definition(self) -> Optional[List[Dict[str, Any]]]:
"""
- Build the schema context that Instructor adds for structured output.
+ Build the tools definition that Instructor sends for TOOLS mode.
- This replicates how Instructor serializes the output schema for the LLM,
- allowing accurate token counting before making API calls.
+ This uses Instructor's actual schema generation to create the exact
+ tools parameter that would be sent to the LLM for TOOLS mode.
+ For JSON modes, returns None as the schema is embedded in messages.
Returns:
- str: JSON string of the output schema in Instructor's format.
+ Optional[List[Dict[str, Any]]]: Tools definition for TOOLS mode, or None for JSON modes.
+ """
+ from instructor.processing.schema import generate_openai_schema
+
+ # Only return tools for TOOLS-based modes
+ tools_modes = {Mode.TOOLS, Mode.TOOLS_STRICT, Mode.PARALLEL_TOOLS}
+ if self.mode in tools_modes:
+ return [
+ {
+ "type": "function",
+ "function": generate_openai_schema(self.output_schema),
+ }
+ ]
+ return None
+
+ def _build_schema_for_json_mode(self) -> str:
"""
+ Build the schema context for JSON modes (appended to system message).
+
+ This matches exactly how Instructor formats the schema for JSON/MD_JSON modes.
+
+ Returns:
+ str: JSON schema string formatted as Instructor does.
+ """
+ from textwrap import dedent
+
schema = self.output_schema.model_json_schema()
- # Instructor typically sends the schema in a format like this for JSON mode
- # For TOOLS mode, it would be in the tools parameter, but the token count is similar
- return json.dumps(schema, ensure_ascii=False)
+ return dedent(f"""
+ As a genius expert, your task is to understand the content and provide
+ the parsed objects in json that match the following json_schema:
+
+ {json.dumps(schema, indent=2, ensure_ascii=False)}
+
+ Make sure to return an instance of the JSON, not the schema itself
+ """).strip()
def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]:
"""
@@ -290,11 +322,16 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
# Multimodal object - use instructor's to_openai method
try:
serialized_content.append(item.to_openai(Mode.JSON))
- except Exception:
- # Fallback: just include a placeholder for the media
- media_type = type(item).__name__.lower()
+ except Exception as e:
+ # Log the error and use placeholder for token estimation
+ logger = logging.getLogger(__name__)
+ media_type = type(item).__name__
+ logger.warning(
+ f"Failed to serialize {media_type} for token counting: {e}. "
+ f"Using placeholder for estimation."
+ )
serialized_content.append(
- {"type": "text", "text": f"[{media_type} content]"}
+ {"type": "text", "text": f"[{media_type.lower()} content]"}
)
else:
# Unknown type - convert to string
@@ -313,8 +350,11 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
This method computes the token count by serializing the context exactly
as Instructor does, including:
- System prompt
- - Conversation history (with Pydantic schemas serialized as JSON)
- - Output schema overhead (the schema Instructor sends for structured output)
+ - Conversation history (with multimodal content serialized properly)
+ - Tools/schema overhead (using Instructor's actual schema generation)
+
+ For TOOLS mode: Uses the actual tools parameter that Instructor sends.
+ For JSON modes: Appends the schema to the system message as Instructor does.
Works with any model supported by LiteLLM including OpenAI, Anthropic,
Google, and 100+ other providers.
@@ -322,8 +362,9 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
Returns:
TokenCountResult: A named tuple containing:
- total: Total tokens in the context (including schema overhead)
- - system_prompt: Tokens in the system prompt + schema overhead
+ - system_prompt: Tokens in the system prompt
- history: Tokens in the conversation history
+ - tools: Tokens in the tools/function definitions (TOOLS mode only)
- model: The model used for counting
- max_tokens: Maximum context window (if known)
- utilization: Percentage of context used (if max_tokens known)
@@ -335,8 +376,9 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
# Get accurate token count at any time
result = agent.get_context_token_count()
print(f"Total: {result.total} tokens")
- print(f"System (with schema): {result.system_prompt} tokens")
+ print(f"System: {result.system_prompt} tokens")
print(f"History: {result.history} tokens")
+ print(f"Tools: {result.tools} tokens")
if result.utilization:
print(f"Context usage: {result.utilization:.1%}")
```
@@ -345,29 +387,32 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
The 'token:counted' hook event is dispatched, allowing for
monitoring and logging of token usage.
"""
- counter = TokenCounter()
+ counter = get_token_counter()
- # Build system messages with schema overhead appended
+ # Build system messages
system_messages = self._build_system_messages()
- schema_context = self._build_schema_context()
- # Add schema as part of system context (Instructor adds this for structured output)
- if system_messages:
- # Append schema to existing system message
- system_messages = [
- {
- "role": system_messages[0]["role"],
- "content": system_messages[0]["content"] + "\n\n" + schema_context,
- }
- ]
- else:
- # No system message, add schema as its own message
- system_messages = [{"role": "system", "content": schema_context}]
+ # Handle schema serialization based on mode
+ tools = self._build_tools_definition()
+
+ if tools is None:
+ # JSON mode - append schema to system message like Instructor does
+ schema_context = self._build_schema_for_json_mode()
+ if system_messages:
+ system_messages = [
+ {
+ "role": system_messages[0]["role"],
+ "content": system_messages[0]["content"] + "\n\n" + schema_context,
+ }
+ ]
+ else:
+ system_messages = [{"role": "system", "content": schema_context}]
result = counter.count_context(
model=self.model,
system_messages=system_messages,
history_messages=self._serialize_history_for_token_count(),
+ tools=tools,
)
# Dispatch hook for monitoring
diff --git a/atomic-agents/atomic_agents/utils/__init__.py b/atomic-agents/atomic_agents/utils/__init__.py
index a0198a7..30fa77d 100644
--- a/atomic-agents/atomic_agents/utils/__init__.py
+++ b/atomic-agents/atomic_agents/utils/__init__.py
@@ -1,10 +1,12 @@
"""Utility functions."""
from .format_tool_message import format_tool_message
-from .token_counter import TokenCounter, TokenCountResult
+from .token_counter import TokenCounter, TokenCountResult, TokenCountError, get_token_counter
__all__ = [
"format_tool_message",
"TokenCounter",
"TokenCountResult",
+ "TokenCountError",
+ "get_token_counter",
]
diff --git a/atomic-agents/atomic_agents/utils/token_counter.py b/atomic-agents/atomic_agents/utils/token_counter.py
index f1440fe..764c135 100644
--- a/atomic-agents/atomic_agents/utils/token_counter.py
+++ b/atomic-agents/atomic_agents/utils/token_counter.py
@@ -6,14 +6,21 @@ from typing import Any, Dict, List, NamedTuple, Optional
logger = logging.getLogger(__name__)
+class TokenCountError(Exception):
+ """Exception raised when token counting fails."""
+
+ pass
+
+
class TokenCountResult(NamedTuple):
"""
Result of a token count operation.
Attributes:
- total: Total number of tokens in the context.
+ total: Total number of tokens in the context (messages + tools).
system_prompt: Tokens in the system prompt (0 if no system prompt).
history: Tokens in the conversation history.
+ tools: Tokens in the tools/function definitions (0 if no tools).
model: The model used for tokenization.
max_tokens: Maximum context window for the model (None if unknown).
utilization: Percentage of context window used (None if max_tokens unknown).
@@ -22,17 +29,30 @@ class TokenCountResult(NamedTuple):
total: int
system_prompt: int
history: int
+ tools: int
model: str
max_tokens: Optional[int] = None
utilization: Optional[float] = None
+# Module-level singleton for efficiency
+_token_counter_instance: Optional["TokenCounter"] = None
+
+
+def get_token_counter() -> "TokenCounter":
+ """Get the singleton TokenCounter instance."""
+ global _token_counter_instance
+ if _token_counter_instance is None:
+ _token_counter_instance = TokenCounter()
+ return _token_counter_instance
+
+
class TokenCounter:
"""
Utility class for counting tokens using LiteLLM's provider-agnostic tokenizer.
- This class provides methods for counting tokens in messages, text, and
- retrieving model context limits. It uses LiteLLM's token_counter which
+ This class provides methods for counting tokens in messages, text, tools,
+ and retrieving model context limits. It uses LiteLLM's token_counter which
automatically selects the appropriate tokenizer based on the model.
Works with any model supported by LiteLLM including:
@@ -49,25 +69,51 @@ class TokenCounter:
messages = [{"role": "user", "content": "Hello, world!"}]
count = counter.count_messages("gpt-4", messages)
+ # Count tokens with tools (for TOOLS mode)
+ tools = [{"type": "function", "function": {...}}]
+ count = counter.count_messages("gpt-4", messages, tools=tools)
+
# Get max tokens for a model
max_tokens = counter.get_max_tokens("gpt-4")
```
"""
- def count_messages(self, model: str, messages: List[Dict[str, Any]]) -> int:
+ def count_messages(
+ self,
+ model: str,
+ messages: List[Dict[str, Any]],
+ tools: Optional[List[Dict[str, Any]]] = None,
+ ) -> int:
"""
- Count the number of tokens in a list of messages.
+ Count the number of tokens in a list of messages and optional tools.
Args:
model: The model identifier (e.g., "gpt-4", "anthropic/claude-3-opus").
messages: List of message dictionaries with 'role' and 'content' keys.
+ tools: Optional list of tool definitions (for TOOLS mode).
Returns:
- The number of tokens in the messages.
+ The number of tokens in the messages (and tools if provided).
+
+ Raises:
+ TokenCountError: If token counting fails.
"""
- from litellm import token_counter
+ if not model:
+ raise ValueError("model is required for token counting")
- return token_counter(model=model, messages=messages)
+ try:
+ from litellm import token_counter
+
+ if tools:
+ return token_counter(model=model, messages=messages, tools=tools)
+ return token_counter(model=model, messages=messages)
+ except ImportError as e:
+ raise ImportError(
+ "litellm is required for token counting. "
+ "Install it with: pip install litellm"
+ ) from e
+ except Exception as e:
+ raise TokenCountError(f"Failed to count tokens for model '{model}': {e}") from e
def count_text(self, model: str, text: str) -> int:
"""
@@ -79,11 +125,12 @@ class TokenCounter:
Returns:
The number of tokens in the text.
- """
- from litellm import token_counter
+ Raises:
+ TokenCountError: If token counting fails.
+ """
messages = [{"role": "user", "content": text}]
- return token_counter(model=model, messages=messages)
+ return self.count_messages(model, messages)
def get_max_tokens(self, model: str) -> Optional[int]:
"""
@@ -94,10 +141,23 @@ class TokenCounter:
Returns:
The maximum number of tokens, or None if unknown.
+
+ Raises:
+ TypeError: If model is None or not a string.
+ ImportError: If litellm is not installed.
"""
+ if not isinstance(model, str):
+ raise TypeError(f"model must be a string, got {type(model).__name__}")
+
try:
from litellm import get_max_tokens
+ except ImportError as e:
+ raise ImportError(
+ "litellm is required for token counting. "
+ "Install it with: pip install litellm"
+ ) from e
+ try:
return get_max_tokens(model)
except Exception as e:
logger.warning(f"Could not determine max tokens for model '{model}': {e}")
@@ -108,29 +168,50 @@ class TokenCounter:
model: str,
system_messages: List[Dict[str, Any]],
history_messages: List[Dict[str, Any]],
+ tools: Optional[List[Dict[str, Any]]] = None,
) -> TokenCountResult:
"""
- Count tokens with breakdown by system prompt and history.
+ Count tokens with breakdown by system prompt, history, and tools.
Args:
model: The model identifier.
system_messages: System prompt messages (may be empty).
history_messages: Conversation history messages.
+ tools: Optional list of tool definitions (for TOOLS mode).
Returns:
TokenCountResult with breakdown and utilization metrics.
+
+ Raises:
+ TokenCountError: If token counting fails.
"""
system_tokens = self.count_messages(model, system_messages) if system_messages else 0
history_tokens = self.count_messages(model, history_messages) if history_messages else 0
- total_tokens = system_tokens + history_tokens
+
+ # Count tool tokens separately if provided
+ tools_tokens = 0
+ if tools:
+ # To count just the tools overhead, we count empty messages with tools
+ # and subtract the base overhead
+ empty_with_tools = self.count_messages(
+ model, [{"role": "user", "content": ""}], tools=tools
+ )
+ empty_without_tools = self.count_messages(
+ model, [{"role": "user", "content": ""}]
+ )
+ tools_tokens = empty_with_tools - empty_without_tools
+
+ total_tokens = system_tokens + history_tokens + tools_tokens
max_tokens = self.get_max_tokens(model)
- utilization = (total_tokens / max_tokens) if max_tokens else None
+ # Prevent division by zero
+ utilization = (total_tokens / max_tokens) if max_tokens and max_tokens > 0 else None
return TokenCountResult(
total=total_tokens,
system_prompt=system_tokens,
history=history_tokens,
+ tools=tools_tokens,
model=model,
max_tokens=max_tokens,
utilization=utilization,
diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.py
index 070c23c..14688e8 100644
--- a/atomic-agents/tests/agents/test_atomic_agent.py
+++ b/atomic-agents/tests/agents/test_atomic_agent.py
@@ -767,8 +767,8 @@ def test_backward_compatibility_no_breaking_changes(mock_instructor, mock_histor
# Token Counting Tests
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_basic(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_basic(mock_get_token_counter, mock_instructor, mock_history, mock_system_prompt_generator):
"""Test basic token counting functionality."""
mock_history.get_history.return_value = [
{"role": "user", "content": "Hello"},
@@ -777,9 +777,9 @@ def test_get_context_token_count_basic(mock_token_counter_class, mock_instructor
mock_system_prompt_generator.generate_prompt.return_value = "You are a helpful assistant."
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=100, system_prompt=30, history=70, model="gpt-5-mini", max_tokens=8192, utilization=0.0122
+ total=100, system_prompt=30, history=70, tools=0, model="gpt-5-mini", max_tokens=8192, utilization=0.0122
)
config = AgentConfig(
@@ -795,22 +795,25 @@ def test_get_context_token_count_basic(mock_token_counter_class, mock_instructor
assert result.total == 100
assert result.system_prompt == 30
assert result.history == 70
+ assert result.tools == 0
assert result.model == "gpt-5-mini"
assert result.max_tokens == 8192
assert result.utilization is not None # Should have utilization since max_tokens is known
mock_counter_instance.count_context.assert_called_once()
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_includes_schema_overhead(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):
- """Test that token counting includes the output schema overhead."""
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_includes_schema_overhead(mock_get_token_counter, mock_instructor, mock_history, mock_system_prompt_generator):
+ """Test that token counting includes the output schema overhead for JSON mode."""
+ from instructor import Mode
+
mock_history.get_history.return_value = []
mock_system_prompt_generator.generate_prompt.return_value = "System prompt"
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=50, system_prompt=50, history=0, model="gpt-5-mini"
+ total=50, system_prompt=50, history=0, tools=0, model="gpt-5-mini"
)
config = AgentConfig(
@@ -818,6 +821,7 @@ def test_get_context_token_count_includes_schema_overhead(mock_token_counter_cla
model="gpt-5-mini",
history=mock_history,
system_prompt_generator=mock_system_prompt_generator,
+ mode=Mode.JSON, # JSON mode appends schema to system message
)
agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)
@@ -827,20 +831,22 @@ def test_get_context_token_count_includes_schema_overhead(mock_token_counter_cla
call_args = mock_counter_instance.count_context.call_args
system_messages = call_args.kwargs["system_messages"]
assert len(system_messages) == 1
- # System message should contain both the prompt AND the schema
+ # System message should contain both the prompt AND the schema (for JSON mode)
assert "System prompt" in system_messages[0]["content"]
assert "chat_message" in system_messages[0]["content"] # Schema field from BasicChatOutputSchema
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_no_system_prompt(mock_token_counter_class, mock_instructor, mock_history):
- """Test token counting when system_role is None (schema still included)."""
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_no_system_prompt(mock_get_token_counter, mock_instructor, mock_history):
+ """Test token counting when system_role is None (schema still included for JSON mode)."""
+ from instructor import Mode
+
mock_history.get_history.return_value = [{"role": "user", "content": "Hello"}]
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=25, system_prompt=20, history=5, model="gpt-5-mini"
+ total=25, system_prompt=20, history=5, tools=0, model="gpt-5-mini"
)
config = AgentConfig(
@@ -848,27 +854,28 @@ def test_get_context_token_count_no_system_prompt(mock_token_counter_class, mock
model="gpt-5-mini",
history=mock_history,
system_role=None, # No system prompt
+ mode=Mode.JSON, # JSON mode to test schema in system message
)
agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)
result = agent.get_context_token_count()
- # Even without system prompt, schema should be included
+ # Even without system prompt, schema should be included (for JSON mode)
call_args = mock_counter_instance.count_context.call_args
system_messages = call_args.kwargs["system_messages"]
assert len(system_messages) == 1 # Schema is added as system message
assert "chat_message" in system_messages[0]["content"] # Schema content
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_uses_configured_model(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_uses_configured_model(mock_get_token_counter, mock_instructor, mock_history, mock_system_prompt_generator):
"""Test that token counting uses the agent's configured model."""
mock_history.get_history.return_value = []
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=15, system_prompt=15, history=0, model="claude-3-opus-20240229"
+ total=15, system_prompt=15, history=0, tools=0, model="claude-3-opus-20240229"
)
config = AgentConfig(
@@ -885,13 +892,13 @@ def test_get_context_token_count_uses_configured_model(mock_token_counter_class,
assert call_args.kwargs["model"] == "claude-3-opus-20240229"
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_returns_token_count_result(mock_token_counter_class, agent):
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_returns_token_count_result(mock_get_token_counter, agent):
"""Test that get_context_token_count returns a TokenCountResult."""
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=50, system_prompt=20, history=30, model="gpt-5-mini"
+ total=50, system_prompt=20, history=30, tools=0, model="gpt-5-mini"
)
result = agent.get_context_token_count()
@@ -900,17 +907,18 @@ def test_get_context_token_count_returns_token_count_result(mock_token_counter_c
assert hasattr(result, "total")
assert hasattr(result, "system_prompt")
assert hasattr(result, "history")
+ assert hasattr(result, "tools")
assert hasattr(result, "model")
assert hasattr(result, "max_tokens")
assert hasattr(result, "utilization")
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_hook_dispatch(mock_token_counter_class, agent):
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_hook_dispatch(mock_get_token_counter, agent):
"""Test that token:counted hook is dispatched."""
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
- expected_result = TokenCountResult(total=100, system_prompt=30, history=70, model="gpt-5-mini")
+ mock_get_token_counter.return_value = mock_counter_instance
+ expected_result = TokenCountResult(total=100, system_prompt=30, history=70, tools=0, model="gpt-5-mini")
mock_counter_instance.count_context.return_value = expected_result
hook_called = []
@@ -927,13 +935,13 @@ def test_get_context_token_count_hook_dispatch(mock_token_counter_class, agent):
assert hook_called[0].total == 100
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_hook_not_called_when_disabled(mock_token_counter_class, agent):
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_hook_not_called_when_disabled(mock_get_token_counter, agent):
"""Test that token:counted hook is not called when hooks are disabled."""
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=100, system_prompt=30, history=70, model="gpt-5-mini"
+ total=100, system_prompt=30, history=70, tools=0, model="gpt-5-mini"
)
hook_called = []
@@ -949,16 +957,16 @@ def test_get_context_token_count_hook_not_called_when_disabled(mock_token_counte
assert len(hook_called) == 0
-@patch("atomic_agents.agents.atomic_agent.TokenCounter")
-def test_get_context_token_count_multimodal_content(mock_token_counter_class, mock_instructor):
+@patch("atomic_agents.agents.atomic_agent.get_token_counter")
+def test_get_context_token_count_multimodal_content(mock_get_token_counter, mock_instructor):
"""Test that multimodal content is properly serialized for token counting."""
from instructor.processing.multimodal import Image
from instructor import Mode
mock_counter_instance = Mock()
- mock_token_counter_class.return_value = mock_counter_instance
+ mock_get_token_counter.return_value = mock_counter_instance
mock_counter_instance.count_context.return_value = TokenCountResult(
- total=150, system_prompt=50, history=100, model="gpt-4-vision-preview"
+ total=150, system_prompt=50, history=100, tools=0, model="gpt-4-vision-preview"
)
# Create a multimodal input schema
diff --git a/atomic-agents/tests/utils/test_token_counter.py b/atomic-agents/tests/utils/test_token_counter.py
index d8d22b6..63c668d 100644
--- a/atomic-agents/tests/utils/test_token_counter.py
+++ b/atomic-agents/tests/utils/test_token_counter.py
@@ -1,6 +1,11 @@
import pytest
from unittest.mock import Mock, patch
-from atomic_agents.utils.token_counter import TokenCounter, TokenCountResult
+from atomic_agents.utils.token_counter import (
+ TokenCounter,
+ TokenCountResult,
+ TokenCountError,
+ get_token_counter,
+)
class TestTokenCountResult:
@@ -10,14 +15,16 @@ class TestTokenCountResult:
result = TokenCountResult(
total=100,
system_prompt=30,
- history=70,
+ history=50,
+ tools=20,
model="gpt-4",
max_tokens=8192,
utilization=0.0122,
)
assert result.total == 100
assert result.system_prompt == 30
- assert result.history == 70
+ assert result.history == 50
+ assert result.tools == 20
assert result.model == "gpt-4"
assert result.max_tokens == 8192
assert result.utilization == 0.0122
@@ -26,7 +33,8 @@ class TestTokenCountResult:
result = TokenCountResult(
total=100,
system_prompt=30,
- history=70,
+ history=50,
+ tools=20,
model="gpt-4",
)
assert result.max_tokens is None
@@ -36,13 +44,15 @@ class TestTokenCountResult:
result = TokenCountResult(
total=100,
system_prompt=30,
- history=70,
+ history=50,
+ tools=20,
model="gpt-4",
)
- total, system_prompt, history, model, max_tokens, utilization = result
+ total, system_prompt, history, tools, model, max_tokens, utilization = result
assert total == 100
assert system_prompt == 30
- assert history == 70
+ assert history == 50
+ assert tools == 20
assert model == "gpt-4"
assert max_tokens is None
assert utilization is None
@@ -51,15 +61,17 @@ class TestTokenCountResult:
result = TokenCountResult(
total=100,
system_prompt=30,
- history=70,
+ history=50,
+ tools=20,
model="gpt-4",
)
assert result[0] == 100 # total
assert result[1] == 30 # system_prompt
- assert result[2] == 70 # history
- assert result[3] == "gpt-4" # model
- assert result[4] is None # max_tokens
- assert result[5] is None # utilization
+ assert result[2] == 50 # history
+ assert result[3] == 20 # tools
+ assert result[4] == "gpt-4" # model
+ assert result[5] is None # max_tokens
+ assert result[6] is None # utilization
class TestTokenCounter:
@@ -123,6 +135,37 @@ class TestTokenCounter:
assert result is None
+ @patch("litellm.token_counter")
+ def test_count_messages_with_tools(self, mock_token_counter):
+ mock_token_counter.return_value = 150
+
+ counter = TokenCounter()
+ messages = [{"role": "user", "content": "Hello"}]
+ tools = [{"type": "function", "function": {"name": "test_fn"}}]
+ result = counter.count_messages("gpt-4", messages, tools=tools)
+
+ assert result == 150
+ mock_token_counter.assert_called_once_with(
+ model="gpt-4", messages=messages, tools=tools
+ )
+
+ @patch("litellm.token_counter")
+ def test_count_messages_raises_token_count_error(self, mock_token_counter):
+ mock_token_counter.side_effect = Exception("API error")
+
+ counter = TokenCounter()
+ with pytest.raises(TokenCountError) as exc_info:
+ counter.count_messages("gpt-4", [{"role": "user", "content": "test"}])
+
+ assert "Failed to count tokens for model 'gpt-4'" in str(exc_info.value)
+
+ def test_count_messages_raises_value_error_for_empty_model(self):
+ counter = TokenCounter()
+ with pytest.raises(ValueError) as exc_info:
+ counter.count_messages("", [{"role": "user", "content": "test"}])
+
+ assert "model is required" in str(exc_info.value)
+
@patch("litellm.get_max_tokens")
@patch("litellm.token_counter")
def test_count_context(self, mock_token_counter, mock_get_max_tokens):
@@ -139,10 +182,33 @@ class TestTokenCounter:
assert result.total == 100
assert result.system_prompt == 30
assert result.history == 70
+ assert result.tools == 0
assert result.model == "gpt-4"
assert result.max_tokens == 8192
assert result.utilization == pytest.approx(100 / 8192)
+ @patch("litellm.get_max_tokens")
+ @patch("litellm.token_counter")
+ def test_count_context_with_tools(self, mock_token_counter, mock_get_max_tokens):
+ # system=30, history=70, empty_with_tools=60, empty_without_tools=10 -> tools=50
+ mock_token_counter.side_effect = [30, 70, 60, 10]
+ mock_get_max_tokens.return_value = 8192
+
+ counter = TokenCounter()
+ tools = [{"type": "function", "function": {"name": "test_fn"}}]
+ result = counter.count_context(
+ model="gpt-4",
+ system_messages=[{"role": "system", "content": "You are helpful"}],
+ history_messages=[{"role": "user", "content": "Hello"}],
+ tools=tools,
+ )
+
+ assert result.system_prompt == 30
+ assert result.history == 70
+ assert result.tools == 50
+ assert result.total == 150 # 30 + 70 + 50
+ assert result.model == "gpt-4"
+
@patch("litellm.get_max_tokens")
@patch("litellm.token_counter")
def test_count_context_empty_system(self, mock_token_counter, mock_get_max_tokens):
@@ -202,15 +268,91 @@ class TestTokenCounter:
assert mock_token_counter.call_count == len(models)
+ @patch("litellm.get_max_tokens")
+ @patch("litellm.token_counter")
+ def test_count_context_division_by_zero_prevention(
+ self, mock_token_counter, mock_get_max_tokens
+ ):
+ """Test that division by zero is prevented when max_tokens is 0."""
+ mock_token_counter.side_effect = [20, 30]
+ mock_get_max_tokens.return_value = 0 # Edge case: max_tokens = 0
+
+ counter = TokenCounter()
+ result = counter.count_context(
+ model="custom-model",
+ system_messages=[{"role": "system", "content": "Test"}],
+ history_messages=[{"role": "user", "content": "Test"}],
+ )
+
+ assert result.total == 50
+ assert result.max_tokens == 0
+ assert result.utilization is None # Should be None, not raise ZeroDivisionError
+
+ @patch("litellm.get_max_tokens")
+ @patch("litellm.token_counter")
+ def test_count_context_empty_history(self, mock_token_counter, mock_get_max_tokens):
+ """Test counting context with empty history messages."""
+ mock_token_counter.return_value = 30 # Only system
+ mock_get_max_tokens.return_value = 4096
+
+ counter = TokenCounter()
+ result = counter.count_context(
+ model="gpt-4",
+ system_messages=[{"role": "system", "content": "You are helpful"}],
+ history_messages=[], # Empty history
+ )
+
+ assert result.total == 30
+ assert result.system_prompt == 30
+ assert result.history == 0
+ assert result.tools == 0
+
+
+class TestGetTokenCounter:
+ """Tests for the get_token_counter singleton function."""
+
+ def test_get_token_counter_returns_instance(self):
+ """Test that get_token_counter returns a TokenCounter instance."""
+ counter = get_token_counter()
+ assert isinstance(counter, TokenCounter)
+
+ def test_get_token_counter_returns_same_instance(self):
+ """Test that get_token_counter returns the same singleton instance."""
+ counter1 = get_token_counter()
+ counter2 = get_token_counter()
+ assert counter1 is counter2
+
+
+class TestTokenCountError:
+ """Tests for TokenCountError exception."""
+
+ def test_token_count_error_is_exception(self):
+ """Test that TokenCountError is an Exception."""
+ error = TokenCountError("test error")
+ assert isinstance(error, Exception)
+
+ def test_token_count_error_message(self):
+ """Test that TokenCountError preserves the error message."""
+ error = TokenCountError("Custom error message")
+ assert str(error) == "Custom error message"
+
+
class TestTokenCounterIntegration:
"""Integration tests that verify the module structure."""
def test_import_from_utils(self):
- """Test that TokenCounter can be imported from utils."""
- from atomic_agents.utils import TokenCounter, TokenCountResult
+ """Test that all exports can be imported from utils."""
+ from atomic_agents.utils import (
+ TokenCounter,
+ TokenCountResult,
+ TokenCountError,
+ get_token_counter,
+ )
assert TokenCounter is not None
assert TokenCountResult is not None
+ assert TokenCountError is not None
+ assert get_token_counter is not None
def test_token_counter_instantiation(self):
"""Test that TokenCounter can be instantiated without arguments."""