Building AI agents, atomically
fix: make token counting accurate with proper error handling
KennyVaneetveldea978204f72f3
diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.pyindex a5c2687..f32acb2 100644--- a/atomic-agents/atomic_agents/agents/atomic_agent.py+++ b/atomic-agents/atomic_agents/agents/atomic_agent.py@@ -10,7 +10,7 @@ from atomic_agents.context.system_prompt_generator import (SystemPromptGenerator,)from atomic_agents.base.base_io_schema import BaseIOSchema-from atomic_agents.utils.token_counter import TokenCounter, TokenCountResult+from atomic_agents.utils.token_counter import get_token_counter, TokenCountResult, TokenCountErrorimport jsonfrom instructor.dsl.partial import PartialBase@@ -77,6 +77,7 @@ class AgentConfig(BaseModel):description="The role of the assistant in the conversation. Use 'model' for Gemini, 'assistant' for OpenAI/Anthropic.",)model_config = {"arbitrary_types_allowed": True}+ mode: Mode = Field(default=Mode.TOOLS, description="The Instructor mode used for structured outputs (TOOLS, JSON, etc.).")model_api_parameters: Optional[dict] = Field(None, description="Additional parameters passed to the API provider.")@@ -171,6 +172,7 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:self.assistant_role = config.assistant_roleself.initial_history = self.history.copy()self.current_user_input = None+ self.mode = config.modeself.model_api_parameters = config.model_api_parameters or {}# Hook management attributes@@ -247,20 +249,50 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:self.messages = self._build_system_messages()self.messages += self.history.get_history()- def _build_schema_context(self) -> str:+ def _build_tools_definition(self) -> Optional[List[Dict[str, Any]]]:"""- Build the schema context that Instructor adds for structured output.+ Build the tools definition that Instructor sends for TOOLS mode.- This replicates how Instructor serializes the output schema for the LLM,- allowing accurate token counting before making API calls.+ This uses Instructor's actual schema generation to create the exact+ tools parameter that would be sent to the LLM for TOOLS mode.+ For JSON modes, returns None as the schema is embedded in messages.Returns:- str: JSON string of the output schema in Instructor's format.+ Optional[List[Dict[str, Any]]]: Tools definition for TOOLS mode, or None for JSON modes.+ """+ from instructor.processing.schema import generate_openai_schema++ # Only return tools for TOOLS-based modes+ tools_modes = {Mode.TOOLS, Mode.TOOLS_STRICT, Mode.PARALLEL_TOOLS}+ if self.mode in tools_modes:+ return [+ {+ "type": "function",+ "function": generate_openai_schema(self.output_schema),+ }+ ]+ return None++ def _build_schema_for_json_mode(self) -> str:"""+ Build the schema context for JSON modes (appended to system message).++ This matches exactly how Instructor formats the schema for JSON/MD_JSON modes.++ Returns:+ str: JSON schema string formatted as Instructor does.+ """+ from textwrap import dedent+schema = self.output_schema.model_json_schema()- # Instructor typically sends the schema in a format like this for JSON mode- # For TOOLS mode, it would be in the tools parameter, but the token count is similar- return json.dumps(schema, ensure_ascii=False)+ return dedent(f"""+ As a genius expert, your task is to understand the content and provide+ the parsed objects in json that match the following json_schema:++ {json.dumps(schema, indent=2, ensure_ascii=False)}++ Make sure to return an instance of the JSON, not the schema itself+ """).strip()def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]:"""@@ -290,11 +322,16 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:# Multimodal object - use instructor's to_openai methodtry:serialized_content.append(item.to_openai(Mode.JSON))- except Exception:- # Fallback: just include a placeholder for the media- media_type = type(item).__name__.lower()+ except Exception as e:+ # Log the error and use placeholder for token estimation+ logger = logging.getLogger(__name__)+ media_type = type(item).__name__+ logger.warning(+ f"Failed to serialize {media_type} for token counting: {e}. "+ f"Using placeholder for estimation."+ )serialized_content.append(- {"type": "text", "text": f"[{media_type} content]"}+ {"type": "text", "text": f"[{media_type.lower()} content]"})else:# Unknown type - convert to string@@ -313,8 +350,11 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:This method computes the token count by serializing the context exactlyas Instructor does, including:- System prompt- - Conversation history (with Pydantic schemas serialized as JSON)- - Output schema overhead (the schema Instructor sends for structured output)+ - Conversation history (with multimodal content serialized properly)+ - Tools/schema overhead (using Instructor's actual schema generation)++ For TOOLS mode: Uses the actual tools parameter that Instructor sends.+ For JSON modes: Appends the schema to the system message as Instructor does.Works with any model supported by LiteLLM including OpenAI, Anthropic,Google, and 100+ other providers.@@ -322,8 +362,9 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:Returns:TokenCountResult: A named tuple containing:- total: Total tokens in the context (including schema overhead)- - system_prompt: Tokens in the system prompt + schema overhead+ - system_prompt: Tokens in the system prompt- history: Tokens in the conversation history+ - tools: Tokens in the tools/function definitions (TOOLS mode only)- model: The model used for counting- max_tokens: Maximum context window (if known)- utilization: Percentage of context used (if max_tokens known)@@ -335,8 +376,9 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:# Get accurate token count at any timeresult = agent.get_context_token_count()print(f"Total: {result.total} tokens")- print(f"System (with schema): {result.system_prompt} tokens")+ print(f"System: {result.system_prompt} tokens")print(f"History: {result.history} tokens")+ print(f"Tools: {result.tools} tokens")if result.utilization:print(f"Context usage: {result.utilization:.1%}")```@@ -345,29 +387,32 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:The 'token:counted' hook event is dispatched, allowing formonitoring and logging of token usage."""- counter = TokenCounter()+ counter = get_token_counter()- # Build system messages with schema overhead appended+ # Build system messagessystem_messages = self._build_system_messages()- schema_context = self._build_schema_context()- # Add schema as part of system context (Instructor adds this for structured output)- if system_messages:- # Append schema to existing system message- system_messages = [- {- "role": system_messages[0]["role"],- "content": system_messages[0]["content"] + "\n\n" + schema_context,- }- ]- else:- # No system message, add schema as its own message- system_messages = [{"role": "system", "content": schema_context}]+ # Handle schema serialization based on mode+ tools = self._build_tools_definition()++ if tools is None:+ # JSON mode - append schema to system message like Instructor does+ schema_context = self._build_schema_for_json_mode()+ if system_messages:+ system_messages = [+ {+ "role": system_messages[0]["role"],+ "content": system_messages[0]["content"] + "\n\n" + schema_context,+ }+ ]+ else:+ system_messages = [{"role": "system", "content": schema_context}]result = counter.count_context(model=self.model,system_messages=system_messages,history_messages=self._serialize_history_for_token_count(),+ tools=tools,)# Dispatch hook for monitoringdiff --git a/atomic-agents/atomic_agents/utils/__init__.py b/atomic-agents/atomic_agents/utils/__init__.pyindex a0198a7..30fa77d 100644--- a/atomic-agents/atomic_agents/utils/__init__.py+++ b/atomic-agents/atomic_agents/utils/__init__.py@@ -1,10 +1,12 @@"""Utility functions."""from .format_tool_message import format_tool_message-from .token_counter import TokenCounter, TokenCountResult+from .token_counter import TokenCounter, TokenCountResult, TokenCountError, get_token_counter__all__ = ["format_tool_message","TokenCounter","TokenCountResult",+ "TokenCountError",+ "get_token_counter",]diff --git a/atomic-agents/atomic_agents/utils/token_counter.py b/atomic-agents/atomic_agents/utils/token_counter.pyindex f1440fe..764c135 100644--- a/atomic-agents/atomic_agents/utils/token_counter.py+++ b/atomic-agents/atomic_agents/utils/token_counter.py@@ -6,14 +6,21 @@ from typing import Any, Dict, List, NamedTuple, Optionallogger = logging.getLogger(__name__)+class TokenCountError(Exception):+ """Exception raised when token counting fails."""++ pass++class TokenCountResult(NamedTuple):"""Result of a token count operation.Attributes:- total: Total number of tokens in the context.+ total: Total number of tokens in the context (messages + tools).system_prompt: Tokens in the system prompt (0 if no system prompt).history: Tokens in the conversation history.+ tools: Tokens in the tools/function definitions (0 if no tools).model: The model used for tokenization.max_tokens: Maximum context window for the model (None if unknown).utilization: Percentage of context window used (None if max_tokens unknown).@@ -22,17 +29,30 @@ class TokenCountResult(NamedTuple):total: intsystem_prompt: inthistory: int+ tools: intmodel: strmax_tokens: Optional[int] = Noneutilization: Optional[float] = None+# Module-level singleton for efficiency+_token_counter_instance: Optional["TokenCounter"] = None+++def get_token_counter() -> "TokenCounter":+ """Get the singleton TokenCounter instance."""+ global _token_counter_instance+ if _token_counter_instance is None:+ _token_counter_instance = TokenCounter()+ return _token_counter_instance++class TokenCounter:"""Utility class for counting tokens using LiteLLM's provider-agnostic tokenizer.- This class provides methods for counting tokens in messages, text, and- retrieving model context limits. It uses LiteLLM's token_counter which+ This class provides methods for counting tokens in messages, text, tools,+ and retrieving model context limits. It uses LiteLLM's token_counter whichautomatically selects the appropriate tokenizer based on the model.Works with any model supported by LiteLLM including:@@ -49,25 +69,51 @@ class TokenCounter:messages = [{"role": "user", "content": "Hello, world!"}]count = counter.count_messages("gpt-4", messages)+ # Count tokens with tools (for TOOLS mode)+ tools = [{"type": "function", "function": {...}}]+ count = counter.count_messages("gpt-4", messages, tools=tools)+# Get max tokens for a modelmax_tokens = counter.get_max_tokens("gpt-4")```"""- def count_messages(self, model: str, messages: List[Dict[str, Any]]) -> int:+ def count_messages(+ self,+ model: str,+ messages: List[Dict[str, Any]],+ tools: Optional[List[Dict[str, Any]]] = None,+ ) -> int:"""- Count the number of tokens in a list of messages.+ Count the number of tokens in a list of messages and optional tools.Args:model: The model identifier (e.g., "gpt-4", "anthropic/claude-3-opus").messages: List of message dictionaries with 'role' and 'content' keys.+ tools: Optional list of tool definitions (for TOOLS mode).Returns:- The number of tokens in the messages.+ The number of tokens in the messages (and tools if provided).++ Raises:+ TokenCountError: If token counting fails."""- from litellm import token_counter+ if not model:+ raise ValueError("model is required for token counting")- return token_counter(model=model, messages=messages)+ try:+ from litellm import token_counter++ if tools:+ return token_counter(model=model, messages=messages, tools=tools)+ return token_counter(model=model, messages=messages)+ except ImportError as e:+ raise ImportError(+ "litellm is required for token counting. "+ "Install it with: pip install litellm"+ ) from e+ except Exception as e:+ raise TokenCountError(f"Failed to count tokens for model '{model}': {e}") from edef count_text(self, model: str, text: str) -> int:"""@@ -79,11 +125,12 @@ class TokenCounter:Returns:The number of tokens in the text.- """- from litellm import token_counter+ Raises:+ TokenCountError: If token counting fails.+ """messages = [{"role": "user", "content": text}]- return token_counter(model=model, messages=messages)+ return self.count_messages(model, messages)def get_max_tokens(self, model: str) -> Optional[int]:"""@@ -94,10 +141,23 @@ class TokenCounter:Returns:The maximum number of tokens, or None if unknown.++ Raises:+ TypeError: If model is None or not a string.+ ImportError: If litellm is not installed."""+ if not isinstance(model, str):+ raise TypeError(f"model must be a string, got {type(model).__name__}")+try:from litellm import get_max_tokens+ except ImportError as e:+ raise ImportError(+ "litellm is required for token counting. "+ "Install it with: pip install litellm"+ ) from e+ try:return get_max_tokens(model)except Exception as e:logger.warning(f"Could not determine max tokens for model '{model}': {e}")@@ -108,29 +168,50 @@ class TokenCounter:model: str,system_messages: List[Dict[str, Any]],history_messages: List[Dict[str, Any]],+ tools: Optional[List[Dict[str, Any]]] = None,) -> TokenCountResult:"""- Count tokens with breakdown by system prompt and history.+ Count tokens with breakdown by system prompt, history, and tools.Args:model: The model identifier.system_messages: System prompt messages (may be empty).history_messages: Conversation history messages.+ tools: Optional list of tool definitions (for TOOLS mode).Returns:TokenCountResult with breakdown and utilization metrics.++ Raises:+ TokenCountError: If token counting fails."""system_tokens = self.count_messages(model, system_messages) if system_messages else 0history_tokens = self.count_messages(model, history_messages) if history_messages else 0- total_tokens = system_tokens + history_tokens++ # Count tool tokens separately if provided+ tools_tokens = 0+ if tools:+ # To count just the tools overhead, we count empty messages with tools+ # and subtract the base overhead+ empty_with_tools = self.count_messages(+ model, [{"role": "user", "content": ""}], tools=tools+ )+ empty_without_tools = self.count_messages(+ model, [{"role": "user", "content": ""}]+ )+ tools_tokens = empty_with_tools - empty_without_tools++ total_tokens = system_tokens + history_tokens + tools_tokensmax_tokens = self.get_max_tokens(model)- utilization = (total_tokens / max_tokens) if max_tokens else None+ # Prevent division by zero+ utilization = (total_tokens / max_tokens) if max_tokens and max_tokens > 0 else Nonereturn TokenCountResult(total=total_tokens,system_prompt=system_tokens,history=history_tokens,+ tools=tools_tokens,model=model,max_tokens=max_tokens,utilization=utilization,diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.pyindex 070c23c..14688e8 100644--- a/atomic-agents/tests/agents/test_atomic_agent.py+++ b/atomic-agents/tests/agents/test_atomic_agent.py@@ -767,8 +767,8 @@ def test_backward_compatibility_no_breaking_changes(mock_instructor, mock_histor# Token Counting Tests-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_basic(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_basic(mock_get_token_counter, mock_instructor, mock_history, mock_system_prompt_generator):"""Test basic token counting functionality."""mock_history.get_history.return_value = [{"role": "user", "content": "Hello"},@@ -777,9 +777,9 @@ def test_get_context_token_count_basic(mock_token_counter_class, mock_instructormock_system_prompt_generator.generate_prompt.return_value = "You are a helpful assistant."mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=100, system_prompt=30, history=70, model="gpt-5-mini", max_tokens=8192, utilization=0.0122+ total=100, system_prompt=30, history=70, tools=0, model="gpt-5-mini", max_tokens=8192, utilization=0.0122)config = AgentConfig(@@ -795,22 +795,25 @@ def test_get_context_token_count_basic(mock_token_counter_class, mock_instructorassert result.total == 100assert result.system_prompt == 30assert result.history == 70+ assert result.tools == 0assert result.model == "gpt-5-mini"assert result.max_tokens == 8192assert result.utilization is not None # Should have utilization since max_tokens is knownmock_counter_instance.count_context.assert_called_once()-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_includes_schema_overhead(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):- """Test that token counting includes the output schema overhead."""+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_includes_schema_overhead(mock_get_token_counter, mock_instructor, mock_history, mock_system_prompt_generator):+ """Test that token counting includes the output schema overhead for JSON mode."""+ from instructor import Mode+mock_history.get_history.return_value = []mock_system_prompt_generator.generate_prompt.return_value = "System prompt"mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=50, system_prompt=50, history=0, model="gpt-5-mini"+ total=50, system_prompt=50, history=0, tools=0, model="gpt-5-mini")config = AgentConfig(@@ -818,6 +821,7 @@ def test_get_context_token_count_includes_schema_overhead(mock_token_counter_clamodel="gpt-5-mini",history=mock_history,system_prompt_generator=mock_system_prompt_generator,+ mode=Mode.JSON, # JSON mode appends schema to system message)agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)@@ -827,20 +831,22 @@ def test_get_context_token_count_includes_schema_overhead(mock_token_counter_clacall_args = mock_counter_instance.count_context.call_argssystem_messages = call_args.kwargs["system_messages"]assert len(system_messages) == 1- # System message should contain both the prompt AND the schema+ # System message should contain both the prompt AND the schema (for JSON mode)assert "System prompt" in system_messages[0]["content"]assert "chat_message" in system_messages[0]["content"] # Schema field from BasicChatOutputSchema-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_no_system_prompt(mock_token_counter_class, mock_instructor, mock_history):- """Test token counting when system_role is None (schema still included)."""+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_no_system_prompt(mock_get_token_counter, mock_instructor, mock_history):+ """Test token counting when system_role is None (schema still included for JSON mode)."""+ from instructor import Mode+mock_history.get_history.return_value = [{"role": "user", "content": "Hello"}]mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=25, system_prompt=20, history=5, model="gpt-5-mini"+ total=25, system_prompt=20, history=5, tools=0, model="gpt-5-mini")config = AgentConfig(@@ -848,27 +854,28 @@ def test_get_context_token_count_no_system_prompt(mock_token_counter_class, mockmodel="gpt-5-mini",history=mock_history,system_role=None, # No system prompt+ mode=Mode.JSON, # JSON mode to test schema in system message)agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)result = agent.get_context_token_count()- # Even without system prompt, schema should be included+ # Even without system prompt, schema should be included (for JSON mode)call_args = mock_counter_instance.count_context.call_argssystem_messages = call_args.kwargs["system_messages"]assert len(system_messages) == 1 # Schema is added as system messageassert "chat_message" in system_messages[0]["content"] # Schema content-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_uses_configured_model(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_uses_configured_model(mock_get_token_counter, mock_instructor, mock_history, mock_system_prompt_generator):"""Test that token counting uses the agent's configured model."""mock_history.get_history.return_value = []mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=15, system_prompt=15, history=0, model="claude-3-opus-20240229"+ total=15, system_prompt=15, history=0, tools=0, model="claude-3-opus-20240229")config = AgentConfig(@@ -885,13 +892,13 @@ def test_get_context_token_count_uses_configured_model(mock_token_counter_class,assert call_args.kwargs["model"] == "claude-3-opus-20240229"-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_returns_token_count_result(mock_token_counter_class, agent):+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_returns_token_count_result(mock_get_token_counter, agent):"""Test that get_context_token_count returns a TokenCountResult."""mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=50, system_prompt=20, history=30, model="gpt-5-mini"+ total=50, system_prompt=20, history=30, tools=0, model="gpt-5-mini")result = agent.get_context_token_count()@@ -900,17 +907,18 @@ def test_get_context_token_count_returns_token_count_result(mock_token_counter_cassert hasattr(result, "total")assert hasattr(result, "system_prompt")assert hasattr(result, "history")+ assert hasattr(result, "tools")assert hasattr(result, "model")assert hasattr(result, "max_tokens")assert hasattr(result, "utilization")-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_hook_dispatch(mock_token_counter_class, agent):+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_hook_dispatch(mock_get_token_counter, agent):"""Test that token:counted hook is dispatched."""mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance- expected_result = TokenCountResult(total=100, system_prompt=30, history=70, model="gpt-5-mini")+ mock_get_token_counter.return_value = mock_counter_instance+ expected_result = TokenCountResult(total=100, system_prompt=30, history=70, tools=0, model="gpt-5-mini")mock_counter_instance.count_context.return_value = expected_resulthook_called = []@@ -927,13 +935,13 @@ def test_get_context_token_count_hook_dispatch(mock_token_counter_class, agent):assert hook_called[0].total == 100-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_hook_not_called_when_disabled(mock_token_counter_class, agent):+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_hook_not_called_when_disabled(mock_get_token_counter, agent):"""Test that token:counted hook is not called when hooks are disabled."""mock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=100, system_prompt=30, history=70, model="gpt-5-mini"+ total=100, system_prompt=30, history=70, tools=0, model="gpt-5-mini")hook_called = []@@ -949,16 +957,16 @@ def test_get_context_token_count_hook_not_called_when_disabled(mock_token_counteassert len(hook_called) == 0-@patch("atomic_agents.agents.atomic_agent.TokenCounter")-def test_get_context_token_count_multimodal_content(mock_token_counter_class, mock_instructor):+@patch("atomic_agents.agents.atomic_agent.get_token_counter")+def test_get_context_token_count_multimodal_content(mock_get_token_counter, mock_instructor):"""Test that multimodal content is properly serialized for token counting."""from instructor.processing.multimodal import Imagefrom instructor import Modemock_counter_instance = Mock()- mock_token_counter_class.return_value = mock_counter_instance+ mock_get_token_counter.return_value = mock_counter_instancemock_counter_instance.count_context.return_value = TokenCountResult(- total=150, system_prompt=50, history=100, model="gpt-4-vision-preview"+ total=150, system_prompt=50, history=100, tools=0, model="gpt-4-vision-preview")# Create a multimodal input schemadiff --git a/atomic-agents/tests/utils/test_token_counter.py b/atomic-agents/tests/utils/test_token_counter.pyindex d8d22b6..63c668d 100644--- a/atomic-agents/tests/utils/test_token_counter.py+++ b/atomic-agents/tests/utils/test_token_counter.py@@ -1,6 +1,11 @@import pytestfrom unittest.mock import Mock, patch-from atomic_agents.utils.token_counter import TokenCounter, TokenCountResult+from atomic_agents.utils.token_counter import (+ TokenCounter,+ TokenCountResult,+ TokenCountError,+ get_token_counter,+)class TestTokenCountResult:@@ -10,14 +15,16 @@ class TestTokenCountResult:result = TokenCountResult(total=100,system_prompt=30,- history=70,+ history=50,+ tools=20,model="gpt-4",max_tokens=8192,utilization=0.0122,)assert result.total == 100assert result.system_prompt == 30- assert result.history == 70+ assert result.history == 50+ assert result.tools == 20assert result.model == "gpt-4"assert result.max_tokens == 8192assert result.utilization == 0.0122@@ -26,7 +33,8 @@ class TestTokenCountResult:result = TokenCountResult(total=100,system_prompt=30,- history=70,+ history=50,+ tools=20,model="gpt-4",)assert result.max_tokens is None@@ -36,13 +44,15 @@ class TestTokenCountResult:result = TokenCountResult(total=100,system_prompt=30,- history=70,+ history=50,+ tools=20,model="gpt-4",)- total, system_prompt, history, model, max_tokens, utilization = result+ total, system_prompt, history, tools, model, max_tokens, utilization = resultassert total == 100assert system_prompt == 30- assert history == 70+ assert history == 50+ assert tools == 20assert model == "gpt-4"assert max_tokens is Noneassert utilization is None@@ -51,15 +61,17 @@ class TestTokenCountResult:result = TokenCountResult(total=100,system_prompt=30,- history=70,+ history=50,+ tools=20,model="gpt-4",)assert result[0] == 100 # totalassert result[1] == 30 # system_prompt- assert result[2] == 70 # history- assert result[3] == "gpt-4" # model- assert result[4] is None # max_tokens- assert result[5] is None # utilization+ assert result[2] == 50 # history+ assert result[3] == 20 # tools+ assert result[4] == "gpt-4" # model+ assert result[5] is None # max_tokens+ assert result[6] is None # utilizationclass TestTokenCounter:@@ -123,6 +135,37 @@ class TestTokenCounter:assert result is None+ @patch("litellm.token_counter")+ def test_count_messages_with_tools(self, mock_token_counter):+ mock_token_counter.return_value = 150++ counter = TokenCounter()+ messages = [{"role": "user", "content": "Hello"}]+ tools = [{"type": "function", "function": {"name": "test_fn"}}]+ result = counter.count_messages("gpt-4", messages, tools=tools)++ assert result == 150+ mock_token_counter.assert_called_once_with(+ model="gpt-4", messages=messages, tools=tools+ )++ @patch("litellm.token_counter")+ def test_count_messages_raises_token_count_error(self, mock_token_counter):+ mock_token_counter.side_effect = Exception("API error")++ counter = TokenCounter()+ with pytest.raises(TokenCountError) as exc_info:+ counter.count_messages("gpt-4", [{"role": "user", "content": "test"}])++ assert "Failed to count tokens for model 'gpt-4'" in str(exc_info.value)++ def test_count_messages_raises_value_error_for_empty_model(self):+ counter = TokenCounter()+ with pytest.raises(ValueError) as exc_info:+ counter.count_messages("", [{"role": "user", "content": "test"}])++ assert "model is required" in str(exc_info.value)+@patch("litellm.get_max_tokens")@patch("litellm.token_counter")def test_count_context(self, mock_token_counter, mock_get_max_tokens):@@ -139,10 +182,33 @@ class TestTokenCounter:assert result.total == 100assert result.system_prompt == 30assert result.history == 70+ assert result.tools == 0assert result.model == "gpt-4"assert result.max_tokens == 8192assert result.utilization == pytest.approx(100 / 8192)+ @patch("litellm.get_max_tokens")+ @patch("litellm.token_counter")+ def test_count_context_with_tools(self, mock_token_counter, mock_get_max_tokens):+ # system=30, history=70, empty_with_tools=60, empty_without_tools=10 -> tools=50+ mock_token_counter.side_effect = [30, 70, 60, 10]+ mock_get_max_tokens.return_value = 8192++ counter = TokenCounter()+ tools = [{"type": "function", "function": {"name": "test_fn"}}]+ result = counter.count_context(+ model="gpt-4",+ system_messages=[{"role": "system", "content": "You are helpful"}],+ history_messages=[{"role": "user", "content": "Hello"}],+ tools=tools,+ )++ assert result.system_prompt == 30+ assert result.history == 70+ assert result.tools == 50+ assert result.total == 150 # 30 + 70 + 50+ assert result.model == "gpt-4"+@patch("litellm.get_max_tokens")@patch("litellm.token_counter")def test_count_context_empty_system(self, mock_token_counter, mock_get_max_tokens):@@ -202,15 +268,91 @@ class TestTokenCounter:assert mock_token_counter.call_count == len(models)+ @patch("litellm.get_max_tokens")+ @patch("litellm.token_counter")+ def test_count_context_division_by_zero_prevention(+ self, mock_token_counter, mock_get_max_tokens+ ):+ """Test that division by zero is prevented when max_tokens is 0."""+ mock_token_counter.side_effect = [20, 30]+ mock_get_max_tokens.return_value = 0 # Edge case: max_tokens = 0++ counter = TokenCounter()+ result = counter.count_context(+ model="custom-model",+ system_messages=[{"role": "system", "content": "Test"}],+ history_messages=[{"role": "user", "content": "Test"}],+ )++ assert result.total == 50+ assert result.max_tokens == 0+ assert result.utilization is None # Should be None, not raise ZeroDivisionError++ @patch("litellm.get_max_tokens")+ @patch("litellm.token_counter")+ def test_count_context_empty_history(self, mock_token_counter, mock_get_max_tokens):+ """Test counting context with empty history messages."""+ mock_token_counter.return_value = 30 # Only system+ mock_get_max_tokens.return_value = 4096++ counter = TokenCounter()+ result = counter.count_context(+ model="gpt-4",+ system_messages=[{"role": "system", "content": "You are helpful"}],+ history_messages=[], # Empty history+ )++ assert result.total == 30+ assert result.system_prompt == 30+ assert result.history == 0+ assert result.tools == 0+++class TestGetTokenCounter:+ """Tests for the get_token_counter singleton function."""++ def test_get_token_counter_returns_instance(self):+ """Test that get_token_counter returns a TokenCounter instance."""+ counter = get_token_counter()+ assert isinstance(counter, TokenCounter)++ def test_get_token_counter_returns_same_instance(self):+ """Test that get_token_counter returns the same singleton instance."""+ counter1 = get_token_counter()+ counter2 = get_token_counter()+ assert counter1 is counter2+++class TestTokenCountError:+ """Tests for TokenCountError exception."""++ def test_token_count_error_is_exception(self):+ """Test that TokenCountError is an Exception."""+ error = TokenCountError("test error")+ assert isinstance(error, Exception)++ def test_token_count_error_message(self):+ """Test that TokenCountError preserves the error message."""+ error = TokenCountError("Custom error message")+ assert str(error) == "Custom error message"++class TestTokenCounterIntegration:"""Integration tests that verify the module structure."""def test_import_from_utils(self):- """Test that TokenCounter can be imported from utils."""- from atomic_agents.utils import TokenCounter, TokenCountResult+ """Test that all exports can be imported from utils."""+ from atomic_agents.utils import (+ TokenCounter,+ TokenCountResult,+ TokenCountError,+ get_token_counter,+ )assert TokenCounter is not Noneassert TokenCountResult is not None+ assert TokenCountError is not None+ assert get_token_counter is not Nonedef test_token_counter_instantiation(self):"""Test that TokenCounter can be instantiated without arguments."""