Skip to content

Building AI agents, atomically

feat: add provider-agnostic token counting utilities

KennyVaneetvelde0c74f7c8332b
diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py
index cd815c1..a5c2687 100644
--- a/atomic-agents/atomic_agents/agents/atomic_agent.py
+++ b/atomic-agents/atomic_agents/agents/atomic_agent.py
@@ -1,6 +1,8 @@
import instructor
+from instructor import Mode
+from instructor.processing.multimodal import Image, Audio, PDF
from pydantic import BaseModel, Field
-from typing import Optional, Type, Generator, AsyncGenerator, get_args, get_origin, Dict, List, Callable
+from typing import Optional, Type, Generator, AsyncGenerator, get_args, get_origin, Dict, List, Callable, Any, Union
import logging
from atomic_agents.context.chat_history import ChatHistory
from atomic_agents.context.system_prompt_generator import (
@@ -8,6 +10,8 @@ from atomic_agents.context.system_prompt_generator import (
SystemPromptGenerator,
)
from atomic_agents.base.base_io_schema import BaseIOSchema
+from atomic_agents.utils.token_counter import TokenCounter, TokenCountResult
+import json
from instructor.dsl.partial import PartialBase
from jiter import from_json
@@ -223,18 +227,153 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
# No type info available
return BasicChatOutputSchema
- def _prepare_messages(self):
+ def _build_system_messages(self) -> List[Dict]:
+ """
+ Builds the system message(s) based on the configured system role.
+
+ Returns:
+ List[Dict]: A list containing the system message, or an empty list if system_role is None.
+ """
if self.system_role is None:
- self.messages = []
- else:
- self.messages = [
+ return []
+ return [
+ {
+ "role": self.system_role,
+ "content": self.system_prompt_generator.generate_prompt(),
+ }
+ ]
+
+ def _prepare_messages(self):
+ self.messages = self._build_system_messages()
+ self.messages += self.history.get_history()
+
+ def _build_schema_context(self) -> str:
+ """
+ Build the schema context that Instructor adds for structured output.
+
+ This replicates how Instructor serializes the output schema for the LLM,
+ allowing accurate token counting before making API calls.
+
+ Returns:
+ str: JSON string of the output schema in Instructor's format.
+ """
+ schema = self.output_schema.model_json_schema()
+ # Instructor typically sends the schema in a format like this for JSON mode
+ # For TOOLS mode, it would be in the tools parameter, but the token count is similar
+ return json.dumps(schema, ensure_ascii=False)
+
+ def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]:
+ """
+ Serialize conversation history for token counting, handling multimodal content.
+
+ This method converts instructor multimodal objects (Image, Audio, PDF) to the
+ OpenAI format that LiteLLM's token counter expects. Text content is also
+ converted to the proper multimodal text format when mixed with media.
+
+ Returns:
+ List[Dict[str, Any]]: History messages in LiteLLM-compatible format.
+ """
+ history = self.history.get_history()
+ serialized = []
+
+ for message in history:
+ content = message.get("content")
+
+ if isinstance(content, list):
+ # Multimodal content - convert to OpenAI format
+ serialized_content = []
+ for item in content:
+ if isinstance(item, str):
+ # Text content - wrap in OpenAI text format
+ serialized_content.append({"type": "text", "text": item})
+ elif isinstance(item, (Image, Audio, PDF)):
+ # Multimodal object - use instructor's to_openai method
+ try:
+ serialized_content.append(item.to_openai(Mode.JSON))
+ except Exception:
+ # Fallback: just include a placeholder for the media
+ media_type = type(item).__name__.lower()
+ serialized_content.append(
+ {"type": "text", "text": f"[{media_type} content]"}
+ )
+ else:
+ # Unknown type - convert to string
+ serialized_content.append({"type": "text", "text": str(item)})
+ serialized.append({"role": message["role"], "content": serialized_content})
+ else:
+ # Simple text content - keep as is
+ serialized.append(message)
+
+ return serialized
+
+ def get_context_token_count(self) -> TokenCountResult:
+ """
+ Get the accurate token count for the current context.
+
+ This method computes the token count by serializing the context exactly
+ as Instructor does, including:
+ - System prompt
+ - Conversation history (with Pydantic schemas serialized as JSON)
+ - Output schema overhead (the schema Instructor sends for structured output)
+
+ Works with any model supported by LiteLLM including OpenAI, Anthropic,
+ Google, and 100+ other providers.
+
+ Returns:
+ TokenCountResult: A named tuple containing:
+ - total: Total tokens in the context (including schema overhead)
+ - system_prompt: Tokens in the system prompt + schema overhead
+ - history: Tokens in the conversation history
+ - model: The model used for counting
+ - max_tokens: Maximum context window (if known)
+ - utilization: Percentage of context used (if max_tokens known)
+
+ Example:
+ ```python
+ agent = AtomicAgent[InputSchema, OutputSchema](config)
+
+ # Get accurate token count at any time
+ result = agent.get_context_token_count()
+ print(f"Total: {result.total} tokens")
+ print(f"System (with schema): {result.system_prompt} tokens")
+ print(f"History: {result.history} tokens")
+ if result.utilization:
+ print(f"Context usage: {result.utilization:.1%}")
+ ```
+
+ Note:
+ The 'token:counted' hook event is dispatched, allowing for
+ monitoring and logging of token usage.
+ """
+ counter = TokenCounter()
+
+ # Build system messages with schema overhead appended
+ system_messages = self._build_system_messages()
+ schema_context = self._build_schema_context()
+
+ # Add schema as part of system context (Instructor adds this for structured output)
+ if system_messages:
+ # Append schema to existing system message
+ system_messages = [
{
- "role": self.system_role,
- "content": self.system_prompt_generator.generate_prompt(),
+ "role": system_messages[0]["role"],
+ "content": system_messages[0]["content"] + "\n\n" + schema_context,
}
]
+ else:
+ # No system message, add schema as its own message
+ system_messages = [{"role": "system", "content": schema_context}]
- self.messages += self.history.get_history()
+ result = counter.count_context(
+ model=self.model,
+ system_messages=system_messages,
+ history_messages=self._serialize_history_for_token_count(),
+ )
+
+ # Dispatch hook for monitoring
+ self._dispatch_hook("token:counted", result)
+
+ return result
def run(self, user_input: Optional[InputSchema] = None) -> OutputSchema:
"""
diff --git a/atomic-agents/atomic_agents/utils/__init__.py b/atomic-agents/atomic_agents/utils/__init__.py
index 73d63d0..a0198a7 100644
--- a/atomic-agents/atomic_agents/utils/__init__.py
+++ b/atomic-agents/atomic_agents/utils/__init__.py
@@ -1,7 +1,10 @@
"""Utility functions."""
from .format_tool_message import format_tool_message
+from .token_counter import TokenCounter, TokenCountResult
__all__ = [
"format_tool_message",
+ "TokenCounter",
+ "TokenCountResult",
]
diff --git a/atomic-agents/atomic_agents/utils/token_counter.py b/atomic-agents/atomic_agents/utils/token_counter.py
new file mode 100644
index 0000000..f1440fe
--- /dev/null
+++ b/atomic-agents/atomic_agents/utils/token_counter.py
@@ -0,0 +1,137 @@
+"""Token counting utilities for provider-agnostic context measurement."""
+
+import logging
+from typing import Any, Dict, List, NamedTuple, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class TokenCountResult(NamedTuple):
+ """
+ Result of a token count operation.
+
+ Attributes:
+ total: Total number of tokens in the context.
+ system_prompt: Tokens in the system prompt (0 if no system prompt).
+ history: Tokens in the conversation history.
+ model: The model used for tokenization.
+ max_tokens: Maximum context window for the model (None if unknown).
+ utilization: Percentage of context window used (None if max_tokens unknown).
+ """
+
+ total: int
+ system_prompt: int
+ history: int
+ model: str
+ max_tokens: Optional[int] = None
+ utilization: Optional[float] = None
+
+
+class TokenCounter:
+ """
+ Utility class for counting tokens using LiteLLM's provider-agnostic tokenizer.
+
+ This class provides methods for counting tokens in messages, text, and
+ retrieving model context limits. It uses LiteLLM's token_counter which
+ automatically selects the appropriate tokenizer based on the model.
+
+ Works with any model supported by LiteLLM including:
+ - OpenAI (gpt-4, gpt-3.5-turbo, etc.)
+ - Anthropic (claude-3-opus, claude-3-sonnet, etc.)
+ - Google (gemini-pro, gemini-1.5-pro, etc.)
+ - And 100+ other providers
+
+ Example:
+ ```python
+ counter = TokenCounter()
+
+ # Count tokens in messages
+ messages = [{"role": "user", "content": "Hello, world!"}]
+ count = counter.count_messages("gpt-4", messages)
+
+ # Get max tokens for a model
+ max_tokens = counter.get_max_tokens("gpt-4")
+ ```
+ """
+
+ def count_messages(self, model: str, messages: List[Dict[str, Any]]) -> int:
+ """
+ Count the number of tokens in a list of messages.
+
+ Args:
+ model: The model identifier (e.g., "gpt-4", "anthropic/claude-3-opus").
+ messages: List of message dictionaries with 'role' and 'content' keys.
+
+ Returns:
+ The number of tokens in the messages.
+ """
+ from litellm import token_counter
+
+ return token_counter(model=model, messages=messages)
+
+ def count_text(self, model: str, text: str) -> int:
+ """
+ Count the number of tokens in a text string.
+
+ Args:
+ model: The model identifier.
+ text: The text to tokenize.
+
+ Returns:
+ The number of tokens in the text.
+ """
+ from litellm import token_counter
+
+ messages = [{"role": "user", "content": text}]
+ return token_counter(model=model, messages=messages)
+
+ def get_max_tokens(self, model: str) -> Optional[int]:
+ """
+ Get the maximum context window size for a model.
+
+ Args:
+ model: The model identifier.
+
+ Returns:
+ The maximum number of tokens, or None if unknown.
+ """
+ try:
+ from litellm import get_max_tokens
+
+ return get_max_tokens(model)
+ except Exception as e:
+ logger.warning(f"Could not determine max tokens for model '{model}': {e}")
+ return None
+
+ def count_context(
+ self,
+ model: str,
+ system_messages: List[Dict[str, Any]],
+ history_messages: List[Dict[str, Any]],
+ ) -> TokenCountResult:
+ """
+ Count tokens with breakdown by system prompt and history.
+
+ Args:
+ model: The model identifier.
+ system_messages: System prompt messages (may be empty).
+ history_messages: Conversation history messages.
+
+ Returns:
+ TokenCountResult with breakdown and utilization metrics.
+ """
+ system_tokens = self.count_messages(model, system_messages) if system_messages else 0
+ history_tokens = self.count_messages(model, history_messages) if history_messages else 0
+ total_tokens = system_tokens + history_tokens
+
+ max_tokens = self.get_max_tokens(model)
+ utilization = (total_tokens / max_tokens) if max_tokens else None
+
+ return TokenCountResult(
+ total=total_tokens,
+ system_prompt=system_tokens,
+ history=history_tokens,
+ model=model,
+ max_tokens=max_tokens,
+ utilization=utilization,
+ )
diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.py
index 41cc40c..070c23c 100644
--- a/atomic-agents/tests/agents/test_atomic_agent.py
+++ b/atomic-agents/tests/agents/test_atomic_agent.py
@@ -1,6 +1,6 @@
import pytest
from unittest.mock import Mock, call, patch
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
import instructor
from atomic_agents import (
BaseIOSchema,
@@ -10,6 +10,7 @@ from atomic_agents import (
BasicChatOutputSchema,
)
from atomic_agents.context import ChatHistory, SystemPromptGenerator, BaseDynamicContextProvider
+from atomic_agents.utils.token_counter import TokenCountResult
from instructor.dsl.partial import PartialBase
@@ -761,3 +762,248 @@ def test_backward_compatibility_no_breaking_changes(mock_instructor, mock_histor
# Should raise KeyError for non-existent provider
with pytest.raises(KeyError):
agent.get_context_provider("test")
+
+
+# Token Counting Tests
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_basic(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):
+ """Test basic token counting functionality."""
+ mock_history.get_history.return_value = [
+ {"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi there!"},
+ ]
+ mock_system_prompt_generator.generate_prompt.return_value = "You are a helpful assistant."
+
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=100, system_prompt=30, history=70, model="gpt-5-mini", max_tokens=8192, utilization=0.0122
+ )
+
+ config = AgentConfig(
+ client=mock_instructor,
+ model="gpt-5-mini",
+ history=mock_history,
+ system_prompt_generator=mock_system_prompt_generator,
+ )
+ agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)
+
+ result = agent.get_context_token_count()
+
+ assert result.total == 100
+ assert result.system_prompt == 30
+ assert result.history == 70
+ assert result.model == "gpt-5-mini"
+ assert result.max_tokens == 8192
+ assert result.utilization is not None # Should have utilization since max_tokens is known
+ mock_counter_instance.count_context.assert_called_once()
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_includes_schema_overhead(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):
+ """Test that token counting includes the output schema overhead."""
+ mock_history.get_history.return_value = []
+ mock_system_prompt_generator.generate_prompt.return_value = "System prompt"
+
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=50, system_prompt=50, history=0, model="gpt-5-mini"
+ )
+
+ config = AgentConfig(
+ client=mock_instructor,
+ model="gpt-5-mini",
+ history=mock_history,
+ system_prompt_generator=mock_system_prompt_generator,
+ )
+ agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)
+
+ result = agent.get_context_token_count()
+
+ # Verify count_context was called with system message that includes schema
+ call_args = mock_counter_instance.count_context.call_args
+ system_messages = call_args.kwargs["system_messages"]
+ assert len(system_messages) == 1
+ # System message should contain both the prompt AND the schema
+ assert "System prompt" in system_messages[0]["content"]
+ assert "chat_message" in system_messages[0]["content"] # Schema field from BasicChatOutputSchema
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_no_system_prompt(mock_token_counter_class, mock_instructor, mock_history):
+ """Test token counting when system_role is None (schema still included)."""
+ mock_history.get_history.return_value = [{"role": "user", "content": "Hello"}]
+
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=25, system_prompt=20, history=5, model="gpt-5-mini"
+ )
+
+ config = AgentConfig(
+ client=mock_instructor,
+ model="gpt-5-mini",
+ history=mock_history,
+ system_role=None, # No system prompt
+ )
+ agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)
+
+ result = agent.get_context_token_count()
+
+ # Even without system prompt, schema should be included
+ call_args = mock_counter_instance.count_context.call_args
+ system_messages = call_args.kwargs["system_messages"]
+ assert len(system_messages) == 1 # Schema is added as system message
+ assert "chat_message" in system_messages[0]["content"] # Schema content
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_uses_configured_model(mock_token_counter_class, mock_instructor, mock_history, mock_system_prompt_generator):
+ """Test that token counting uses the agent's configured model."""
+ mock_history.get_history.return_value = []
+
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=15, system_prompt=15, history=0, model="claude-3-opus-20240229"
+ )
+
+ config = AgentConfig(
+ client=mock_instructor,
+ model="claude-3-opus-20240229", # Different model
+ history=mock_history,
+ system_prompt_generator=mock_system_prompt_generator,
+ )
+ agent = AtomicAgent[BasicChatInputSchema, BasicChatOutputSchema](config)
+
+ agent.get_context_token_count()
+
+ call_args = mock_counter_instance.count_context.call_args
+ assert call_args.kwargs["model"] == "claude-3-opus-20240229"
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_returns_token_count_result(mock_token_counter_class, agent):
+ """Test that get_context_token_count returns a TokenCountResult."""
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=50, system_prompt=20, history=30, model="gpt-5-mini"
+ )
+
+ result = agent.get_context_token_count()
+
+ assert isinstance(result, TokenCountResult)
+ assert hasattr(result, "total")
+ assert hasattr(result, "system_prompt")
+ assert hasattr(result, "history")
+ assert hasattr(result, "model")
+ assert hasattr(result, "max_tokens")
+ assert hasattr(result, "utilization")
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_hook_dispatch(mock_token_counter_class, agent):
+ """Test that token:counted hook is dispatched."""
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ expected_result = TokenCountResult(total=100, system_prompt=30, history=70, model="gpt-5-mini")
+ mock_counter_instance.count_context.return_value = expected_result
+
+ hook_called = []
+
+ def hook_handler(result):
+ hook_called.append(result)
+
+ agent.register_hook("token:counted", hook_handler)
+
+ result = agent.get_context_token_count()
+
+ assert len(hook_called) == 1
+ assert hook_called[0] == expected_result
+ assert hook_called[0].total == 100
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_hook_not_called_when_disabled(mock_token_counter_class, agent):
+ """Test that token:counted hook is not called when hooks are disabled."""
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=100, system_prompt=30, history=70, model="gpt-5-mini"
+ )
+
+ hook_called = []
+
+ def hook_handler(result):
+ hook_called.append(result)
+
+ agent.register_hook("token:counted", hook_handler)
+ agent.disable_hooks()
+
+ agent.get_context_token_count()
+
+ assert len(hook_called) == 0
+
+
+@patch("atomic_agents.agents.atomic_agent.TokenCounter")
+def test_get_context_token_count_multimodal_content(mock_token_counter_class, mock_instructor):
+ """Test that multimodal content is properly serialized for token counting."""
+ from instructor.processing.multimodal import Image
+ from instructor import Mode
+
+ mock_counter_instance = Mock()
+ mock_token_counter_class.return_value = mock_counter_instance
+ mock_counter_instance.count_context.return_value = TokenCountResult(
+ total=150, system_prompt=50, history=100, model="gpt-4-vision-preview"
+ )
+
+ # Create a multimodal input schema
+ class MultimodalInputSchema(BaseIOSchema):
+ """Input with image."""
+
+ text: str = Field(..., description="Text input")
+ image: instructor.Image = Field(..., description="Image to analyze")
+
+ # Create agent with multimodal schema
+ config = AgentConfig(
+ client=mock_instructor,
+ model="gpt-4-vision-preview",
+ )
+ agent = AtomicAgent[MultimodalInputSchema, BasicChatOutputSchema](config)
+
+ # Add multimodal message to history
+ test_image = Image(source="https://example.com/test.png", media_type="image/png")
+ multimodal_input = MultimodalInputSchema(text="Describe this image", image=test_image)
+ agent.history.add_message("user", multimodal_input)
+
+ # Get token count
+ result = agent.get_context_token_count()
+
+ # Verify count_context was called
+ assert mock_counter_instance.count_context.called
+
+ # Get the history_messages passed to count_context
+ call_args = mock_counter_instance.count_context.call_args
+ history_messages = call_args.kwargs["history_messages"]
+
+ # Verify multimodal content was serialized properly
+ assert len(history_messages) == 1
+ assert history_messages[0]["role"] == "user"
+
+ # Content should be a list with text and image
+ content = history_messages[0]["content"]
+ assert isinstance(content, list)
+
+ # Should have text and image entries
+ content_types = [item.get("type") for item in content]
+ assert "text" in content_types
+ assert "image_url" in content_types
+
+ # Verify image was converted to OpenAI format
+ image_entry = next(item for item in content if item.get("type") == "image_url")
+ assert "image_url" in image_entry
+ assert image_entry["image_url"]["url"] == "https://example.com/test.png"
diff --git a/atomic-agents/tests/utils/test_token_counter.py b/atomic-agents/tests/utils/test_token_counter.py
new file mode 100644
index 0000000..d8d22b6
--- /dev/null
+++ b/atomic-agents/tests/utils/test_token_counter.py
@@ -0,0 +1,222 @@
+import pytest
+from unittest.mock import Mock, patch
+from atomic_agents.utils.token_counter import TokenCounter, TokenCountResult
+
+
+class TestTokenCountResult:
+ """Tests for TokenCountResult named tuple."""
+
+ def test_creation_with_all_fields(self):
+ result = TokenCountResult(
+ total=100,
+ system_prompt=30,
+ history=70,
+ model="gpt-4",
+ max_tokens=8192,
+ utilization=0.0122,
+ )
+ assert result.total == 100
+ assert result.system_prompt == 30
+ assert result.history == 70
+ assert result.model == "gpt-4"
+ assert result.max_tokens == 8192
+ assert result.utilization == 0.0122
+
+ def test_optional_fields_default_to_none(self):
+ result = TokenCountResult(
+ total=100,
+ system_prompt=30,
+ history=70,
+ model="gpt-4",
+ )
+ assert result.max_tokens is None
+ assert result.utilization is None
+
+ def test_named_tuple_unpacking(self):
+ result = TokenCountResult(
+ total=100,
+ system_prompt=30,
+ history=70,
+ model="gpt-4",
+ )
+ total, system_prompt, history, model, max_tokens, utilization = result
+ assert total == 100
+ assert system_prompt == 30
+ assert history == 70
+ assert model == "gpt-4"
+ assert max_tokens is None
+ assert utilization is None
+
+ def test_access_by_index(self):
+ result = TokenCountResult(
+ total=100,
+ system_prompt=30,
+ history=70,
+ model="gpt-4",
+ )
+ assert result[0] == 100 # total
+ assert result[1] == 30 # system_prompt
+ assert result[2] == 70 # history
+ assert result[3] == "gpt-4" # model
+ assert result[4] is None # max_tokens
+ assert result[5] is None # utilization
+
+
+class TestTokenCounter:
+ """Tests for TokenCounter class."""
+
+ @patch("litellm.token_counter")
+ def test_count_messages(self, mock_token_counter):
+ mock_token_counter.return_value = 42
+
+ counter = TokenCounter()
+ messages = [{"role": "user", "content": "Hello"}]
+ result = counter.count_messages("gpt-4", messages)
+
+ assert result == 42
+ mock_token_counter.assert_called_once_with(model="gpt-4", messages=messages)
+
+ @patch("litellm.token_counter")
+ def test_count_messages_multiple(self, mock_token_counter):
+ mock_token_counter.return_value = 100
+
+ counter = TokenCounter()
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi there!"},
+ ]
+ result = counter.count_messages("gpt-4", messages)
+
+ assert result == 100
+ mock_token_counter.assert_called_once()
+
+ @patch("litellm.token_counter")
+ def test_count_text(self, mock_token_counter):
+ mock_token_counter.return_value = 5
+
+ counter = TokenCounter()
+ result = counter.count_text("gpt-4", "Hello world")
+
+ assert result == 5
+ # Should wrap text in a message
+ mock_token_counter.assert_called_once_with(
+ model="gpt-4", messages=[{"role": "user", "content": "Hello world"}]
+ )
+
+ @patch("litellm.get_max_tokens")
+ def test_get_max_tokens(self, mock_get_max_tokens):
+ mock_get_max_tokens.return_value = 8192
+
+ counter = TokenCounter()
+ result = counter.get_max_tokens("gpt-4")
+
+ assert result == 8192
+ mock_get_max_tokens.assert_called_once_with("gpt-4")
+
+ @patch("litellm.get_max_tokens")
+ def test_get_max_tokens_unknown_model(self, mock_get_max_tokens):
+ mock_get_max_tokens.side_effect = Exception("Unknown model")
+
+ counter = TokenCounter()
+ result = counter.get_max_tokens("unknown-model")
+
+ assert result is None
+
+ @patch("litellm.get_max_tokens")
+ @patch("litellm.token_counter")
+ def test_count_context(self, mock_token_counter, mock_get_max_tokens):
+ mock_token_counter.side_effect = [30, 70] # system, then history
+ mock_get_max_tokens.return_value = 8192
+
+ counter = TokenCounter()
+ result = counter.count_context(
+ model="gpt-4",
+ system_messages=[{"role": "system", "content": "You are helpful"}],
+ history_messages=[{"role": "user", "content": "Hello"}],
+ )
+
+ assert result.total == 100
+ assert result.system_prompt == 30
+ assert result.history == 70
+ assert result.model == "gpt-4"
+ assert result.max_tokens == 8192
+ assert result.utilization == pytest.approx(100 / 8192)
+
+ @patch("litellm.get_max_tokens")
+ @patch("litellm.token_counter")
+ def test_count_context_empty_system(self, mock_token_counter, mock_get_max_tokens):
+ mock_token_counter.return_value = 50
+ mock_get_max_tokens.return_value = 4096
+
+ counter = TokenCounter()
+ result = counter.count_context(
+ model="gpt-3.5-turbo",
+ system_messages=[], # No system prompt
+ history_messages=[{"role": "user", "content": "Hello"}],
+ )
+
+ assert result.total == 50
+ assert result.system_prompt == 0
+ assert result.history == 50
+ assert result.model == "gpt-3.5-turbo"
+ assert result.max_tokens == 4096
+
+ @patch("litellm.get_max_tokens")
+ @patch("litellm.token_counter")
+ def test_count_context_no_max_tokens(self, mock_token_counter, mock_get_max_tokens):
+ mock_token_counter.side_effect = [20, 30]
+ mock_get_max_tokens.return_value = None # Unknown model
+
+ counter = TokenCounter()
+ result = counter.count_context(
+ model="custom-model",
+ system_messages=[{"role": "system", "content": "Test"}],
+ history_messages=[{"role": "user", "content": "Test"}],
+ )
+
+ assert result.total == 50
+ assert result.max_tokens is None
+ assert result.utilization is None
+
+ @patch("litellm.token_counter")
+ def test_count_messages_different_models(self, mock_token_counter):
+ mock_token_counter.return_value = 10
+ counter = TokenCounter()
+
+ # Test various model formats
+ models = [
+ "gpt-4",
+ "gpt-3.5-turbo",
+ "claude-3-opus-20240229",
+ "anthropic/claude-3-sonnet",
+ "gemini-pro",
+ "gemini/gemini-1.5-pro",
+ ]
+
+ for model in models:
+ result = counter.count_messages(model, [{"role": "user", "content": "test"}])
+ assert result == 10
+
+ # Verify all models were called
+ assert mock_token_counter.call_count == len(models)
+
+
+class TestTokenCounterIntegration:
+ """Integration tests that verify the module structure."""
+
+ def test_import_from_utils(self):
+ """Test that TokenCounter can be imported from utils."""
+ from atomic_agents.utils import TokenCounter, TokenCountResult
+
+ assert TokenCounter is not None
+ assert TokenCountResult is not None
+
+ def test_token_counter_instantiation(self):
+ """Test that TokenCounter can be instantiated without arguments."""
+ counter = TokenCounter()
+ assert counter is not None
+
+
+if __name__ == "__main__":
+ pytest.main()
diff --git a/atomic-examples/quickstart/quickstart/1_0_basic_chatbot.py b/atomic-examples/quickstart/quickstart/1_0_basic_chatbot.py
index f4550b8..f1d97c2 100644
--- a/atomic-examples/quickstart/quickstart/1_0_basic_chatbot.py
+++ b/atomic-examples/quickstart/quickstart/1_0_basic_chatbot.py
@@ -57,6 +57,16 @@ while True:
if user_input.lower() in ["/exit", "/quit"]:
console.print("Exiting chat...")
break
+ # Check if the user wants to see token count
+ if user_input.lower() == "/tokens":
+ token_info = agent.get_context_token_count()
+ console.print("[bold magenta]Token Usage:[/bold magenta]")
+ console.print(f" Total: {token_info.total} tokens")
+ console.print(f" System prompt: {token_info.system_prompt} tokens")
+ console.print(f" History: {token_info.history} tokens")
+ if token_info.utilization:
+ console.print(f" Context utilization: {token_info.utilization:.1%}")
+ continue
# Process the user's input through the agent and get the response
input_schema = BasicChatInputSchema(chat_message=user_input)
diff --git a/atomic-examples/quickstart/quickstart/4_basic_chatbot_different_providers.py b/atomic-examples/quickstart/quickstart/4_basic_chatbot_different_providers.py
index 99bc1c9..3948cec 100644
--- a/atomic-examples/quickstart/quickstart/4_basic_chatbot_different_providers.py
+++ b/atomic-examples/quickstart/quickstart/4_basic_chatbot_different_providers.py
@@ -124,6 +124,18 @@ while True:
if user_input.lower() in ["/exit", "/quit"]:
console.print("Exiting chat...")
break
+ # Check if the user wants to see token count (works with any provider!)
+ if user_input.lower() == "/tokens":
+ token_info = agent.get_context_token_count()
+ console.print(f"[bold magenta]Token Usage ({model}):[/bold magenta]")
+ console.print(f" Total: {token_info.total} tokens")
+ console.print(f" System prompt: {token_info.system_prompt} tokens")
+ console.print(f" History: {token_info.history} tokens")
+ if token_info.max_tokens:
+ console.print(f" Max context: {token_info.max_tokens} tokens")
+ if token_info.utilization:
+ console.print(f" Context utilization: {token_info.utilization:.1%}")
+ continue
# Process the user's input through the agent and get the response
input_schema = BasicChatInputSchema(chat_message=user_input)
diff --git a/docs/api/agents.md b/docs/api/agents.md
index 0cc5c89..a794d75 100644
--- a/docs/api/agents.md
+++ b/docs/api/agents.md
@@ -167,6 +167,7 @@ class BasicChatOutputSchema(BaseIOSchema):
- `get_context_provider(provider_name: str)`: Get a registered context provider
- `register_context_provider(provider_name: str, provider: BaseDynamicContextProvider)`: Register a new context provider
- `unregister_context_provider(provider_name: str)`: Remove a context provider
+- `get_context_token_count() -> TokenCountResult`: Get token count for current context (system prompt + history)
### Context Providers
@@ -220,6 +221,33 @@ serialized = agent.history.dump()
agent.history.load(serialized)
```
+### Token Counting
+
+Monitor context usage with the `get_context_token_count()` method. Token counts are computed accurately on-demand by serializing the context exactly as Instructor does, including the output schema overhead. This works with any provider (OpenAI, Anthropic, Google, etc.) and supports multimodal content:
+
+```python
+# Get accurate token count at any time - always returns a result
+token_info = agent.get_context_token_count()
+print(f"Total tokens: {token_info.total}")
+print(f"System prompt (with schema): {token_info.system_prompt} tokens")
+print(f"History: {token_info.history} tokens")
+print(f"Model: {token_info.model}")
+
+# Check context utilization if max tokens is known
+if token_info.max_tokens:
+ print(f"Max context: {token_info.max_tokens} tokens")
+if token_info.utilization:
+ print(f"Context utilization: {token_info.utilization:.1%}")
+```
+
+The `TokenCountResult` contains:
+- `total`: Total tokens in context (system + history + schema overhead)
+- `system_prompt`: Tokens used by system prompt and output schema
+- `history`: Tokens used by conversation history (including multimodal content)
+- `model`: The model name used for counting
+- `max_tokens`: Maximum context window (if known)
+- `utilization`: Percentage of context used (if max_tokens known)
+
### Custom Schemas
You can use custom input/output schemas for structured interactions:
diff --git a/docs/api/utils.md b/docs/api/utils.md
index a85a0ed..efb4135 100644
--- a/docs/api/utils.md
+++ b/docs/api/utils.md
@@ -1,5 +1,142 @@
# Utilities
+## Token Counting
+
+The `TokenCounter` utility provides provider-agnostic token counting for any model supported by LiteLLM. This allows you to monitor context usage regardless of whether you're using OpenAI, Anthropic, Google, or any other supported provider.
+
+### TokenCountResult
+
+A named tuple containing token count information:
+
+```{eval-rst}
+.. py:class:: TokenCountResult
+
+ Named tuple containing token count information.
+
+ .. py:attribute:: total
+ :type: int
+
+ Total tokens in the context (system prompt + history + schema overhead).
+
+ .. py:attribute:: system_prompt
+ :type: int
+
+ Tokens used by the system prompt and output schema.
+
+ .. py:attribute:: history
+ :type: int
+
+ Tokens used by conversation history (including multimodal content).
+
+ .. py:attribute:: model
+ :type: str
+
+ The model used for token counting.
+
+ .. py:attribute:: max_tokens
+ :type: Optional[int]
+
+ Maximum context window for the model (if known).
+
+ .. py:attribute:: utilization
+ :type: Optional[float]
+
+ Context utilization percentage (0.0 to 1.0) if max_tokens is known.
+```
+
+### TokenCounter
+
+The main utility class for counting tokens:
+
+```{eval-rst}
+.. py:class:: TokenCounter
+
+ Utility class for counting tokens in messages using LiteLLM.
+
+ .. py:method:: count_messages(model: str, messages: List[Dict[str, Any]]) -> int
+
+ Count tokens in a list of messages.
+
+ :param model: The model name (e.g., "gpt-4", "claude-3-opus-20240229")
+ :param messages: List of message dictionaries with "role" and "content" keys
+ :return: Number of tokens
+
+ .. py:method:: count_text(model: str, text: str) -> int
+
+ Count tokens in a text string.
+
+ :param model: The model name
+ :param text: The text to count tokens for
+ :return: Number of tokens
+
+ .. py:method:: get_max_tokens(model: str) -> Optional[int]
+
+ Get the maximum context window for a model.
+
+ :param model: The model name
+ :return: Maximum tokens, or None if unknown
+
+ .. py:method:: count_context(model: str, system_messages: List[Dict], history_messages: List[Dict]) -> TokenCountResult
+
+ Count tokens for a complete context (system prompt + history).
+
+ :param model: The model name
+ :param system_messages: System prompt messages
+ :param history_messages: Conversation history messages
+ :return: TokenCountResult with detailed breakdown
+```
+
+### Usage Example
+
+```python
+from atomic_agents.utils import TokenCounter, TokenCountResult
+
+# Direct usage
+counter = TokenCounter()
+messages = [
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Hello!"},
+ {"role": "assistant", "content": "Hi there! How can I help?"},
+]
+
+# Count tokens in messages
+token_count = counter.count_messages("gpt-4", messages)
+
+# Get max context window
+max_tokens = counter.get_max_tokens("gpt-4")
+
+# Count complete context with breakdown
+result = counter.count_context(
+ model="gpt-4",
+ system_messages=[{"role": "system", "content": "You are helpful."}],
+ history_messages=[{"role": "user", "content": "Hello!"}],
+)
+print(f"Total: {result.total}, System: {result.system_prompt}, History: {result.history}")
+if result.utilization:
+ print(f"Context utilization: {result.utilization:.1%}")
+```
+
+### Using with AtomicAgent
+
+The easiest way to get token counts is through the agent's `get_context_token_count()` method. The agent computes accurate token counts on-demand by serializing the context exactly as Instructor does, including output schema overhead and multimodal content:
+
+```python
+# Get accurate token count at any time - always returns a result
+token_info = agent.get_context_token_count()
+print(f"Total tokens: {token_info.total}")
+print(f"System prompt (with schema): {token_info.system_prompt} tokens")
+print(f"History: {token_info.history} tokens")
+if token_info.utilization:
+ print(f"Context utilization: {token_info.utilization:.1%}")
+```
+
+The token count includes:
+- System prompt content
+- Output schema overhead (the JSON schema Instructor sends for structured output)
+- Conversation history (including multimodal content like images, PDFs, audio)
+
+This gives you an accurate count that matches what would be sent to the API.
+
## Tool Message Formatting
```{eval-rst}
diff --git a/docs/guides/quickstart.md b/docs/guides/quickstart.md
index 93ab843..8387e24 100644
--- a/docs/guides/quickstart.md
+++ b/docs/guides/quickstart.md
@@ -67,6 +67,50 @@ while True:
console.print("Agent: ", response.chat_message)
```
+## Token Counting
+
+Monitor your context usage with the `get_context_token_count()` method. Token counts are computed accurately on-demand by serializing the context exactly as Instructor does, including the output schema overhead. This works with any provider (OpenAI, Anthropic, Google, Groq, etc.) and supports multimodal content (images, PDFs, audio):
+
+```python
+# Get accurate token count at any time - no need to make an API call first
+token_info = agent.get_context_token_count()
+print(f"Total tokens: {token_info.total}")
+print(f"System prompt (with schema): {token_info.system_prompt} tokens")
+print(f"History: {token_info.history} tokens")
+
+# Check context utilization (if model's max tokens is known)
+if token_info.max_tokens:
+ print(f"Max context: {token_info.max_tokens} tokens")
+if token_info.utilization:
+ print(f"Context utilization: {token_info.utilization:.1%}")
+```
+
+You can add a `/tokens` command to your chatbot for easy monitoring:
+
+```python
+while True:
+ user_input = console.input("[bold blue]You:[/bold blue] ")
+
+ if user_input.lower() in ["/exit", "/quit"]:
+ break
+
+ # Add token counting command
+ if user_input.lower() == "/tokens":
+ token_info = agent.get_context_token_count()
+ console.print(f"[bold magenta]Token Usage:[/bold magenta]")
+ console.print(f" Total: {token_info.total} tokens")
+ console.print(f" System prompt: {token_info.system_prompt} tokens")
+ console.print(f" History: {token_info.history} tokens")
+ if token_info.utilization:
+ console.print(f" Context utilization: {token_info.utilization:.1%}")
+ continue
+
+ # Process normal input
+ input_schema = BasicChatInputSchema(chat_message=user_input)
+ response = agent.run(input_schema)
+ console.print("Agent: ", response.chat_message)
+```
+
## Streaming Responses
For a more interactive experience, you can use streaming with async processing:
diff --git a/pyproject.toml b/pyproject.toml
index 7bac577..f4ecaa0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,6 +33,7 @@ dependencies = [
"pyyaml>=6.0.2,<7.0.0",
"requests>=2.32.3,<3.0.0",
"mcp[cli]>=1.6.0",
+ "litellm>=1.50.0,<2.0.0",
]
[project.urls]
diff --git a/uv.lock b/uv.lock
index b9e77c0..7685946 100644
--- a/uv.lock
+++ b/uv.lock
@@ -197,6 +197,7 @@ source = { editable = "." }
dependencies = [
{ name = "gitpython" },
{ name = "instructor" },
+ { name = "litellm" },
{ name = "mcp", extra = ["cli"] },
{ name = "pydantic" },
{ name = "pyfiglet" },
@@ -229,6 +230,7 @@ dev = [
requires-dist = [
{ name = "gitpython", specifier = ">=3.1.43,<4.0.0" },
{ name = "instructor", specifier = "==1.13.0" },
+ { name = "litellm", specifier = ">=1.50.0,<2.0.0" },
{ name = "mcp", extras = ["cli"], specifier = ">=1.6.0" },
{ name = "pydantic", specifier = ">=2.11.0,<3.0.0" },
{ name = "pyfiglet", specifier = ">=1.0.2,<2.0.0" },