Building AI agents, atomically
fix: use max_input_tokens instead of max_tokens for context window size
KennyVaneetvelded5bfb221e0f3
diff --git a/atomic-agents/atomic_agents/utils/token_counter.py b/atomic-agents/atomic_agents/utils/token_counter.pyindex 1427f82..f562ca5 100644--- a/atomic-agents/atomic_agents/utils/token_counter.py+++ b/atomic-agents/atomic_agents/utils/token_counter.py@@ -147,12 +147,14 @@ class TokenCounter:raise TypeError(f"model must be a string, got {type(model).__name__}")try:- from litellm import get_max_tokens+ from litellm import get_model_infoexcept ImportError as e:raise ImportError("litellm is required for token counting. " "Install it with: pip install litellm") from etry:- return get_max_tokens(model)+ info = get_model_info(model)+ # Use max_input_tokens (context window) not max_tokens (output limit)+ return info.get("max_input_tokens") or info.get("max_tokens")except Exception as e:logger.warning(f"Could not determine max tokens for model '{model}': {e}")return Nonediff --git a/atomic-agents/tests/utils/test_token_counter.py b/atomic-agents/tests/utils/test_token_counter.pyindex 9534e7a..7308690 100644--- a/atomic-agents/tests/utils/test_token_counter.py+++ b/atomic-agents/tests/utils/test_token_counter.py@@ -114,19 +114,28 @@ class TestTokenCounter:# Should wrap text in a messagemock_token_counter.assert_called_once_with(model="gpt-4", messages=[{"role": "user", "content": "Hello world"}])- @patch("litellm.get_max_tokens")- def test_get_max_tokens(self, mock_get_max_tokens):- mock_get_max_tokens.return_value = 8192+ @patch("litellm.get_model_info")+ def test_get_max_tokens(self, mock_get_model_info):+ mock_get_model_info.return_value = {"max_input_tokens": 128000, "max_tokens": 16384}++ counter = TokenCounter()+ result = counter.get_max_tokens("gpt-4")++ assert result == 128000+ mock_get_model_info.assert_called_once_with("gpt-4")++ @patch("litellm.get_model_info")+ def test_get_max_tokens_falls_back_to_max_tokens(self, mock_get_model_info):+ mock_get_model_info.return_value = {"max_tokens": 8192}counter = TokenCounter()result = counter.get_max_tokens("gpt-4")assert result == 8192- mock_get_max_tokens.assert_called_once_with("gpt-4")- @patch("litellm.get_max_tokens")- def test_get_max_tokens_unknown_model(self, mock_get_max_tokens):- mock_get_max_tokens.side_effect = Exception("Unknown model")+ @patch("litellm.get_model_info")+ def test_get_max_tokens_unknown_model(self, mock_get_model_info):+ mock_get_model_info.side_effect = Exception("Unknown model")counter = TokenCounter()result = counter.get_max_tokens("unknown-model")@@ -162,11 +171,11 @@ class TestTokenCounter:assert "model is required" in str(exc_info.value)- @patch("litellm.get_max_tokens")+ @patch("litellm.get_model_info")@patch("litellm.token_counter")- def test_count_context(self, mock_token_counter, mock_get_max_tokens):+ def test_count_context(self, mock_token_counter, mock_get_model_info):mock_token_counter.side_effect = [30, 70] # system, then history- mock_get_max_tokens.return_value = 8192+ mock_get_model_info.return_value = {"max_input_tokens": 8192, "max_tokens": 4096}counter = TokenCounter()result = counter.count_context(@@ -183,12 +192,12 @@ class TestTokenCounter:assert result.max_tokens == 8192assert result.utilization == pytest.approx(100 / 8192)- @patch("litellm.get_max_tokens")+ @patch("litellm.get_model_info")@patch("litellm.token_counter")- def test_count_context_with_tools(self, mock_token_counter, mock_get_max_tokens):+ def test_count_context_with_tools(self, mock_token_counter, mock_get_model_info):# system=30, history=70, empty_with_tools=60, empty_without_tools=10 -> tools=50mock_token_counter.side_effect = [30, 70, 60, 10]- mock_get_max_tokens.return_value = 8192+ mock_get_model_info.return_value = {"max_input_tokens": 8192, "max_tokens": 4096}counter = TokenCounter()tools = [{"type": "function", "function": {"name": "test_fn"}}]@@ -205,11 +214,11 @@ class TestTokenCounter:assert result.total == 150 # 30 + 70 + 50assert result.model == "gpt-4"- @patch("litellm.get_max_tokens")+ @patch("litellm.get_model_info")@patch("litellm.token_counter")- def test_count_context_empty_system(self, mock_token_counter, mock_get_max_tokens):+ def test_count_context_empty_system(self, mock_token_counter, mock_get_model_info):mock_token_counter.return_value = 50- mock_get_max_tokens.return_value = 4096+ mock_get_model_info.return_value = {"max_input_tokens": 4096, "max_tokens": 2048}counter = TokenCounter()result = counter.count_context(@@ -224,11 +233,11 @@ class TestTokenCounter:assert result.model == "gpt-3.5-turbo"assert result.max_tokens == 4096- @patch("litellm.get_max_tokens")+ @patch("litellm.get_model_info")@patch("litellm.token_counter")- def test_count_context_no_max_tokens(self, mock_token_counter, mock_get_max_tokens):+ def test_count_context_no_max_tokens(self, mock_token_counter, mock_get_model_info):mock_token_counter.side_effect = [20, 30]- mock_get_max_tokens.return_value = None # Unknown model+ mock_get_model_info.side_effect = Exception("Unknown model")counter = TokenCounter()result = counter.count_context(@@ -263,12 +272,12 @@ class TestTokenCounter:# Verify all models were calledassert mock_token_counter.call_count == len(models)- @patch("litellm.get_max_tokens")+ @patch("litellm.get_model_info")@patch("litellm.token_counter")- def test_count_context_division_by_zero_prevention(self, mock_token_counter, mock_get_max_tokens):+ def test_count_context_division_by_zero_prevention(self, mock_token_counter, mock_get_model_info):"""Test that division by zero is prevented when max_tokens is 0."""mock_token_counter.side_effect = [20, 30]- mock_get_max_tokens.return_value = 0 # Edge case: max_tokens = 0+ mock_get_model_info.return_value = {"max_input_tokens": 0, "max_tokens": 0} # Edge casecounter = TokenCounter()result = counter.count_context(@@ -281,12 +290,12 @@ class TestTokenCounter:assert result.max_tokens == 0assert result.utilization is None # Should be None, not raise ZeroDivisionError- @patch("litellm.get_max_tokens")+ @patch("litellm.get_model_info")@patch("litellm.token_counter")- def test_count_context_empty_history(self, mock_token_counter, mock_get_max_tokens):+ def test_count_context_empty_history(self, mock_token_counter, mock_get_model_info):"""Test counting context with empty history messages."""mock_token_counter.return_value = 30 # Only system- mock_get_max_tokens.return_value = 4096+ mock_get_model_info.return_value = {"max_input_tokens": 4096, "max_tokens": 2048}counter = TokenCounter()result = counter.count_context(