Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): resolve SmartDecisionMaker ChatCompletionMessage error and enhance tool call token counting (#11059)

Zamil Majdy8b4eb6f87cbf
diff --git a/autogpt_platform/backend/backend/blocks/smart_decision_maker.py b/autogpt_platform/backend/backend/blocks/smart_decision_maker.py
index b1432a954f..991bd1f566 100644
--- a/autogpt_platform/backend/backend/blocks/smart_decision_maker.py
+++ b/autogpt_platform/backend/backend/blocks/smart_decision_maker.py
@@ -98,6 +98,22 @@ def _create_tool_response(call_id: str, output: Any) -> dict[str, Any]:
return {"role": "tool", "tool_call_id": call_id, "content": content}
+def _convert_raw_response_to_dict(raw_response: Any) -> dict[str, Any]:
+ """
+ Safely convert raw_response to dictionary format for conversation history.
+ Handles different response types from different LLM providers.
+ """
+ if isinstance(raw_response, str):
+ # Ollama returns a string, convert to dict format
+ return {"role": "assistant", "content": raw_response}
+ elif isinstance(raw_response, dict):
+ # Already a dict (from tests or some providers)
+ return raw_response
+ else:
+ # OpenAI/Anthropic return objects, convert with json.to_dict
+ return json.to_dict(raw_response)
+
+
def get_pending_tool_calls(conversation_history: list[Any]) -> dict[str, int]:
"""
All the tool calls entry in the conversation history requires a response.
@@ -605,7 +621,7 @@ class SmartDecisionMakerBlock(Block):
# If validation failed, add feedback and raise for retry
if validation_errors:
# Add the failed response to conversation
- prompt.append(response.raw_response)
+ prompt.append(_convert_raw_response_to_dict(response.raw_response))
# Add error feedback for retry
error_feedback = (
@@ -661,5 +677,6 @@ class SmartDecisionMakerBlock(Block):
{"role": "assistant", "content": f"[Reasoning]: {response.reasoning}"}
)
- prompt.append(response.raw_response)
+ # Add the successful response to conversation
+ prompt.append(_convert_raw_response_to_dict(response.raw_response))
yield "conversations", prompt
diff --git a/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py b/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py
index d8c011149c..564e42057d 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py
@@ -478,3 +478,207 @@ async def test_smart_decision_maker_parameter_validation():
assert outputs["tools_^_search_keywords_~_query"] == "test"
assert outputs["tools_^_search_keywords_~_max_keyword_difficulty"] == 50
assert outputs["tools_^_search_keywords_~_optional_param"] == "custom_value"
+
+
+async def test_smart_decision_maker_raw_response_conversion():
+ """Test that SmartDecisionMaker correctly handles different raw_response types with retry mechanism."""
+ from unittest.mock import MagicMock, patch
+
+ import backend.blocks.llm as llm_module
+ from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
+
+ block = SmartDecisionMakerBlock()
+
+ # Mock tool functions
+ mock_tool_functions = [
+ {
+ "type": "function",
+ "function": {
+ "name": "test_tool",
+ "parameters": {
+ "type": "object",
+ "properties": {"param": {"type": "string"}},
+ "required": ["param"],
+ },
+ },
+ }
+ ]
+
+ # Test case 1: Simulate ChatCompletionMessage raw_response that caused the original error
+ class MockChatCompletionMessage:
+ """Simulate OpenAI's ChatCompletionMessage object that lacks .get() method"""
+
+ def __init__(self, role, content, tool_calls=None):
+ self.role = role
+ self.content = content
+ self.tool_calls = tool_calls or []
+
+ # This is what caused the error - no .get() method
+ # def get(self, key, default=None): # Intentionally missing
+
+ # First response: has invalid parameter name (triggers retry)
+ mock_tool_call_invalid = MagicMock()
+ mock_tool_call_invalid.function.name = "test_tool"
+ mock_tool_call_invalid.function.arguments = (
+ '{"wrong_param": "test_value"}' # Invalid parameter name
+ )
+
+ mock_response_retry = MagicMock()
+ mock_response_retry.response = None
+ mock_response_retry.tool_calls = [mock_tool_call_invalid]
+ mock_response_retry.prompt_tokens = 50
+ mock_response_retry.completion_tokens = 25
+ mock_response_retry.reasoning = None
+ # This would cause the original error without our fix
+ mock_response_retry.raw_response = MockChatCompletionMessage(
+ role="assistant", content=None, tool_calls=[mock_tool_call_invalid]
+ )
+
+ # Second response: successful (correct parameter name)
+ mock_tool_call_valid = MagicMock()
+ mock_tool_call_valid.function.name = "test_tool"
+ mock_tool_call_valid.function.arguments = (
+ '{"param": "test_value"}' # Correct parameter name
+ )
+
+ mock_response_success = MagicMock()
+ mock_response_success.response = None
+ mock_response_success.tool_calls = [mock_tool_call_valid]
+ mock_response_success.prompt_tokens = 50
+ mock_response_success.completion_tokens = 25
+ mock_response_success.reasoning = None
+ mock_response_success.raw_response = MockChatCompletionMessage(
+ role="assistant", content=None, tool_calls=[mock_tool_call_valid]
+ )
+
+ # Mock llm_call to return different responses on different calls
+ with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch.object(
+ SmartDecisionMakerBlock,
+ "_create_function_signature",
+ return_value=mock_tool_functions,
+ ):
+ # First call returns response that will trigger retry due to validation error
+ # Second call returns successful response
+ mock_llm_call.side_effect = [mock_response_retry, mock_response_success]
+
+ input_data = SmartDecisionMakerBlock.Input(
+ prompt="Test prompt",
+ model=llm_module.LlmModel.GPT4O,
+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore
+ retry=2,
+ )
+
+ # Should succeed after retry, demonstrating our helper function works
+ outputs = {}
+ async for output_name, output_data in block.run(
+ input_data,
+ credentials=llm_module.TEST_CREDENTIALS,
+ graph_id="test-graph-id",
+ node_id="test-node-id",
+ graph_exec_id="test-exec-id",
+ node_exec_id="test-node-exec-id",
+ user_id="test-user-id",
+ ):
+ outputs[output_name] = output_data
+
+ # Verify the tool output was generated successfully
+ assert "tools_^_test_tool_~_param" in outputs
+ assert outputs["tools_^_test_tool_~_param"] == "test_value"
+
+ # Verify conversation history was properly maintained
+ assert "conversations" in outputs
+ conversations = outputs["conversations"]
+ assert len(conversations) > 0
+
+ # The conversations should contain properly converted raw_response objects as dicts
+ # This would have failed with the original bug due to ChatCompletionMessage.get() error
+ for msg in conversations:
+ assert isinstance(msg, dict), f"Expected dict, got {type(msg)}"
+ if msg.get("role") == "assistant":
+ # Should have been converted from ChatCompletionMessage to dict
+ assert "role" in msg
+
+ # Verify LLM was called twice (initial + 1 retry)
+ assert mock_llm_call.call_count == 2
+
+ # Test case 2: Test with different raw_response types (Ollama string, dict)
+ # Test Ollama string response
+ mock_response_ollama = MagicMock()
+ mock_response_ollama.response = "I'll help you with that."
+ mock_response_ollama.tool_calls = None
+ mock_response_ollama.prompt_tokens = 30
+ mock_response_ollama.completion_tokens = 15
+ mock_response_ollama.reasoning = None
+ mock_response_ollama.raw_response = (
+ "I'll help you with that." # Ollama returns string
+ )
+
+ with patch(
+ "backend.blocks.llm.llm_call", return_value=mock_response_ollama
+ ), patch.object(
+ SmartDecisionMakerBlock,
+ "_create_function_signature",
+ return_value=[], # No tools for this test
+ ):
+ input_data = SmartDecisionMakerBlock.Input(
+ prompt="Simple prompt",
+ model=llm_module.LlmModel.GPT4O,
+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore
+ )
+
+ outputs = {}
+ async for output_name, output_data in block.run(
+ input_data,
+ credentials=llm_module.TEST_CREDENTIALS,
+ graph_id="test-graph-id",
+ node_id="test-node-id",
+ graph_exec_id="test-exec-id",
+ node_exec_id="test-node-exec-id",
+ user_id="test-user-id",
+ ):
+ outputs[output_name] = output_data
+
+ # Should finish since no tool calls
+ assert "finished" in outputs
+ assert outputs["finished"] == "I'll help you with that."
+
+ # Test case 3: Test with dict raw_response (some providers/tests)
+ mock_response_dict = MagicMock()
+ mock_response_dict.response = "Test response"
+ mock_response_dict.tool_calls = None
+ mock_response_dict.prompt_tokens = 25
+ mock_response_dict.completion_tokens = 10
+ mock_response_dict.reasoning = None
+ mock_response_dict.raw_response = {
+ "role": "assistant",
+ "content": "Test response",
+ } # Dict format
+
+ with patch(
+ "backend.blocks.llm.llm_call", return_value=mock_response_dict
+ ), patch.object(
+ SmartDecisionMakerBlock,
+ "_create_function_signature",
+ return_value=[],
+ ):
+ input_data = SmartDecisionMakerBlock.Input(
+ prompt="Another test",
+ model=llm_module.LlmModel.GPT4O,
+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore
+ )
+
+ outputs = {}
+ async for output_name, output_data in block.run(
+ input_data,
+ credentials=llm_module.TEST_CREDENTIALS,
+ graph_id="test-graph-id",
+ node_id="test-node-id",
+ graph_exec_id="test-exec-id",
+ node_exec_id="test-node-exec-id",
+ user_id="test-user-id",
+ ):
+ outputs[output_name] = output_data
+
+ assert "finished" in outputs
+ assert outputs["finished"] == "Test response"
diff --git a/autogpt_platform/backend/backend/blocks/time_blocks.py b/autogpt_platform/backend/backend/blocks/time_blocks.py
index df5c34af5c..61fd7606f3 100644
--- a/autogpt_platform/backend/backend/blocks/time_blocks.py
+++ b/autogpt_platform/backend/backend/blocks/time_blocks.py
@@ -270,13 +270,17 @@ class GetCurrentDateBlock(Block):
test_output=[
(
"date",
- lambda t: abs(datetime.now() - datetime.strptime(t, "%Y-%m-%d"))
- < timedelta(days=8), # 7 days difference + 1 day error margin.
+ lambda t: abs(
+ datetime.now().date() - datetime.strptime(t, "%Y-%m-%d").date()
+ )
+ <= timedelta(days=8), # 7 days difference + 1 day error margin.
),
(
"date",
- lambda t: abs(datetime.now() - datetime.strptime(t, "%m/%d/%Y"))
- < timedelta(days=8),
+ lambda t: abs(
+ datetime.now().date() - datetime.strptime(t, "%m/%d/%Y").date()
+ )
+ <= timedelta(days=8),
# 7 days difference + 1 day error margin.
),
(
@@ -382,7 +386,7 @@ class GetCurrentDateAndTimeBlock(Block):
lambda t: abs(
datetime.now().date() - datetime.strptime(t, "%Y/%m/%d").date()
)
- < timedelta(days=1), # Date format only, no time component
+ <= timedelta(days=1), # Date format only, no time component
),
(
"date_time",
diff --git a/autogpt_platform/backend/backend/util/prompt.py b/autogpt_platform/backend/backend/util/prompt.py
index 2640e038dc..a39f0367dd 100644
--- a/autogpt_platform/backend/backend/util/prompt.py
+++ b/autogpt_platform/backend/backend/util/prompt.py
@@ -19,9 +19,48 @@ def _msg_tokens(msg: dict, enc) -> int:
"""
OpenAI counts ≈3 wrapper tokens per chat message, plus 1 if "name"
is present, plus the tokenised content length.
+ For tool calls, we need to count tokens in tool_calls and content fields.
"""
WRAPPER = 3 + (1 if "name" in msg else 0)
- return WRAPPER + _tok_len(msg.get("content") or "", enc)
+
+ # Count content tokens
+ content_tokens = _tok_len(msg.get("content") or "", enc)
+
+ # Count tool call tokens for both OpenAI and Anthropic formats
+ tool_call_tokens = 0
+
+ # OpenAI format: tool_calls array at message level
+ if "tool_calls" in msg and isinstance(msg["tool_calls"], list):
+ for tool_call in msg["tool_calls"]:
+ # Count the tool call structure tokens
+ tool_call_tokens += _tok_len(tool_call.get("id", ""), enc)
+ tool_call_tokens += _tok_len(tool_call.get("type", ""), enc)
+ if "function" in tool_call:
+ tool_call_tokens += _tok_len(tool_call["function"].get("name", ""), enc)
+ tool_call_tokens += _tok_len(
+ tool_call["function"].get("arguments", ""), enc
+ )
+
+ # Anthropic format: tool_use within content array
+ content = msg.get("content")
+ if isinstance(content, list):
+ for item in content:
+ if isinstance(item, dict) and item.get("type") == "tool_use":
+ # Count the tool use structure tokens
+ tool_call_tokens += _tok_len(item.get("id", ""), enc)
+ tool_call_tokens += _tok_len(item.get("name", ""), enc)
+ tool_call_tokens += _tok_len(json.dumps(item.get("input", {})), enc)
+ elif isinstance(item, dict) and item.get("type") == "tool_result":
+ # Count tool result tokens
+ tool_call_tokens += _tok_len(item.get("tool_use_id", ""), enc)
+ tool_call_tokens += _tok_len(item.get("content", ""), enc)
+ elif isinstance(item, dict) and "content" in item:
+ # Other content types with content field
+ tool_call_tokens += _tok_len(item.get("content", ""), enc)
+ # For list content, override content_tokens since we counted everything above
+ content_tokens = 0
+
+ return WRAPPER + content_tokens + tool_call_tokens
def _truncate_middle_tokens(text: str, enc, max_tok: int) -> str:
diff --git a/autogpt_platform/backend/backend/util/prompt_test.py b/autogpt_platform/backend/backend/util/prompt_test.py
new file mode 100644
index 0000000000..af6b230f8f
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/prompt_test.py
@@ -0,0 +1,278 @@
+"""Tests for prompt utility functions, especially tool call token counting."""
+
+import pytest
+from tiktoken import encoding_for_model
+
+from backend.util import json
+from backend.util.prompt import _msg_tokens, estimate_token_count
+
+
+class TestMsgTokens:
+ """Test the _msg_tokens function with various message types."""
+
+ @pytest.fixture
+ def enc(self):
+ """Get the encoding for gpt-4o model."""
+ return encoding_for_model("gpt-4o")
+
+ def test_regular_message_token_counting(self, enc):
+ """Test that regular messages are counted correctly (backward compatibility)."""
+ msg = {"role": "user", "content": "What's the weather like in San Francisco?"}
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should be wrapper (3) + content tokens
+ expected = 3 + len(enc.encode(msg["content"]))
+ assert tokens == expected
+ assert tokens > 3 # Has content
+
+ def test_regular_message_with_name(self, enc):
+ """Test that messages with name field get extra wrapper token."""
+ msg = {"role": "user", "name": "test_user", "content": "Hello!"}
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should be wrapper (3 + 1 for name) + content tokens
+ expected = 4 + len(enc.encode(msg["content"]))
+ assert tokens == expected
+
+ def test_openai_tool_call_token_counting(self, enc):
+ """Test OpenAI format tool call token counting."""
+ msg = {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_abc123",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": '{"location": "San Francisco", "unit": "celsius"}',
+ },
+ }
+ ],
+ }
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should count wrapper + all tool call components
+ expected_tool_tokens = (
+ len(enc.encode("call_abc123"))
+ + len(enc.encode("function"))
+ + len(enc.encode("get_weather"))
+ + len(enc.encode('{"location": "San Francisco", "unit": "celsius"}'))
+ )
+ expected = 3 + expected_tool_tokens # wrapper + tool tokens
+
+ assert tokens == expected
+ assert tokens > 8 # Should be significantly more than just wrapper
+
+ def test_openai_multiple_tool_calls(self, enc):
+ """Test OpenAI format with multiple tool calls."""
+ msg = {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "func1", "arguments": '{"arg": "value1"}'},
+ },
+ {
+ "id": "call_2",
+ "type": "function",
+ "function": {"name": "func2", "arguments": '{"arg": "value2"}'},
+ },
+ ],
+ }
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should count all tool calls
+ assert tokens > 20 # Should be more than single tool call
+
+ def test_anthropic_tool_use_token_counting(self, enc):
+ """Test Anthropic format tool use token counting."""
+ msg = {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_xyz456",
+ "name": "get_weather",
+ "input": {"location": "San Francisco", "unit": "celsius"},
+ }
+ ],
+ }
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should count wrapper + tool use components
+ expected_tool_tokens = (
+ len(enc.encode("toolu_xyz456"))
+ + len(enc.encode("get_weather"))
+ + len(
+ enc.encode(json.dumps({"location": "San Francisco", "unit": "celsius"}))
+ )
+ )
+ expected = 3 + expected_tool_tokens # wrapper + tool tokens
+
+ assert tokens == expected
+ assert tokens > 8 # Should be significantly more than just wrapper
+
+ def test_anthropic_tool_result_token_counting(self, enc):
+ """Test Anthropic format tool result token counting."""
+ msg = {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_xyz456",
+ "content": "The weather in San Francisco is 22°C and sunny.",
+ }
+ ],
+ }
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should count wrapper + tool result components
+ expected_tool_tokens = len(enc.encode("toolu_xyz456")) + len(
+ enc.encode("The weather in San Francisco is 22°C and sunny.")
+ )
+ expected = 3 + expected_tool_tokens # wrapper + tool tokens
+
+ assert tokens == expected
+ assert tokens > 8 # Should be significantly more than just wrapper
+
+ def test_anthropic_mixed_content(self, enc):
+ """Test Anthropic format with mixed content types."""
+ msg = {
+ "role": "assistant",
+ "content": [
+ {"type": "text", "content": "I'll check the weather for you."},
+ {
+ "type": "tool_use",
+ "id": "toolu_123",
+ "name": "get_weather",
+ "input": {"location": "SF"},
+ },
+ ],
+ }
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should count all content items
+ assert tokens > 15 # Should count both text and tool use
+
+ def test_empty_content(self, enc):
+ """Test message with empty or None content."""
+ msg = {"role": "assistant", "content": None}
+
+ tokens = _msg_tokens(msg, enc)
+ assert tokens == 3 # Just wrapper tokens
+
+ msg["content"] = ""
+ tokens = _msg_tokens(msg, enc)
+ assert tokens == 3 # Just wrapper tokens
+
+ def test_string_content_with_tool_calls(self, enc):
+ """Test OpenAI format where content is string but tool_calls exist."""
+ msg = {
+ "role": "assistant",
+ "content": "Let me check that for you.",
+ "tool_calls": [
+ {
+ "id": "call_123",
+ "type": "function",
+ "function": {"name": "test_func", "arguments": "{}"},
+ }
+ ],
+ }
+
+ tokens = _msg_tokens(msg, enc)
+
+ # Should count both content and tool calls
+ content_tokens = len(enc.encode("Let me check that for you."))
+ tool_tokens = (
+ len(enc.encode("call_123"))
+ + len(enc.encode("function"))
+ + len(enc.encode("test_func"))
+ + len(enc.encode("{}"))
+ )
+ expected = 3 + content_tokens + tool_tokens
+
+ assert tokens == expected
+
+
+class TestEstimateTokenCount:
+ """Test the estimate_token_count function with conversations containing tool calls."""
+
+ def test_conversation_with_tool_calls(self):
+ """Test token counting for a complete conversation with tool calls."""
+ conversation = [
+ {"role": "user", "content": "What's the weather like in San Francisco?"},
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_123",
+ "name": "get_weather",
+ "input": {"location": "San Francisco"},
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_123",
+ "content": "22°C and sunny",
+ }
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": "The weather in San Francisco is 22°C and sunny.",
+ },
+ ]
+
+ total_tokens = estimate_token_count(conversation)
+
+ # Verify total equals sum of individual messages
+ enc = encoding_for_model("gpt-4o")
+ expected_total = sum(_msg_tokens(msg, enc) for msg in conversation)
+
+ assert total_tokens == expected_total
+ assert total_tokens > 40 # Should be substantial for this conversation
+
+ def test_openai_conversation(self):
+ """Test token counting for OpenAI format conversation."""
+ conversation = [
+ {"role": "user", "content": "Calculate 2 + 2"},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "call_calc",
+ "type": "function",
+ "function": {
+ "name": "calculate",
+ "arguments": '{"expression": "2 + 2"}',
+ },
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_calc", "content": "4"},
+ {"role": "assistant", "content": "The result is 4."},
+ ]
+
+ total_tokens = estimate_token_count(conversation)
+
+ # Verify total equals sum of individual messages
+ enc = encoding_for_model("gpt-4o")
+ expected_total = sum(_msg_tokens(msg, enc) for msg in conversation)
+
+ assert total_tokens == expected_total
+ assert total_tokens > 20 # Should be substantial