Skip to content

A programming framework for agentic AI

Add ToolCallEvent and log it from all builtin tools (#5859)

Eric Zhu740afe5b611c
diff --git a/python/packages/autogen-core/src/autogen_core/logging.py b/python/packages/autogen-core/src/autogen_core/logging.py
index 057458e18..6ea430f27 100644
--- a/python/packages/autogen-core/src/autogen_core/logging.py
+++ b/python/packages/autogen-core/src/autogen_core/logging.py
@@ -1,8 +1,9 @@
import json
from enum import Enum
-from typing import Any, Dict, cast
+from typing import Any, Dict, List, cast
from ._agent_id import AgentId
+from ._message_handler_context import MessageHandlerContext
from ._topic import TopicId
@@ -10,21 +11,19 @@ class LLMCallEvent:
def __init__(
self,
*,
- messages: Dict[str, Any],
+ messages: List[Dict[str, Any]],
response: Dict[str, Any],
prompt_tokens: int,
completion_tokens: int,
- agent_id: AgentId | None = None,
**kwargs: Any,
) -> None:
"""To be used by model clients to log the call to the LLM.
Args:
- messages (Dict[str, Any]): The messages of the call. Must be json serializable.
+ messages (List[Dict[str, Any]]): The messages used in the call. Must be json serializable.
response (Dict[str, Any]): The response of the call. Must be json serializable.
prompt_tokens (int): Number of tokens used in the prompt.
completion_tokens (int): Number of tokens used in the completion.
- agent_id (AgentId | None, optional): The agent id of the model. Defaults to None.
Example:
@@ -43,8 +42,11 @@ class LLMCallEvent:
self.kwargs["response"] = response
self.kwargs["prompt_tokens"] = prompt_tokens
self.kwargs["completion_tokens"] = completion_tokens
+ try:
+ agent_id = MessageHandlerContext.agent_id()
+ except RuntimeError:
+ agent_id = None
self.kwargs["agent_id"] = None if agent_id is None else str(agent_id)
- self.kwargs["type"] = "LLMCall"
@property
def prompt_tokens(self) -> int:
@@ -59,6 +61,48 @@ class LLMCallEvent:
return json.dumps(self.kwargs)
+class ToolCallEvent:
+ def __init__(
+ self,
+ *,
+ tool_name: str,
+ arguments: Dict[str, Any],
+ result: str,
+ ) -> None:
+ """Used by subclasses of :class:`~autogen_core.tools.BaseTool` to log executions of tools.
+
+ Args:
+ tool_name (str): The name of the tool.
+ arguments (Dict[str, Any]): The arguments of the tool. Must be json serializable.
+ result (str): The result of the tool. Must be a string.
+
+ Example:
+
+ .. code-block:: python
+
+ from autogen_core import EVENT_LOGGER_NAME
+ from autogen_core.logging import ToolCallEvent
+
+ logger = logging.getLogger(EVENT_LOGGER_NAME)
+ logger.info(ToolCallEvent(tool_name="Tool1", call_id="123", arguments={"arg1": "value1"}))
+
+ """
+ self.kwargs: Dict[str, Any] = {}
+ self.kwargs["type"] = "ToolCall"
+ self.kwargs["tool_name"] = tool_name
+ self.kwargs["arguments"] = arguments
+ self.kwargs["result"] = result
+ try:
+ agent_id = MessageHandlerContext.agent_id()
+ except RuntimeError:
+ agent_id = None
+ self.kwargs["agent_id"] = None if agent_id is None else str(agent_id)
+
+ # This must output the event in a json serializable format
+ def __str__(self) -> str:
+ return json.dumps(self.kwargs)
+
+
class MessageKind(Enum):
DIRECT = 1
PUBLISH = 2
diff --git a/python/packages/autogen-core/src/autogen_core/tools/_base.py b/python/packages/autogen-core/src/autogen_core/tools/_base.py
index 8b6489731..814463e36 100644
--- a/python/packages/autogen-core/src/autogen_core/tools/_base.py
+++ b/python/packages/autogen-core/src/autogen_core/tools/_base.py
@@ -1,4 +1,5 @@
import json
+import logging
from abc import ABC, abstractmethod
from collections.abc import Sequence
from typing import Any, Dict, Generic, Mapping, Protocol, Type, TypedDict, TypeVar, cast, runtime_checkable
@@ -7,12 +8,15 @@ import jsonref
from pydantic import BaseModel
from typing_extensions import NotRequired
-from .. import CancellationToken
+from .. import EVENT_LOGGER_NAME, CancellationToken
from .._component_config import ComponentBase
from .._function_utils import normalize_annotated_type
+from ..logging import ToolCallEvent
T = TypeVar("T", bound=BaseModel, contravariant=True)
+logger = logging.getLogger(EVENT_LOGGER_NAME)
+
class ParametersSchema(TypedDict):
type: str
@@ -143,7 +147,17 @@ class BaseTool(ABC, Tool, Generic[ArgsT, ReturnT], ComponentBase[BaseModel]):
async def run(self, args: ArgsT, cancellation_token: CancellationToken) -> ReturnT: ...
async def run_json(self, args: Mapping[str, Any], cancellation_token: CancellationToken) -> Any:
+ # Execute the tool's run method
return_value = await self.run(self._args_type.model_validate(args), cancellation_token)
+
+ # Log the tool call event
+ event = ToolCallEvent(
+ tool_name=self.name,
+ arguments=dict(args), # Using the raw args passed to run_json
+ result=self.return_value_as_string(return_value),
+ )
+ logger.info(event)
+
return return_value
def save_state_json(self) -> Mapping[str, Any]:
diff --git a/python/packages/autogen-core/tests/test_tool_agent.py b/python/packages/autogen-core/tests/test_tool_agent.py
index 715d5519c..5ef312e7b 100644
--- a/python/packages/autogen-core/tests/test_tool_agent.py
+++ b/python/packages/autogen-core/tests/test_tool_agent.py
@@ -1,9 +1,10 @@
import asyncio
import json
+import logging
from typing import Any, AsyncGenerator, List, Mapping, Optional, Sequence, Union
import pytest
-from autogen_core import AgentId, CancellationToken, FunctionCall, SingleThreadedAgentRuntime
+from autogen_core import EVENT_LOGGER_NAME, AgentId, CancellationToken, FunctionCall, SingleThreadedAgentRuntime
from autogen_core.models import (
AssistantMessage,
ChatCompletionClient,
@@ -25,6 +26,8 @@ from autogen_core.tool_agent import (
)
from autogen_core.tools import FunctionTool, Tool, ToolSchema
+logging.getLogger(EVENT_LOGGER_NAME).setLevel(logging.INFO)
+
def _pass_function(input: str) -> str:
return "pass"
@@ -40,7 +43,7 @@ async def _async_sleep_function(input: str) -> str:
@pytest.mark.asyncio
-async def test_tool_agent() -> None:
+async def test_tool_agent(caplog: pytest.LogCaptureFixture) -> None:
runtime = SingleThreadedAgentRuntime()
await ToolAgent.register(
runtime,
@@ -63,6 +66,9 @@ async def test_tool_agent() -> None:
)
assert result == FunctionExecutionResult(call_id="1", content="pass", is_error=False, name="pass")
+ # Check log.
+ assert any(("ToolCall" in record.message and str(agent) in record.message) for record in caplog.records)
+
# Test raise function
with pytest.raises(ToolExecutionException):
await runtime.send_message(FunctionCall(id="2", arguments=json.dumps({"input": "raise"}), name="raise"), agent)
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py
index 2ed797abd..ea21fa28e 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py
@@ -44,7 +44,6 @@ from autogen_core import (
Component,
FunctionCall,
Image,
- MessageHandlerContext,
)
from autogen_core.logging import LLMCallEvent
from autogen_core.models import (
@@ -503,19 +502,12 @@ class BaseAnthropicChatCompletionClient(ChatCompletionClient):
completion_tokens=result.usage.output_tokens,
)
- # Log the event if in a handler context
- try:
- agent_id = MessageHandlerContext.agent_id()
- except RuntimeError:
- agent_id = None
-
logger.info(
LLMCallEvent(
- messages=cast(Dict[str, Any], anthropic_messages),
+ messages=cast(List[Dict[str, Any]], anthropic_messages),
response=result.model_dump(),
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
- agent_id=agent_id,
)
)
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py
index 1b22abe42..b6257bd9a 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py
@@ -1,10 +1,12 @@
import asyncio
+import logging
import re
from asyncio import Task
from inspect import getfullargspec
from typing import Any, Dict, List, Mapping, Optional, Sequence, cast
-from autogen_core import CancellationToken, FunctionCall, Image
+from autogen_core import EVENT_LOGGER_NAME, CancellationToken, FunctionCall, Image
+from autogen_core.logging import LLMCallEvent
from autogen_core.models import (
AssistantMessage,
ChatCompletionClient,
@@ -62,6 +64,8 @@ from .._utils.parse_r1_content import parse_r1_content
create_kwargs = set(getfullargspec(ChatCompletionsClient.complete).kwonlyargs)
AzureMessage = Union[AzureSystemMessage, AzureUserMessage, AzureAssistantMessage, AzureToolMessage]
+logger = logging.getLogger(EVENT_LOGGER_NAME)
+
def _is_github_model(endpoint: str) -> bool:
return endpoint == GITHUB_MODELS_ENDPOINT
@@ -339,6 +343,15 @@ class AzureAIChatCompletionClient(ChatCompletionClient):
completion_tokens=result.usage.completion_tokens if result.usage else 0,
)
+ logger.info(
+ LLMCallEvent(
+ messages=[m.as_dict() for m in azure_messages],
+ response=result.as_dict(),
+ prompt_tokens=usage.prompt_tokens,
+ completion_tokens=usage.completion_tokens,
+ )
+ )
+
choice = result.choices[0]
if choice.finish_reason == CompletionsFinishReason.TOOL_CALLS:
assert choice.message.tool_calls is not None
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py b/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py
index 761863eeb..f817a17ae 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py
@@ -26,7 +26,6 @@ from autogen_core import (
Component,
FunctionCall,
Image,
- MessageHandlerContext,
)
from autogen_core.logging import LLMCallEvent
from autogen_core.models import (
@@ -483,19 +482,12 @@ class BaseOllamaChatCompletionClient(ChatCompletionClient):
completion_tokens=(result.eval_count if result.eval_count is not None else 0),
)
- # If we are running in the context of a handler we can get the agent_id
- try:
- agent_id = MessageHandlerContext.agent_id()
- except RuntimeError:
- agent_id = None
-
logger.info(
LLMCallEvent(
- messages=cast(Dict[str, Any], ollama_messages),
+ messages=cast(List[Dict[str, Any]], ollama_messages),
response=result.model_dump(),
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
- agent_id=agent_id,
)
)
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
index 34279acc2..05557e19f 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
@@ -29,7 +29,6 @@ from autogen_core import (
Component,
FunctionCall,
Image,
- MessageHandlerContext,
)
from autogen_core.logging import LLMCallEvent
from autogen_core.models import (
@@ -531,19 +530,12 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
completion_tokens=(result.usage.completion_tokens if result.usage is not None else 0),
)
- # If we are running in the context of a handler we can get the agent_id
- try:
- agent_id = MessageHandlerContext.agent_id()
- except RuntimeError:
- agent_id = None
-
logger.info(
LLMCallEvent(
- messages=cast(Dict[str, Any], oai_messages),
+ messages=cast(List[Dict[str, Any]], oai_messages),
response=result.model_dump(),
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
- agent_id=agent_id,
)
)
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py b/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py
index aa58830a9..7df5dd59b 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py
@@ -1,9 +1,11 @@
import json
+import logging
import warnings
from typing import Any, Literal, Mapping, Optional, Sequence
-from autogen_core import FunctionCall
+from autogen_core import EVENT_LOGGER_NAME, FunctionCall
from autogen_core._cancellation_token import CancellationToken
+from autogen_core.logging import LLMCallEvent
from autogen_core.models import (
ChatCompletionClient,
CreateResult,
@@ -32,6 +34,8 @@ from autogen_ext.tools.semantic_kernel import KernelFunctionFromTool
from .._utils.parse_r1_content import parse_r1_content
+logger = logging.getLogger(EVENT_LOGGER_NAME)
+
class SKChatCompletionAdapter(ChatCompletionClient):
"""
@@ -465,6 +469,15 @@ class SKChatCompletionAdapter(ChatCompletionClient):
prompt_tokens = getattr(usage, "prompt_tokens", 0)
completion_tokens = getattr(usage, "completion_tokens", 0)
+ logger.info(
+ LLMCallEvent(
+ messages=[msg.model_dump() for msg in chat_history],
+ response=result[0].model_dump(),
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ )
+ )
+
self._total_prompt_tokens += prompt_tokens
self._total_completion_tokens += completion_tokens
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py b/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py
index 14086b63b..fd6d51a77 100644
--- a/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py
+++ b/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py
@@ -83,7 +83,6 @@ class PythonCodeExecutionTool(
result = await self._executor.execute_code_blocks(
code_blocks=code_blocks, cancellation_token=cancellation_token
)
-
return CodeExecutionResult(success=result.exit_code == 0, output=result.output)
def _to_config(self) -> PythonCodeExecutionToolConfig:
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py
index 445832040..1a8915ee9 100644
--- a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py
+++ b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py
@@ -177,9 +177,9 @@ class GlobalSearchTool(BaseTool[GlobalSearchToolArgs, GlobalSearchToolReturn]):
)
async def run(self, args: GlobalSearchToolArgs, cancellation_token: CancellationToken) -> GlobalSearchToolReturn:
- result = await self._search_engine.asearch(args.query)
- assert isinstance(result.response, str), "Expected response to be a string"
- return GlobalSearchToolReturn(answer=result.response)
+ search_result = await self._search_engine.asearch(args.query)
+ assert isinstance(search_result.response, str), "Expected response to be a string"
+ return GlobalSearchToolReturn(answer=search_result.response)
@classmethod
def from_settings(cls, settings_path: str | Path) -> "GlobalSearchTool":
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py
index 0031d1d1b..625c7a4e1 100644
--- a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py
+++ b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py
@@ -188,9 +188,9 @@ class LocalSearchTool(BaseTool[LocalSearchToolArgs, LocalSearchToolReturn]):
)
async def run(self, args: LocalSearchToolArgs, cancellation_token: CancellationToken) -> LocalSearchToolReturn:
- result = await self._search_engine.asearch(args.query) # type: ignore
- assert isinstance(result.response, str), "Expected response to be a string"
- return LocalSearchToolReturn(answer=result.response)
+ search_result = await self._search_engine.asearch(args.query) # type: ignore
+ assert isinstance(search_result.response, str), "Expected response to be a string"
+ return LocalSearchToolReturn(answer=search_result.response)
@classmethod
def from_settings(cls, settings_path: str | Path) -> "LocalSearchTool":
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py b/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py
index 81dc1d8f7..c4cba4174 100644
--- a/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py
+++ b/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py
@@ -189,12 +189,10 @@ class LangChainToolAdapter(BaseTool[BaseModel, Any]):
# Determine if the callable is asynchronous
if inspect.iscoroutinefunction(self._callable):
- result = await self._callable(**kwargs)
+ return await self._callable(**kwargs)
else:
# Run in a thread to avoid blocking the event loop
- result = await asyncio.to_thread(self._call_sync, kwargs)
-
- return result
+ return await asyncio.to_thread(self._call_sync, kwargs)
def _call_sync(self, kwargs: Dict[str, Any]) -> Any:
return self._callable(**kwargs)
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py b/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py
index cf905800d..a8bb3e939 100644
--- a/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py
+++ b/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py
@@ -67,7 +67,6 @@ class McpToolAdapter(BaseTool[BaseModel, Any], ABC, Generic[TServerParams]):
if result.isError:
raise Exception(f"MCP tool execution failed: {result.content}")
-
return result.content
except Exception as e:
raise Exception(str(e)) from e
diff --git a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py
index 8e652663d..74092b33f 100644
--- a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py
+++ b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py
@@ -1,4 +1,5 @@
import asyncio
+import logging
import os
from typing import List, Sequence
@@ -28,7 +29,7 @@ def _add_numbers(a: int, b: int) -> int:
@pytest.mark.asyncio
-async def test_anthropic_basic_completion() -> None:
+async def test_anthropic_basic_completion(caplog: pytest.LogCaptureFixture) -> None:
"""Test basic message completion with Claude."""
api_key = os.getenv("ANTHROPIC_API_KEY")
if not api_key:
@@ -42,16 +43,18 @@ async def test_anthropic_basic_completion() -> None:
)
# Test basic completion
- result = await client.create(
- messages=[
- SystemMessage(content="You are a helpful assistant."),
- UserMessage(content="What's 2+2? Answer with just the number.", source="user"),
- ]
- )
+ with caplog.at_level(logging.INFO):
+ result = await client.create(
+ messages=[
+ SystemMessage(content="You are a helpful assistant."),
+ UserMessage(content="What's 2+2? Answer with just the number.", source="user"),
+ ]
+ )
- assert isinstance(result.content, str)
- assert "4" in result.content
- assert result.finish_reason == "stop"
+ assert isinstance(result.content, str)
+ assert "4" in result.content
+ assert result.finish_reason == "stop"
+ assert "LLMCall" in caplog.text and result.content in caplog.text
# Test JSON output - add to existing test
json_result = await client.create(
diff --git a/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py b/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py
index 4545928ed..986ae27ba 100644
--- a/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py
+++ b/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py
@@ -1,4 +1,5 @@
import asyncio
+import logging
import os
from datetime import datetime
from typing import Any, AsyncGenerator, List
@@ -170,9 +171,13 @@ async def test_azure_ai_chat_completion_client(azure_client: AzureAIChatCompleti
@pytest.mark.asyncio
-async def test_azure_ai_chat_completion_client_create(azure_client: AzureAIChatCompletionClient) -> None:
- result = await azure_client.create(messages=[UserMessage(content="Hello", source="user")])
- assert result.content == "Hello"
+async def test_azure_ai_chat_completion_client_create(
+ azure_client: AzureAIChatCompletionClient, caplog: pytest.LogCaptureFixture
+) -> None:
+ with caplog.at_level(logging.INFO):
+ result = await azure_client.create(messages=[UserMessage(content="Hello", source="user")])
+ assert result.content == "Hello"
+ assert "LLMCall" in caplog.text and "Hello" in caplog.text
@pytest.mark.asyncio
diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py
index 4eea3164c..2f51b6e20 100644
--- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py
+++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py
@@ -1,5 +1,6 @@
import asyncio
import json
+import logging
import os
from typing import Annotated, Any, AsyncGenerator, Dict, List, Literal, Tuple, TypeVar
from unittest.mock import MagicMock
@@ -222,11 +223,15 @@ async def test_azure_openai_chat_completion_client() -> None:
@pytest.mark.asyncio
-async def test_openai_chat_completion_client_create(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_openai_chat_completion_client_create(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
monkeypatch.setattr(AsyncCompletions, "create", _mock_create)
- client = OpenAIChatCompletionClient(model="gpt-4o", api_key="api_key")
- result = await client.create(messages=[UserMessage(content="Hello", source="user")])
- assert result.content == "Hello"
+ with caplog.at_level(logging.INFO):
+ client = OpenAIChatCompletionClient(model="gpt-4o", api_key="api_key")
+ result = await client.create(messages=[UserMessage(content="Hello", source="user")])
+ assert result.content == "Hello"
+ assert "LLMCall" in caplog.text and "Hello" in caplog.text
@pytest.mark.asyncio
diff --git a/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py b/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py
index 66052259e..3ec2bcfaf 100644
--- a/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py
+++ b/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py
@@ -1,3 +1,4 @@
+import logging
import os
from typing import Any, AsyncGenerator
from unittest.mock import AsyncMock
@@ -335,7 +336,9 @@ async def test_sk_chat_completion_with_tools(sk_client: AzureChatCompletion) ->
@pytest.mark.asyncio
-async def test_sk_chat_completion_without_tools(sk_client: AzureChatCompletion) -> None:
+async def test_sk_chat_completion_without_tools(
+ sk_client: AzureChatCompletion, caplog: pytest.LogCaptureFixture
+) -> None:
# Create adapter and kernel
adapter = SKChatCompletionAdapter(sk_client)
kernel = Kernel(memory=NullMemory())
@@ -346,15 +349,19 @@ async def test_sk_chat_completion_without_tools(sk_client: AzureChatCompletion)
UserMessage(content="Say hello!", source="user"),
]
- # Call create without tools
- result = await adapter.create(messages=messages, extra_create_args={"kernel": kernel})
+ with caplog.at_level(logging.INFO):
+ # Call create without tools
+ result = await adapter.create(messages=messages, extra_create_args={"kernel": kernel})
- # Verify response
- assert isinstance(result.content, str)
- assert result.finish_reason == "stop"
- assert result.usage.prompt_tokens >= 0
- assert result.usage.completion_tokens >= 0
- assert not result.cached
+ # Verify response
+ assert isinstance(result.content, str)
+ assert result.finish_reason == "stop"
+ assert result.usage.prompt_tokens >= 0
+ assert result.usage.completion_tokens >= 0
+ assert not result.cached
+
+ # Check log output
+ assert "LLMCall" in caplog.text and result.content in caplog.text
@pytest.mark.asyncio
diff --git a/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py b/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py
index 1c6f662a2..f77eca627 100644
--- a/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py
+++ b/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py
@@ -91,6 +91,7 @@ async def test_global_search_tool(
entity_df_fixture: pd.DataFrame,
report_df_fixture: pd.DataFrame,
entity_embedding_fixture: pd.DataFrame,
+ caplog: pytest.LogCaptureFixture,
) -> None:
# Create a temporary directory to simulate the data config
with tempfile.TemporaryDirectory() as tempdir:
@@ -120,12 +121,16 @@ async def test_global_search_tool(
global_search_tool = GlobalSearchTool(token_encoder=token_encoder, llm=llm, data_config=data_config)
- # Example of running the tool and checking the result
- query = "What is the overall sentiment of the community reports?"
- cancellation_token = CancellationToken()
- result = await global_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)
- assert isinstance(result, GlobalSearchToolReturn)
- assert isinstance(result.answer, str)
+ with caplog.at_level("INFO"):
+ # Example of running the tool and checking the result
+ query = "What is the overall sentiment of the community reports?"
+ cancellation_token = CancellationToken()
+ result = await global_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)
+ assert isinstance(result, GlobalSearchToolReturn)
+ assert isinstance(result.answer, str)
+
+ # Check if the log contains the expected message
+ assert result.answer in caplog.text
@pytest.mark.asyncio
@@ -135,6 +140,7 @@ async def test_local_search_tool(
text_unit_df_fixture: pd.DataFrame,
entity_embedding_fixture: pd.DataFrame,
monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
) -> None:
# Create a temporary directory to simulate the data config
with tempfile.TemporaryDirectory() as tempdir:
@@ -176,9 +182,13 @@ async def test_local_search_tool(
token_encoder=token_encoder, llm=llm, embedder=embedder, data_config=data_config
)
- # Example of running the tool and checking the result
- query = "What are the relationships between Dr. Becher and the station-master?"
- cancellation_token = CancellationToken()
- result = await local_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)
- assert isinstance(result, LocalSearchToolReturn)
- assert isinstance(result.answer, str)
+ with caplog.at_level("INFO"):
+ # Example of running the tool and checking the result
+ query = "What are the relationships between Dr. Becher and the station-master?"
+ cancellation_token = CancellationToken()
+ result = await local_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)
+ assert isinstance(result, LocalSearchToolReturn)
+ assert isinstance(result.answer, str)
+
+ # Check if the log contains the expected message
+ assert result.answer in caplog.text
diff --git a/python/packages/autogen-ext/tests/tools/http/test_http_tool.py b/python/packages/autogen-ext/tests/tools/http/test_http_tool.py
index 3756c4847..9d2898bae 100644
--- a/python/packages/autogen-ext/tests/tools/http/test_http_tool.py
+++ b/python/packages/autogen-ext/tests/tools/http/test_http_tool.py
@@ -1,4 +1,5 @@
import json
+import logging
import httpx
import pytest
@@ -45,12 +46,16 @@ def test_component_base_class(test_config: ComponentModel) -> None:
@pytest.mark.asyncio
-async def test_post_request(test_config: ComponentModel, test_server: None) -> None:
+async def test_post_request(test_config: ComponentModel, test_server: None, caplog: pytest.LogCaptureFixture) -> None:
tool = HttpTool.load_component(test_config)
- result = await tool.run_json({"query": "test query", "value": 42}, CancellationToken())
- assert isinstance(result, str)
- assert json.loads(result)["result"] == "Received: test query with value 42"
+ with caplog.at_level(logging.INFO):
+ result = await tool.run_json({"query": "test query", "value": 42}, CancellationToken())
+
+ assert isinstance(result, str)
+ assert json.loads(result)["result"] == "Received: test query with value 42"
+
+ assert "Received: test query with value 42" in caplog.text
@pytest.mark.asyncio
diff --git a/python/packages/autogen-ext/tests/tools/test_langchain_tools.py b/python/packages/autogen-ext/tests/tools/test_langchain_tools.py
index 58896642a..bd9cfea6f 100644
--- a/python/packages/autogen-ext/tests/tools/test_langchain_tools.py
+++ b/python/packages/autogen-ext/tests/tools/test_langchain_tools.py
@@ -1,3 +1,4 @@
+import logging
from typing import Optional, Type, cast
import pytest
@@ -42,7 +43,7 @@ class CustomCalculatorTool(LangChainTool):
@pytest.mark.asyncio
-async def test_langchain_tool_adapter() -> None:
+async def test_langchain_tool_adapter(caplog: pytest.LogCaptureFixture) -> None:
# Create a LangChain tool
langchain_tool = add # type: ignore
@@ -66,9 +67,12 @@ async def test_langchain_tool_adapter() -> None:
assert set(schema["parameters"]["required"]) == {"a", "b"}
assert len(schema["parameters"]["properties"]) == 2
- # Test run method
- result = await adapter.run_json({"a": 2, "b": 3}, CancellationToken())
- assert result == 5
+ # Check log.
+ with caplog.at_level(logging.INFO):
+ # Test run method
+ result = await adapter.run_json({"a": 2, "b": 3}, CancellationToken())
+ assert result == 5
+ assert str(result) in caplog.text
# Test that the adapter's run method can be called multiple times
result = await adapter.run_json({"a": 5, "b": 7}, CancellationToken())
diff --git a/python/packages/autogen-ext/tests/tools/test_mcp_tools.py b/python/packages/autogen-ext/tests/tools/test_mcp_tools.py
index 7af9933d3..215521aa9 100644
--- a/python/packages/autogen-ext/tests/tools/test_mcp_tools.py
+++ b/python/packages/autogen-ext/tests/tools/test_mcp_tools.py
@@ -1,3 +1,4 @@
+import logging
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -109,6 +110,7 @@ async def test_mcp_tool_execution(
mock_tool_response: MagicMock,
cancellation_token: CancellationToken,
monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
) -> None:
"""Test that adapter properly executes tools through ClientSession."""
mock_context = AsyncMock()
@@ -120,15 +122,19 @@ async def test_mcp_tool_execution(
mock_session.call_tool.return_value = mock_tool_response
- adapter = StdioMcpToolAdapter(server_params=sample_server_params, tool=sample_tool)
- result = await adapter.run(
- args=create_model(sample_tool.inputSchema)(**{"test_param": "test"}),
- cancellation_token=cancellation_token,
- )
+ with caplog.at_level(logging.INFO):
+ adapter = StdioMcpToolAdapter(server_params=sample_server_params, tool=sample_tool)
+ result = await adapter.run_json(
+ args=create_model(sample_tool.inputSchema)(**{"test_param": "test"}).model_dump(),
+ cancellation_token=cancellation_token,
+ )
+
+ assert result == mock_tool_response.content
+ mock_session.initialize.assert_called_once()
+ mock_session.call_tool.assert_called_once()
- assert result == mock_tool_response.content
- mock_session.initialize.assert_called_once()
- mock_session.call_tool.assert_called_once()
+ # Check log.
+ assert "test_output" in caplog.text
@pytest.mark.asyncio
@@ -206,6 +212,7 @@ async def test_sse_tool_execution(
sample_sse_tool: Tool,
mock_sse_session: AsyncMock,
monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
) -> None:
"""Test that SSE adapter properly executes tools through ClientSession."""
params = SseServerParams(url="http://test-url")
@@ -219,15 +226,19 @@ async def test_sse_tool_execution(
lambda *args, **kwargs: mock_context, # type: ignore
)
- adapter = SseMcpToolAdapter(server_params=params, tool=sample_sse_tool)
- result = await adapter.run(
- args=create_model(sample_sse_tool.inputSchema)(**{"test_param": "test"}),
- cancellation_token=CancellationToken(),
- )
+ with caplog.at_level(logging.INFO):
+ adapter = SseMcpToolAdapter(server_params=params, tool=sample_sse_tool)
+ result = await adapter.run_json(
+ args=create_model(sample_sse_tool.inputSchema)(**{"test_param": "test"}).model_dump(),
+ cancellation_token=CancellationToken(),
+ )
+
+ assert result == mock_sse_session.call_tool.return_value.content
+ mock_sse_session.initialize.assert_called_once()
+ mock_sse_session.call_tool.assert_called_once()
- assert result == mock_sse_session.call_tool.return_value.content
- mock_sse_session.initialize.assert_called_once()
- mock_sse_session.call_tool.assert_called_once()
+ # Check log.
+ assert "test_output" in caplog.text
@pytest.mark.asyncio
diff --git a/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py b/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py
index 2593a4475..8ecb1700e 100644
--- a/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py
+++ b/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py
@@ -1,3 +1,4 @@
+import logging
import tempfile
import pytest
@@ -7,7 +8,7 @@ from autogen_ext.tools.code_execution import CodeExecutionInput, PythonCodeExecu
@pytest.mark.asyncio
-async def test_python_code_execution_tool() -> None:
+async def test_python_code_execution_tool(caplog: pytest.LogCaptureFixture) -> None:
"""Test basic functionality of PythonCodeExecutionTool."""
# Create a temporary directory for the executor
with tempfile.TemporaryDirectory() as temp_dir:
@@ -15,9 +16,12 @@ async def test_python_code_execution_tool() -> None:
executor = LocalCommandLineCodeExecutor(work_dir=temp_dir)
tool = PythonCodeExecutionTool(executor=executor)
- # Test simple code execution
- code = "print('hello world!')"
- result = await tool.run(args=CodeExecutionInput(code=code), cancellation_token=CancellationToken())
+ with caplog.at_level(logging.INFO):
+ # Test simple code execution
+ code = "print('hello world!')"
+ result = await tool.run_json(args={"code": code}, cancellation_token=CancellationToken())
+ # Check log output
+ assert "hello world!" in caplog.text
# Verify successful execution
assert result.success is True