A programming framework for agentic AI
Add ToolCallEvent and log it from all builtin tools (#5859)
Eric Zhu740afe5b611c
diff --git a/python/packages/autogen-core/src/autogen_core/logging.py b/python/packages/autogen-core/src/autogen_core/logging.pyindex 057458e18..6ea430f27 100644--- a/python/packages/autogen-core/src/autogen_core/logging.py+++ b/python/packages/autogen-core/src/autogen_core/logging.py@@ -1,8 +1,9 @@import jsonfrom enum import Enum-from typing import Any, Dict, cast+from typing import Any, Dict, List, castfrom ._agent_id import AgentId+from ._message_handler_context import MessageHandlerContextfrom ._topic import TopicId@@ -10,21 +11,19 @@ class LLMCallEvent:def __init__(self,*,- messages: Dict[str, Any],+ messages: List[Dict[str, Any]],response: Dict[str, Any],prompt_tokens: int,completion_tokens: int,- agent_id: AgentId | None = None,**kwargs: Any,) -> None:"""To be used by model clients to log the call to the LLM.Args:- messages (Dict[str, Any]): The messages of the call. Must be json serializable.+ messages (List[Dict[str, Any]]): The messages used in the call. Must be json serializable.response (Dict[str, Any]): The response of the call. Must be json serializable.prompt_tokens (int): Number of tokens used in the prompt.completion_tokens (int): Number of tokens used in the completion.- agent_id (AgentId | None, optional): The agent id of the model. Defaults to None.Example:@@ -43,8 +42,11 @@ class LLMCallEvent:self.kwargs["response"] = responseself.kwargs["prompt_tokens"] = prompt_tokensself.kwargs["completion_tokens"] = completion_tokens+ try:+ agent_id = MessageHandlerContext.agent_id()+ except RuntimeError:+ agent_id = Noneself.kwargs["agent_id"] = None if agent_id is None else str(agent_id)- self.kwargs["type"] = "LLMCall"@propertydef prompt_tokens(self) -> int:@@ -59,6 +61,48 @@ class LLMCallEvent:return json.dumps(self.kwargs)+class ToolCallEvent:+ def __init__(+ self,+ *,+ tool_name: str,+ arguments: Dict[str, Any],+ result: str,+ ) -> None:+ """Used by subclasses of :class:`~autogen_core.tools.BaseTool` to log executions of tools.++ Args:+ tool_name (str): The name of the tool.+ arguments (Dict[str, Any]): The arguments of the tool. Must be json serializable.+ result (str): The result of the tool. Must be a string.++ Example:++ .. code-block:: python++ from autogen_core import EVENT_LOGGER_NAME+ from autogen_core.logging import ToolCallEvent++ logger = logging.getLogger(EVENT_LOGGER_NAME)+ logger.info(ToolCallEvent(tool_name="Tool1", call_id="123", arguments={"arg1": "value1"}))++ """+ self.kwargs: Dict[str, Any] = {}+ self.kwargs["type"] = "ToolCall"+ self.kwargs["tool_name"] = tool_name+ self.kwargs["arguments"] = arguments+ self.kwargs["result"] = result+ try:+ agent_id = MessageHandlerContext.agent_id()+ except RuntimeError:+ agent_id = None+ self.kwargs["agent_id"] = None if agent_id is None else str(agent_id)++ # This must output the event in a json serializable format+ def __str__(self) -> str:+ return json.dumps(self.kwargs)++class MessageKind(Enum):DIRECT = 1PUBLISH = 2diff --git a/python/packages/autogen-core/src/autogen_core/tools/_base.py b/python/packages/autogen-core/src/autogen_core/tools/_base.pyindex 8b6489731..814463e36 100644--- a/python/packages/autogen-core/src/autogen_core/tools/_base.py+++ b/python/packages/autogen-core/src/autogen_core/tools/_base.py@@ -1,4 +1,5 @@import json+import loggingfrom abc import ABC, abstractmethodfrom collections.abc import Sequencefrom typing import Any, Dict, Generic, Mapping, Protocol, Type, TypedDict, TypeVar, cast, runtime_checkable@@ -7,12 +8,15 @@ import jsonreffrom pydantic import BaseModelfrom typing_extensions import NotRequired-from .. import CancellationToken+from .. import EVENT_LOGGER_NAME, CancellationTokenfrom .._component_config import ComponentBasefrom .._function_utils import normalize_annotated_type+from ..logging import ToolCallEventT = TypeVar("T", bound=BaseModel, contravariant=True)+logger = logging.getLogger(EVENT_LOGGER_NAME)+class ParametersSchema(TypedDict):type: str@@ -143,7 +147,17 @@ class BaseTool(ABC, Tool, Generic[ArgsT, ReturnT], ComponentBase[BaseModel]):async def run(self, args: ArgsT, cancellation_token: CancellationToken) -> ReturnT: ...async def run_json(self, args: Mapping[str, Any], cancellation_token: CancellationToken) -> Any:+ # Execute the tool's run methodreturn_value = await self.run(self._args_type.model_validate(args), cancellation_token)++ # Log the tool call event+ event = ToolCallEvent(+ tool_name=self.name,+ arguments=dict(args), # Using the raw args passed to run_json+ result=self.return_value_as_string(return_value),+ )+ logger.info(event)+return return_valuedef save_state_json(self) -> Mapping[str, Any]:diff --git a/python/packages/autogen-core/tests/test_tool_agent.py b/python/packages/autogen-core/tests/test_tool_agent.pyindex 715d5519c..5ef312e7b 100644--- a/python/packages/autogen-core/tests/test_tool_agent.py+++ b/python/packages/autogen-core/tests/test_tool_agent.py@@ -1,9 +1,10 @@import asyncioimport json+import loggingfrom typing import Any, AsyncGenerator, List, Mapping, Optional, Sequence, Unionimport pytest-from autogen_core import AgentId, CancellationToken, FunctionCall, SingleThreadedAgentRuntime+from autogen_core import EVENT_LOGGER_NAME, AgentId, CancellationToken, FunctionCall, SingleThreadedAgentRuntimefrom autogen_core.models import (AssistantMessage,ChatCompletionClient,@@ -25,6 +26,8 @@ from autogen_core.tool_agent import ()from autogen_core.tools import FunctionTool, Tool, ToolSchema+logging.getLogger(EVENT_LOGGER_NAME).setLevel(logging.INFO)+def _pass_function(input: str) -> str:return "pass"@@ -40,7 +43,7 @@ async def _async_sleep_function(input: str) -> str:@pytest.mark.asyncio-async def test_tool_agent() -> None:+async def test_tool_agent(caplog: pytest.LogCaptureFixture) -> None:runtime = SingleThreadedAgentRuntime()await ToolAgent.register(runtime,@@ -63,6 +66,9 @@ async def test_tool_agent() -> None:)assert result == FunctionExecutionResult(call_id="1", content="pass", is_error=False, name="pass")+ # Check log.+ assert any(("ToolCall" in record.message and str(agent) in record.message) for record in caplog.records)+# Test raise functionwith pytest.raises(ToolExecutionException):await runtime.send_message(FunctionCall(id="2", arguments=json.dumps({"input": "raise"}), name="raise"), agent)diff --git a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.pyindex 2ed797abd..ea21fa28e 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_anthropic_client.py@@ -44,7 +44,6 @@ from autogen_core import (Component,FunctionCall,Image,- MessageHandlerContext,)from autogen_core.logging import LLMCallEventfrom autogen_core.models import (@@ -503,19 +502,12 @@ class BaseAnthropicChatCompletionClient(ChatCompletionClient):completion_tokens=result.usage.output_tokens,)- # Log the event if in a handler context- try:- agent_id = MessageHandlerContext.agent_id()- except RuntimeError:- agent_id = None-logger.info(LLMCallEvent(- messages=cast(Dict[str, Any], anthropic_messages),+ messages=cast(List[Dict[str, Any]], anthropic_messages),response=result.model_dump(),prompt_tokens=usage.prompt_tokens,completion_tokens=usage.completion_tokens,- agent_id=agent_id,))diff --git a/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.pyindex 1b22abe42..b6257bd9a 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/azure/_azure_ai_client.py@@ -1,10 +1,12 @@import asyncio+import loggingimport refrom asyncio import Taskfrom inspect import getfullargspecfrom typing import Any, Dict, List, Mapping, Optional, Sequence, cast-from autogen_core import CancellationToken, FunctionCall, Image+from autogen_core import EVENT_LOGGER_NAME, CancellationToken, FunctionCall, Image+from autogen_core.logging import LLMCallEventfrom autogen_core.models import (AssistantMessage,ChatCompletionClient,@@ -62,6 +64,8 @@ from .._utils.parse_r1_content import parse_r1_contentcreate_kwargs = set(getfullargspec(ChatCompletionsClient.complete).kwonlyargs)AzureMessage = Union[AzureSystemMessage, AzureUserMessage, AzureAssistantMessage, AzureToolMessage]+logger = logging.getLogger(EVENT_LOGGER_NAME)+def _is_github_model(endpoint: str) -> bool:return endpoint == GITHUB_MODELS_ENDPOINT@@ -339,6 +343,15 @@ class AzureAIChatCompletionClient(ChatCompletionClient):completion_tokens=result.usage.completion_tokens if result.usage else 0,)+ logger.info(+ LLMCallEvent(+ messages=[m.as_dict() for m in azure_messages],+ response=result.as_dict(),+ prompt_tokens=usage.prompt_tokens,+ completion_tokens=usage.completion_tokens,+ )+ )+choice = result.choices[0]if choice.finish_reason == CompletionsFinishReason.TOOL_CALLS:assert choice.message.tool_calls is not Nonediff --git a/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py b/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.pyindex 761863eeb..f817a17ae 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/ollama/_ollama_client.py@@ -26,7 +26,6 @@ from autogen_core import (Component,FunctionCall,Image,- MessageHandlerContext,)from autogen_core.logging import LLMCallEventfrom autogen_core.models import (@@ -483,19 +482,12 @@ class BaseOllamaChatCompletionClient(ChatCompletionClient):completion_tokens=(result.eval_count if result.eval_count is not None else 0),)- # If we are running in the context of a handler we can get the agent_id- try:- agent_id = MessageHandlerContext.agent_id()- except RuntimeError:- agent_id = None-logger.info(LLMCallEvent(- messages=cast(Dict[str, Any], ollama_messages),+ messages=cast(List[Dict[str, Any]], ollama_messages),response=result.model_dump(),prompt_tokens=usage.prompt_tokens,completion_tokens=usage.completion_tokens,- agent_id=agent_id,))diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.pyindex 34279acc2..05557e19f 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py@@ -29,7 +29,6 @@ from autogen_core import (Component,FunctionCall,Image,- MessageHandlerContext,)from autogen_core.logging import LLMCallEventfrom autogen_core.models import (@@ -531,19 +530,12 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):completion_tokens=(result.usage.completion_tokens if result.usage is not None else 0),)- # If we are running in the context of a handler we can get the agent_id- try:- agent_id = MessageHandlerContext.agent_id()- except RuntimeError:- agent_id = None-logger.info(LLMCallEvent(- messages=cast(Dict[str, Any], oai_messages),+ messages=cast(List[Dict[str, Any]], oai_messages),response=result.model_dump(),prompt_tokens=usage.prompt_tokens,completion_tokens=usage.completion_tokens,- agent_id=agent_id,))diff --git a/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py b/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.pyindex aa58830a9..7df5dd59b 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/semantic_kernel/_sk_chat_completion_adapter.py@@ -1,9 +1,11 @@import json+import loggingimport warningsfrom typing import Any, Literal, Mapping, Optional, Sequence-from autogen_core import FunctionCall+from autogen_core import EVENT_LOGGER_NAME, FunctionCallfrom autogen_core._cancellation_token import CancellationToken+from autogen_core.logging import LLMCallEventfrom autogen_core.models import (ChatCompletionClient,CreateResult,@@ -32,6 +34,8 @@ from autogen_ext.tools.semantic_kernel import KernelFunctionFromToolfrom .._utils.parse_r1_content import parse_r1_content+logger = logging.getLogger(EVENT_LOGGER_NAME)+class SKChatCompletionAdapter(ChatCompletionClient):"""@@ -465,6 +469,15 @@ class SKChatCompletionAdapter(ChatCompletionClient):prompt_tokens = getattr(usage, "prompt_tokens", 0)completion_tokens = getattr(usage, "completion_tokens", 0)+ logger.info(+ LLMCallEvent(+ messages=[msg.model_dump() for msg in chat_history],+ response=result[0].model_dump(),+ prompt_tokens=prompt_tokens,+ completion_tokens=completion_tokens,+ )+ )+self._total_prompt_tokens += prompt_tokensself._total_completion_tokens += completion_tokensdiff --git a/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py b/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.pyindex 14086b63b..fd6d51a77 100644--- a/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py+++ b/python/packages/autogen-ext/src/autogen_ext/tools/code_execution/_code_execution.py@@ -83,7 +83,6 @@ class PythonCodeExecutionTool(result = await self._executor.execute_code_blocks(code_blocks=code_blocks, cancellation_token=cancellation_token)-return CodeExecutionResult(success=result.exit_code == 0, output=result.output)def _to_config(self) -> PythonCodeExecutionToolConfig:diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.pyindex 445832040..1a8915ee9 100644--- a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py+++ b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_global_search.py@@ -177,9 +177,9 @@ class GlobalSearchTool(BaseTool[GlobalSearchToolArgs, GlobalSearchToolReturn]):)async def run(self, args: GlobalSearchToolArgs, cancellation_token: CancellationToken) -> GlobalSearchToolReturn:- result = await self._search_engine.asearch(args.query)- assert isinstance(result.response, str), "Expected response to be a string"- return GlobalSearchToolReturn(answer=result.response)+ search_result = await self._search_engine.asearch(args.query)+ assert isinstance(search_result.response, str), "Expected response to be a string"+ return GlobalSearchToolReturn(answer=search_result.response)@classmethoddef from_settings(cls, settings_path: str | Path) -> "GlobalSearchTool":diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.pyindex 0031d1d1b..625c7a4e1 100644--- a/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py+++ b/python/packages/autogen-ext/src/autogen_ext/tools/graphrag/_local_search.py@@ -188,9 +188,9 @@ class LocalSearchTool(BaseTool[LocalSearchToolArgs, LocalSearchToolReturn]):)async def run(self, args: LocalSearchToolArgs, cancellation_token: CancellationToken) -> LocalSearchToolReturn:- result = await self._search_engine.asearch(args.query) # type: ignore- assert isinstance(result.response, str), "Expected response to be a string"- return LocalSearchToolReturn(answer=result.response)+ search_result = await self._search_engine.asearch(args.query) # type: ignore+ assert isinstance(search_result.response, str), "Expected response to be a string"+ return LocalSearchToolReturn(answer=search_result.response)@classmethoddef from_settings(cls, settings_path: str | Path) -> "LocalSearchTool":diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py b/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.pyindex 81dc1d8f7..c4cba4174 100644--- a/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py+++ b/python/packages/autogen-ext/src/autogen_ext/tools/langchain/_langchain_adapter.py@@ -189,12 +189,10 @@ class LangChainToolAdapter(BaseTool[BaseModel, Any]):# Determine if the callable is asynchronousif inspect.iscoroutinefunction(self._callable):- result = await self._callable(**kwargs)+ return await self._callable(**kwargs)else:# Run in a thread to avoid blocking the event loop- result = await asyncio.to_thread(self._call_sync, kwargs)-- return result+ return await asyncio.to_thread(self._call_sync, kwargs)def _call_sync(self, kwargs: Dict[str, Any]) -> Any:return self._callable(**kwargs)diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py b/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.pyindex cf905800d..a8bb3e939 100644--- a/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py+++ b/python/packages/autogen-ext/src/autogen_ext/tools/mcp/_base.py@@ -67,7 +67,6 @@ class McpToolAdapter(BaseTool[BaseModel, Any], ABC, Generic[TServerParams]):if result.isError:raise Exception(f"MCP tool execution failed: {result.content}")-return result.contentexcept Exception as e:raise Exception(str(e)) from ediff --git a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.pyindex 8e652663d..74092b33f 100644--- a/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py+++ b/python/packages/autogen-ext/tests/models/test_anthropic_model_client.py@@ -1,4 +1,5 @@import asyncio+import loggingimport osfrom typing import List, Sequence@@ -28,7 +29,7 @@ def _add_numbers(a: int, b: int) -> int:@pytest.mark.asyncio-async def test_anthropic_basic_completion() -> None:+async def test_anthropic_basic_completion(caplog: pytest.LogCaptureFixture) -> None:"""Test basic message completion with Claude."""api_key = os.getenv("ANTHROPIC_API_KEY")if not api_key:@@ -42,16 +43,18 @@ async def test_anthropic_basic_completion() -> None:)# Test basic completion- result = await client.create(- messages=[- SystemMessage(content="You are a helpful assistant."),- UserMessage(content="What's 2+2? Answer with just the number.", source="user"),- ]- )+ with caplog.at_level(logging.INFO):+ result = await client.create(+ messages=[+ SystemMessage(content="You are a helpful assistant."),+ UserMessage(content="What's 2+2? Answer with just the number.", source="user"),+ ]+ )- assert isinstance(result.content, str)- assert "4" in result.content- assert result.finish_reason == "stop"+ assert isinstance(result.content, str)+ assert "4" in result.content+ assert result.finish_reason == "stop"+ assert "LLMCall" in caplog.text and result.content in caplog.text# Test JSON output - add to existing testjson_result = await client.create(diff --git a/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py b/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.pyindex 4545928ed..986ae27ba 100644--- a/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py+++ b/python/packages/autogen-ext/tests/models/test_azure_ai_model_client.py@@ -1,4 +1,5 @@import asyncio+import loggingimport osfrom datetime import datetimefrom typing import Any, AsyncGenerator, List@@ -170,9 +171,13 @@ async def test_azure_ai_chat_completion_client(azure_client: AzureAIChatCompleti@pytest.mark.asyncio-async def test_azure_ai_chat_completion_client_create(azure_client: AzureAIChatCompletionClient) -> None:- result = await azure_client.create(messages=[UserMessage(content="Hello", source="user")])- assert result.content == "Hello"+async def test_azure_ai_chat_completion_client_create(+ azure_client: AzureAIChatCompletionClient, caplog: pytest.LogCaptureFixture+) -> None:+ with caplog.at_level(logging.INFO):+ result = await azure_client.create(messages=[UserMessage(content="Hello", source="user")])+ assert result.content == "Hello"+ assert "LLMCall" in caplog.text and "Hello" in caplog.text@pytest.mark.asynciodiff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.pyindex 4eea3164c..2f51b6e20 100644--- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py+++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py@@ -1,5 +1,6 @@import asyncioimport json+import loggingimport osfrom typing import Annotated, Any, AsyncGenerator, Dict, List, Literal, Tuple, TypeVarfrom unittest.mock import MagicMock@@ -222,11 +223,15 @@ async def test_azure_openai_chat_completion_client() -> None:@pytest.mark.asyncio-async def test_openai_chat_completion_client_create(monkeypatch: pytest.MonkeyPatch) -> None:+async def test_openai_chat_completion_client_create(+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:monkeypatch.setattr(AsyncCompletions, "create", _mock_create)- client = OpenAIChatCompletionClient(model="gpt-4o", api_key="api_key")- result = await client.create(messages=[UserMessage(content="Hello", source="user")])- assert result.content == "Hello"+ with caplog.at_level(logging.INFO):+ client = OpenAIChatCompletionClient(model="gpt-4o", api_key="api_key")+ result = await client.create(messages=[UserMessage(content="Hello", source="user")])+ assert result.content == "Hello"+ assert "LLMCall" in caplog.text and "Hello" in caplog.text@pytest.mark.asynciodiff --git a/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py b/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.pyindex 66052259e..3ec2bcfaf 100644--- a/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py+++ b/python/packages/autogen-ext/tests/models/test_sk_chat_completion_adapter.py@@ -1,3 +1,4 @@+import loggingimport osfrom typing import Any, AsyncGeneratorfrom unittest.mock import AsyncMock@@ -335,7 +336,9 @@ async def test_sk_chat_completion_with_tools(sk_client: AzureChatCompletion) ->@pytest.mark.asyncio-async def test_sk_chat_completion_without_tools(sk_client: AzureChatCompletion) -> None:+async def test_sk_chat_completion_without_tools(+ sk_client: AzureChatCompletion, caplog: pytest.LogCaptureFixture+) -> None:# Create adapter and kerneladapter = SKChatCompletionAdapter(sk_client)kernel = Kernel(memory=NullMemory())@@ -346,15 +349,19 @@ async def test_sk_chat_completion_without_tools(sk_client: AzureChatCompletion)UserMessage(content="Say hello!", source="user"),]- # Call create without tools- result = await adapter.create(messages=messages, extra_create_args={"kernel": kernel})+ with caplog.at_level(logging.INFO):+ # Call create without tools+ result = await adapter.create(messages=messages, extra_create_args={"kernel": kernel})- # Verify response- assert isinstance(result.content, str)- assert result.finish_reason == "stop"- assert result.usage.prompt_tokens >= 0- assert result.usage.completion_tokens >= 0- assert not result.cached+ # Verify response+ assert isinstance(result.content, str)+ assert result.finish_reason == "stop"+ assert result.usage.prompt_tokens >= 0+ assert result.usage.completion_tokens >= 0+ assert not result.cached++ # Check log output+ assert "LLMCall" in caplog.text and result.content in caplog.text@pytest.mark.asynciodiff --git a/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py b/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.pyindex 1c6f662a2..f77eca627 100644--- a/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py+++ b/python/packages/autogen-ext/tests/tools/graphrag/test_graphrag_tools.py@@ -91,6 +91,7 @@ async def test_global_search_tool(entity_df_fixture: pd.DataFrame,report_df_fixture: pd.DataFrame,entity_embedding_fixture: pd.DataFrame,+ caplog: pytest.LogCaptureFixture,) -> None:# Create a temporary directory to simulate the data configwith tempfile.TemporaryDirectory() as tempdir:@@ -120,12 +121,16 @@ async def test_global_search_tool(global_search_tool = GlobalSearchTool(token_encoder=token_encoder, llm=llm, data_config=data_config)- # Example of running the tool and checking the result- query = "What is the overall sentiment of the community reports?"- cancellation_token = CancellationToken()- result = await global_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)- assert isinstance(result, GlobalSearchToolReturn)- assert isinstance(result.answer, str)+ with caplog.at_level("INFO"):+ # Example of running the tool and checking the result+ query = "What is the overall sentiment of the community reports?"+ cancellation_token = CancellationToken()+ result = await global_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)+ assert isinstance(result, GlobalSearchToolReturn)+ assert isinstance(result.answer, str)++ # Check if the log contains the expected message+ assert result.answer in caplog.text@pytest.mark.asyncio@@ -135,6 +140,7 @@ async def test_local_search_tool(text_unit_df_fixture: pd.DataFrame,entity_embedding_fixture: pd.DataFrame,monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,) -> None:# Create a temporary directory to simulate the data configwith tempfile.TemporaryDirectory() as tempdir:@@ -176,9 +182,13 @@ async def test_local_search_tool(token_encoder=token_encoder, llm=llm, embedder=embedder, data_config=data_config)- # Example of running the tool and checking the result- query = "What are the relationships between Dr. Becher and the station-master?"- cancellation_token = CancellationToken()- result = await local_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)- assert isinstance(result, LocalSearchToolReturn)- assert isinstance(result.answer, str)+ with caplog.at_level("INFO"):+ # Example of running the tool and checking the result+ query = "What are the relationships between Dr. Becher and the station-master?"+ cancellation_token = CancellationToken()+ result = await local_search_tool.run_json(args={"query": query}, cancellation_token=cancellation_token)+ assert isinstance(result, LocalSearchToolReturn)+ assert isinstance(result.answer, str)++ # Check if the log contains the expected message+ assert result.answer in caplog.textdiff --git a/python/packages/autogen-ext/tests/tools/http/test_http_tool.py b/python/packages/autogen-ext/tests/tools/http/test_http_tool.pyindex 3756c4847..9d2898bae 100644--- a/python/packages/autogen-ext/tests/tools/http/test_http_tool.py+++ b/python/packages/autogen-ext/tests/tools/http/test_http_tool.py@@ -1,4 +1,5 @@import json+import loggingimport httpximport pytest@@ -45,12 +46,16 @@ def test_component_base_class(test_config: ComponentModel) -> None:@pytest.mark.asyncio-async def test_post_request(test_config: ComponentModel, test_server: None) -> None:+async def test_post_request(test_config: ComponentModel, test_server: None, caplog: pytest.LogCaptureFixture) -> None:tool = HttpTool.load_component(test_config)- result = await tool.run_json({"query": "test query", "value": 42}, CancellationToken())- assert isinstance(result, str)- assert json.loads(result)["result"] == "Received: test query with value 42"+ with caplog.at_level(logging.INFO):+ result = await tool.run_json({"query": "test query", "value": 42}, CancellationToken())++ assert isinstance(result, str)+ assert json.loads(result)["result"] == "Received: test query with value 42"++ assert "Received: test query with value 42" in caplog.text@pytest.mark.asynciodiff --git a/python/packages/autogen-ext/tests/tools/test_langchain_tools.py b/python/packages/autogen-ext/tests/tools/test_langchain_tools.pyindex 58896642a..bd9cfea6f 100644--- a/python/packages/autogen-ext/tests/tools/test_langchain_tools.py+++ b/python/packages/autogen-ext/tests/tools/test_langchain_tools.py@@ -1,3 +1,4 @@+import loggingfrom typing import Optional, Type, castimport pytest@@ -42,7 +43,7 @@ class CustomCalculatorTool(LangChainTool):@pytest.mark.asyncio-async def test_langchain_tool_adapter() -> None:+async def test_langchain_tool_adapter(caplog: pytest.LogCaptureFixture) -> None:# Create a LangChain toollangchain_tool = add # type: ignore@@ -66,9 +67,12 @@ async def test_langchain_tool_adapter() -> None:assert set(schema["parameters"]["required"]) == {"a", "b"}assert len(schema["parameters"]["properties"]) == 2- # Test run method- result = await adapter.run_json({"a": 2, "b": 3}, CancellationToken())- assert result == 5+ # Check log.+ with caplog.at_level(logging.INFO):+ # Test run method+ result = await adapter.run_json({"a": 2, "b": 3}, CancellationToken())+ assert result == 5+ assert str(result) in caplog.text# Test that the adapter's run method can be called multiple timesresult = await adapter.run_json({"a": 5, "b": 7}, CancellationToken())diff --git a/python/packages/autogen-ext/tests/tools/test_mcp_tools.py b/python/packages/autogen-ext/tests/tools/test_mcp_tools.pyindex 7af9933d3..215521aa9 100644--- a/python/packages/autogen-ext/tests/tools/test_mcp_tools.py+++ b/python/packages/autogen-ext/tests/tools/test_mcp_tools.py@@ -1,3 +1,4 @@+import loggingfrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -109,6 +110,7 @@ async def test_mcp_tool_execution(mock_tool_response: MagicMock,cancellation_token: CancellationToken,monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,) -> None:"""Test that adapter properly executes tools through ClientSession."""mock_context = AsyncMock()@@ -120,15 +122,19 @@ async def test_mcp_tool_execution(mock_session.call_tool.return_value = mock_tool_response- adapter = StdioMcpToolAdapter(server_params=sample_server_params, tool=sample_tool)- result = await adapter.run(- args=create_model(sample_tool.inputSchema)(**{"test_param": "test"}),- cancellation_token=cancellation_token,- )+ with caplog.at_level(logging.INFO):+ adapter = StdioMcpToolAdapter(server_params=sample_server_params, tool=sample_tool)+ result = await adapter.run_json(+ args=create_model(sample_tool.inputSchema)(**{"test_param": "test"}).model_dump(),+ cancellation_token=cancellation_token,+ )++ assert result == mock_tool_response.content+ mock_session.initialize.assert_called_once()+ mock_session.call_tool.assert_called_once()- assert result == mock_tool_response.content- mock_session.initialize.assert_called_once()- mock_session.call_tool.assert_called_once()+ # Check log.+ assert "test_output" in caplog.text@pytest.mark.asyncio@@ -206,6 +212,7 @@ async def test_sse_tool_execution(sample_sse_tool: Tool,mock_sse_session: AsyncMock,monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,) -> None:"""Test that SSE adapter properly executes tools through ClientSession."""params = SseServerParams(url="http://test-url")@@ -219,15 +226,19 @@ async def test_sse_tool_execution(lambda *args, **kwargs: mock_context, # type: ignore)- adapter = SseMcpToolAdapter(server_params=params, tool=sample_sse_tool)- result = await adapter.run(- args=create_model(sample_sse_tool.inputSchema)(**{"test_param": "test"}),- cancellation_token=CancellationToken(),- )+ with caplog.at_level(logging.INFO):+ adapter = SseMcpToolAdapter(server_params=params, tool=sample_sse_tool)+ result = await adapter.run_json(+ args=create_model(sample_sse_tool.inputSchema)(**{"test_param": "test"}).model_dump(),+ cancellation_token=CancellationToken(),+ )++ assert result == mock_sse_session.call_tool.return_value.content+ mock_sse_session.initialize.assert_called_once()+ mock_sse_session.call_tool.assert_called_once()- assert result == mock_sse_session.call_tool.return_value.content- mock_sse_session.initialize.assert_called_once()- mock_sse_session.call_tool.assert_called_once()+ # Check log.+ assert "test_output" in caplog.text@pytest.mark.asynciodiff --git a/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py b/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.pyindex 2593a4475..8ecb1700e 100644--- a/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py+++ b/python/packages/autogen-ext/tests/tools/test_python_code_executor_tool.py@@ -1,3 +1,4 @@+import loggingimport tempfileimport pytest@@ -7,7 +8,7 @@ from autogen_ext.tools.code_execution import CodeExecutionInput, PythonCodeExecu@pytest.mark.asyncio-async def test_python_code_execution_tool() -> None:+async def test_python_code_execution_tool(caplog: pytest.LogCaptureFixture) -> None:"""Test basic functionality of PythonCodeExecutionTool."""# Create a temporary directory for the executorwith tempfile.TemporaryDirectory() as temp_dir:@@ -15,9 +16,12 @@ async def test_python_code_execution_tool() -> None:executor = LocalCommandLineCodeExecutor(work_dir=temp_dir)tool = PythonCodeExecutionTool(executor=executor)- # Test simple code execution- code = "print('hello world!')"- result = await tool.run(args=CodeExecutionInput(code=code), cancellation_token=CancellationToken())+ with caplog.at_level(logging.INFO):+ # Test simple code execution+ code = "print('hello world!')"+ result = await tool.run_json(args={"code": code}, cancellation_token=CancellationToken())+ # Check log output+ assert "hello world!" in caplog.text# Verify successful executionassert result.success is True