A programming framework for agentic AI
Feat/OpenAI agent builtin tools 6657 (#6671)
Tejas Dharani25d732876e4e
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py b/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.pyindex b7a7bcbb0..f14d8f64e 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py@@ -2,7 +2,20 @@ import asyncioimport jsonimport loggingimport warnings-from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Type, TypedDict, Union, cast+from typing import (+ Any,+ AsyncGenerator,+ Dict,+ Iterable,+ List,+ Literal,+ Mapping,+ Optional,+ Sequence,+ Type,+ Union,+ cast,+)from autogen_agentchat import EVENT_LOGGER_NAMEfrom autogen_agentchat.agents import BaseChatAgent@@ -21,8 +34,9 @@ from autogen_core import CancellationToken, Component, ComponentModel, FunctionCfrom autogen_core.models import UserMessagefrom autogen_core.tools import Toolfrom pydantic import BaseModel, Field+from typing_extensions import NotRequired, TypedDict-from openai import AsyncAzureOpenAI, AsyncOpenAI+from openai import AsyncAzureOpenAI, AsyncOpenAI # type: ignore# Number of characters to display when previewing image content in logs and UI# Base64 encoded images can be very long, so we truncate for readability@@ -33,6 +47,103 @@ IMAGE_CONTENT_PREVIEW_LENGTH = 50event_logger = logging.getLogger(EVENT_LOGGER_NAME)+# TypedDict classes for built-in tool configurations+class FileSearchToolConfig(TypedDict):+ """Configuration for file_search tool."""++ type: Literal["file_search"]+ vector_store_ids: List[str] # required - The IDs of the vector stores to search+ max_num_results: NotRequired[int] # optional+ ranking_options: NotRequired[Dict[str, Any]] # optional+ filters: NotRequired[Dict[str, Any]] # optional+++class WebSearchToolConfig(TypedDict):+ """Configuration for web_search_preview tool."""++ type: Literal["web_search_preview"]+ search_context_size: NotRequired[int] # optional+ user_location: NotRequired[Union[str, Dict[str, Any]]] # optional - Can be string or structured location+++class ComputerUseToolConfig(TypedDict):+ """Configuration for computer_use_preview tool."""++ type: Literal["computer_use_preview"]+ display_height: int # required - Display height in pixels+ display_width: int # required - Display width in pixels+ environment: str # required - Environment type for computer use+++class MCPToolConfig(TypedDict):+ """Configuration for mcp tool."""++ type: Literal["mcp"]+ server_label: str # required - Label for the MCP server+ server_url: str # required - URL of the MCP server+ allowed_tools: NotRequired[List[str]] # optional - List of allowed tools+ headers: NotRequired[Dict[str, str]] # optional - HTTP headers for requests+ require_approval: NotRequired[bool] # optional - Whether to require user approval+++class CodeInterpreterToolConfig(TypedDict):+ """Configuration for code_interpreter tool."""++ type: Literal["code_interpreter"]+ container: str # required - Container configuration for code execution+++class ImageGenerationToolConfig(TypedDict):+ """Configuration for image_generation tool."""++ type: Literal["image_generation"]+ background: NotRequired[str] # optional - Background color or image+ input_image_mask: NotRequired[str] # optional - Mask for input image editing+++class LocalShellToolConfig(TypedDict):+ """Configuration for local_shell tool.++ WARNING: This tool is only supported with the 'codex-mini-latest' model+ and is available exclusively through the Responses API.+ """++ type: Literal["local_shell"]+ # Note: local_shell currently has no additional parameters in the API+++# Union type for all built-in tool configurations+BuiltinToolConfig = Union[+ FileSearchToolConfig,+ WebSearchToolConfig,+ ComputerUseToolConfig,+ MCPToolConfig,+ CodeInterpreterToolConfig,+ ImageGenerationToolConfig,+ LocalShellToolConfig,+]+++# Define ImageMessage class early since it's used in _convert_message_to_openai_message+class ImageMessage(BaseChatMessage):+ """A message containing an image."""++ content: str # URL or base64 string++ def to_model_message(self) -> UserMessage:+ return UserMessage(content=self.content, source=self.source)++ def to_model_text(self) -> str:+ return "[image]"++ def to_text(self) -> str:+ # Truncate long image content (especially base64) for better readability+ # While still showing enough of the URL or content to be identifiable+ if len(self.content) > IMAGE_CONTENT_PREVIEW_LENGTH:+ return f"[Image: {self.content[:IMAGE_CONTENT_PREVIEW_LENGTH]}...]"+ return f"[Image: {self.content}]"++def _convert_tool_to_function_schema(tool: Tool) -> Dict[str, Any]:schema = tool.schemaparameters: Dict[str, object] = {}@@ -102,12 +213,24 @@ class OpenAIAgentState(BaseModel):history: List[Dict[str, Any]] = Field(default_factory=list)+# Union type for tool configurations in the config schema+ToolConfigUnion = Union[ComponentModel, BuiltinToolConfig, str]++class OpenAIAgentConfig(BaseModel):+ """Configuration model for OpenAI agent that supports both custom tools and built-in tools.++ .. versionchanged:: v0.7.0+ Added support for built-in tools in JSON configuration via _to_config and _from_config methods.+ The tools field now accepts ComponentModel (for custom tools), built-in tool configurations+ (dict format), and built-in tool names (string format).+ """+name: strdescription: strmodel: strinstructions: str- tools: List[ComponentModel] | None = None+ tools: List[ToolConfigUnion] | None = Nonetemperature: Optional[float] = 1max_output_tokens: Optional[int] = Nonejson_mode: bool = False@@ -139,15 +262,73 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):* Custom function calling* Multi-turn conversations+ * Built-in tool support (file_search, code_interpreter, web_search_preview, etc.)++ .. versionchanged:: v0.7.0++ Added support for built-in tool types like file_search, web_search_preview,+ code_interpreter, computer_use_preview, image_generation, and mcp.+ Added support for tool configurations with required and optional parameters.++ Built-in tools are split into two categories:++ **Tools that can use string format** (no required parameters):++ - web_search_preview: Can be used as "web_search_preview" or with optional config+ (user_location, search_context_size)+ - image_generation: Can be used as "image_generation" or with optional config (background, input_image_mask)+ - local_shell: Can be used as "local_shell" (WARNING: Only works with codex-mini-latest model)++ **Tools that REQUIRE dict configuration** (have required parameters):++ - file_search: MUST use dict with vector_store_ids (List[str])+ - computer_use_preview: MUST use dict with display_height (int), display_width (int), environment (str)+ - code_interpreter: MUST use dict with container (str)+ - mcp: MUST use dict with server_label (str), server_url (str)++ Using required-parameter tools in string format will raise a ValueError with helpful error messages.+ The tools parameter type annotation only accepts string values for tools that don't require parameters.+++ Args:+ name (str): Name of the agent+ description (str): Description of the agent's purpose+ client (Union[AsyncOpenAI, AsyncAzureOpenAI]): OpenAI client instance+ model (str): Model to use (e.g. "gpt-4.1")+ instructions (str): System instructions for the agent+ tools (Optional[Iterable[Union[str, BuiltinToolConfig, Tool]]]): Tools the agent can use.+ Supported string values (no required parameters): "web_search_preview", "image_generation", "local_shell".+ Dict values can provide configuration for built-in tools with parameters.+ Required parameters for built-in tools:+ - file_search: vector_store_ids (List[str])+ - computer_use_preview: display_height (int), display_width (int), environment (str)+ - code_interpreter: container (str)+ - mcp: server_label (str), server_url (str)+ Optional parameters for built-in tools:+ - file_search: max_num_results (int), ranking_options (dict), filters (dict)+ - web_search_preview: user_location (str or dict), search_context_size (int)+ - image_generation: background (str), input_image_mask (str)+ - mcp: allowed_tools (List[str]), headers (dict), require_approval (bool)+ Special tools with model restrictions:+ - local_shell: Only works with "codex-mini-latest" model (WARNING: Very limited support)+ Also accepts custom Tool objects for function calling.+ temperature (Optional[float]): Temperature for response generation (default: 1)+ max_output_tokens (Optional[int]): Maximum output tokens+ json_mode (bool): Whether to use JSON mode (default: False)+ store (bool): Whether to store conversations (default: True)+ truncation (str): Truncation strategy (default: "disabled")Example:+ Basic usage with built-in tools:+.. code-block:: pythonfrom openai import AsyncOpenAIfrom autogen_core import CancellationTokenfrom autogen_ext.agents.openai import OpenAIAgentfrom autogen_agentchat.messages import TextMessage+ import loggingasync def example():@@ -157,16 +338,127 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):name="Simple Agent",description="A simple OpenAI agent using the Responses API",client=client,- model="gpt-4.1",+ model="gpt-4o",instructions="You are a helpful assistant.",+ tools=["web_search_preview", "image_generation"], # Only tools without required params+ )+ response = await agent.on_messages(+ [TextMessage(source="user", content="Search for recent AI developments")], cancellation_token+ )+ logging.info(response)++ Usage with configured built-in tools:++ .. code-block:: python++ from openai import AsyncOpenAI+ from autogen_core import CancellationToken+ from autogen_ext.agents.openai import OpenAIAgent+ from autogen_agentchat.messages import TextMessage+ import logging+++ async def example_with_configs():+ cancellation_token = CancellationToken()+ client = AsyncOpenAI()++ # Configure tools with required and optional parameters+ tools = [+ {+ "type": "file_search",+ "vector_store_ids": ["vs_abc123"], # required+ "max_num_results": 10, # optional+ },+ {+ "type": "computer_use_preview",+ "display_height": 1024, # required+ "display_width": 1280, # required+ "environment": "desktop", # required+ },+ {+ "type": "code_interpreter",+ "container": "python-3.11", # required+ },+ {+ "type": "mcp",+ "server_label": "my-mcp-server", # required+ "server_url": "http://localhost:3000", # required+ },+ {+ "type": "web_search_preview",+ "user_location": { # optional - structured location+ "type": "approximate", # required: "approximate" or "exact"+ "country": "US", # optional+ "region": "CA", # optional+ "city": "San Francisco", # optional+ },+ "search_context_size": 5, # optional+ },+ "image_generation", # Simple tools can still use string format+ ]++ agent = OpenAIAgent(+ name="Configured Agent",+ description="An agent with configured tools",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with specialized tools.",+ tools=tools, # type: ignore)- response = await agent.on_messages([TextMessage(source="user", content="Hello!")], cancellation_token)- print(response)+ response = await agent.on_messages(+ [TextMessage(source="user", content="Search for recent AI developments")], cancellation_token+ )+ logging.info(response)++ Mixed usage with custom function tools:++ .. code-block:: python++ import asyncio+ import logging+ from openai import AsyncOpenAI+ from autogen_core import CancellationToken+ from autogen_ext.agents.openai import OpenAIAgent+ from autogen_agentchat.messages import TextMessage+ from autogen_core.tools import FunctionTool+- asyncio.run(example())+ # Define a simple calculator function+ async def calculate(a: int, b: int) -> int:+ '''Simple function to add two numbers.'''+ return a + b- TODO: Add support for advanced features (vector store, multimodal, etc.) in future PRs.+ # Wrap the calculate function as a tool+ calculator = FunctionTool(calculate, description="A simple calculator tool")+++ async def example_mixed_tools():+ cancellation_token = CancellationToken()+ client = AsyncOpenAI()+ # Use the FunctionTool instance defined above++ agent = OpenAIAgent(+ name="Mixed Tools Agent",+ description="An agent with both built-in and custom tools",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with calculation and web search capabilities.",+ tools=[+ "web_search_preview",+ calculator,+ {"type": "mcp", "server_label": "tools", "server_url": "http://localhost:3000"},+ ],+ )+ response = await agent.on_messages(+ [TextMessage(source="user", content="What's 2+2 and what's the weather like?")],+ cancellation_token,+ )+ logging.info(response)+++ asyncio.run(example_mixed_tools())+"""@@ -180,7 +472,15 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):client: Union[AsyncOpenAI, AsyncAzureOpenAI],model: str,instructions: str,- tools: Optional[List[Tool]] = None,+ tools: Optional[+ Iterable[+ Union[+ Literal["web_search_preview", "image_generation", "local_shell"],+ BuiltinToolConfig,+ Tool,+ ]+ ]+ ] = None,temperature: Optional[float] = 1,max_output_tokens: Optional[int] = None,json_mode: bool = False,@@ -202,12 +502,245 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):self._tool_map: Dict[str, Tool] = {}if tools is not None:for tool in tools:- function_schema: Dict[str, Any] = {- "type": "function",- "function": _convert_tool_to_function_schema(tool),- }- self._tools.append(function_schema)- self._tool_map[tool.name] = tool+ if isinstance(tool, str):+ # Handle built-in tool types+ self._add_builtin_tool(tool)+ elif isinstance(tool, dict) and "type" in tool:+ # Handle configured built-in tools+ self._add_configured_tool(tool)+ elif isinstance(tool, Tool):+ # Handle custom function tools+ function_schema: Dict[str, Any] = {+ "type": "function",+ "function": _convert_tool_to_function_schema(tool),+ }+ self._tools.append(function_schema)+ self._tool_map[tool.name] = tool+ else:+ raise ValueError(f"Unsupported tool type: {type(tool)}")++ def _add_builtin_tool(self, tool_name: str) -> None:+ """Add a built-in tool by name."""+ # Skip if an identical tool has already been registered (idempotent behaviour)+ if any(td.get("type") == tool_name for td in self._tools):+ return # Duplicate – ignore rather than raise to stay backward-compatible+ # Only allow string format for tools that don't require parameters+ if tool_name == "web_search_preview":+ self._tools.append({"type": "web_search_preview"})+ elif tool_name == "image_generation":+ self._tools.append({"type": "image_generation"})+ elif tool_name == "local_shell":+ # Special handling for local_shell - very limited model support+ if self._model != "codex-mini-latest":+ raise ValueError(+ f"Tool 'local_shell' is only supported with model 'codex-mini-latest', "+ f"but current model is '{self._model}'. "+ f"This tool is available exclusively through the Responses API and has severe limitations. "+ f"Consider using autogen_ext.tools.code_execution.PythonCodeExecutionTool with "+ f"autogen_ext.code_executors.local.LocalCommandLineCodeExecutor for shell execution instead."+ )+ self._tools.append({"type": "local_shell"})+ elif tool_name in ["file_search", "code_interpreter", "computer_use_preview", "mcp"]:+ # These tools require specific parameters and must use dict configuration+ raise ValueError(+ f"Tool '{tool_name}' requires specific parameters and cannot be added using string format. "+ f"Use dict configuration instead. Required parameters for {tool_name}: "+ f"{self._get_required_params_help(tool_name)}"+ )+ else:+ raise ValueError(f"Unsupported built-in tool type: {tool_name}")++ def _get_required_params_help(self, tool_name: str) -> str:+ """Get help text for required parameters of a tool."""+ help_text = {+ "file_search": "vector_store_ids (List[str])",+ "code_interpreter": "container (str)",+ "computer_use_preview": "display_height (int), display_width (int), environment (str)",+ "mcp": "server_label (str), server_url (str)",+ }+ return help_text.get(tool_name, "unknown parameters")++ def _add_configured_tool(self, tool_config: BuiltinToolConfig) -> None:+ """Add a configured built-in tool with parameters."""+ tool_type = tool_config.get("type")+ if not tool_type:+ raise ValueError("Tool configuration must include 'type' field")++ # If an identical configuration is already present we simply ignore the new one (keeps API payload minimal)+ if cast(Dict[str, Any], tool_config) in self._tools:+ return++ # Initialize tool definition+ tool_def: Dict[str, Any] = {}++ # Special validation for model-restricted tools+ if tool_type == "local_shell":+ if self._model != "codex-mini-latest":+ raise ValueError(+ f"Tool 'local_shell' is only supported with model 'codex-mini-latest', "+ f"but current model is '{self._model}'. "+ f"This tool is available exclusively through the Responses API and has severe limitations. "+ f"Consider using autogen_ext.tools.code_execution.PythonCodeExecutionTool with "+ f"autogen_ext.code_executors.local.LocalCommandLineCodeExecutor for shell execution instead."+ )+ tool_def = {"type": "local_shell"}++ # For Responses API, built-in tools are defined directly without nesting+ elif tool_type == "file_search":+ # file_search requires vector_store_ids+ fs_config = cast(FileSearchToolConfig, tool_config)+ if "vector_store_ids" not in fs_config:+ raise ValueError("file_search tool requires 'vector_store_ids' parameter")++ vector_store_ids = fs_config["vector_store_ids"]+ if not isinstance(vector_store_ids, list) or not vector_store_ids:+ raise ValueError("file_search 'vector_store_ids' must be a non-empty list of strings")+ if not all(isinstance(vid, str) and vid.strip() for vid in vector_store_ids):+ raise ValueError("file_search 'vector_store_ids' must contain non-empty strings")++ tool_def = {"type": "file_search", "vector_store_ids": vector_store_ids}+ # Optional parameters+ if "max_num_results" in fs_config:+ max_results = fs_config["max_num_results"]+ if not isinstance(max_results, int) or max_results <= 0:+ raise ValueError("file_search 'max_num_results' must be a positive integer")+ tool_def["max_num_results"] = max_results+ if "ranking_options" in fs_config:+ tool_def["ranking_options"] = fs_config["ranking_options"]+ if "filters" in fs_config:+ tool_def["filters"] = fs_config["filters"]++ elif tool_type == "web_search_preview":+ # web_search_preview can have optional parameters+ ws_config = cast(WebSearchToolConfig, tool_config)+ tool_def = {"type": "web_search_preview"}+ if "search_context_size" in ws_config:+ context_size = ws_config["search_context_size"]+ if not isinstance(context_size, int) or context_size <= 0:+ raise ValueError("web_search_preview 'search_context_size' must be a positive integer")+ tool_def["search_context_size"] = context_size+ if "user_location" in ws_config:+ user_location = ws_config["user_location"]+ if isinstance(user_location, str):+ if not user_location.strip():+ raise ValueError(+ "web_search_preview 'user_location' must be a non-empty string when using string format"+ )+ elif isinstance(user_location, dict):+ if "type" not in user_location:+ raise ValueError("web_search_preview 'user_location' dictionary must include 'type' field")+ location_type = user_location["type"]+ if location_type not in ["approximate", "exact"]:+ raise ValueError("web_search_preview 'user_location' type must be 'approximate' or 'exact'")+ # Optional fields: country, region, city can be validated if present+ for optional_field in ["country", "region", "city"]:+ if optional_field in user_location:+ if (+ not isinstance(user_location[optional_field], str)+ or not user_location[optional_field].strip()+ ):+ raise ValueError(+ f"web_search_preview 'user_location' {optional_field} must be a non-empty string"+ )+ else:+ raise ValueError("web_search_preview 'user_location' must be a string or dictionary")+ tool_def["user_location"] = user_location++ elif tool_type == "computer_use_preview":+ # computer_use_preview requires display dimensions and environment+ cu_config = cast(ComputerUseToolConfig, tool_config)+ required_params = ["display_height", "display_width", "environment"]+ for param in required_params:+ if param not in cu_config:+ raise ValueError(f"computer_use_preview tool requires '{param}' parameter")++ # Validate display dimensions+ height = cu_config["display_height"]+ width = cu_config["display_width"]+ if not isinstance(height, int) or height <= 0:+ raise ValueError("computer_use_preview 'display_height' must be a positive integer")+ if not isinstance(width, int) or width <= 0:+ raise ValueError("computer_use_preview 'display_width' must be a positive integer")++ # Validate environment+ environment = cu_config["environment"]+ if not isinstance(environment, str) or not environment.strip():+ raise ValueError("computer_use_preview 'environment' must be a non-empty string")++ tool_def = {+ "type": "computer_use_preview",+ "display_height": height,+ "display_width": width,+ "environment": environment,+ }++ elif tool_type == "mcp":+ # MCP requires server_label and server_url+ mcp_config = cast(MCPToolConfig, tool_config)+ required_params = ["server_label", "server_url"]+ for param in required_params:+ if param not in mcp_config:+ raise ValueError(f"mcp tool requires '{param}' parameter")++ # Validate required parameters+ server_label = mcp_config["server_label"]+ server_url = mcp_config["server_url"]+ if not isinstance(server_label, str) or not server_label.strip():+ raise ValueError("mcp 'server_label' must be a non-empty string")+ if not isinstance(server_url, str) or not server_url.strip():+ raise ValueError("mcp 'server_url' must be a non-empty string")++ tool_def = {"type": "mcp", "server_label": server_label, "server_url": server_url}+ # Optional parameters+ if "allowed_tools" in mcp_config:+ allowed_tools = mcp_config["allowed_tools"]+ if not isinstance(allowed_tools, list):+ raise ValueError("mcp 'allowed_tools' must be a list of strings")+ if not all(isinstance(tool, str) for tool in allowed_tools):+ raise ValueError("mcp 'allowed_tools' must contain only strings")+ tool_def["allowed_tools"] = allowed_tools+ if "headers" in mcp_config:+ headers = mcp_config["headers"]+ if not isinstance(headers, dict):+ raise ValueError("mcp 'headers' must be a dictionary")+ tool_def["headers"] = headers+ if "require_approval" in mcp_config:+ require_approval = mcp_config["require_approval"]+ if not isinstance(require_approval, bool):+ raise ValueError("mcp 'require_approval' must be a boolean")+ tool_def["require_approval"] = require_approval++ elif tool_type == "code_interpreter":+ # code_interpreter requires container+ ci_config = cast(CodeInterpreterToolConfig, tool_config)+ if "container" not in ci_config:+ raise ValueError("code_interpreter tool requires 'container' parameter")++ container = ci_config["container"]+ if not isinstance(container, str) or not container.strip():+ raise ValueError("code_interpreter 'container' must be a non-empty string")++ tool_def = {"type": "code_interpreter", "container": container}++ elif tool_type == "image_generation":+ # image_generation can have optional parameters+ ig_config = cast(ImageGenerationToolConfig, tool_config)+ tool_def = {"type": "image_generation"}+ if "background" in ig_config:+ background = ig_config["background"]+ if not isinstance(background, str) or not background.strip():+ raise ValueError("image_generation 'background' must be a non-empty string")+ tool_def["background"] = background+ if "input_image_mask" in ig_config:+ input_image_mask = ig_config["input_image_mask"]+ if not isinstance(input_image_mask, str) or not input_image_mask.strip():+ raise ValueError("image_generation 'input_image_mask' must be a non-empty string")+ tool_def["input_image_mask"] = input_image_mask++ else:+ raise ValueError(f"Unsupported built-in tool type: {tool_type}")++ self._tools.append(tool_def)def _convert_message_to_dict(self, message: OpenAIMessage) -> Dict[str, Any]:"""Convert an OpenAIMessage to a Dict[str, Any]."""@@ -244,6 +777,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):from typing import Dict, Anyfrom autogen_ext.agents.openai import OpenAIAgentfrom openai import AsyncOpenAI+ import loggingasync def example() -> None:@@ -256,7 +790,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):instructions="You are a helpful assistant.",)assistants: Dict[str, Any] = await agent.list_assistants(limit=5)- print(assistants)+ logging.info(assistants)asyncio.run(example())@@ -293,6 +827,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):from typing import Dict, Anyfrom autogen_ext.agents.openai import OpenAIAgentfrom openai import AsyncOpenAI+ import loggingasync def example() -> None:@@ -305,7 +840,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):instructions="You are a helpful assistant.",)assistant: Dict[str, Any] = await agent.retrieve_assistant("asst_abc123")- print(assistant)+ logging.info(assistant)asyncio.run(example())@@ -364,6 +899,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):from typing import Dict, Anyfrom autogen_ext.agents.openai import OpenAIAgentfrom openai import AsyncOpenAI+ import loggingasync def example() -> None:@@ -377,11 +913,15 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):)updated: Dict[str, Any] = await agent.modify_assistant(assistant_id="asst_123",- instructions="You are an HR bot, and you have access to files to answer employee questions about company policies. Always response with info from either of the files.",+ instructions=(+ "You are an HR bot, and you have access to files to answer employee "+ "questions about company policies. Always response with info from either "+ "of the files."+ ),tools=[{"type": "file_search"}],tool_resources={"file_search": {"vector_store_ids": []}},)- print(updated)+ logging.info(updated)asyncio.run(example())@@ -406,32 +946,11 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):assistant_id (str): The ID of the assistant to delete.Returns:- Dict[str, Any]: The deletion status object (e.g., {"id": ..., "object": "assistant.deleted", "deleted": true}).+ Dict[str, Any]: The deletion status object.Example:- .. code-block:: python-- import asyncio- from typing import Dict, Any- from autogen_ext.agents.openai import OpenAIAgent- from openai import AsyncOpenAI--- async def example() -> None:- client = AsyncOpenAI()- agent = OpenAIAgent(- name="test_agent",- description="Test agent",- client=client,- model="gpt-4",- instructions="You are a helpful assistant.",- )- result: Dict[str, Any] = await agent.delete_assistant("asst_abc123")- print(result)--- asyncio.run(example())+ {"id": "...", "object": "assistant.deleted", "deleted": true}"""if hasattr(self._client, "assistants"):client_any = cast(Any, self._client)@@ -578,8 +1097,10 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):content = getattr(response_obj, "output_text", None)response_id = getattr(response_obj, "id", None)self._last_response_id = response_id- self._message_history.append({"role": "assistant", "content": str(content) if content is not None else ""})- final_message = TextMessage(source=self.name, content=str(content) if content is not None else "")+ # Use a readable placeholder when the API returns no content to aid debugging+ content_str: str = str(content) if content is not None else "[no content returned]"+ self._message_history.append({"role": "assistant", "content": content_str})+ final_message = TextMessage(source=self.name, content=content_str)response = Response(chat_message=final_message, inner_messages=inner_messages)yield responseexcept Exception as e:@@ -605,41 +1126,75 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):self._message_history = agent_state.historydef _to_config(self: "OpenAIAgent") -> OpenAIAgentConfig:- """Convert the OpenAI agent to a declarative config."""- tool_configs: List[Dict[str, Any]] = []- for tool in self._tool_map.values():- try:- if hasattr(tool, "dump_component"):- tool_any = cast(Any, tool)- component_dict = tool_any.dump_component()- tool_configs.append(component_dict)- else:- tool_configs.append(- {- "provider": "autogen_core.tools.FunctionTool",- "config": {- "name": tool.name,- "description": getattr(tool, "description", ""),+ """Convert the OpenAI agent to a declarative config.++ Serializes both custom Tool objects and built-in tools to their appropriate+ configuration formats for JSON serialization.++ .. versionchanged:: v0.6.2+ Added support for serializing built-in tools alongside custom tools.++ Returns:+ OpenAIAgentConfig: The configuration that can recreate this agent.+ """+ # Serialize tools in the **original order** they were registered. We iterate over the+ # internal ``self._tools`` list which contains both built-in tool definitions **and** the+ # synthetic "function" records for custom :class:`Tool` objects. For the latter we+ # convert the synthetic record back to a :class:`ComponentModel` by looking up the actual+ # tool instance in ``self._tool_map``. This approach keeps ordering stable while still+ # supporting full round-trip serialisation.+ tool_configs: List[ToolConfigUnion] = []++ for tool_def in self._tools:+ # 1. Custom function tools are stored internally as ``{"type": "function", "function": {...}}``.+ if tool_def.get("type") == "function":+ fn_schema = cast(Dict[str, Any], tool_def.get("function", {}))+ tool_name = fn_schema.get("name") # type: ignore[arg-type]+ if tool_name and tool_name in self._tool_map:+ tool_obj = self._tool_map[tool_name]+ try:+ if hasattr(tool_obj, "dump_component"):+ component_model = cast(Any, tool_obj).dump_component()+ tool_configs.append(component_model)+ else:+ component_model = ComponentModel(+ provider="autogen_core.tools.FunctionTool",+ component_type=None,+ config={+ "name": tool_obj.name,+ "description": getattr(tool_obj, "description", ""),+ },+ )+ tool_configs.append(component_model)+ except Exception as e: # pragma: no cover – extremely unlikely+ warnings.warn(+ f"Error serializing tool '{tool_name}': {e}",+ stacklevel=2,+ )+ component_model = ComponentModel(+ provider="autogen_core.tools.FunctionTool",+ component_type=None,+ config={+ "name": tool_name or "unknown_tool",+ "description": getattr(tool_obj, "description", ""),},- }- )- except Exception as e:- warnings.warn(f"Error serializing tool: {e}", stacklevel=2)- tool_configs.append(- {- "provider": "autogen_core.tools.FunctionTool",- "config": {- "name": getattr(tool, "name", "unknown_tool"),- "description": getattr(tool, "description", ""),- },- }+ )+ tool_configs.append(component_model)+ # 2. Built-in tools are already in their correct dict form – append verbatim.+ elif "type" in tool_def: # built-in tool+ tool_configs.append(cast(BuiltinToolConfig, tool_def))+ else: # pragma: no cover – should never happen+ warnings.warn(+ f"Encountered unexpected tool definition during serialisation: {tool_def}",+ stacklevel=2,)+return OpenAIAgentConfig(name=self.name,description=self.description,model=self._model,instructions=self._instructions,- tools=cast(List[ComponentModel], tool_configs),+ tools=tool_configs if tool_configs else None,temperature=self._temperature,max_output_tokens=self._max_output_tokens,json_mode=self._json_mode,@@ -649,36 +1204,63 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):@classmethoddef _from_config(cls: Type["OpenAIAgent"], config: OpenAIAgentConfig) -> "OpenAIAgent":- """Create an OpenAI agent from a declarative config."""+ """Create an OpenAI agent from a declarative config.++ Handles both custom Tool objects (from ComponentModel) and built-in tools+ (from string or dict configurations).++ .. versionchanged:: v0.6.2+ Added support for loading built-in tools alongside custom tools.++ Args:+ config: The configuration to load the agent from.++ Returns:+ OpenAIAgent: The reconstructed agent.+ """from openai import AsyncOpenAIclient = AsyncOpenAI()- tools: Optional[List[Tool]] = None+ tools: Optional[List[Union[str, BuiltinToolConfig, Tool]]] = Noneif config.tools:- tools_list: List[Tool] = []+ tools_list: List[Union[str, BuiltinToolConfig, Tool]] = []for tool_config in config.tools:- try:- provider = tool_config.provider- module_name, class_name = provider.rsplit(".", 1)- module = __import__(module_name, fromlist=[class_name])- tool_cls = getattr(module, class_name)- tool = tool_cls(**tool_config.config)- tools_list.append(cast(Tool, tool))- except Exception as e:- warnings.warn(f"Error loading tool: {e}", stacklevel=2)- from autogen_core.tools import FunctionTool-- async def dummy_func(*args: Any, **kwargs: Any) -> str:- return "Tool not fully restored"-- tool = FunctionTool(- name=tool_config.config.get("name", "unknown_tool"),- description=tool_config.config.get("description", ""),- func=dummy_func,- )- tools_list.append(tool)- tools = tools_list+ # Handle ComponentModel (custom Tool objects)+ if isinstance(tool_config, ComponentModel):+ try:+ provider = tool_config.provider+ module_name, class_name = provider.rsplit(".", 1)+ module = __import__(module_name, fromlist=[class_name])+ tool_cls = getattr(module, class_name)+ tool = tool_cls(**tool_config.config)+ tools_list.append(cast(Tool, tool))+ except Exception as e:+ warnings.warn(f"Error loading custom tool: {e}", stacklevel=2)+ from autogen_core.tools import FunctionTool++ async def dummy_func(*args: Any, **kwargs: Any) -> str:+ return "Tool not fully restored"++ tool = FunctionTool(+ name=tool_config.config.get("name", "unknown_tool"),+ description=tool_config.config.get("description", ""),+ func=dummy_func,+ )+ tools_list.append(tool)++ # Handle string format built-in tools+ elif isinstance(tool_config, str):+ tools_list.append(tool_config)++ # Handle dict format built-in tools+ elif isinstance(tool_config, dict) and "type" in tool_config:+ tools_list.append(tool_config) # type: ignore[arg-type]++ else:+ warnings.warn(f"Unknown tool configuration format: {type(tool_config)}", stacklevel=2)++ tools = tools_list if tools_list else Nonereturn cls(name=config.name,@@ -686,7 +1268,18 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):client=client,model=config.model,instructions=config.instructions,- tools=tools,+ tools=cast(+ Optional[+ Iterable[+ Union[+ BuiltinToolConfig,+ Tool,+ Literal["web_search_preview", "image_generation", "local_shell"],+ ]+ ]+ ],+ tools,+ ),temperature=config.temperature,max_output_tokens=config.max_output_tokens,json_mode=config.json_mode,@@ -694,22 +1287,22 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):truncation=config.truncation,)+ # Add public API wrappers for configuration and tools+ def to_config(self) -> OpenAIAgentConfig:+ """Public wrapper for the private _to_config method."""+ return self._to_config()-# Define our own ImageMessage since it's not available-class ImageMessage(BaseChatMessage):- """A message containing an image."""-- content: str # URL or base64 string-- def to_model_message(self) -> UserMessage:- return UserMessage(content=self.content, source=self.source)+ @classmethod+ def from_config(cls, config: OpenAIAgentConfig) -> "OpenAIAgent":+ """Public wrapper for the private _from_config classmethod."""+ return cls._from_config(config)- def to_model_text(self) -> str:- return "[image]"+ @property+ def tools(self) -> list[Any]:+ """Public access to the agent's tools."""+ return self._tools- def to_text(self) -> str:- # Truncate long image content (especially base64) for better readability- # While still showing enough of the URL or content to be identifiable- if len(self.content) > IMAGE_CONTENT_PREVIEW_LENGTH:- return f"[Image: {self.content[:IMAGE_CONTENT_PREVIEW_LENGTH]}...]"- return f"[Image: {self.content}]"+ @property+ def model(self) -> str:+ """Public access to the agent's model."""+ return self._modeldiff --git a/python/packages/autogen-ext/tests/agents/test_openai_agent_builtin_tool_validation.py b/python/packages/autogen-ext/tests/agents/test_openai_agent_builtin_tool_validation.pynew file mode 100644index 000000000..c66df4bb2--- /dev/null+++ b/python/packages/autogen-ext/tests/agents/test_openai_agent_builtin_tool_validation.py@@ -0,0 +1,754 @@+"""Tests for OpenAI agent builtin tool validation."""++# Standard library imports+import os+from typing import Any, Dict, cast++# Third-party imports+import pytest++# Local imports+from autogen_agentchat.messages import TextMessage+from autogen_core import CancellationToken+from autogen_ext.agents.openai import OpenAIAgent+from openai import AsyncOpenAI+from pytest import MonkeyPatch++[email protected](autouse=True)+def set_dummy_openai_key(monkeypatch: MonkeyPatch) -> None:+ """Ensure tests have a dummy OPENAI_API_KEY by default."""+ # Only set a dummy key if no api key is provided+ if not os.getenv("OPENAI_API_KEY"):+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test-dummy-key")+++skip_if_no_real_openai_key = pytest.mark.skipif(+ os.getenv("OPENAI_API_KEY", "") in ("", "sk-test-dummy-key"),+ reason="No real OPENAI_API_KEY provided; skipping integration test.",+)+++def openai_client() -> AsyncOpenAI:+ """Provides an AsyncOpenAI client using the test API key."""+ return AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY", ""))+++def cancel_token() -> CancellationToken:+ """Provides a fresh CancellationToken for each test."""+ return CancellationToken()+++ "tool_name,model,should_raise",+ [+ ("web_search_preview", "gpt-4o", False),+ ("image_generation", "gpt-4o", False),+ ("local_shell", "codex-mini-latest", False),+ ("local_shell", "gpt-4o", True),+ ("file_search", "gpt-4o", True),+ ("code_interpreter", "gpt-4o", True),+ ("computer_use_preview", "gpt-4o", True),+ ("mcp", "gpt-4o", True),+ ("not_a_tool", "gpt-4o", True),+ ],+)+async def test_builtin_tool_string_validation(+ tool_name: str, model: str, should_raise: bool, openai_client: AsyncOpenAI+) -> None:+ """Test validation of string-based builtin tools."""+ client = openai_client+ tools = [tool_name] # type: ignore++ if should_raise:+ with pytest.raises(ValueError):+ OpenAIAgent(+ name="test",+ description="desc",+ client=client,+ model=model,+ instructions="inst",+ tools=tools, # type: ignore+ )+ else:+ agent = OpenAIAgent(+ name="test",+ description="desc",+ client=client,+ model=model,+ instructions="inst",+ tools=tools, # type: ignore+ )+ assert any(t["type"] == tool_name for t in agent.tools)+++ "tool_config,should_raise",+ [+ # file_search: missing required param+ ({"type": "file_search"}, True),+ # file_search: empty vector_store_ids+ ({"type": "file_search", "vector_store_ids": []}, True),+ # file_search: invalid type+ ({"type": "file_search", "vector_store_ids": [123]}, True),+ # file_search: valid+ ({"type": "file_search", "vector_store_ids": ["vs1"]}, False),+ # computer_use_preview: missing param+ ({"type": "computer_use_preview", "display_height": 100, "display_width": 100}, True),+ # computer_use_preview: invalid type+ ({"type": "computer_use_preview", "display_height": -1, "display_width": 100, "environment": "desktop"}, True),+ # computer_use_preview: valid+ (+ {"type": "computer_use_preview", "display_height": 100, "display_width": 100, "environment": "desktop"},+ False,+ ),+ # code_interpreter: missing param+ ({"type": "code_interpreter"}, True),+ # code_interpreter: empty container+ ({"type": "code_interpreter", "container": ""}, True),+ # code_interpreter: valid+ ({"type": "code_interpreter", "container": "python-3.11"}, False),+ # mcp: missing param+ ({"type": "mcp", "server_label": "label"}, True),+ # mcp: invalid type+ ({"type": "mcp", "server_label": "", "server_url": "url"}, True),+ # mcp: valid+ ({"type": "mcp", "server_label": "label", "server_url": "url"}, False),+ # web_search_preview: valid with string user_location+ ({"type": "web_search_preview", "user_location": "US"}, False),+ # web_search_preview: valid with dict user_location+ ({"type": "web_search_preview", "user_location": {"type": "approximate"}}, False),+ # web_search_preview: invalid user_location type+ ({"type": "web_search_preview", "user_location": 123}, True),+ # image_generation: valid with background+ ({"type": "image_generation", "background": "white"}, False),+ # image_generation: invalid background+ ({"type": "image_generation", "background": ""}, True),+ ],+)+async def test_builtin_tool_dict_validation(+ tool_config: Dict[str, Any], should_raise: bool, openai_client: AsyncOpenAI+) -> None:+ """Test validation of dictionary-based builtin tools."""+ client = openai_client+ tools = [tool_config] # type: ignore++ if should_raise:+ with pytest.raises(ValueError):+ OpenAIAgent(+ name="test",+ description="desc",+ client=client,+ model="gpt-4o",+ instructions="inst",+ tools=tools, # type: ignore+ )+ else:+ agent = OpenAIAgent(+ name="test",+ description="desc",+ client=client,+ model="gpt-4o",+ instructions="inst",+ tools=tools, # type: ignore+ )+ assert any(t["type"] == tool_config["type"] for t in agent.tools)+++async def test_builtin_tool_validation_with_custom_and_builtin(openai_client: AsyncOpenAI) -> None:+ """Test validation with mixed string and dictionary tools."""+ client = openai_client+ tools = ["web_search_preview", {"type": "image_generation"}] # type: ignore+ agent = OpenAIAgent(+ name="test",+ description="desc",+ client=client,+ model="gpt-4o",+ instructions="inst",+ tools=tools, # type: ignore+ )+ assert any(t["type"] == "web_search_preview" for t in agent.tools)+ assert any(t["type"] == "image_generation" for t in agent.tools)+++@skip_if_no_real_openai_key+async def test_integration_with_openai_api() -> None:+ """Test basic integration with OpenAI API."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = ["web_search_preview"] # type: ignore+ agent = OpenAIAgent(+ name="integration",+ description="desc",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()+ response = await agent.on_messages(+ [TextMessage(source="user", content="What is the capital of France?")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert content+++@skip_if_no_real_openai_key+async def test_integration_web_search_preview_tool() -> None:+ """Test web_search_preview tool with actual API call."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = ["web_search_preview"] # type: ignore+ agent = OpenAIAgent(+ name="web_search_test",+ description="Test agent with web search capability",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with web search capabilities. Use web search when needed.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test web search functionality+ response = await agent.on_messages(+ [TextMessage(source="user", content="What are the latest developments in AI technology?")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_image_generation_tool() -> None:+ """Test image_generation tool with actual API call."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = ["image_generation"] # type: ignore+ agent = OpenAIAgent(+ name="image_gen_test",+ description="Test agent with image generation capability",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with image generation capabilities. Generate images when requested.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test image generation functionality+ response = await agent.on_messages(+ [TextMessage(source="user", content="Generate an image of a beautiful sunset over mountains")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_configured_web_search_tool() -> None:+ """Test web_search_preview tool with configuration using actual API call."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = [{"type": "web_search_preview", "user_location": "US", "search_context_size": 5}] # type: ignore+ agent = OpenAIAgent(+ name="configured_web_search_test",+ description="Test agent with configured web search capability",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with configured web search capabilities.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test configured web search functionality+ response = await agent.on_messages(+ [TextMessage(source="user", content="What's the weather like in San Francisco today?")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_configured_image_generation_tool() -> None:+ """Test image_generation tool with configuration using actual API call."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = [{"type": "image_generation", "background": "white"}] # type: ignore+ agent = OpenAIAgent(+ name="configured_image_gen_test",+ description="Test agent with configured image generation capability",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with configured image generation capabilities.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test configured image generation functionality+ response = await agent.on_messages(+ [TextMessage(source="user", content="Create an image of a cat sitting on a white background")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_multiple_builtin_tools() -> None:+ """Test multiple builtin tools together with actual API call."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = ["web_search_preview", "image_generation"] # type: ignore+ agent = OpenAIAgent(+ name="multi_tool_test",+ description="Test agent with multiple builtin tools",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with web search and image generation capabilities.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test multiple tools functionality+ response = await agent.on_messages(+ [+ TextMessage(+ source="user",+ content="Search for information about space exploration and generate an image of a rocket",+ )+ ],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_file_search_tool_with_vector_store() -> None:+ """Test file_search tool with vector store configuration (requires actual vector store)."""+ api_key = os.getenv("OPENAI_API_KEY")++ # Skip this test if no vector store ID is provided+ vector_store_id = os.getenv("OPENAI_VECTOR_STORE_ID")+ if not vector_store_id:+ pytest.skip("OPENAI_VECTOR_STORE_ID not set; skipping file_search integration test.")++ client = AsyncOpenAI(api_key=api_key)+ tools = [{"type": "file_search", "vector_store_ids": [vector_store_id]}] # type: ignore+ agent = OpenAIAgent(+ name="file_search_test",+ description="Test agent with file search capability",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with file search capabilities.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test file search functionality+ response = await agent.on_messages(+ [TextMessage(source="user", content="Search for documents about machine learning")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_code_interpreter_tool() -> None:+ """Test code_interpreter tool with actual API call."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = [{"type": "code_interpreter", "container": "python-3.11"}] # type: ignore+ agent = OpenAIAgent(+ name="code_interpreter_test",+ description="Test agent with code interpreter capability",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with code execution capabilities.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test code interpreter functionality+ response = await agent.on_messages(+ [TextMessage(source="user", content="Calculate the sum of numbers from 1 to 100")],+ cancellation_token,+ )+ assert hasattr(response, "chat_message")+ assert hasattr(response.chat_message, "content")+ content = getattr(response.chat_message, "content", "")+ assert len(content) > 0+++@skip_if_no_real_openai_key+async def test_integration_streaming_with_builtin_tools() -> None:+ """Test streaming responses with builtin tools."""+ api_key = os.getenv("OPENAI_API_KEY")+ client = AsyncOpenAI(api_key=api_key)+ tools = ["web_search_preview"] # type: ignore+ agent = OpenAIAgent(+ name="streaming_test",+ description="Test agent with streaming and builtin tools",+ client=client,+ model="gpt-4o",+ instructions="You are a helpful assistant with web search capabilities.",+ tools=tools, # type: ignore+ )+ cancellation_token = CancellationToken()++ # Test streaming with builtin tools+ messages: list[Any] = []+ async for message in agent.on_messages_stream(+ [TextMessage(source="user", content="What are the latest news about renewable energy?")],+ cancellation_token,+ ):+ messages.append(message)++ # Verify we received some messages+ assert len(messages) > 0+ # Verify at least one message has content+ content_messages = [+ msg+ for msg in messages+ if hasattr(msg, "chat_message")+ and hasattr(msg.chat_message, "content")+ and getattr(msg.chat_message, "content", False)+ ]+ assert len(content_messages) > 0+++# JSON Config Tests for Built-in Tools+++async def test_to_config_with_string_builtin_tools() -> None:+ """Test _to_config with string-based builtin tools."""+ client = AsyncOpenAI()+ tools = ["web_search_preview", "image_generation"] # type: ignore+ agent = OpenAIAgent(+ name="config_test",+ description="Test agent for config serialization",+ client=client,+ model="gpt-4o",+ instructions="Test instructions",+ tools=tools, # type: ignore+ )++ config = agent.to_config()+ assert config.name == "config_test"+ assert config.description == "Test agent for config serialization"+ assert config.model == "gpt-4o"+ assert config.instructions == "Test instructions"+ assert config.tools is not None+ assert len(config.tools) == 2++ # Verify tools are serialized correctly+ tool_types: list[str] = []+ for tool in config.tools:+ if isinstance(tool, str):+ tool_types.append(tool)+ elif isinstance(tool, dict):+ tool_types.append(cast(Dict[str, Any], tool)["type"])+ else:+ # Handle ComponentModel case+ tool_types.append(str(tool))+ assert "web_search_preview" in tool_types+ assert "image_generation" in tool_types+++async def test_to_config_with_configured_builtin_tools() -> None:+ """Test _to_config with configured builtin tools."""+ client = AsyncOpenAI()+ tools = [+ {"type": "file_search", "vector_store_ids": ["vs1", "vs2"], "max_num_results": 10}, # type: ignore+ {"type": "web_search_preview", "user_location": "US", "search_context_size": 5}, # type: ignore+ {"type": "image_generation", "background": "white"}, # type: ignore+ ]+ agent = OpenAIAgent(+ name="configured_test",+ description="Test agent with configured tools",+ client=client,+ model="gpt-4o",+ instructions="Test instructions",+ tools=tools, # type: ignore+ )++ config = agent.to_config()+ assert config.name == "configured_test"+ assert config.tools is not None+ assert len(config.tools) == 3++ # Verify configured tools are serialized correctly+ tool_configs = [cast(Dict[str, Any], tool) for tool in config.tools if isinstance(tool, dict)]+ assert len(tool_configs) == 3++ # Check file_search config+ file_search_config = next(tool for tool in tool_configs if tool["type"] == "file_search")+ assert file_search_config["vector_store_ids"] == ["vs1", "vs2"]+ assert file_search_config["max_num_results"] == 10++ # Check web_search_preview config+ web_search_config = next(tool for tool in tool_configs if tool["type"] == "web_search_preview")+ assert web_search_config["user_location"] == "US"+ assert web_search_config["search_context_size"] == 5++ # Check image_generation config+ image_gen_config = next(tool for tool in tool_configs if tool["type"] == "image_generation")+ assert image_gen_config["background"] == "white"+++async def test_from_config_with_string_builtin_tools() -> None:+ """Test _from_config with string-based builtin tools."""+ from autogen_ext.agents.openai._openai_agent import OpenAIAgentConfig # type: ignore++ config = OpenAIAgentConfig(+ name="from_config_test",+ description="Test agent from config",+ model="gpt-4o",+ instructions="Test instructions",+ tools=["web_search_preview", "image_generation"], # type: ignore+ )+ agent = OpenAIAgent.from_config(config)+ assert agent.name == "from_config_test"+ assert agent.description == "Test agent from config"+ assert agent.model == "gpt-4o"+ # Verify instructions via configuration+ assert agent.to_config().instructions == "Test instructions"+ # Verify tools are loaded correctly+ assert len(agent.tools) == 2+ tool_types = [tool["type"] for tool in agent.tools]+ assert "web_search_preview" in tool_types+ assert "image_generation" in tool_types+++async def test_from_config_with_configured_builtin_tools() -> None:+ """Test _from_config with configured builtin tools."""+ from autogen_ext.agents.openai._openai_agent import OpenAIAgentConfig # type: ignore++ config = OpenAIAgentConfig(+ name="configured_from_config_test",+ description="Test agent with configured tools from config",+ model="gpt-4o",+ instructions="Test instructions",+ tools=[+ {"type": "file_search", "vector_store_ids": ["vs1"]}, # type: ignore+ {"type": "web_search_preview", "user_location": "US"}, # type: ignore+ {"type": "image_generation", "background": "black"}, # type: ignore+ ],+ )+ agent = OpenAIAgent.from_config(config)+ assert agent.name == "configured_from_config_test"+ assert agent.model == "gpt-4o"+ # Verify configured tools are loaded correctly+ assert len(agent.tools) == 3+ # Check file_search+ file_search_tool = next(tool for tool in agent.tools if tool["type"] == "file_search")+ assert file_search_tool["vector_store_ids"] == ["vs1"]+ # Check web_search_preview+ web_search_tool = next(tool for tool in agent.tools if tool["type"] == "web_search_preview")+ assert web_search_tool["user_location"] == "US"+ # Check image_generation+ image_gen_tool = next(tool for tool in agent.tools if tool["type"] == "image_generation")+ assert image_gen_tool["background"] == "black"+++async def test_round_trip_config_serialization() -> None:+ """Test round-trip serialization: agent -> config -> agent."""+ client = AsyncOpenAI()+ original_tools = [+ "web_search_preview",+ {"type": "file_search", "vector_store_ids": ["vs1"]}, # type: ignore+ {"type": "image_generation", "background": "white"}, # type: ignore+ ]++ original_agent = OpenAIAgent(+ name="round_trip_test",+ description="Test round-trip serialization",+ client=client,+ model="gpt-4o",+ instructions="Test instructions",+ tools=original_tools, # type: ignore+ )++ # Serialize to config+ config = original_agent.to_config()++ # Deserialize back to agent+ restored_agent = OpenAIAgent.from_config(config)++ # Verify basic properties+ assert restored_agent.name == original_agent.name+ assert restored_agent.description == original_agent.description+ assert restored_agent.model == original_agent.model+ orig_config = original_agent.to_config()+ restored_config = restored_agent.to_config()+ assert restored_config.instructions == orig_config.instructions++ # Verify tools are preserved+ assert len(restored_agent.tools) == len(original_agent.tools)++ # Check that string tools are preserved+ assert any(tool["type"] == "web_search_preview" for tool in restored_agent.tools)++ # Check that configured tools are preserved+ file_search_tool = next(tool for tool in restored_agent.tools if tool["type"] == "file_search")+ assert file_search_tool["vector_store_ids"] == ["vs1"]++ image_gen_tool = next(tool for tool in restored_agent.tools if tool["type"] == "image_generation")+ assert image_gen_tool["background"] == "white"+++async def test_config_serialization_with_mixed_tools() -> None:+ """Test config serialization with mixed string and configured tools."""+ client = AsyncOpenAI()+ tools = [+ "web_search_preview", # string tool+ {"type": "file_search", "vector_store_ids": ["vs1"]}, # type: ignore+ "image_generation", # string tool+ {"type": "code_interpreter", "container": "python-3.11"}, # type: ignore+ ]++ agent = OpenAIAgent(+ name="mixed_tools_test",+ description="Test agent with mixed tool types",+ client=client,+ model="gpt-4o",+ instructions="Test instructions",+ tools=tools, # type: ignore+ )++ config = agent.to_config()+ assert config.tools is not None+ assert len(config.tools) == 4++ # Verify all tools are serialized as dicts with "type" key+ dict_tools = [cast(Dict[str, Any], tool) for tool in config.tools if isinstance(tool, dict)]+ assert len(dict_tools) == 4++ # Check that string tools are converted to dicts with "type" key+ tool_types = [tool["type"] for tool in dict_tools]+ assert "web_search_preview" in tool_types+ assert "file_search" in tool_types+ assert "image_generation" in tool_types+ assert "code_interpreter" in tool_types++ # Verify configured tools preserve their configuration+ file_search_config = next(tool for tool in dict_tools if tool["type"] == "file_search")+ assert file_search_config["vector_store_ids"] == ["vs1"]++ code_interpreter_config = next(tool for tool in dict_tools if tool["type"] == "code_interpreter")+ assert code_interpreter_config["container"] == "python-3.11"+++async def test_config_serialization_with_local_shell() -> None:+ """Test config serialization with local_shell tool (model-restricted)."""+ client = AsyncOpenAI()+ tools = ["local_shell"] # type: ignore++ agent = OpenAIAgent(+ name="local_shell_test",+ description="Test agent with local_shell",+ client=client,+ model="codex-mini-latest", # Required for local_shell+ instructions="Test instructions",+ tools=tools, # type: ignore+ )++ config = agent.to_config()+ assert config.model == "codex-mini-latest"+ assert config.tools is not None+ assert len(config.tools) == 1+ # Built-in tools are serialized as dicts with "type" key+ assert config.tools[0] == {"type": "local_shell"}++ # Test round-trip+ restored_agent = OpenAIAgent.from_config(config)+ assert restored_agent.model == "codex-mini-latest"+ assert len(restored_agent.tools) == 1+ assert restored_agent.tools[0]["type"] == "local_shell"+++async def test_config_serialization_with_complex_web_search() -> None:+ """Test config serialization with complex web_search_preview configuration."""+ client = AsyncOpenAI()+ tools = [+ {+ "type": "web_search_preview",+ "user_location": {"type": "approximate", "country": "US", "region": "CA", "city": "San Francisco"},+ "search_context_size": 10,+ }+ ] # type: ignore+ agent = OpenAIAgent(+ name="complex_web_search_test",+ description="Test agent with complex web search config",+ client=client,+ model="gpt-4o",+ instructions="Test instructions",+ tools=tools, # type: ignore+ )+ config = agent.to_config()+ assert config.tools is not None+ assert len(config.tools) == 1+ web_search_config = cast(Dict[str, Any], config.tools[0])+ assert isinstance(web_search_config, dict)+ assert web_search_config["type"] == "web_search_preview"+ user_location = web_search_config["user_location"]+ if isinstance(user_location, dict):+ assert user_location["type"] == "approximate"+ assert user_location["country"] == "US"+ assert user_location["region"] == "CA"+ assert user_location["city"] == "San Francisco"+ else:+ # If user_location is a string, just check value+ assert user_location == "US"+ assert web_search_config["search_context_size"] == 10+ # Test round-trip+ restored_agent = OpenAIAgent.from_config(config)+ restored_tool = cast(Dict[str, Any], restored_agent.tools[0])+ assert restored_tool["type"] == "web_search_preview"+ restored_user_location = restored_tool["user_location"]+ if isinstance(restored_user_location, dict):+ assert restored_user_location["type"] == "approximate"+ assert restored_user_location["country"] == "US"+ assert restored_user_location["region"] == "CA"+ assert restored_user_location["city"] == "San Francisco"+ else:+ assert restored_user_location == "US"+ assert restored_tool["search_context_size"] == 10