Skip to content

A programming framework for agentic AI

Feat/OpenAI agent builtin tools 6657 (#6671)

Tejas Dharani25d732876e4e
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py b/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py
index b7a7bcbb0..f14d8f64e 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/openai/_openai_agent.py
@@ -2,7 +2,20 @@ import asyncio
import json
import logging
import warnings
-from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Type, TypedDict, Union, cast
+from typing import (
+ Any,
+ AsyncGenerator,
+ Dict,
+ Iterable,
+ List,
+ Literal,
+ Mapping,
+ Optional,
+ Sequence,
+ Type,
+ Union,
+ cast,
+)
from autogen_agentchat import EVENT_LOGGER_NAME
from autogen_agentchat.agents import BaseChatAgent
@@ -21,8 +34,9 @@ from autogen_core import CancellationToken, Component, ComponentModel, FunctionC
from autogen_core.models import UserMessage
from autogen_core.tools import Tool
from pydantic import BaseModel, Field
+from typing_extensions import NotRequired, TypedDict
-from openai import AsyncAzureOpenAI, AsyncOpenAI
+from openai import AsyncAzureOpenAI, AsyncOpenAI # type: ignore
# Number of characters to display when previewing image content in logs and UI
# Base64 encoded images can be very long, so we truncate for readability
@@ -33,6 +47,103 @@ IMAGE_CONTENT_PREVIEW_LENGTH = 50
event_logger = logging.getLogger(EVENT_LOGGER_NAME)
+# TypedDict classes for built-in tool configurations
+class FileSearchToolConfig(TypedDict):
+ """Configuration for file_search tool."""
+
+ type: Literal["file_search"]
+ vector_store_ids: List[str] # required - The IDs of the vector stores to search
+ max_num_results: NotRequired[int] # optional
+ ranking_options: NotRequired[Dict[str, Any]] # optional
+ filters: NotRequired[Dict[str, Any]] # optional
+
+
+class WebSearchToolConfig(TypedDict):
+ """Configuration for web_search_preview tool."""
+
+ type: Literal["web_search_preview"]
+ search_context_size: NotRequired[int] # optional
+ user_location: NotRequired[Union[str, Dict[str, Any]]] # optional - Can be string or structured location
+
+
+class ComputerUseToolConfig(TypedDict):
+ """Configuration for computer_use_preview tool."""
+
+ type: Literal["computer_use_preview"]
+ display_height: int # required - Display height in pixels
+ display_width: int # required - Display width in pixels
+ environment: str # required - Environment type for computer use
+
+
+class MCPToolConfig(TypedDict):
+ """Configuration for mcp tool."""
+
+ type: Literal["mcp"]
+ server_label: str # required - Label for the MCP server
+ server_url: str # required - URL of the MCP server
+ allowed_tools: NotRequired[List[str]] # optional - List of allowed tools
+ headers: NotRequired[Dict[str, str]] # optional - HTTP headers for requests
+ require_approval: NotRequired[bool] # optional - Whether to require user approval
+
+
+class CodeInterpreterToolConfig(TypedDict):
+ """Configuration for code_interpreter tool."""
+
+ type: Literal["code_interpreter"]
+ container: str # required - Container configuration for code execution
+
+
+class ImageGenerationToolConfig(TypedDict):
+ """Configuration for image_generation tool."""
+
+ type: Literal["image_generation"]
+ background: NotRequired[str] # optional - Background color or image
+ input_image_mask: NotRequired[str] # optional - Mask for input image editing
+
+
+class LocalShellToolConfig(TypedDict):
+ """Configuration for local_shell tool.
+
+ WARNING: This tool is only supported with the 'codex-mini-latest' model
+ and is available exclusively through the Responses API.
+ """
+
+ type: Literal["local_shell"]
+ # Note: local_shell currently has no additional parameters in the API
+
+
+# Union type for all built-in tool configurations
+BuiltinToolConfig = Union[
+ FileSearchToolConfig,
+ WebSearchToolConfig,
+ ComputerUseToolConfig,
+ MCPToolConfig,
+ CodeInterpreterToolConfig,
+ ImageGenerationToolConfig,
+ LocalShellToolConfig,
+]
+
+
+# Define ImageMessage class early since it's used in _convert_message_to_openai_message
+class ImageMessage(BaseChatMessage):
+ """A message containing an image."""
+
+ content: str # URL or base64 string
+
+ def to_model_message(self) -> UserMessage:
+ return UserMessage(content=self.content, source=self.source)
+
+ def to_model_text(self) -> str:
+ return "[image]"
+
+ def to_text(self) -> str:
+ # Truncate long image content (especially base64) for better readability
+ # While still showing enough of the URL or content to be identifiable
+ if len(self.content) > IMAGE_CONTENT_PREVIEW_LENGTH:
+ return f"[Image: {self.content[:IMAGE_CONTENT_PREVIEW_LENGTH]}...]"
+ return f"[Image: {self.content}]"
+
+
def _convert_tool_to_function_schema(tool: Tool) -> Dict[str, Any]:
schema = tool.schema
parameters: Dict[str, object] = {}
@@ -102,12 +213,24 @@ class OpenAIAgentState(BaseModel):
history: List[Dict[str, Any]] = Field(default_factory=list)
+# Union type for tool configurations in the config schema
+ToolConfigUnion = Union[ComponentModel, BuiltinToolConfig, str]
+
+
class OpenAIAgentConfig(BaseModel):
+ """Configuration model for OpenAI agent that supports both custom tools and built-in tools.
+
+ .. versionchanged:: v0.7.0
+ Added support for built-in tools in JSON configuration via _to_config and _from_config methods.
+ The tools field now accepts ComponentModel (for custom tools), built-in tool configurations
+ (dict format), and built-in tool names (string format).
+ """
+
name: str
description: str
model: str
instructions: str
- tools: List[ComponentModel] | None = None
+ tools: List[ToolConfigUnion] | None = None
temperature: Optional[float] = 1
max_output_tokens: Optional[int] = None
json_mode: bool = False
@@ -139,15 +262,73 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
* Custom function calling
* Multi-turn conversations
+ * Built-in tool support (file_search, code_interpreter, web_search_preview, etc.)
+
+ .. versionchanged:: v0.7.0
+
+ Added support for built-in tool types like file_search, web_search_preview,
+ code_interpreter, computer_use_preview, image_generation, and mcp.
+ Added support for tool configurations with required and optional parameters.
+
+ Built-in tools are split into two categories:
+
+ **Tools that can use string format** (no required parameters):
+
+ - web_search_preview: Can be used as "web_search_preview" or with optional config
+ (user_location, search_context_size)
+ - image_generation: Can be used as "image_generation" or with optional config (background, input_image_mask)
+ - local_shell: Can be used as "local_shell" (WARNING: Only works with codex-mini-latest model)
+
+ **Tools that REQUIRE dict configuration** (have required parameters):
+
+ - file_search: MUST use dict with vector_store_ids (List[str])
+ - computer_use_preview: MUST use dict with display_height (int), display_width (int), environment (str)
+ - code_interpreter: MUST use dict with container (str)
+ - mcp: MUST use dict with server_label (str), server_url (str)
+
+ Using required-parameter tools in string format will raise a ValueError with helpful error messages.
+ The tools parameter type annotation only accepts string values for tools that don't require parameters.
+
+
+ Args:
+ name (str): Name of the agent
+ description (str): Description of the agent's purpose
+ client (Union[AsyncOpenAI, AsyncAzureOpenAI]): OpenAI client instance
+ model (str): Model to use (e.g. "gpt-4.1")
+ instructions (str): System instructions for the agent
+ tools (Optional[Iterable[Union[str, BuiltinToolConfig, Tool]]]): Tools the agent can use.
+ Supported string values (no required parameters): "web_search_preview", "image_generation", "local_shell".
+ Dict values can provide configuration for built-in tools with parameters.
+ Required parameters for built-in tools:
+ - file_search: vector_store_ids (List[str])
+ - computer_use_preview: display_height (int), display_width (int), environment (str)
+ - code_interpreter: container (str)
+ - mcp: server_label (str), server_url (str)
+ Optional parameters for built-in tools:
+ - file_search: max_num_results (int), ranking_options (dict), filters (dict)
+ - web_search_preview: user_location (str or dict), search_context_size (int)
+ - image_generation: background (str), input_image_mask (str)
+ - mcp: allowed_tools (List[str]), headers (dict), require_approval (bool)
+ Special tools with model restrictions:
+ - local_shell: Only works with "codex-mini-latest" model (WARNING: Very limited support)
+ Also accepts custom Tool objects for function calling.
+ temperature (Optional[float]): Temperature for response generation (default: 1)
+ max_output_tokens (Optional[int]): Maximum output tokens
+ json_mode (bool): Whether to use JSON mode (default: False)
+ store (bool): Whether to store conversations (default: True)
+ truncation (str): Truncation strategy (default: "disabled")
Example:
+ Basic usage with built-in tools:
+
.. code-block:: python
from openai import AsyncOpenAI
from autogen_core import CancellationToken
from autogen_ext.agents.openai import OpenAIAgent
from autogen_agentchat.messages import TextMessage
+ import logging
async def example():
@@ -157,16 +338,127 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
name="Simple Agent",
description="A simple OpenAI agent using the Responses API",
client=client,
- model="gpt-4.1",
+ model="gpt-4o",
instructions="You are a helpful assistant.",
+ tools=["web_search_preview", "image_generation"], # Only tools without required params
+ )
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="Search for recent AI developments")], cancellation_token
+ )
+ logging.info(response)
+
+ Usage with configured built-in tools:
+
+ .. code-block:: python
+
+ from openai import AsyncOpenAI
+ from autogen_core import CancellationToken
+ from autogen_ext.agents.openai import OpenAIAgent
+ from autogen_agentchat.messages import TextMessage
+ import logging
+
+
+ async def example_with_configs():
+ cancellation_token = CancellationToken()
+ client = AsyncOpenAI()
+
+ # Configure tools with required and optional parameters
+ tools = [
+ {
+ "type": "file_search",
+ "vector_store_ids": ["vs_abc123"], # required
+ "max_num_results": 10, # optional
+ },
+ {
+ "type": "computer_use_preview",
+ "display_height": 1024, # required
+ "display_width": 1280, # required
+ "environment": "desktop", # required
+ },
+ {
+ "type": "code_interpreter",
+ "container": "python-3.11", # required
+ },
+ {
+ "type": "mcp",
+ "server_label": "my-mcp-server", # required
+ "server_url": "http://localhost:3000", # required
+ },
+ {
+ "type": "web_search_preview",
+ "user_location": { # optional - structured location
+ "type": "approximate", # required: "approximate" or "exact"
+ "country": "US", # optional
+ "region": "CA", # optional
+ "city": "San Francisco", # optional
+ },
+ "search_context_size": 5, # optional
+ },
+ "image_generation", # Simple tools can still use string format
+ ]
+
+ agent = OpenAIAgent(
+ name="Configured Agent",
+ description="An agent with configured tools",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with specialized tools.",
+ tools=tools, # type: ignore
)
- response = await agent.on_messages([TextMessage(source="user", content="Hello!")], cancellation_token)
- print(response)
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="Search for recent AI developments")], cancellation_token
+ )
+ logging.info(response)
+
+ Mixed usage with custom function tools:
+
+ .. code-block:: python
+
+ import asyncio
+ import logging
+ from openai import AsyncOpenAI
+ from autogen_core import CancellationToken
+ from autogen_ext.agents.openai import OpenAIAgent
+ from autogen_agentchat.messages import TextMessage
+ from autogen_core.tools import FunctionTool
+
- asyncio.run(example())
+ # Define a simple calculator function
+ async def calculate(a: int, b: int) -> int:
+ '''Simple function to add two numbers.'''
+ return a + b
- TODO: Add support for advanced features (vector store, multimodal, etc.) in future PRs.
+ # Wrap the calculate function as a tool
+ calculator = FunctionTool(calculate, description="A simple calculator tool")
+
+
+ async def example_mixed_tools():
+ cancellation_token = CancellationToken()
+ client = AsyncOpenAI()
+ # Use the FunctionTool instance defined above
+
+ agent = OpenAIAgent(
+ name="Mixed Tools Agent",
+ description="An agent with both built-in and custom tools",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with calculation and web search capabilities.",
+ tools=[
+ "web_search_preview",
+ calculator,
+ {"type": "mcp", "server_label": "tools", "server_url": "http://localhost:3000"},
+ ],
+ )
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="What's 2+2 and what's the weather like?")],
+ cancellation_token,
+ )
+ logging.info(response)
+
+
+ asyncio.run(example_mixed_tools())
+
"""
@@ -180,7 +472,15 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
client: Union[AsyncOpenAI, AsyncAzureOpenAI],
model: str,
instructions: str,
- tools: Optional[List[Tool]] = None,
+ tools: Optional[
+ Iterable[
+ Union[
+ Literal["web_search_preview", "image_generation", "local_shell"],
+ BuiltinToolConfig,
+ Tool,
+ ]
+ ]
+ ] = None,
temperature: Optional[float] = 1,
max_output_tokens: Optional[int] = None,
json_mode: bool = False,
@@ -202,12 +502,245 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
self._tool_map: Dict[str, Tool] = {}
if tools is not None:
for tool in tools:
- function_schema: Dict[str, Any] = {
- "type": "function",
- "function": _convert_tool_to_function_schema(tool),
- }
- self._tools.append(function_schema)
- self._tool_map[tool.name] = tool
+ if isinstance(tool, str):
+ # Handle built-in tool types
+ self._add_builtin_tool(tool)
+ elif isinstance(tool, dict) and "type" in tool:
+ # Handle configured built-in tools
+ self._add_configured_tool(tool)
+ elif isinstance(tool, Tool):
+ # Handle custom function tools
+ function_schema: Dict[str, Any] = {
+ "type": "function",
+ "function": _convert_tool_to_function_schema(tool),
+ }
+ self._tools.append(function_schema)
+ self._tool_map[tool.name] = tool
+ else:
+ raise ValueError(f"Unsupported tool type: {type(tool)}")
+
+ def _add_builtin_tool(self, tool_name: str) -> None:
+ """Add a built-in tool by name."""
+ # Skip if an identical tool has already been registered (idempotent behaviour)
+ if any(td.get("type") == tool_name for td in self._tools):
+ return # Duplicate – ignore rather than raise to stay backward-compatible
+ # Only allow string format for tools that don't require parameters
+ if tool_name == "web_search_preview":
+ self._tools.append({"type": "web_search_preview"})
+ elif tool_name == "image_generation":
+ self._tools.append({"type": "image_generation"})
+ elif tool_name == "local_shell":
+ # Special handling for local_shell - very limited model support
+ if self._model != "codex-mini-latest":
+ raise ValueError(
+ f"Tool 'local_shell' is only supported with model 'codex-mini-latest', "
+ f"but current model is '{self._model}'. "
+ f"This tool is available exclusively through the Responses API and has severe limitations. "
+ f"Consider using autogen_ext.tools.code_execution.PythonCodeExecutionTool with "
+ f"autogen_ext.code_executors.local.LocalCommandLineCodeExecutor for shell execution instead."
+ )
+ self._tools.append({"type": "local_shell"})
+ elif tool_name in ["file_search", "code_interpreter", "computer_use_preview", "mcp"]:
+ # These tools require specific parameters and must use dict configuration
+ raise ValueError(
+ f"Tool '{tool_name}' requires specific parameters and cannot be added using string format. "
+ f"Use dict configuration instead. Required parameters for {tool_name}: "
+ f"{self._get_required_params_help(tool_name)}"
+ )
+ else:
+ raise ValueError(f"Unsupported built-in tool type: {tool_name}")
+
+ def _get_required_params_help(self, tool_name: str) -> str:
+ """Get help text for required parameters of a tool."""
+ help_text = {
+ "file_search": "vector_store_ids (List[str])",
+ "code_interpreter": "container (str)",
+ "computer_use_preview": "display_height (int), display_width (int), environment (str)",
+ "mcp": "server_label (str), server_url (str)",
+ }
+ return help_text.get(tool_name, "unknown parameters")
+
+ def _add_configured_tool(self, tool_config: BuiltinToolConfig) -> None:
+ """Add a configured built-in tool with parameters."""
+ tool_type = tool_config.get("type")
+ if not tool_type:
+ raise ValueError("Tool configuration must include 'type' field")
+
+ # If an identical configuration is already present we simply ignore the new one (keeps API payload minimal)
+ if cast(Dict[str, Any], tool_config) in self._tools:
+ return
+
+ # Initialize tool definition
+ tool_def: Dict[str, Any] = {}
+
+ # Special validation for model-restricted tools
+ if tool_type == "local_shell":
+ if self._model != "codex-mini-latest":
+ raise ValueError(
+ f"Tool 'local_shell' is only supported with model 'codex-mini-latest', "
+ f"but current model is '{self._model}'. "
+ f"This tool is available exclusively through the Responses API and has severe limitations. "
+ f"Consider using autogen_ext.tools.code_execution.PythonCodeExecutionTool with "
+ f"autogen_ext.code_executors.local.LocalCommandLineCodeExecutor for shell execution instead."
+ )
+ tool_def = {"type": "local_shell"}
+
+ # For Responses API, built-in tools are defined directly without nesting
+ elif tool_type == "file_search":
+ # file_search requires vector_store_ids
+ fs_config = cast(FileSearchToolConfig, tool_config)
+ if "vector_store_ids" not in fs_config:
+ raise ValueError("file_search tool requires 'vector_store_ids' parameter")
+
+ vector_store_ids = fs_config["vector_store_ids"]
+ if not isinstance(vector_store_ids, list) or not vector_store_ids:
+ raise ValueError("file_search 'vector_store_ids' must be a non-empty list of strings")
+ if not all(isinstance(vid, str) and vid.strip() for vid in vector_store_ids):
+ raise ValueError("file_search 'vector_store_ids' must contain non-empty strings")
+
+ tool_def = {"type": "file_search", "vector_store_ids": vector_store_ids}
+ # Optional parameters
+ if "max_num_results" in fs_config:
+ max_results = fs_config["max_num_results"]
+ if not isinstance(max_results, int) or max_results <= 0:
+ raise ValueError("file_search 'max_num_results' must be a positive integer")
+ tool_def["max_num_results"] = max_results
+ if "ranking_options" in fs_config:
+ tool_def["ranking_options"] = fs_config["ranking_options"]
+ if "filters" in fs_config:
+ tool_def["filters"] = fs_config["filters"]
+
+ elif tool_type == "web_search_preview":
+ # web_search_preview can have optional parameters
+ ws_config = cast(WebSearchToolConfig, tool_config)
+ tool_def = {"type": "web_search_preview"}
+ if "search_context_size" in ws_config:
+ context_size = ws_config["search_context_size"]
+ if not isinstance(context_size, int) or context_size <= 0:
+ raise ValueError("web_search_preview 'search_context_size' must be a positive integer")
+ tool_def["search_context_size"] = context_size
+ if "user_location" in ws_config:
+ user_location = ws_config["user_location"]
+ if isinstance(user_location, str):
+ if not user_location.strip():
+ raise ValueError(
+ "web_search_preview 'user_location' must be a non-empty string when using string format"
+ )
+ elif isinstance(user_location, dict):
+ if "type" not in user_location:
+ raise ValueError("web_search_preview 'user_location' dictionary must include 'type' field")
+ location_type = user_location["type"]
+ if location_type not in ["approximate", "exact"]:
+ raise ValueError("web_search_preview 'user_location' type must be 'approximate' or 'exact'")
+ # Optional fields: country, region, city can be validated if present
+ for optional_field in ["country", "region", "city"]:
+ if optional_field in user_location:
+ if (
+ not isinstance(user_location[optional_field], str)
+ or not user_location[optional_field].strip()
+ ):
+ raise ValueError(
+ f"web_search_preview 'user_location' {optional_field} must be a non-empty string"
+ )
+ else:
+ raise ValueError("web_search_preview 'user_location' must be a string or dictionary")
+ tool_def["user_location"] = user_location
+
+ elif tool_type == "computer_use_preview":
+ # computer_use_preview requires display dimensions and environment
+ cu_config = cast(ComputerUseToolConfig, tool_config)
+ required_params = ["display_height", "display_width", "environment"]
+ for param in required_params:
+ if param not in cu_config:
+ raise ValueError(f"computer_use_preview tool requires '{param}' parameter")
+
+ # Validate display dimensions
+ height = cu_config["display_height"]
+ width = cu_config["display_width"]
+ if not isinstance(height, int) or height <= 0:
+ raise ValueError("computer_use_preview 'display_height' must be a positive integer")
+ if not isinstance(width, int) or width <= 0:
+ raise ValueError("computer_use_preview 'display_width' must be a positive integer")
+
+ # Validate environment
+ environment = cu_config["environment"]
+ if not isinstance(environment, str) or not environment.strip():
+ raise ValueError("computer_use_preview 'environment' must be a non-empty string")
+
+ tool_def = {
+ "type": "computer_use_preview",
+ "display_height": height,
+ "display_width": width,
+ "environment": environment,
+ }
+
+ elif tool_type == "mcp":
+ # MCP requires server_label and server_url
+ mcp_config = cast(MCPToolConfig, tool_config)
+ required_params = ["server_label", "server_url"]
+ for param in required_params:
+ if param not in mcp_config:
+ raise ValueError(f"mcp tool requires '{param}' parameter")
+
+ # Validate required parameters
+ server_label = mcp_config["server_label"]
+ server_url = mcp_config["server_url"]
+ if not isinstance(server_label, str) or not server_label.strip():
+ raise ValueError("mcp 'server_label' must be a non-empty string")
+ if not isinstance(server_url, str) or not server_url.strip():
+ raise ValueError("mcp 'server_url' must be a non-empty string")
+
+ tool_def = {"type": "mcp", "server_label": server_label, "server_url": server_url}
+ # Optional parameters
+ if "allowed_tools" in mcp_config:
+ allowed_tools = mcp_config["allowed_tools"]
+ if not isinstance(allowed_tools, list):
+ raise ValueError("mcp 'allowed_tools' must be a list of strings")
+ if not all(isinstance(tool, str) for tool in allowed_tools):
+ raise ValueError("mcp 'allowed_tools' must contain only strings")
+ tool_def["allowed_tools"] = allowed_tools
+ if "headers" in mcp_config:
+ headers = mcp_config["headers"]
+ if not isinstance(headers, dict):
+ raise ValueError("mcp 'headers' must be a dictionary")
+ tool_def["headers"] = headers
+ if "require_approval" in mcp_config:
+ require_approval = mcp_config["require_approval"]
+ if not isinstance(require_approval, bool):
+ raise ValueError("mcp 'require_approval' must be a boolean")
+ tool_def["require_approval"] = require_approval
+
+ elif tool_type == "code_interpreter":
+ # code_interpreter requires container
+ ci_config = cast(CodeInterpreterToolConfig, tool_config)
+ if "container" not in ci_config:
+ raise ValueError("code_interpreter tool requires 'container' parameter")
+
+ container = ci_config["container"]
+ if not isinstance(container, str) or not container.strip():
+ raise ValueError("code_interpreter 'container' must be a non-empty string")
+
+ tool_def = {"type": "code_interpreter", "container": container}
+
+ elif tool_type == "image_generation":
+ # image_generation can have optional parameters
+ ig_config = cast(ImageGenerationToolConfig, tool_config)
+ tool_def = {"type": "image_generation"}
+ if "background" in ig_config:
+ background = ig_config["background"]
+ if not isinstance(background, str) or not background.strip():
+ raise ValueError("image_generation 'background' must be a non-empty string")
+ tool_def["background"] = background
+ if "input_image_mask" in ig_config:
+ input_image_mask = ig_config["input_image_mask"]
+ if not isinstance(input_image_mask, str) or not input_image_mask.strip():
+ raise ValueError("image_generation 'input_image_mask' must be a non-empty string")
+ tool_def["input_image_mask"] = input_image_mask
+
+ else:
+ raise ValueError(f"Unsupported built-in tool type: {tool_type}")
+
+ self._tools.append(tool_def)
def _convert_message_to_dict(self, message: OpenAIMessage) -> Dict[str, Any]:
"""Convert an OpenAIMessage to a Dict[str, Any]."""
@@ -244,6 +777,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
from typing import Dict, Any
from autogen_ext.agents.openai import OpenAIAgent
from openai import AsyncOpenAI
+ import logging
async def example() -> None:
@@ -256,7 +790,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
instructions="You are a helpful assistant.",
)
assistants: Dict[str, Any] = await agent.list_assistants(limit=5)
- print(assistants)
+ logging.info(assistants)
asyncio.run(example())
@@ -293,6 +827,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
from typing import Dict, Any
from autogen_ext.agents.openai import OpenAIAgent
from openai import AsyncOpenAI
+ import logging
async def example() -> None:
@@ -305,7 +840,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
instructions="You are a helpful assistant.",
)
assistant: Dict[str, Any] = await agent.retrieve_assistant("asst_abc123")
- print(assistant)
+ logging.info(assistant)
asyncio.run(example())
@@ -364,6 +899,7 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
from typing import Dict, Any
from autogen_ext.agents.openai import OpenAIAgent
from openai import AsyncOpenAI
+ import logging
async def example() -> None:
@@ -377,11 +913,15 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
)
updated: Dict[str, Any] = await agent.modify_assistant(
assistant_id="asst_123",
- instructions="You are an HR bot, and you have access to files to answer employee questions about company policies. Always response with info from either of the files.",
+ instructions=(
+ "You are an HR bot, and you have access to files to answer employee "
+ "questions about company policies. Always response with info from either "
+ "of the files."
+ ),
tools=[{"type": "file_search"}],
tool_resources={"file_search": {"vector_store_ids": []}},
)
- print(updated)
+ logging.info(updated)
asyncio.run(example())
@@ -406,32 +946,11 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
assistant_id (str): The ID of the assistant to delete.
Returns:
- Dict[str, Any]: The deletion status object (e.g., {"id": ..., "object": "assistant.deleted", "deleted": true}).
+ Dict[str, Any]: The deletion status object.
Example:
- .. code-block:: python
-
- import asyncio
- from typing import Dict, Any
- from autogen_ext.agents.openai import OpenAIAgent
- from openai import AsyncOpenAI
-
-
- async def example() -> None:
- client = AsyncOpenAI()
- agent = OpenAIAgent(
- name="test_agent",
- description="Test agent",
- client=client,
- model="gpt-4",
- instructions="You are a helpful assistant.",
- )
- result: Dict[str, Any] = await agent.delete_assistant("asst_abc123")
- print(result)
-
-
- asyncio.run(example())
+ {"id": "...", "object": "assistant.deleted", "deleted": true}
"""
if hasattr(self._client, "assistants"):
client_any = cast(Any, self._client)
@@ -578,8 +1097,10 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
content = getattr(response_obj, "output_text", None)
response_id = getattr(response_obj, "id", None)
self._last_response_id = response_id
- self._message_history.append({"role": "assistant", "content": str(content) if content is not None else ""})
- final_message = TextMessage(source=self.name, content=str(content) if content is not None else "")
+ # Use a readable placeholder when the API returns no content to aid debugging
+ content_str: str = str(content) if content is not None else "[no content returned]"
+ self._message_history.append({"role": "assistant", "content": content_str})
+ final_message = TextMessage(source=self.name, content=content_str)
response = Response(chat_message=final_message, inner_messages=inner_messages)
yield response
except Exception as e:
@@ -605,41 +1126,75 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
self._message_history = agent_state.history
def _to_config(self: "OpenAIAgent") -> OpenAIAgentConfig:
- """Convert the OpenAI agent to a declarative config."""
- tool_configs: List[Dict[str, Any]] = []
- for tool in self._tool_map.values():
- try:
- if hasattr(tool, "dump_component"):
- tool_any = cast(Any, tool)
- component_dict = tool_any.dump_component()
- tool_configs.append(component_dict)
- else:
- tool_configs.append(
- {
- "provider": "autogen_core.tools.FunctionTool",
- "config": {
- "name": tool.name,
- "description": getattr(tool, "description", ""),
+ """Convert the OpenAI agent to a declarative config.
+
+ Serializes both custom Tool objects and built-in tools to their appropriate
+ configuration formats for JSON serialization.
+
+ .. versionchanged:: v0.6.2
+ Added support for serializing built-in tools alongside custom tools.
+
+ Returns:
+ OpenAIAgentConfig: The configuration that can recreate this agent.
+ """
+ # Serialize tools in the **original order** they were registered. We iterate over the
+ # internal ``self._tools`` list which contains both built-in tool definitions **and** the
+ # synthetic "function" records for custom :class:`Tool` objects. For the latter we
+ # convert the synthetic record back to a :class:`ComponentModel` by looking up the actual
+ # tool instance in ``self._tool_map``. This approach keeps ordering stable while still
+ # supporting full round-trip serialisation.
+ tool_configs: List[ToolConfigUnion] = []
+
+ for tool_def in self._tools:
+ # 1. Custom function tools are stored internally as ``{"type": "function", "function": {...}}``.
+ if tool_def.get("type") == "function":
+ fn_schema = cast(Dict[str, Any], tool_def.get("function", {}))
+ tool_name = fn_schema.get("name") # type: ignore[arg-type]
+ if tool_name and tool_name in self._tool_map:
+ tool_obj = self._tool_map[tool_name]
+ try:
+ if hasattr(tool_obj, "dump_component"):
+ component_model = cast(Any, tool_obj).dump_component()
+ tool_configs.append(component_model)
+ else:
+ component_model = ComponentModel(
+ provider="autogen_core.tools.FunctionTool",
+ component_type=None,
+ config={
+ "name": tool_obj.name,
+ "description": getattr(tool_obj, "description", ""),
+ },
+ )
+ tool_configs.append(component_model)
+ except Exception as e: # pragma: no cover – extremely unlikely
+ warnings.warn(
+ f"Error serializing tool '{tool_name}': {e}",
+ stacklevel=2,
+ )
+ component_model = ComponentModel(
+ provider="autogen_core.tools.FunctionTool",
+ component_type=None,
+ config={
+ "name": tool_name or "unknown_tool",
+ "description": getattr(tool_obj, "description", ""),
},
- }
- )
- except Exception as e:
- warnings.warn(f"Error serializing tool: {e}", stacklevel=2)
- tool_configs.append(
- {
- "provider": "autogen_core.tools.FunctionTool",
- "config": {
- "name": getattr(tool, "name", "unknown_tool"),
- "description": getattr(tool, "description", ""),
- },
- }
+ )
+ tool_configs.append(component_model)
+ # 2. Built-in tools are already in their correct dict form – append verbatim.
+ elif "type" in tool_def: # built-in tool
+ tool_configs.append(cast(BuiltinToolConfig, tool_def))
+ else: # pragma: no cover – should never happen
+ warnings.warn(
+ f"Encountered unexpected tool definition during serialisation: {tool_def}",
+ stacklevel=2,
)
+
return OpenAIAgentConfig(
name=self.name,
description=self.description,
model=self._model,
instructions=self._instructions,
- tools=cast(List[ComponentModel], tool_configs),
+ tools=tool_configs if tool_configs else None,
temperature=self._temperature,
max_output_tokens=self._max_output_tokens,
json_mode=self._json_mode,
@@ -649,36 +1204,63 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
@classmethod
def _from_config(cls: Type["OpenAIAgent"], config: OpenAIAgentConfig) -> "OpenAIAgent":
- """Create an OpenAI agent from a declarative config."""
+ """Create an OpenAI agent from a declarative config.
+
+ Handles both custom Tool objects (from ComponentModel) and built-in tools
+ (from string or dict configurations).
+
+ .. versionchanged:: v0.6.2
+ Added support for loading built-in tools alongside custom tools.
+
+ Args:
+ config: The configuration to load the agent from.
+
+ Returns:
+ OpenAIAgent: The reconstructed agent.
+ """
from openai import AsyncOpenAI
client = AsyncOpenAI()
- tools: Optional[List[Tool]] = None
+ tools: Optional[List[Union[str, BuiltinToolConfig, Tool]]] = None
if config.tools:
- tools_list: List[Tool] = []
+ tools_list: List[Union[str, BuiltinToolConfig, Tool]] = []
for tool_config in config.tools:
- try:
- provider = tool_config.provider
- module_name, class_name = provider.rsplit(".", 1)
- module = __import__(module_name, fromlist=[class_name])
- tool_cls = getattr(module, class_name)
- tool = tool_cls(**tool_config.config)
- tools_list.append(cast(Tool, tool))
- except Exception as e:
- warnings.warn(f"Error loading tool: {e}", stacklevel=2)
- from autogen_core.tools import FunctionTool
-
- async def dummy_func(*args: Any, **kwargs: Any) -> str:
- return "Tool not fully restored"
-
- tool = FunctionTool(
- name=tool_config.config.get("name", "unknown_tool"),
- description=tool_config.config.get("description", ""),
- func=dummy_func,
- )
- tools_list.append(tool)
- tools = tools_list
+ # Handle ComponentModel (custom Tool objects)
+ if isinstance(tool_config, ComponentModel):
+ try:
+ provider = tool_config.provider
+ module_name, class_name = provider.rsplit(".", 1)
+ module = __import__(module_name, fromlist=[class_name])
+ tool_cls = getattr(module, class_name)
+ tool = tool_cls(**tool_config.config)
+ tools_list.append(cast(Tool, tool))
+ except Exception as e:
+ warnings.warn(f"Error loading custom tool: {e}", stacklevel=2)
+ from autogen_core.tools import FunctionTool
+
+ async def dummy_func(*args: Any, **kwargs: Any) -> str:
+ return "Tool not fully restored"
+
+ tool = FunctionTool(
+ name=tool_config.config.get("name", "unknown_tool"),
+ description=tool_config.config.get("description", ""),
+ func=dummy_func,
+ )
+ tools_list.append(tool)
+
+ # Handle string format built-in tools
+ elif isinstance(tool_config, str):
+ tools_list.append(tool_config)
+
+ # Handle dict format built-in tools
+ elif isinstance(tool_config, dict) and "type" in tool_config:
+ tools_list.append(tool_config) # type: ignore[arg-type]
+
+ else:
+ warnings.warn(f"Unknown tool configuration format: {type(tool_config)}", stacklevel=2)
+
+ tools = tools_list if tools_list else None
return cls(
name=config.name,
@@ -686,7 +1268,18 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
client=client,
model=config.model,
instructions=config.instructions,
- tools=tools,
+ tools=cast(
+ Optional[
+ Iterable[
+ Union[
+ BuiltinToolConfig,
+ Tool,
+ Literal["web_search_preview", "image_generation", "local_shell"],
+ ]
+ ]
+ ],
+ tools,
+ ),
temperature=config.temperature,
max_output_tokens=config.max_output_tokens,
json_mode=config.json_mode,
@@ -694,22 +1287,22 @@ class OpenAIAgent(BaseChatAgent, Component[OpenAIAgentConfig]):
truncation=config.truncation,
)
+ # Add public API wrappers for configuration and tools
+ def to_config(self) -> OpenAIAgentConfig:
+ """Public wrapper for the private _to_config method."""
+ return self._to_config()
-# Define our own ImageMessage since it's not available
-class ImageMessage(BaseChatMessage):
- """A message containing an image."""
-
- content: str # URL or base64 string
-
- def to_model_message(self) -> UserMessage:
- return UserMessage(content=self.content, source=self.source)
+ @classmethod
+ def from_config(cls, config: OpenAIAgentConfig) -> "OpenAIAgent":
+ """Public wrapper for the private _from_config classmethod."""
+ return cls._from_config(config)
- def to_model_text(self) -> str:
- return "[image]"
+ @property
+ def tools(self) -> list[Any]:
+ """Public access to the agent's tools."""
+ return self._tools
- def to_text(self) -> str:
- # Truncate long image content (especially base64) for better readability
- # While still showing enough of the URL or content to be identifiable
- if len(self.content) > IMAGE_CONTENT_PREVIEW_LENGTH:
- return f"[Image: {self.content[:IMAGE_CONTENT_PREVIEW_LENGTH]}...]"
- return f"[Image: {self.content}]"
+ @property
+ def model(self) -> str:
+ """Public access to the agent's model."""
+ return self._model
diff --git a/python/packages/autogen-ext/tests/agents/test_openai_agent_builtin_tool_validation.py b/python/packages/autogen-ext/tests/agents/test_openai_agent_builtin_tool_validation.py
new file mode 100644
index 000000000..c66df4bb2
--- /dev/null
+++ b/python/packages/autogen-ext/tests/agents/test_openai_agent_builtin_tool_validation.py
@@ -0,0 +1,754 @@
+"""Tests for OpenAI agent builtin tool validation."""
+
+# Standard library imports
+import os
+from typing import Any, Dict, cast
+
+# Third-party imports
+import pytest
+
+# Local imports
+from autogen_agentchat.messages import TextMessage
+from autogen_core import CancellationToken
+from autogen_ext.agents.openai import OpenAIAgent
+from openai import AsyncOpenAI
+from pytest import MonkeyPatch
+
+
[email protected](autouse=True)
+def set_dummy_openai_key(monkeypatch: MonkeyPatch) -> None:
+ """Ensure tests have a dummy OPENAI_API_KEY by default."""
+ # Only set a dummy key if no api key is provided
+ if not os.getenv("OPENAI_API_KEY"):
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test-dummy-key")
+
+
+skip_if_no_real_openai_key = pytest.mark.skipif(
+ os.getenv("OPENAI_API_KEY", "") in ("", "sk-test-dummy-key"),
+ reason="No real OPENAI_API_KEY provided; skipping integration test.",
+)
+
+
+def openai_client() -> AsyncOpenAI:
+ """Provides an AsyncOpenAI client using the test API key."""
+ return AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY", ""))
+
+
+def cancel_token() -> CancellationToken:
+ """Provides a fresh CancellationToken for each test."""
+ return CancellationToken()
+
+
+ "tool_name,model,should_raise",
+ [
+ ("web_search_preview", "gpt-4o", False),
+ ("image_generation", "gpt-4o", False),
+ ("local_shell", "codex-mini-latest", False),
+ ("local_shell", "gpt-4o", True),
+ ("file_search", "gpt-4o", True),
+ ("code_interpreter", "gpt-4o", True),
+ ("computer_use_preview", "gpt-4o", True),
+ ("mcp", "gpt-4o", True),
+ ("not_a_tool", "gpt-4o", True),
+ ],
+)
+async def test_builtin_tool_string_validation(
+ tool_name: str, model: str, should_raise: bool, openai_client: AsyncOpenAI
+) -> None:
+ """Test validation of string-based builtin tools."""
+ client = openai_client
+ tools = [tool_name] # type: ignore
+
+ if should_raise:
+ with pytest.raises(ValueError):
+ OpenAIAgent(
+ name="test",
+ description="desc",
+ client=client,
+ model=model,
+ instructions="inst",
+ tools=tools, # type: ignore
+ )
+ else:
+ agent = OpenAIAgent(
+ name="test",
+ description="desc",
+ client=client,
+ model=model,
+ instructions="inst",
+ tools=tools, # type: ignore
+ )
+ assert any(t["type"] == tool_name for t in agent.tools)
+
+
+ "tool_config,should_raise",
+ [
+ # file_search: missing required param
+ ({"type": "file_search"}, True),
+ # file_search: empty vector_store_ids
+ ({"type": "file_search", "vector_store_ids": []}, True),
+ # file_search: invalid type
+ ({"type": "file_search", "vector_store_ids": [123]}, True),
+ # file_search: valid
+ ({"type": "file_search", "vector_store_ids": ["vs1"]}, False),
+ # computer_use_preview: missing param
+ ({"type": "computer_use_preview", "display_height": 100, "display_width": 100}, True),
+ # computer_use_preview: invalid type
+ ({"type": "computer_use_preview", "display_height": -1, "display_width": 100, "environment": "desktop"}, True),
+ # computer_use_preview: valid
+ (
+ {"type": "computer_use_preview", "display_height": 100, "display_width": 100, "environment": "desktop"},
+ False,
+ ),
+ # code_interpreter: missing param
+ ({"type": "code_interpreter"}, True),
+ # code_interpreter: empty container
+ ({"type": "code_interpreter", "container": ""}, True),
+ # code_interpreter: valid
+ ({"type": "code_interpreter", "container": "python-3.11"}, False),
+ # mcp: missing param
+ ({"type": "mcp", "server_label": "label"}, True),
+ # mcp: invalid type
+ ({"type": "mcp", "server_label": "", "server_url": "url"}, True),
+ # mcp: valid
+ ({"type": "mcp", "server_label": "label", "server_url": "url"}, False),
+ # web_search_preview: valid with string user_location
+ ({"type": "web_search_preview", "user_location": "US"}, False),
+ # web_search_preview: valid with dict user_location
+ ({"type": "web_search_preview", "user_location": {"type": "approximate"}}, False),
+ # web_search_preview: invalid user_location type
+ ({"type": "web_search_preview", "user_location": 123}, True),
+ # image_generation: valid with background
+ ({"type": "image_generation", "background": "white"}, False),
+ # image_generation: invalid background
+ ({"type": "image_generation", "background": ""}, True),
+ ],
+)
+async def test_builtin_tool_dict_validation(
+ tool_config: Dict[str, Any], should_raise: bool, openai_client: AsyncOpenAI
+) -> None:
+ """Test validation of dictionary-based builtin tools."""
+ client = openai_client
+ tools = [tool_config] # type: ignore
+
+ if should_raise:
+ with pytest.raises(ValueError):
+ OpenAIAgent(
+ name="test",
+ description="desc",
+ client=client,
+ model="gpt-4o",
+ instructions="inst",
+ tools=tools, # type: ignore
+ )
+ else:
+ agent = OpenAIAgent(
+ name="test",
+ description="desc",
+ client=client,
+ model="gpt-4o",
+ instructions="inst",
+ tools=tools, # type: ignore
+ )
+ assert any(t["type"] == tool_config["type"] for t in agent.tools)
+
+
+async def test_builtin_tool_validation_with_custom_and_builtin(openai_client: AsyncOpenAI) -> None:
+ """Test validation with mixed string and dictionary tools."""
+ client = openai_client
+ tools = ["web_search_preview", {"type": "image_generation"}] # type: ignore
+ agent = OpenAIAgent(
+ name="test",
+ description="desc",
+ client=client,
+ model="gpt-4o",
+ instructions="inst",
+ tools=tools, # type: ignore
+ )
+ assert any(t["type"] == "web_search_preview" for t in agent.tools)
+ assert any(t["type"] == "image_generation" for t in agent.tools)
+
+
+@skip_if_no_real_openai_key
+async def test_integration_with_openai_api() -> None:
+ """Test basic integration with OpenAI API."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = ["web_search_preview"] # type: ignore
+ agent = OpenAIAgent(
+ name="integration",
+ description="desc",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="What is the capital of France?")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert content
+
+
+@skip_if_no_real_openai_key
+async def test_integration_web_search_preview_tool() -> None:
+ """Test web_search_preview tool with actual API call."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = ["web_search_preview"] # type: ignore
+ agent = OpenAIAgent(
+ name="web_search_test",
+ description="Test agent with web search capability",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with web search capabilities. Use web search when needed.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test web search functionality
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="What are the latest developments in AI technology?")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_image_generation_tool() -> None:
+ """Test image_generation tool with actual API call."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = ["image_generation"] # type: ignore
+ agent = OpenAIAgent(
+ name="image_gen_test",
+ description="Test agent with image generation capability",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with image generation capabilities. Generate images when requested.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test image generation functionality
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="Generate an image of a beautiful sunset over mountains")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_configured_web_search_tool() -> None:
+ """Test web_search_preview tool with configuration using actual API call."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = [{"type": "web_search_preview", "user_location": "US", "search_context_size": 5}] # type: ignore
+ agent = OpenAIAgent(
+ name="configured_web_search_test",
+ description="Test agent with configured web search capability",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with configured web search capabilities.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test configured web search functionality
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="What's the weather like in San Francisco today?")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_configured_image_generation_tool() -> None:
+ """Test image_generation tool with configuration using actual API call."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = [{"type": "image_generation", "background": "white"}] # type: ignore
+ agent = OpenAIAgent(
+ name="configured_image_gen_test",
+ description="Test agent with configured image generation capability",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with configured image generation capabilities.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test configured image generation functionality
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="Create an image of a cat sitting on a white background")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_multiple_builtin_tools() -> None:
+ """Test multiple builtin tools together with actual API call."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = ["web_search_preview", "image_generation"] # type: ignore
+ agent = OpenAIAgent(
+ name="multi_tool_test",
+ description="Test agent with multiple builtin tools",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with web search and image generation capabilities.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test multiple tools functionality
+ response = await agent.on_messages(
+ [
+ TextMessage(
+ source="user",
+ content="Search for information about space exploration and generate an image of a rocket",
+ )
+ ],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_file_search_tool_with_vector_store() -> None:
+ """Test file_search tool with vector store configuration (requires actual vector store)."""
+ api_key = os.getenv("OPENAI_API_KEY")
+
+ # Skip this test if no vector store ID is provided
+ vector_store_id = os.getenv("OPENAI_VECTOR_STORE_ID")
+ if not vector_store_id:
+ pytest.skip("OPENAI_VECTOR_STORE_ID not set; skipping file_search integration test.")
+
+ client = AsyncOpenAI(api_key=api_key)
+ tools = [{"type": "file_search", "vector_store_ids": [vector_store_id]}] # type: ignore
+ agent = OpenAIAgent(
+ name="file_search_test",
+ description="Test agent with file search capability",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with file search capabilities.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test file search functionality
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="Search for documents about machine learning")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_code_interpreter_tool() -> None:
+ """Test code_interpreter tool with actual API call."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = [{"type": "code_interpreter", "container": "python-3.11"}] # type: ignore
+ agent = OpenAIAgent(
+ name="code_interpreter_test",
+ description="Test agent with code interpreter capability",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with code execution capabilities.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test code interpreter functionality
+ response = await agent.on_messages(
+ [TextMessage(source="user", content="Calculate the sum of numbers from 1 to 100")],
+ cancellation_token,
+ )
+ assert hasattr(response, "chat_message")
+ assert hasattr(response.chat_message, "content")
+ content = getattr(response.chat_message, "content", "")
+ assert len(content) > 0
+
+
+@skip_if_no_real_openai_key
+async def test_integration_streaming_with_builtin_tools() -> None:
+ """Test streaming responses with builtin tools."""
+ api_key = os.getenv("OPENAI_API_KEY")
+ client = AsyncOpenAI(api_key=api_key)
+ tools = ["web_search_preview"] # type: ignore
+ agent = OpenAIAgent(
+ name="streaming_test",
+ description="Test agent with streaming and builtin tools",
+ client=client,
+ model="gpt-4o",
+ instructions="You are a helpful assistant with web search capabilities.",
+ tools=tools, # type: ignore
+ )
+ cancellation_token = CancellationToken()
+
+ # Test streaming with builtin tools
+ messages: list[Any] = []
+ async for message in agent.on_messages_stream(
+ [TextMessage(source="user", content="What are the latest news about renewable energy?")],
+ cancellation_token,
+ ):
+ messages.append(message)
+
+ # Verify we received some messages
+ assert len(messages) > 0
+ # Verify at least one message has content
+ content_messages = [
+ msg
+ for msg in messages
+ if hasattr(msg, "chat_message")
+ and hasattr(msg.chat_message, "content")
+ and getattr(msg.chat_message, "content", False)
+ ]
+ assert len(content_messages) > 0
+
+
+# JSON Config Tests for Built-in Tools
+
+
+async def test_to_config_with_string_builtin_tools() -> None:
+ """Test _to_config with string-based builtin tools."""
+ client = AsyncOpenAI()
+ tools = ["web_search_preview", "image_generation"] # type: ignore
+ agent = OpenAIAgent(
+ name="config_test",
+ description="Test agent for config serialization",
+ client=client,
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=tools, # type: ignore
+ )
+
+ config = agent.to_config()
+ assert config.name == "config_test"
+ assert config.description == "Test agent for config serialization"
+ assert config.model == "gpt-4o"
+ assert config.instructions == "Test instructions"
+ assert config.tools is not None
+ assert len(config.tools) == 2
+
+ # Verify tools are serialized correctly
+ tool_types: list[str] = []
+ for tool in config.tools:
+ if isinstance(tool, str):
+ tool_types.append(tool)
+ elif isinstance(tool, dict):
+ tool_types.append(cast(Dict[str, Any], tool)["type"])
+ else:
+ # Handle ComponentModel case
+ tool_types.append(str(tool))
+ assert "web_search_preview" in tool_types
+ assert "image_generation" in tool_types
+
+
+async def test_to_config_with_configured_builtin_tools() -> None:
+ """Test _to_config with configured builtin tools."""
+ client = AsyncOpenAI()
+ tools = [
+ {"type": "file_search", "vector_store_ids": ["vs1", "vs2"], "max_num_results": 10}, # type: ignore
+ {"type": "web_search_preview", "user_location": "US", "search_context_size": 5}, # type: ignore
+ {"type": "image_generation", "background": "white"}, # type: ignore
+ ]
+ agent = OpenAIAgent(
+ name="configured_test",
+ description="Test agent with configured tools",
+ client=client,
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=tools, # type: ignore
+ )
+
+ config = agent.to_config()
+ assert config.name == "configured_test"
+ assert config.tools is not None
+ assert len(config.tools) == 3
+
+ # Verify configured tools are serialized correctly
+ tool_configs = [cast(Dict[str, Any], tool) for tool in config.tools if isinstance(tool, dict)]
+ assert len(tool_configs) == 3
+
+ # Check file_search config
+ file_search_config = next(tool for tool in tool_configs if tool["type"] == "file_search")
+ assert file_search_config["vector_store_ids"] == ["vs1", "vs2"]
+ assert file_search_config["max_num_results"] == 10
+
+ # Check web_search_preview config
+ web_search_config = next(tool for tool in tool_configs if tool["type"] == "web_search_preview")
+ assert web_search_config["user_location"] == "US"
+ assert web_search_config["search_context_size"] == 5
+
+ # Check image_generation config
+ image_gen_config = next(tool for tool in tool_configs if tool["type"] == "image_generation")
+ assert image_gen_config["background"] == "white"
+
+
+async def test_from_config_with_string_builtin_tools() -> None:
+ """Test _from_config with string-based builtin tools."""
+ from autogen_ext.agents.openai._openai_agent import OpenAIAgentConfig # type: ignore
+
+ config = OpenAIAgentConfig(
+ name="from_config_test",
+ description="Test agent from config",
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=["web_search_preview", "image_generation"], # type: ignore
+ )
+ agent = OpenAIAgent.from_config(config)
+ assert agent.name == "from_config_test"
+ assert agent.description == "Test agent from config"
+ assert agent.model == "gpt-4o"
+ # Verify instructions via configuration
+ assert agent.to_config().instructions == "Test instructions"
+ # Verify tools are loaded correctly
+ assert len(agent.tools) == 2
+ tool_types = [tool["type"] for tool in agent.tools]
+ assert "web_search_preview" in tool_types
+ assert "image_generation" in tool_types
+
+
+async def test_from_config_with_configured_builtin_tools() -> None:
+ """Test _from_config with configured builtin tools."""
+ from autogen_ext.agents.openai._openai_agent import OpenAIAgentConfig # type: ignore
+
+ config = OpenAIAgentConfig(
+ name="configured_from_config_test",
+ description="Test agent with configured tools from config",
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=[
+ {"type": "file_search", "vector_store_ids": ["vs1"]}, # type: ignore
+ {"type": "web_search_preview", "user_location": "US"}, # type: ignore
+ {"type": "image_generation", "background": "black"}, # type: ignore
+ ],
+ )
+ agent = OpenAIAgent.from_config(config)
+ assert agent.name == "configured_from_config_test"
+ assert agent.model == "gpt-4o"
+ # Verify configured tools are loaded correctly
+ assert len(agent.tools) == 3
+ # Check file_search
+ file_search_tool = next(tool for tool in agent.tools if tool["type"] == "file_search")
+ assert file_search_tool["vector_store_ids"] == ["vs1"]
+ # Check web_search_preview
+ web_search_tool = next(tool for tool in agent.tools if tool["type"] == "web_search_preview")
+ assert web_search_tool["user_location"] == "US"
+ # Check image_generation
+ image_gen_tool = next(tool for tool in agent.tools if tool["type"] == "image_generation")
+ assert image_gen_tool["background"] == "black"
+
+
+async def test_round_trip_config_serialization() -> None:
+ """Test round-trip serialization: agent -> config -> agent."""
+ client = AsyncOpenAI()
+ original_tools = [
+ "web_search_preview",
+ {"type": "file_search", "vector_store_ids": ["vs1"]}, # type: ignore
+ {"type": "image_generation", "background": "white"}, # type: ignore
+ ]
+
+ original_agent = OpenAIAgent(
+ name="round_trip_test",
+ description="Test round-trip serialization",
+ client=client,
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=original_tools, # type: ignore
+ )
+
+ # Serialize to config
+ config = original_agent.to_config()
+
+ # Deserialize back to agent
+ restored_agent = OpenAIAgent.from_config(config)
+
+ # Verify basic properties
+ assert restored_agent.name == original_agent.name
+ assert restored_agent.description == original_agent.description
+ assert restored_agent.model == original_agent.model
+ orig_config = original_agent.to_config()
+ restored_config = restored_agent.to_config()
+ assert restored_config.instructions == orig_config.instructions
+
+ # Verify tools are preserved
+ assert len(restored_agent.tools) == len(original_agent.tools)
+
+ # Check that string tools are preserved
+ assert any(tool["type"] == "web_search_preview" for tool in restored_agent.tools)
+
+ # Check that configured tools are preserved
+ file_search_tool = next(tool for tool in restored_agent.tools if tool["type"] == "file_search")
+ assert file_search_tool["vector_store_ids"] == ["vs1"]
+
+ image_gen_tool = next(tool for tool in restored_agent.tools if tool["type"] == "image_generation")
+ assert image_gen_tool["background"] == "white"
+
+
+async def test_config_serialization_with_mixed_tools() -> None:
+ """Test config serialization with mixed string and configured tools."""
+ client = AsyncOpenAI()
+ tools = [
+ "web_search_preview", # string tool
+ {"type": "file_search", "vector_store_ids": ["vs1"]}, # type: ignore
+ "image_generation", # string tool
+ {"type": "code_interpreter", "container": "python-3.11"}, # type: ignore
+ ]
+
+ agent = OpenAIAgent(
+ name="mixed_tools_test",
+ description="Test agent with mixed tool types",
+ client=client,
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=tools, # type: ignore
+ )
+
+ config = agent.to_config()
+ assert config.tools is not None
+ assert len(config.tools) == 4
+
+ # Verify all tools are serialized as dicts with "type" key
+ dict_tools = [cast(Dict[str, Any], tool) for tool in config.tools if isinstance(tool, dict)]
+ assert len(dict_tools) == 4
+
+ # Check that string tools are converted to dicts with "type" key
+ tool_types = [tool["type"] for tool in dict_tools]
+ assert "web_search_preview" in tool_types
+ assert "file_search" in tool_types
+ assert "image_generation" in tool_types
+ assert "code_interpreter" in tool_types
+
+ # Verify configured tools preserve their configuration
+ file_search_config = next(tool for tool in dict_tools if tool["type"] == "file_search")
+ assert file_search_config["vector_store_ids"] == ["vs1"]
+
+ code_interpreter_config = next(tool for tool in dict_tools if tool["type"] == "code_interpreter")
+ assert code_interpreter_config["container"] == "python-3.11"
+
+
+async def test_config_serialization_with_local_shell() -> None:
+ """Test config serialization with local_shell tool (model-restricted)."""
+ client = AsyncOpenAI()
+ tools = ["local_shell"] # type: ignore
+
+ agent = OpenAIAgent(
+ name="local_shell_test",
+ description="Test agent with local_shell",
+ client=client,
+ model="codex-mini-latest", # Required for local_shell
+ instructions="Test instructions",
+ tools=tools, # type: ignore
+ )
+
+ config = agent.to_config()
+ assert config.model == "codex-mini-latest"
+ assert config.tools is not None
+ assert len(config.tools) == 1
+ # Built-in tools are serialized as dicts with "type" key
+ assert config.tools[0] == {"type": "local_shell"}
+
+ # Test round-trip
+ restored_agent = OpenAIAgent.from_config(config)
+ assert restored_agent.model == "codex-mini-latest"
+ assert len(restored_agent.tools) == 1
+ assert restored_agent.tools[0]["type"] == "local_shell"
+
+
+async def test_config_serialization_with_complex_web_search() -> None:
+ """Test config serialization with complex web_search_preview configuration."""
+ client = AsyncOpenAI()
+ tools = [
+ {
+ "type": "web_search_preview",
+ "user_location": {"type": "approximate", "country": "US", "region": "CA", "city": "San Francisco"},
+ "search_context_size": 10,
+ }
+ ] # type: ignore
+ agent = OpenAIAgent(
+ name="complex_web_search_test",
+ description="Test agent with complex web search config",
+ client=client,
+ model="gpt-4o",
+ instructions="Test instructions",
+ tools=tools, # type: ignore
+ )
+ config = agent.to_config()
+ assert config.tools is not None
+ assert len(config.tools) == 1
+ web_search_config = cast(Dict[str, Any], config.tools[0])
+ assert isinstance(web_search_config, dict)
+ assert web_search_config["type"] == "web_search_preview"
+ user_location = web_search_config["user_location"]
+ if isinstance(user_location, dict):
+ assert user_location["type"] == "approximate"
+ assert user_location["country"] == "US"
+ assert user_location["region"] == "CA"
+ assert user_location["city"] == "San Francisco"
+ else:
+ # If user_location is a string, just check value
+ assert user_location == "US"
+ assert web_search_config["search_context_size"] == 10
+ # Test round-trip
+ restored_agent = OpenAIAgent.from_config(config)
+ restored_tool = cast(Dict[str, Any], restored_agent.tools[0])
+ assert restored_tool["type"] == "web_search_preview"
+ restored_user_location = restored_tool["user_location"]
+ if isinstance(restored_user_location, dict):
+ assert restored_user_location["type"] == "approximate"
+ assert restored_user_location["country"] == "US"
+ assert restored_user_location["region"] == "CA"
+ assert restored_user_location["city"] == "San Francisco"
+ else:
+ assert restored_user_location == "US"
+ assert restored_tool["search_context_size"] == 10