Skip to content

A programming framework for agentic AI

Move local code exec to ext, uplevel components (#4557)

Jack Gerrits9af450a59f8d
diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py
index 77a3581f7..3d55ff154 100644
--- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py
+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py
@@ -2,7 +2,7 @@ import re
from typing import List, Sequence
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import CodeBlock, CodeExecutor
+from autogen_core.code_executor import CodeBlock, CodeExecutor
from ..base import Response
from ..messages import ChatMessage, TextMessage
diff --git a/python/packages/autogen-agentchat/tests/test_code_executor_agent.py b/python/packages/autogen-agentchat/tests/test_code_executor_agent.py
index ca09a203b..89476586a 100644
--- a/python/packages/autogen-agentchat/tests/test_code_executor_agent.py
+++ b/python/packages/autogen-agentchat/tests/test_code_executor_agent.py
@@ -3,7 +3,7 @@ from autogen_agentchat.agents import CodeExecutorAgent
from autogen_agentchat.base import Response
from autogen_agentchat.messages import TextMessage
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import LocalCommandLineCodeExecutor
+from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor
@pytest.mark.asyncio
diff --git a/python/packages/autogen-agentchat/tests/test_group_chat.py b/python/packages/autogen-agentchat/tests/test_group_chat.py
index 12115664e..689be6238 100644
--- a/python/packages/autogen-agentchat/tests/test_group_chat.py
+++ b/python/packages/autogen-agentchat/tests/test_group_chat.py
@@ -33,9 +33,9 @@ from autogen_agentchat.teams._group_chat._selector_group_chat import SelectorGro
from autogen_agentchat.teams._group_chat._swarm_group_chat import SwarmGroupChatManager
from autogen_agentchat.ui import Console
from autogen_core import AgentId, CancellationToken, FunctionCall
-from autogen_core.components.code_executor import LocalCommandLineCodeExecutor
from autogen_core.components.models import FunctionExecutionResult
from autogen_core.components.tools import FunctionTool
+from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor
from autogen_ext.models import OpenAIChatCompletionClient, ReplayChatCompletionClient
from openai.resources.chat.completions import AsyncCompletions
from openai.types.chat.chat_completion import ChatCompletion, Choice
diff --git a/python/packages/autogen-core/docs/src/reference/index.md b/python/packages/autogen-core/docs/src/reference/index.md
index ccdffee61..c3e8dc104 100644
--- a/python/packages/autogen-core/docs/src/reference/index.md
+++ b/python/packages/autogen-core/docs/src/reference/index.md
@@ -26,8 +26,8 @@ python/autogen_agentchat.state
:caption: AutoGen Core
python/autogen_core
+python/autogen_core.code_executor
python/autogen_core.components.models
-python/autogen_core.components.code_executor
python/autogen_core.components.model_context
python/autogen_core.components.tools
python/autogen_core.components.tool_agent
diff --git a/python/packages/autogen-core/docs/src/reference/python/autogen_core.code_executor.rst b/python/packages/autogen-core/docs/src/reference/python/autogen_core.code_executor.rst
new file mode 100644
index 000000000..f05bdd68d
--- /dev/null
+++ b/python/packages/autogen-core/docs/src/reference/python/autogen_core.code_executor.rst
@@ -0,0 +1,7 @@
+autogen\_core.code_executor
+===========================
+
+.. automodule:: autogen_core.code_executor
+ :members:
+ :undoc-members:
+ :show-inheritance:
\ No newline at end of file
diff --git a/python/packages/autogen-core/docs/src/reference/python/autogen_core.components.code_executor.rst b/python/packages/autogen-core/docs/src/reference/python/autogen_core.components.code_executor.rst
deleted file mode 100644
index 54d0cb095..000000000
--- a/python/packages/autogen-core/docs/src/reference/python/autogen_core.components.code_executor.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-autogen\_core.components.code\_executor
-=======================================
-
-
-.. automodule:: autogen_core.components.code_executor
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb b/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb
index 19d0ad489..152079594 100644
--- a/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb
+++ b/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb
@@ -11,7 +11,7 @@
"This means that each code block is executed in a new process. There are two forms of this executor:\n",
"\n",
"- Docker ({py:class}`~autogen_ext.code_executor.docker_executor.DockerCommandLineCodeExecutor`) - this is where all commands are executed in a Docker container\n",
- "- Local ({py:class}`~autogen_core.components.code_executor.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n",
+ "- Local ({py:class}`~autogen_core.code_executor.local.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n",
"\n",
"## Docker\n",
"\n",
@@ -40,7 +40,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {},
"outputs": [
{
@@ -55,7 +55,7 @@
"from pathlib import Path\n",
"\n",
"from autogen_core import CancellationToken\n",
- "from autogen_core.components.code_executor import CodeBlock\n",
+ "from autogen_core.code_executor import CodeBlock\n",
"from autogen_ext.code_executors import DockerCommandLineCodeExecutor\n",
"\n",
"work_dir = Path(\"coding\")\n",
@@ -111,7 +111,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {},
"outputs": [
{
@@ -126,7 +126,8 @@
"from pathlib import Path\n",
"\n",
"from autogen_core import CancellationToken\n",
- "from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor\n",
+ "from autogen_core.code_executor import CodeBlock\n",
+ "from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor\n",
"\n",
"work_dir = Path(\"coding\")\n",
"work_dir.mkdir(exist_ok=True)\n",
@@ -153,7 +154,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
"outputs": [
{
@@ -172,7 +173,8 @@
"from pathlib import Path\n",
"\n",
"from autogen_core import CancellationToken\n",
- "from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor\n",
+ "from autogen_core.code_executor import CodeBlock\n",
+ "from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor\n",
"\n",
"work_dir = Path(\"coding\")\n",
"work_dir.mkdir(exist_ok=True)\n",
@@ -197,20 +199,6 @@
"source": [
"As we can see, the code has executed successfully, and the installation has been isolated to the newly created virtual environment, without affecting our global environment."
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -229,7 +217,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.4"
+ "version": "3.12.5"
}
},
"nbformat": 4,
diff --git a/python/packages/autogen-core/src/autogen_core/code_executor/__init__.py b/python/packages/autogen-core/src/autogen_core/code_executor/__init__.py
new file mode 100644
index 000000000..f1789a546
--- /dev/null
+++ b/python/packages/autogen-core/src/autogen_core/code_executor/__init__.py
@@ -0,0 +1,21 @@
+from ._base import CodeBlock, CodeExecutor, CodeResult
+from ._func_with_reqs import (
+ Alias,
+ FunctionWithRequirements,
+ FunctionWithRequirementsStr,
+ Import,
+ ImportFromModule,
+ with_requirements,
+)
+
+__all__ = [
+ "CodeBlock",
+ "CodeExecutor",
+ "CodeResult",
+ "Alias",
+ "ImportFromModule",
+ "Import",
+ "FunctionWithRequirements",
+ "FunctionWithRequirementsStr",
+ "with_requirements",
+]
diff --git a/python/packages/autogen-core/src/autogen_core/code_executor/_base.py b/python/packages/autogen-core/src/autogen_core/code_executor/_base.py
new file mode 100644
index 000000000..16f023a21
--- /dev/null
+++ b/python/packages/autogen-core/src/autogen_core/code_executor/_base.py
@@ -0,0 +1,59 @@
+# File based from: https://github.com/microsoft/autogen/blob/main/autogen/coding/base.py
+# Credit to original authors
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Protocol, runtime_checkable
+
+from .._cancellation_token import CancellationToken
+
+
+@dataclass
+class CodeBlock:
+ """A code block extracted fromm an agent message."""
+
+ code: str
+ language: str
+
+
+@dataclass
+class CodeResult:
+ """Result of a code execution."""
+
+ exit_code: int
+ output: str
+
+
+@runtime_checkable
+class CodeExecutor(Protocol):
+ """Executes code blocks and returns the result."""
+
+ async def execute_code_blocks(
+ self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken
+ ) -> CodeResult:
+ """Execute code blocks and return the result.
+
+ This method should be implemented by the code executor.
+
+ Args:
+ code_blocks (List[CodeBlock]): The code blocks to execute.
+
+ Returns:
+ CodeResult: The result of the code execution.
+
+ Raises:
+ ValueError: Errors in user inputs
+ asyncio.TimeoutError: Code execution timeouts
+ asyncio.CancelledError: CancellationToken evoked during execution
+ """
+ ...
+
+ async def restart(self) -> None:
+ """Restart the code executor.
+
+ This method should be implemented by the code executor.
+
+ This method is called when the agent is reset.
+ """
+ ...
diff --git a/python/packages/autogen-core/src/autogen_core/code_executor/_func_with_reqs.py b/python/packages/autogen-core/src/autogen_core/code_executor/_func_with_reqs.py
new file mode 100644
index 000000000..b7f4fcaef
--- /dev/null
+++ b/python/packages/autogen-core/src/autogen_core/code_executor/_func_with_reqs.py
@@ -0,0 +1,201 @@
+# File based from: https://github.com/microsoft/autogen/blob/main/autogen/coding/func_with_reqs.py
+# Credit to original authors
+
+from __future__ import annotations
+
+import functools
+import inspect
+from dataclasses import dataclass, field
+from importlib.abc import SourceLoader
+from importlib.util import module_from_spec, spec_from_loader
+from textwrap import dedent, indent
+from typing import Any, Callable, Generic, List, Sequence, Set, TypeVar, Union
+
+from typing_extensions import ParamSpec
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str:
+ if isinstance(func, FunctionWithRequirementsStr):
+ return func.func
+
+ code = inspect.getsource(func)
+ # Strip the decorator
+ if code.startswith("@"):
+ code = code[code.index("\n") + 1 :]
+ return code
+
+
+@dataclass
+class Alias:
+ name: str
+ alias: str
+
+
+@dataclass
+class ImportFromModule:
+ module: str
+ imports: List[Union[str, Alias]]
+
+
+Import = Union[str, ImportFromModule, Alias]
+
+
+def _import_to_str(im: Import) -> str:
+ if isinstance(im, str):
+ return f"import {im}"
+ elif isinstance(im, Alias):
+ return f"import {im.name} as {im.alias}"
+ else:
+
+ def to_str(i: Union[str, Alias]) -> str:
+ if isinstance(i, str):
+ return i
+ else:
+ return f"{i.name} as {i.alias}"
+
+ imports = ", ".join(map(to_str, im.imports))
+ return f"from {im.module} import {imports}"
+
+
+class _StringLoader(SourceLoader):
+ def __init__(self, data: str):
+ self.data = data
+
+ def get_source(self, fullname: str) -> str:
+ return self.data
+
+ def get_data(self, path: str) -> bytes:
+ return self.data.encode("utf-8")
+
+ def get_filename(self, fullname: str) -> str:
+ return "<not a real path>/" + fullname + ".py"
+
+
+@dataclass
+class FunctionWithRequirementsStr:
+ func: str
+ compiled_func: Callable[..., Any]
+ _func_name: str
+ python_packages: Sequence[str] = field(default_factory=list)
+ global_imports: Sequence[Import] = field(default_factory=list)
+
+ def __init__(self, func: str, python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []):
+ self.func = func
+ self.python_packages = python_packages
+ self.global_imports = global_imports
+
+ module_name = "func_module"
+ loader = _StringLoader(func)
+ spec = spec_from_loader(module_name, loader)
+ if spec is None:
+ raise ValueError("Could not create spec")
+ module = module_from_spec(spec)
+ if spec.loader is None:
+ raise ValueError("Could not create loader")
+
+ try:
+ spec.loader.exec_module(module)
+ except Exception as e:
+ raise ValueError(f"Could not compile function: {e}") from e
+
+ functions = inspect.getmembers(module, inspect.isfunction)
+ if len(functions) != 1:
+ raise ValueError("The string must contain exactly one function")
+
+ self._func_name, self.compiled_func = functions[0]
+
+ def __call__(self, *args: Any, **kwargs: Any) -> None:
+ raise NotImplementedError("String based function with requirement objects are not directly callable")
+
+
+@dataclass
+class FunctionWithRequirements(Generic[T, P]):
+ func: Callable[P, T]
+ python_packages: Sequence[str] = field(default_factory=list)
+ global_imports: Sequence[Import] = field(default_factory=list)
+
+ @classmethod
+ def from_callable(
+ cls, func: Callable[P, T], python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []
+ ) -> FunctionWithRequirements[T, P]:
+ return cls(python_packages=python_packages, global_imports=global_imports, func=func)
+
+ @staticmethod
+ def from_str(
+ func: str, python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []
+ ) -> FunctionWithRequirementsStr:
+ return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports)
+
+ # Type this based on F
+ def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T:
+ return self.func(*args, **kwargs)
+
+
+def with_requirements(
+ python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []
+) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]:
+ """Decorate a function with package and import requirements
+
+ Args:
+ python_packages (List[str], optional): Packages required to function. Can include version info.. Defaults to [].
+ global_imports (List[Import], optional): Required imports. Defaults to [].
+
+ Returns:
+ Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: The decorated function
+ """
+
+ def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]:
+ func_with_reqs = FunctionWithRequirements(
+ python_packages=python_packages, global_imports=global_imports, func=func
+ )
+
+ functools.update_wrapper(func_with_reqs, func)
+ return func_with_reqs
+
+ return wrapper
+
+
+def build_python_functions_file(
+ funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]],
+) -> str:
+ """:meta private:"""
+ # First collect all global imports
+ global_imports: Set[Import] = set()
+ for func in funcs:
+ if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)):
+ global_imports.update(func.global_imports)
+
+ content = "\n".join(map(_import_to_str, global_imports)) + "\n\n"
+
+ for func in funcs:
+ content += _to_code(func) + "\n\n"
+
+ return content
+
+
+def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str:
+ """Generate a stub for a function as a string
+
+ Args:
+ func (Callable[..., Any]): The function to generate a stub for
+
+ Returns:
+ str: The stub for the function
+ """
+ if isinstance(func, FunctionWithRequirementsStr):
+ return to_stub(func.compiled_func)
+
+ content = f"def {func.__name__}{inspect.signature(func)}:\n"
+ docstring = func.__doc__
+
+ if docstring:
+ docstring = dedent(docstring)
+ docstring = '"""' + docstring + '"""'
+ docstring = indent(docstring, " ")
+ content += docstring + "\n"
+
+ content += " ..."
+ return content
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py
index 805b718ff..e1265a81c 100644
--- a/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py
+++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py
@@ -1,17 +1,14 @@
-from ._base import CodeBlock, CodeExecutor, CodeResult
+from ._base import CodeBlock, CodeExecutor, CodeResult # type: ignore
from ._func_with_reqs import (
- Alias,
- FunctionWithRequirements,
- FunctionWithRequirementsStr,
+ Alias, # type: ignore
+ FunctionWithRequirements, # type: ignore
+ FunctionWithRequirementsStr, # type: ignore
Import,
- ImportFromModule,
- build_python_functions_file,
- to_stub,
- with_requirements,
+ ImportFromModule, # type: ignore
+ with_requirements, # type: ignore
)
-from ._impl.command_line_code_result import CommandLineCodeResult
-from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor
-from ._impl.utils import get_file_name_from_content, get_required_packages, lang_to_cmd, silence_pip
+from ._impl.command_line_code_result import CommandLineCodeResult # type: ignore
+from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor # type: ignore
__all__ = [
"LocalCommandLineCodeExecutor",
@@ -25,11 +22,4 @@ __all__ = [
"FunctionWithRequirements",
"FunctionWithRequirementsStr",
"with_requirements",
- "to_stub",
- "get_required_packages",
- "build_python_functions_file",
- "get_required_packages",
- "lang_to_cmd",
- "get_file_name_from_content",
- "silence_pip",
]
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py
index 4c12f5735..420a96447 100644
--- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py
+++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py
@@ -6,9 +6,12 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import List, Protocol, runtime_checkable
+from typing_extensions import deprecated
+
from ... import CancellationToken
+@deprecated("Moved to autogen_core.code_executor.CodeBlock. This alias will be removed in 0.4.0.")
@dataclass
class CodeBlock:
"""A code block extracted fromm an agent message."""
@@ -17,6 +20,7 @@ class CodeBlock:
language: str
+@deprecated("Moved to autogen_core.code_executor.CodeResult. This alias will be removed in 0.4.0.")
@dataclass
class CodeResult:
"""Result of a code execution."""
@@ -25,13 +29,16 @@ class CodeResult:
output: str
+@deprecated("Moved to autogen_core.code_executor.CodeExecutor. This alias will be removed in 0.4.0.")
@runtime_checkable
class CodeExecutor(Protocol):
"""Executes code blocks and returns the result."""
async def execute_code_blocks(
- self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken
- ) -> CodeResult:
+ self,
+ code_blocks: List[CodeBlock], # type: ignore
+ cancellation_token: CancellationToken, # type: ignore
+ ) -> CodeResult: # type: ignore
"""Execute code blocks and return the result.
This method should be implemented by the code executor.
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py
index b7f4fcaef..98c39518e 100644
--- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py
+++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py
@@ -11,14 +11,14 @@ from importlib.util import module_from_spec, spec_from_loader
from textwrap import dedent, indent
from typing import Any, Callable, Generic, List, Sequence, Set, TypeVar, Union
-from typing_extensions import ParamSpec
+from typing_extensions import ParamSpec, deprecated
T = TypeVar("T")
P = ParamSpec("P")
-def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str:
- if isinstance(func, FunctionWithRequirementsStr):
+def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str: # type: ignore
+ if isinstance(func, FunctionWithRequirementsStr): # type: ignore
return func.func
code = inspect.getsource(func)
@@ -28,29 +28,31 @@ def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], Functio
return code
+@deprecated("Moved to autogen_core.code_executor.Alias. This alias will be removed in 0.4.0.")
@dataclass
class Alias:
name: str
alias: str
+@deprecated("Moved to autogen_core.code_executor.ImportFromModule. This alias will be removed in 0.4.0.")
@dataclass
class ImportFromModule:
module: str
- imports: List[Union[str, Alias]]
+ imports: List[Union[str, Alias]] # type: ignore
-Import = Union[str, ImportFromModule, Alias]
+Import = Union[str, ImportFromModule, Alias] # type: ignore
def _import_to_str(im: Import) -> str:
if isinstance(im, str):
return f"import {im}"
- elif isinstance(im, Alias):
+ elif isinstance(im, Alias): # type: ignore
return f"import {im.name} as {im.alias}"
else:
- def to_str(i: Union[str, Alias]) -> str:
+ def to_str(i: Union[str, Alias]) -> str: # type: ignore
if isinstance(i, str):
return i
else:
@@ -74,6 +76,7 @@ class _StringLoader(SourceLoader):
return "<not a real path>/" + fullname + ".py"
+@deprecated("Moved to autogen_core.code_executor.CodeBlock. This alias will be removed in 0.4.0.")
@dataclass
class FunctionWithRequirementsStr:
func: str
@@ -111,6 +114,7 @@ class FunctionWithRequirementsStr:
raise NotImplementedError("String based function with requirement objects are not directly callable")
+@deprecated("Moved to autogen_core.code_executor.FunctionWithRequirements. This alias will be removed in 0.4.0.")
@dataclass
class FunctionWithRequirements(Generic[T, P]):
func: Callable[P, T]
@@ -120,23 +124,24 @@ class FunctionWithRequirements(Generic[T, P]):
@classmethod
def from_callable(
cls, func: Callable[P, T], python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []
- ) -> FunctionWithRequirements[T, P]:
+ ) -> FunctionWithRequirements[T, P]: # type: ignore
return cls(python_packages=python_packages, global_imports=global_imports, func=func)
@staticmethod
def from_str(
func: str, python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []
- ) -> FunctionWithRequirementsStr:
- return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports)
+ ) -> FunctionWithRequirementsStr: # type: ignore
+ return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports) # type: ignore
# Type this based on F
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T:
return self.func(*args, **kwargs)
+@deprecated("Moved to autogen_core.code_executor.with_requirements. This alias will be removed in 0.4.0.")
def with_requirements(
python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []
-) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]:
+) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: # type: ignore
"""Decorate a function with package and import requirements
Args:
@@ -147,8 +152,8 @@ def with_requirements(
Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: The decorated function
"""
- def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]:
- func_with_reqs = FunctionWithRequirements(
+ def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]: # type: ignore
+ func_with_reqs = FunctionWithRequirements( # type: ignore
python_packages=python_packages, global_imports=global_imports, func=func
)
@@ -159,13 +164,13 @@ def with_requirements(
def build_python_functions_file(
- funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]],
+ funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]], # type: ignore
) -> str:
""":meta private:"""
# First collect all global imports
global_imports: Set[Import] = set()
for func in funcs:
- if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)):
+ if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)): # type: ignore
global_imports.update(func.global_imports)
content = "\n".join(map(_import_to_str, global_imports)) + "\n\n"
@@ -176,7 +181,7 @@ def build_python_functions_file(
return content
-def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str:
+def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str: # type: ignore
"""Generate a stub for a function as a string
Args:
@@ -185,7 +190,7 @@ def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str
Returns:
str: The stub for the function
"""
- if isinstance(func, FunctionWithRequirementsStr):
+ if isinstance(func, FunctionWithRequirementsStr): # type: ignore
return to_stub(func.compiled_func)
content = f"def {func.__name__}{inspect.signature(func)}:\n"
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/base_code_executor.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/base_code_executor.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py
index cafc73f14..879ef8af3 100644
--- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py
+++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py
@@ -1,9 +1,14 @@
from dataclasses import dataclass
from typing import Optional
-from .._base import CodeResult
+from typing_extensions import deprecated
+from ....code_executor._base import CodeResult
+
+@deprecated(
+ "CommandLineCodeResult moved to autogen_ext.code_executors.CommandLineCodeResult. This alias will be removed in 0.4.0."
+)
@dataclass
class CommandLineCodeResult(CodeResult):
"""A code result class for command line code executor."""
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py
index 01e098881..e6a71d65b 100644
--- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py
+++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py
@@ -12,17 +12,17 @@ from string import Template
from types import SimpleNamespace
from typing import Any, Callable, ClassVar, List, Optional, Sequence, Union
-from typing_extensions import ParamSpec
+from typing_extensions import ParamSpec, deprecated
from .... import CancellationToken
-from .._base import CodeBlock, CodeExecutor
-from .._func_with_reqs import (
+from ....code_executor._base import CodeBlock, CodeExecutor
+from ....code_executor._func_with_reqs import (
FunctionWithRequirements,
FunctionWithRequirementsStr,
build_python_functions_file,
to_stub,
)
-from .command_line_code_result import CommandLineCodeResult
+from .command_line_code_result import CommandLineCodeResult # type: ignore
from .utils import PYTHON_VARIANTS, get_file_name_from_content, lang_to_cmd, silence_pip # type: ignore
__all__ = ("LocalCommandLineCodeExecutor",)
@@ -30,6 +30,9 @@ __all__ = ("LocalCommandLineCodeExecutor",)
A = ParamSpec("A")
+@deprecated(
+ "LocalCommandLineCodeExecutor moved to autogen_ext.code_executors.local.LocalCommandLineCodeExecutor. This alias will be removed in 0.4.0."
+)
class LocalCommandLineCodeExecutor(CodeExecutor):
"""A code executor class that executes code through a local command line
environment.
@@ -70,7 +73,8 @@ class LocalCommandLineCodeExecutor(CodeExecutor):
import asyncio
from autogen_core import CancellationToken
- from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor
+ from autogen_core.code_executor import CodeBlock
+ from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor
async def example():
@@ -243,7 +247,7 @@ $functions"""
async def execute_code_blocks(
self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken
- ) -> CommandLineCodeResult:
+ ) -> CommandLineCodeResult: # type: ignore
"""(Experimental) Execute the code blocks and return the result.
Args:
@@ -260,7 +264,7 @@ $functions"""
async def _execute_code_dont_check_setup(
self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken
- ) -> CommandLineCodeResult:
+ ) -> CommandLineCodeResult: # type: ignore
logs_all: str = ""
file_names: List[Path] = []
exitcode = 0
@@ -283,7 +287,7 @@ $functions"""
# Check if there is a filename comment
filename = get_file_name_from_content(code, self._work_dir)
except ValueError:
- return CommandLineCodeResult(
+ return CommandLineCodeResult( # type: ignore
exit_code=1,
output="Filename is not in the workspace",
code_file=None,
@@ -347,7 +351,7 @@ $functions"""
break
code_file = str(file_names[0]) if len(file_names) > 0 else None
- return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file)
+ return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file) # type: ignore
async def restart(self) -> None:
"""(Experimental) Restart the code executor."""
diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py
index 98ce3e2de..3fc39054a 100644
--- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py
+++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py
@@ -46,24 +46,6 @@ def silence_pip(code: str, lang: str) -> str:
return "\n".join(lines)
-def get_required_packages(code: str, lang: str) -> set[str]:
- ret: set[str] = set()
- if lang == "python":
- regex = r"^! ?pip install(.*)$"
- else:
- return ret
-
- # Find lines that start with pip install and make sure "-qqq" flag is added.
- lines = code.split("\n")
- for _, line in enumerate(lines):
- # use regex to find lines that start with pip install.
- match = re.search(regex, line)
- if match is not None:
- reqs = match.group(1).split(",")
- ret = {req.strip(" ") for req in reqs}
- return ret
-
-
PYTHON_VARIANTS = ["python", "Python", "py"]
diff --git a/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py b/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py
index 5b3f0bcad..eb792a32f 100644
--- a/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py
+++ b/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py
@@ -1,7 +1,7 @@
from pydantic import BaseModel, Field, model_serializer
from ... import CancellationToken
-from ..code_executor import CodeBlock, CodeExecutor
+from ...code_executor import CodeBlock, CodeExecutor
from ._base import BaseTool
diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py
index f984cdf3a..c54d7c54c 100644
--- a/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py
+++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py
@@ -14,18 +14,17 @@ import aiohttp
# async functions shouldn't use open()
from anyio import open_file
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import (
+from autogen_core.code_executor import (
CodeBlock,
CodeExecutor,
CodeResult,
FunctionWithRequirements,
FunctionWithRequirementsStr,
- build_python_functions_file,
- get_required_packages,
- to_stub,
)
from typing_extensions import ParamSpec
+from ._common import build_python_functions_file, get_required_packages, to_stub
+
if TYPE_CHECKING:
from azure.core.credentials import AccessToken
diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py
new file mode 100644
index 000000000..ce656e93f
--- /dev/null
+++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py
@@ -0,0 +1,190 @@
+import inspect
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from textwrap import dedent, indent
+from typing import Any, Callable, Optional, Sequence, Set, TypeVar, Union
+
+from autogen_core.code_executor import Alias, CodeResult, FunctionWithRequirements, FunctionWithRequirementsStr, Import
+from typing_extensions import ParamSpec
+
+
+@dataclass
+class CommandLineCodeResult(CodeResult):
+ """A code result class for command line code executor."""
+
+ code_file: Optional[str]
+
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str:
+ if isinstance(func, FunctionWithRequirementsStr):
+ return func.func
+
+ code = inspect.getsource(func)
+ # Strip the decorator
+ if code.startswith("@"):
+ code = code[code.index("\n") + 1 :]
+ return code
+
+
+def _import_to_str(im: Import) -> str:
+ if isinstance(im, str):
+ return f"import {im}"
+ elif isinstance(im, Alias):
+ return f"import {im.name} as {im.alias}"
+ else:
+
+ def to_str(i: Union[str, Alias]) -> str:
+ if isinstance(i, str):
+ return i
+ else:
+ return f"{i.name} as {i.alias}"
+
+ imports = ", ".join(map(to_str, im.imports))
+ return f"from {im.module} import {imports}"
+
+
+def build_python_functions_file(
+ funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]],
+) -> str:
+ """:meta private:"""
+ # First collect all global imports
+ global_imports: Set[Import] = set()
+ for func in funcs:
+ if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)):
+ global_imports.update(func.global_imports)
+
+ content = "\n".join(map(_import_to_str, global_imports)) + "\n\n"
+
+ for func in funcs:
+ content += _to_code(func) + "\n\n"
+
+ return content
+
+
+def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str:
+ """Generate a stub for a function as a string
+
+ Args:
+ func (Callable[..., Any]): The function to generate a stub for
+
+ Returns:
+ str: The stub for the function
+ """
+ if isinstance(func, FunctionWithRequirementsStr):
+ return to_stub(func.compiled_func)
+
+ content = f"def {func.__name__}{inspect.signature(func)}:\n"
+ docstring = func.__doc__
+
+ if docstring:
+ docstring = dedent(docstring)
+ docstring = '"""' + docstring + '"""'
+ docstring = indent(docstring, " ")
+ content += docstring + "\n"
+
+ content += " ..."
+ return content
+
+
+# Raises ValueError if the file is not in the workspace
+def get_file_name_from_content(code: str, workspace_path: Path) -> Optional[str]:
+ first_line = code.split("\n")[0]
+ # TODO - support other languages
+ if first_line.startswith("# filename:"):
+ filename = first_line.split(":")[1].strip()
+
+ # Handle relative paths in the filename
+ path = Path(filename)
+ if not path.is_absolute():
+ path = workspace_path / path
+ path = path.resolve()
+ # Throws an error if the file is not in the workspace
+ relative = path.relative_to(workspace_path.resolve())
+ return str(relative)
+
+ return None
+
+
+def silence_pip(code: str, lang: str) -> str:
+ """Apply -qqq flag to pip install commands."""
+ if lang == "python":
+ regex = r"^! ?pip install"
+ elif lang in ["bash", "shell", "sh", "pwsh", "powershell", "ps1"]:
+ regex = r"^pip install"
+ else:
+ return code
+
+ # Find lines that start with pip install and make sure "-qqq" flag is added.
+ lines = code.split("\n")
+ for i, line in enumerate(lines):
+ # use regex to find lines that start with pip install.
+ match = re.search(regex, line)
+ if match is not None:
+ if "-qqq" not in line:
+ lines[i] = line.replace(match.group(0), match.group(0) + " -qqq")
+ return "\n".join(lines)
+
+
+def get_required_packages(code: str, lang: str) -> set[str]:
+ ret: set[str] = set()
+ if lang == "python":
+ regex = r"^! ?pip install(.*)$"
+ else:
+ return ret
+
+ # Find lines that start with pip install and make sure "-qqq" flag is added.
+ lines = code.split("\n")
+ for _, line in enumerate(lines):
+ # use regex to find lines that start with pip install.
+ match = re.search(regex, line)
+ if match is not None:
+ reqs = match.group(1).split(",")
+ ret = {req.strip(" ") for req in reqs}
+ return ret
+
+
+PYTHON_VARIANTS = ["python", "Python", "py"]
+
+
+def lang_to_cmd(lang: str) -> str:
+ if lang in PYTHON_VARIANTS:
+ return "python"
+ if lang.startswith("python") or lang in ["bash", "sh"]:
+ return lang
+ if lang in ["shell"]:
+ return "sh"
+ else:
+ raise ValueError(f"Unsupported language: {lang}")
+
+
+# Regular expression for finding a code block
+# ```[ \t]*(\w+)?[ \t]*\r?\n(.*?)[ \t]*\r?\n``` Matches multi-line code blocks.
+# The [ \t]* matches the potential spaces before language name.
+# The (\w+)? matches the language, where the ? indicates it is optional.
+# The [ \t]* matches the potential spaces (not newlines) after language name.
+# The \r?\n makes sure there is a linebreak after ```.
+# The (.*?) matches the code itself (non-greedy).
+# The \r?\n makes sure there is a linebreak before ```.
+# The [ \t]* matches the potential spaces before closing ``` (the spec allows indentation).
+CODE_BLOCK_PATTERN = r"```[ \t]*(\w+)?[ \t]*\r?\n(.*?)\r?\n[ \t]*```"
+
+
+def infer_lang(code: str) -> str:
+ """infer the language for the code.
+ TODO: make it robust.
+ """
+ if code.startswith("python ") or code.startswith("pip") or code.startswith("python3 "):
+ return "sh"
+
+ # check if code is a valid python code
+ try:
+ compile(code, "test", "exec")
+ return "python"
+ except SyntaxError:
+ # not a valid python code
+ return "unknown"
diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py
index 442486a83..170c4f468 100644
--- a/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py
+++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py
@@ -15,12 +15,15 @@ from types import TracebackType
from typing import Any, Callable, ClassVar, List, Optional, ParamSpec, Type, Union
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import (
+from autogen_core.code_executor import (
CodeBlock,
CodeExecutor,
- CommandLineCodeResult,
FunctionWithRequirements,
FunctionWithRequirementsStr,
+)
+
+from ._common import (
+ CommandLineCodeResult,
build_python_functions_file,
get_file_name_from_content,
lang_to_cmd,
diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py
new file mode 100644
index 000000000..9055804d1
--- /dev/null
+++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py
@@ -0,0 +1,358 @@
+# File based from: https://github.com/microsoft/autogen/blob/main/autogen/coding/local_commandline_code_executor.py
+# Credit to original authors
+
+import asyncio
+import logging
+import os
+import sys
+import warnings
+from hashlib import sha256
+from pathlib import Path
+from string import Template
+from types import SimpleNamespace
+from typing import Any, Callable, ClassVar, List, Optional, Sequence, Union
+
+from autogen_core import CancellationToken
+from autogen_core.code_executor import CodeBlock, CodeExecutor, FunctionWithRequirements, FunctionWithRequirementsStr
+from typing_extensions import ParamSpec
+
+from .._common import (
+ PYTHON_VARIANTS,
+ CommandLineCodeResult,
+ build_python_functions_file,
+ get_file_name_from_content,
+ lang_to_cmd,
+ silence_pip,
+ to_stub,
+)
+
+__all__ = ("LocalCommandLineCodeExecutor",)
+
+A = ParamSpec("A")
+
+
+class LocalCommandLineCodeExecutor(CodeExecutor):
+ """A code executor class that executes code through a local command line
+ environment.
+
+ .. danger::
+
+ This will execute code on the local machine. If being used with LLM generated code, caution should be used.
+
+ Each code block is saved as a file and executed in a separate process in
+ the working directory, and a unique file is generated and saved in the
+ working directory for each code block.
+ The code blocks are executed in the order they are received.
+ Command line code is sanitized using regular expression match against a list of dangerous commands in order to prevent self-destructive
+ commands from being executed which may potentially affect the users environment.
+ Currently the only supported languages is Python and shell scripts.
+ For Python code, use the language "python" for the code block.
+ For shell scripts, use the language "bash", "shell", or "sh" for the code
+ block.
+
+ Args:
+ timeout (int): The timeout for the execution of any single code block. Default is 60.
+ work_dir (str): The working directory for the code execution. If None,
+ a default working directory will be used. The default working
+ directory is the current directory ".".
+ functions (List[Union[FunctionWithRequirements[Any, A], Callable[..., Any]]]): A list of functions that are available to the code executor. Default is an empty list.
+ functions_module (str, optional): The name of the module that will be created to store the functions. Defaults to "functions".
+ virtual_env_context (Optional[SimpleNamespace], optional): The virtual environment context. Defaults to None.
+
+ Example:
+
+ How to use `LocalCommandLineCodeExecutor` with a virtual environment different from the one used to run the autogen application:
+ Set up a virtual environment using the `venv` module, and pass its context to the initializer of `LocalCommandLineCodeExecutor`. This way, the executor will run code within the new environment.
+
+ .. code-block:: python
+
+ import venv
+ from pathlib import Path
+ import asyncio
+
+ from autogen_core import CancellationToken
+ from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor
+
+
+ async def example():
+ work_dir = Path("coding")
+ work_dir.mkdir(exist_ok=True)
+
+ venv_dir = work_dir / ".venv"
+ venv_builder = venv.EnvBuilder(with_pip=True)
+ venv_builder.create(venv_dir)
+ venv_context = venv_builder.ensure_directories(venv_dir)
+
+ local_executor = LocalCommandLineCodeExecutor(work_dir=work_dir, virtual_env_context=venv_context)
+ await local_executor.execute_code_blocks(
+ code_blocks=[
+ CodeBlock(language="bash", code="pip install matplotlib"),
+ ],
+ cancellation_token=CancellationToken(),
+ )
+
+
+ asyncio.run(example())
+
+ """
+
+ SUPPORTED_LANGUAGES: ClassVar[List[str]] = [
+ "bash",
+ "shell",
+ "sh",
+ "pwsh",
+ "powershell",
+ "ps1",
+ "python",
+ ]
+ FUNCTION_PROMPT_TEMPLATE: ClassVar[
+ str
+ ] = """You have access to the following user defined functions. They can be accessed from the module called `$module_name` by their function names.
+
+For example, if there was a function called `foo` you could import it by writing `from $module_name import foo`
+
+$functions"""
+
+ def __init__(
+ self,
+ timeout: int = 60,
+ work_dir: Union[Path, str] = Path("."),
+ functions: Sequence[
+ Union[
+ FunctionWithRequirements[Any, A],
+ Callable[..., Any],
+ FunctionWithRequirementsStr,
+ ]
+ ] = [],
+ functions_module: str = "functions",
+ virtual_env_context: Optional[SimpleNamespace] = None,
+ ):
+ if timeout < 1:
+ raise ValueError("Timeout must be greater than or equal to 1.")
+
+ if isinstance(work_dir, str):
+ work_dir = Path(work_dir)
+
+ if not functions_module.isidentifier():
+ raise ValueError("Module name must be a valid Python identifier")
+
+ self._functions_module = functions_module
+
+ work_dir.mkdir(exist_ok=True)
+
+ self._timeout = timeout
+ self._work_dir: Path = work_dir
+
+ self._functions = functions
+ # Setup could take some time so we intentionally wait for the first code block to do it.
+ if len(functions) > 0:
+ self._setup_functions_complete = False
+ else:
+ self._setup_functions_complete = True
+
+ self._virtual_env_context: Optional[SimpleNamespace] = virtual_env_context
+
+ def format_functions_for_prompt(self, prompt_template: str = FUNCTION_PROMPT_TEMPLATE) -> str:
+ """(Experimental) Format the functions for a prompt.
+
+ The template includes two variables:
+ - `$module_name`: The module name.
+ - `$functions`: The functions formatted as stubs with two newlines between each function.
+
+ Args:
+ prompt_template (str): The prompt template. Default is the class default.
+
+ Returns:
+ str: The formatted prompt.
+ """
+
+ template = Template(prompt_template)
+ return template.substitute(
+ module_name=self._functions_module,
+ functions="\n\n".join([to_stub(func) for func in self._functions]),
+ )
+
+ @property
+ def functions_module(self) -> str:
+ """(Experimental) The module name for the functions."""
+ return self._functions_module
+
+ @property
+ def functions(self) -> List[str]:
+ raise NotImplementedError
+
+ @property
+ def timeout(self) -> int:
+ """(Experimental) The timeout for code execution."""
+ return self._timeout
+
+ @property
+ def work_dir(self) -> Path:
+ """(Experimental) The working directory for the code execution."""
+ return self._work_dir
+
+ async def _setup_functions(self, cancellation_token: CancellationToken) -> None:
+ func_file_content = build_python_functions_file(self._functions)
+ func_file = self._work_dir / f"{self._functions_module}.py"
+ func_file.write_text(func_file_content)
+
+ # Collect requirements
+ lists_of_packages = [x.python_packages for x in self._functions if isinstance(x, FunctionWithRequirements)]
+ flattened_packages = [item for sublist in lists_of_packages for item in sublist]
+ required_packages = list(set(flattened_packages))
+ if len(required_packages) > 0:
+ logging.info("Ensuring packages are installed in executor.")
+
+ cmd_args = ["-m", "pip", "install"]
+ cmd_args.extend(required_packages)
+
+ if self._virtual_env_context:
+ py_executable = self._virtual_env_context.env_exe
+ else:
+ py_executable = sys.executable
+
+ task = asyncio.create_task(
+ asyncio.create_subprocess_exec(
+ py_executable,
+ *cmd_args,
+ cwd=self._work_dir,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ )
+ cancellation_token.link_future(task)
+ try:
+ proc = await task
+ stdout, stderr = await asyncio.wait_for(proc.communicate(), self._timeout)
+ except asyncio.TimeoutError as e:
+ raise ValueError("Pip install timed out") from e
+ except asyncio.CancelledError as e:
+ raise ValueError("Pip install was cancelled") from e
+
+ if proc.returncode is not None and proc.returncode != 0:
+ raise ValueError(f"Pip install failed. {stdout.decode()}, {stderr.decode()}")
+
+ # Attempt to load the function file to check for syntax errors, imports etc.
+ exec_result = await self._execute_code_dont_check_setup(
+ [CodeBlock(code=func_file_content, language="python")], cancellation_token
+ )
+
+ if exec_result.exit_code != 0:
+ raise ValueError(f"Functions failed to load: {exec_result.output}")
+
+ self._setup_functions_complete = True
+
+ async def execute_code_blocks(
+ self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken
+ ) -> CommandLineCodeResult:
+ """(Experimental) Execute the code blocks and return the result.
+
+ Args:
+ code_blocks (List[CodeBlock]): The code blocks to execute.
+ cancellation_token (CancellationToken): a token to cancel the operation
+
+ Returns:
+ CommandLineCodeResult: The result of the code execution."""
+
+ if not self._setup_functions_complete:
+ await self._setup_functions(cancellation_token)
+
+ return await self._execute_code_dont_check_setup(code_blocks, cancellation_token)
+
+ async def _execute_code_dont_check_setup(
+ self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken
+ ) -> CommandLineCodeResult:
+ logs_all: str = ""
+ file_names: List[Path] = []
+ exitcode = 0
+ for code_block in code_blocks:
+ lang, code = code_block.language, code_block.code
+ lang = lang.lower()
+
+ code = silence_pip(code, lang)
+
+ if lang in PYTHON_VARIANTS:
+ lang = "python"
+
+ if lang not in self.SUPPORTED_LANGUAGES:
+ # In case the language is not supported, we return an error message.
+ exitcode = 1
+ logs_all += "\n" + f"unknown language {lang}"
+ break
+
+ try:
+ # Check if there is a filename comment
+ filename = get_file_name_from_content(code, self._work_dir)
+ except ValueError:
+ return CommandLineCodeResult(
+ exit_code=1,
+ output="Filename is not in the workspace",
+ code_file=None,
+ )
+
+ if filename is None:
+ # create a file with an automatically generated name
+ code_hash = sha256(code.encode()).hexdigest()
+ filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}"
+
+ written_file = (self._work_dir / filename).resolve()
+ with written_file.open("w", encoding="utf-8") as f:
+ f.write(code)
+ file_names.append(written_file)
+
+ env = os.environ.copy()
+
+ if self._virtual_env_context:
+ virtual_env_exe_abs_path = os.path.abspath(self._virtual_env_context.env_exe)
+ virtual_env_bin_abs_path = os.path.abspath(self._virtual_env_context.bin_path)
+ env["PATH"] = f"{virtual_env_bin_abs_path}{os.pathsep}{env['PATH']}"
+
+ program = virtual_env_exe_abs_path if lang.startswith("python") else lang_to_cmd(lang)
+ else:
+ program = sys.executable if lang.startswith("python") else lang_to_cmd(lang)
+
+ # Wrap in a task to make it cancellable
+ task = asyncio.create_task(
+ asyncio.create_subprocess_exec(
+ program,
+ str(written_file.absolute()),
+ cwd=self._work_dir,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ env=env,
+ )
+ )
+ cancellation_token.link_future(task)
+ try:
+ proc = await task
+ stdout, stderr = await asyncio.wait_for(proc.communicate(), self._timeout)
+ exitcode = proc.returncode or 0
+
+ except asyncio.TimeoutError:
+ logs_all += "\n Timeout"
+ # Same exit code as the timeout command on linux.
+ exitcode = 124
+ break
+ except asyncio.CancelledError:
+ logs_all += "\n Cancelled"
+ # TODO: which exit code? 125 is Operation Canceled
+ exitcode = 125
+ break
+
+ self._running_cmd_task = None
+
+ logs_all += stderr.decode()
+ logs_all += stdout.decode()
+
+ if exitcode != 0:
+ break
+
+ code_file = str(file_names[0]) if len(file_names) > 0 else None
+ return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file)
+
+ async def restart(self) -> None:
+ """(Experimental) Restart the code executor."""
+ warnings.warn(
+ "Restarting local command line code executor is not supported. No action is taken.",
+ stacklevel=2,
+ )
diff --git a/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py b/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py
index df5caf5ce..e46912568 100644
--- a/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py
+++ b/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py
@@ -9,7 +9,7 @@ import tempfile
import pytest
from anyio import open_file
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import CodeBlock
+from autogen_core.code_executor import CodeBlock
from autogen_ext.code_executors import ACADynamicSessionsCodeExecutor
from azure.identity import DefaultAzureCredential
diff --git a/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py b/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py
index 7b8f2ef2a..e826abec0 100644
--- a/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py
+++ b/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py
@@ -6,7 +6,7 @@ import os
import polars
import pytest
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import (
+from autogen_core.code_executor import (
CodeBlock,
FunctionWithRequirements,
with_requirements,
diff --git a/python/packages/autogen-core/tests/execution/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py
similarity index 98%
rename from python/packages/autogen-core/tests/execution/test_commandline_code_executor.py
rename to python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py
index 482330714..e5e4407f6 100644
--- a/python/packages/autogen-core/tests/execution/test_commandline_code_executor.py
+++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py
@@ -14,7 +14,8 @@ import pytest
import pytest_asyncio
from aiofiles import open
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor
+from autogen_core.code_executor import CodeBlock
+from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor
@pytest_asyncio.fixture(scope="function") # type: ignore
diff --git a/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py
index bee4957f3..c9e57aee1 100644
--- a/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py
+++ b/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py
@@ -9,7 +9,7 @@ import pytest
import pytest_asyncio
from aiofiles import open
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import CodeBlock
+from autogen_core.code_executor import CodeBlock
from autogen_ext.code_executors import DockerCommandLineCodeExecutor
diff --git a/python/packages/autogen-core/tests/execution/test_user_defined_functions.py b/python/packages/autogen-ext/tests/code_executors/test_user_defined_functions.py
similarity index 98%
rename from python/packages/autogen-core/tests/execution/test_user_defined_functions.py
rename to python/packages/autogen-ext/tests/code_executors/test_user_defined_functions.py
index 6aa3a9e45..0815e173b 100644
--- a/python/packages/autogen-core/tests/execution/test_user_defined_functions.py
+++ b/python/packages/autogen-ext/tests/code_executors/test_user_defined_functions.py
@@ -7,12 +7,12 @@ import tempfile
import polars
import pytest
from autogen_core import CancellationToken
-from autogen_core.components.code_executor import (
+from autogen_core.code_executor import (
CodeBlock,
FunctionWithRequirements,
- LocalCommandLineCodeExecutor,
with_requirements,
)
+from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor
ENVIRON_KEY_AZURE_POOL_ENDPOINT = "AZURE_POOL_ENDPOINT"
diff --git a/python/packages/autogen-magentic-one/examples/example.py b/python/packages/autogen-magentic-one/examples/example.py
index 2b96db531..fb0a7d9b4 100644
--- a/python/packages/autogen-magentic-one/examples/example.py
+++ b/python/packages/autogen-magentic-one/examples/example.py
@@ -7,7 +7,7 @@ import os
from autogen_core import AgentId, AgentProxy, SingleThreadedAgentRuntime
from autogen_core.application.logging import EVENT_LOGGER_NAME
-from autogen_core.components.code_executor import CodeBlock
+from autogen_core.code_executor import CodeBlock
from autogen_ext.code_executors import DockerCommandLineCodeExecutor
from autogen_magentic_one.agents.coder import Coder, Executor
from autogen_magentic_one.agents.file_surfer import FileSurfer
diff --git a/python/packages/autogen-magentic-one/examples/example_coder.py b/python/packages/autogen-magentic-one/examples/example_coder.py
index b861c5855..6c7bfc6e2 100644
--- a/python/packages/autogen-magentic-one/examples/example_coder.py
+++ b/python/packages/autogen-magentic-one/examples/example_coder.py
@@ -9,7 +9,7 @@ import logging
from autogen_core import AgentId, AgentProxy, SingleThreadedAgentRuntime
from autogen_core.application.logging import EVENT_LOGGER_NAME
-from autogen_core.components.code_executor import CodeBlock
+from autogen_core.code_executor import CodeBlock
from autogen_ext.code_executors import DockerCommandLineCodeExecutor
from autogen_magentic_one.agents.coder import Coder, Executor
from autogen_magentic_one.agents.orchestrator import RoundRobinOrchestrator
diff --git a/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py b/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py
index f30a8da41..427adde86 100644
--- a/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py
+++ b/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py
@@ -2,7 +2,7 @@ import re
from typing import Awaitable, Callable, List, Literal, Tuple, Union
from autogen_core import CancellationToken, default_subscription
-from autogen_core.components.code_executor import CodeBlock, CodeExecutor
+from autogen_core.code_executor import CodeBlock, CodeExecutor
from autogen_core.components.models import (
ChatCompletionClient,
SystemMessage,