A programming framework for agentic AI
Jupyter Code Executor in v0.4 (alternative implementation) (#4885)
Leon De Andrade34bc82e24ff7
diff --git a/python/packages/autogen-core/docs/src/reference/index.md b/python/packages/autogen-core/docs/src/reference/index.mdindex a3e633551..869ffc234 100644--- a/python/packages/autogen-core/docs/src/reference/index.md+++ b/python/packages/autogen-core/docs/src/reference/index.md@@ -58,6 +58,7 @@ python/autogen_ext.tools.code_executionpython/autogen_ext.tools.semantic_kernelpython/autogen_ext.code_executors.localpython/autogen_ext.code_executors.docker+python/autogen_ext.code_executors.jupyterpython/autogen_ext.code_executors.azurepython/autogen_ext.cache_store.diskcachepython/autogen_ext.cache_store.redisdiff --git a/python/packages/autogen-core/docs/src/reference/python/autogen_ext.code_executors.jupyter.rst b/python/packages/autogen-core/docs/src/reference/python/autogen_ext.code_executors.jupyter.rstnew file mode 100644index 000000000..6b44c137f--- /dev/null+++ b/python/packages/autogen-core/docs/src/reference/python/autogen_ext.code_executors.jupyter.rst@@ -0,0 +1,8 @@+autogen\_ext.code\_executors.jupyter+====================================+++.. automodule:: autogen_ext.code_executors.jupyter+ :members:+ :undoc-members:+ :show-inheritance:diff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.tomlindex 5b7d0fa52..20579c99b 100644--- a/python/packages/autogen-ext/pyproject.toml+++ b/python/packages/autogen-ext/pyproject.toml@@ -56,6 +56,10 @@ redis = [grpc = ["grpcio~=1.62.0", # TODO: update this once we have a stable version.]+jupyter-executor = [+ "ipykernel>=6.29.5",+ "nbclient>=0.10.2",+]semantic-kernel-core = ["semantic-kernel>=1.17.1",diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/jupyter/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/jupyter/__init__.pynew file mode 100644index 000000000..1a6ba799d--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/jupyter/__init__.py@@ -0,0 +1,6 @@+from ._jupyter_code_executor import JupyterCodeExecutor, JupyterCodeResult++__all__ = [+ "JupyterCodeExecutor",+ "JupyterCodeResult",+]diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/jupyter/_jupyter_code_executor.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/jupyter/_jupyter_code_executor.pynew file mode 100644index 000000000..bdeb0a78d--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/jupyter/_jupyter_code_executor.py@@ -0,0 +1,263 @@+import asyncio+import base64+import json+import re+import sys+import uuid+from dataclasses import dataclass+from pathlib import Path+from types import TracebackType++if sys.version_info >= (3, 11):+ from typing import Self+else:+ from typing_extensions import Self++from autogen_core import CancellationToken+from autogen_core.code_executor import CodeBlock, CodeExecutor, CodeResult+from nbclient import NotebookClient+from nbformat import NotebookNode+from nbformat import v4 as nbformat++from .._common import silence_pip+++@dataclass+class JupyterCodeResult(CodeResult):+ """A code result class for Jupyter code executor."""++ output_files: list[Path]+++class JupyterCodeExecutor(CodeExecutor):+ """A code executor class that executes code statefully using [nbclient](https://github.com/jupyter/nbclient).++ .. danger::++ This will execute code on the local machine. If being used with LLM generated code, caution should be used.++ Example of using it directly:++ .. code-block:: python++ import asyncio+ from autogen_core import CancellationToken+ from autogen_core.code_executor import CodeBlock+ from autogen_ext.code_executors.jupyter import JupyterCodeExecutor+++ async def main() -> None:+ async with JupyterCodeExecutor() as executor:+ cancel_token = CancellationToken()+ code_blocks = [CodeBlock(code="print('hello world!')", language="python")]+ code_result = await executor.execute_code_blocks(code_blocks, cancel_token)+ print(code_result)+++ asyncio.run(main())++ Example of using it with :class:`~autogen_ext.tools.code_execution.PythonCodeExecutionTool`:++ .. code-block:: python++ import asyncio+ from autogen_agentchat.agents import AssistantAgent+ from autogen_ext.code_executors.jupyter import JupyterCodeExecutor+ from autogen_ext.models.openai import OpenAIChatCompletionClient+ from autogen_ext.tools.code_execution import PythonCodeExecutionTool+++ async def main() -> None:+ async with JupyterCodeExecutor() as executor:+ tool = PythonCodeExecutionTool(executor)+ model_client = OpenAIChatCompletionClient(model="gpt-4o")+ agent = AssistantAgent("assistant", model_client=model_client, tools=[tool])+ result = await agent.run(task="What is the 10th Fibonacci number? Use Python to calculate it.")+ print(result)+++ asyncio.run(main())++ Example of using it inside a :class:`~autogen_agentchat.agents._code_executor_agent.CodeExecutorAgent`:++ .. code-block:: python++ import asyncio+ from autogen_agentchat.agents import CodeExecutorAgent+ from autogen_agentchat.messages import TextMessage+ from autogen_ext.code_executors.jupyter import JupyterCodeExecutor+ from autogen_core import CancellationToken+++ async def main() -> None:+ async with JupyterCodeExecutor() as executor:+ code_executor_agent = CodeExecutorAgent("code_executor", code_executor=executor)+ task = TextMessage(+ content='''Here is some code+ ```python+ print('Hello world')+ ```+ ''',+ source="user",+ )+ response = await code_executor_agent.on_messages([task], CancellationToken())+ print(response.chat_message)+++ asyncio.run(main())+++ Args:+ kernel_name (str): The kernel name to use. By default, "python3".+ timeout (int): The timeout for code execution, by default 60.+ output_dir (Path): The directory to save output files, by default ".".+ """++ def __init__(+ self,+ kernel_name: str = "python3",+ timeout: int = 60,+ output_dir: Path = Path("."),+ ):+ if timeout < 1:+ raise ValueError("Timeout must be greater than or equal to 1.")++ self._kernel_name = kernel_name+ self._timeout = timeout+ self._output_dir = output_dir+ # TODO: Forward arguments perhaps?+ self._client = NotebookClient(+ nb=nbformat.new_notebook(), # type: ignore+ kernel_name=self._kernel_name,+ timeout=self._timeout,+ allow_errors=True,+ )++ async def execute_code_blocks(+ self, code_blocks: list[CodeBlock], cancellation_token: CancellationToken+ ) -> JupyterCodeResult:+ """Execute code blocks and return the result.++ Args:+ code_blocks (list[CodeBlock]): The code blocks to execute.++ Returns:+ JupyterCodeResult: The result of the code execution.+ """+ outputs: list[str] = []+ output_files: list[Path] = []+ exit_code = 0++ for code_block in code_blocks:+ result = await self._execute_code_block(code_block, cancellation_token)+ exit_code = result.exit_code+ outputs.append(result.output)+ output_files.extend(result.output_files)++ # Stop execution if one code block fails+ if exit_code != 0:+ break++ return JupyterCodeResult(exit_code=exit_code, output="\n".join(outputs), output_files=output_files)++ async def _execute_code_block(+ self, code_block: CodeBlock, cancellation_token: CancellationToken+ ) -> JupyterCodeResult:+ """Execute single code block and return the result.++ Args:+ code_block (CodeBlock): The code block to execute.++ Returns:+ JupyterCodeResult: The result of the code execution.+ """+ execute_task = asyncio.create_task(+ self._execute_cell(+ nbformat.new_code_cell(silence_pip(code_block.code, code_block.language)) # type: ignore+ )+ )++ cancellation_token.link_future(execute_task)+ output_cell = await asyncio.wait_for(asyncio.shield(execute_task), timeout=self._timeout)++ outputs: list[str] = []+ output_files: list[Path] = []+ exit_code = 0++ for output in output_cell.get("outputs", []):+ match output.get("output_type"):+ case "stream":+ outputs.append(output.get("text", ""))+ case "error":+ traceback = re.sub(r"\x1b\[[0-9;]*[A-Za-z]", "", "\n".join(output["traceback"]))+ outputs.append(traceback)+ exit_code = 1+ case "execute_result" | "display_data":+ data = output.get("data", {})+ for mime, content in data.items():+ match mime:+ case "text/plain":+ outputs.append(content)+ case "image/png":+ path = self._save_image(content)+ output_files.append(path)+ case "image/jpeg":+ # TODO: Should this also be encoded? Images are encoded as both png and jpg+ pass+ case "text/html":+ path = self._save_html(content)+ output_files.append(path)+ case _:+ outputs.append(json.dumps(content))+ case _:+ pass++ return JupyterCodeResult(exit_code=exit_code, output="\n".join(outputs), output_files=output_files)++ async def _execute_cell(self, cell: NotebookNode) -> NotebookNode:+ # Temporary push cell to nb as async_execute_cell expects it. But then we want to remove it again as cells can take up significant amount of memory (especially with images)+ self._client.nb.cells.append(cell)+ output = await self._client.async_execute_cell(+ cell,+ cell_index=0,+ )+ self._client.nb.cells.pop()+ return output++ def _save_image(self, image_data_base64: str) -> Path:+ """Save image data to a file."""+ image_data = base64.b64decode(image_data_base64)+ path = self._output_dir / f"{uuid.uuid4().hex}.png"+ path.write_bytes(image_data)+ return path.absolute()++ def _save_html(self, html_data: str) -> Path:+ """Save HTML data to a file."""+ path = self._output_dir / f"{uuid.uuid4().hex}.html"+ path.write_text(html_data)+ return path.absolute()++ async def restart(self) -> None:+ """Restart the code executor."""+ await self.stop()+ await self.start()++ async def start(self) -> None:+ self.kernel_context = self._client.async_setup_kernel()+ await self.kernel_context.__aenter__()++ async def stop(self) -> None:+ """Stop the kernel."""+ await self.kernel_context.__aexit__(None, None, None)++ async def __aenter__(self) -> Self:+ await self.start()+ return self++ async def __aexit__(+ self,+ exc_type: type[BaseException] | None,+ exc_val: BaseException | None,+ exc_tb: TracebackType | None,+ ) -> None:+ await self.stop()diff --git a/python/packages/autogen-ext/tests/code_executors/test_jupyter_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_jupyter_code_executor.pynew file mode 100644index 000000000..e8ddace5e--- /dev/null+++ b/python/packages/autogen-ext/tests/code_executors/test_jupyter_code_executor.py@@ -0,0 +1,169 @@+import asyncio+import inspect+from pathlib import Path++import pytest+from autogen_core import CancellationToken+from autogen_core.code_executor import CodeBlock+from autogen_ext.code_executors.jupyter import JupyterCodeExecutor, JupyterCodeResult+++async def test_execute_code(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [CodeBlock(code="import sys; print('hello world!')", language="python")]+ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())+ assert code_result == JupyterCodeResult(exit_code=0, output="hello world!\n", output_files=[])+++async def test_execute_code_error(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [CodeBlock(code="print(undefined_variable)", language="python")]+ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())+ assert code_result == JupyterCodeResult(+ exit_code=1,+ output=inspect.cleandoc("""+ ---------------------------------------------------------------------------+ NameError Traceback (most recent call last)+ Cell In[1], line 1+ ----> 1 print(undefined_variable)++ NameError: name 'undefined_variable' is not defined+ """),+ output_files=[],+ )+++async def test_execute_multiple_code_blocks(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [+ CodeBlock(code="import sys; print('hello world!')", language="python"),+ CodeBlock(code="a = 100 + 100; print(a)", language="python"),+ ]+ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())+ assert code_result == JupyterCodeResult(exit_code=0, output="hello world!\n\n200\n", output_files=[])+++async def test_depedent_executions(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks_1 = [CodeBlock(code="a = 'hello world!'", language="python")]+ code_blocks_2 = [+ CodeBlock(code="print(a)", language="python"),+ ]+ await executor.execute_code_blocks(code_blocks_1, CancellationToken())+ code_result = await executor.execute_code_blocks(code_blocks_2, CancellationToken())+ assert code_result == JupyterCodeResult(exit_code=0, output="hello world!\n", output_files=[])+++async def test_execute_multiple_code_blocks_error(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [+ CodeBlock(code="import sys; print('hello world!')", language="python"),+ CodeBlock(code="a = 100 + 100; print(a); print(undefined_variable)", language="python"),+ ]+ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())+ assert code_result == JupyterCodeResult(+ exit_code=1,+ output=inspect.cleandoc("""+ hello world!++ 200++ ---------------------------------------------------------------------------+ NameError Traceback (most recent call last)+ Cell In[2], line 1+ ----> 1 a = 100 + 100; print(a); print(undefined_variable)++ NameError: name 'undefined_variable' is not defined+ """),+ output_files=[],+ )+++async def test_execute_code_after_restart(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ await executor.restart()++ code_blocks = [CodeBlock(code="import sys; print('hello world!')", language="python")]+ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())+ assert code_result == JupyterCodeResult(exit_code=0, output="hello world!\n", output_files=[])+++async def test_commandline_code_executor_timeout(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path, timeout=2) as executor:+ code_blocks = [CodeBlock(code="import time; time.sleep(10); print('hello world!')", language="python")]++ with pytest.raises(asyncio.TimeoutError):+ await executor.execute_code_blocks(code_blocks, CancellationToken())+++async def test_commandline_code_executor_cancellation(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [CodeBlock(code="import time; time.sleep(10); print('hello world!')", language="python")]++ cancellation_token = CancellationToken()+ code_result_coroutine = executor.execute_code_blocks(code_blocks, cancellation_token)++ await asyncio.sleep(1)+ cancellation_token.cancel()++ with pytest.raises(asyncio.CancelledError):+ await code_result_coroutine+++async def test_execute_code_with_image_output(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [+ CodeBlock(+ code=inspect.cleandoc("""+ from PIL import Image, ImageDraw+ img = Image.new("RGB", (100, 100), color="white")+ draw = ImageDraw.Draw(img)+ draw.rectangle((10, 10, 90, 90), outline="black", fill="blue")+ display(img)+ """),+ language="python",+ )+ ]++ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())++ assert len(code_result.output_files) == 1+ assert code_result == JupyterCodeResult(+ exit_code=0,+ output="<PIL.Image.Image image mode=RGB size=100x100>",+ output_files=code_result.output_files,+ )+ assert code_result.output_files[0].parent == tmp_path+++async def test_execute_code_with_html_output(tmp_path: Path) -> None:+ async with JupyterCodeExecutor(output_dir=tmp_path) as executor:+ code_blocks = [+ CodeBlock(+ code=inspect.cleandoc("""+ from IPython.core.display import HTML+ HTML("<div style='color:blue'>Hello, HTML world!</div>")+ """),+ language="python",+ )+ ]++ code_result = await executor.execute_code_blocks(code_blocks, CancellationToken())++ assert len(code_result.output_files) == 1+ assert code_result == JupyterCodeResult(+ exit_code=0,+ output="<IPython.core.display.HTML object>",+ output_files=code_result.output_files,+ )+ assert code_result.output_files[0].parent == tmp_pathdiff --git a/python/uv.lock b/python/uv.lockindex dcb958f9b..e8a07804e 100644--- a/python/uv.lock+++ b/python/uv.lock@@ -580,6 +580,10 @@ graphrag = [grpc = [{ name = "grpcio" },]+jupyter-executor = [+ { name = "ipykernel" },+ { name = "nbclient" },+]langchain = [{ name = "langchain-core" },]@@ -665,10 +669,12 @@ requires-dist = [{ name = "ffmpeg-python", marker = "extra == 'video-surfer'" },{ name = "graphrag", marker = "extra == 'graphrag'", specifier = ">=1.0.1" },{ name = "grpcio", marker = "extra == 'grpc'", specifier = "~=1.62.0" },+ { name = "ipykernel", marker = "extra == 'jupyter-executor'", specifier = ">=6.29.5" },{ name = "langchain-core", marker = "extra == 'langchain'", specifier = "~=0.3.3" },{ name = "markitdown", marker = "extra == 'file-surfer'", specifier = ">=0.0.1a2" },{ name = "markitdown", marker = "extra == 'magentic-one'", specifier = ">=0.0.1a2" },{ name = "markitdown", marker = "extra == 'web-surfer'", specifier = ">=0.0.1a2" },+ { name = "nbclient", marker = "extra == 'jupyter-executor'", specifier = ">=0.10.2" },{ name = "openai", marker = "extra == 'openai'", specifier = ">=1.52.2" },{ name = "openai-whisper", marker = "extra == 'video-surfer'" },{ name = "opencv-python", marker = "extra == 'video-surfer'", specifier = ">=4.5" },